From bdf7499608bdc101c83bcc2a40b04eebd4f6daef Mon Sep 17 00:00:00 2001 From: forhappy Date: Sun, 1 Nov 2026 17:22:20 -0700 Subject: [PATCH 01/44] fix: own RPC recovery observation domain --- tools/rpc/host/proof/recovery.rs | 70 ++++++++++++++------------ tools/rpc/host/tests/proof_recovery.rs | 11 ++-- 2 files changed, 47 insertions(+), 34 deletions(-) diff --git a/tools/rpc/host/proof/recovery.rs b/tools/rpc/host/proof/recovery.rs index 9746a252..5afb7327 100644 --- a/tools/rpc/host/proof/recovery.rs +++ b/tools/rpc/host/proof/recovery.rs @@ -20,40 +20,48 @@ fn transient_transport(error: &reqwest::Error) -> bool { } pub(crate) async fn read(client: &reqwest::Client, url: &str, target: &str) -> Result { - tokio::time::timeout(Duration::from_secs(20), async { - loop { - // This is a new read observation, never a command retry. A fenced - // route can close transport before its typed refusal arrives. - // Keep builder, decoding and arbitrary application errors fatal. - let response = async { - let response = client - .get(format!("{url}/rust/read?target={target}")) - .send() - .await?; - let status = response.status(); - Ok::<_, reqwest::Error>((status, response.text().await?)) - } - .await; - let (status, text) = match response { - Ok(observation) => observation, - Err(error) if transient_transport(&error) => { - tokio::time::sleep(Duration::from_millis(20)).await; - continue; - } - Err(error) => return Err(error.into()), - }; - if status.is_success() { - return Ok::<_, anyhow::Error>(serde_json::from_str(&text)?); - } - ensure!( + // Recovery must not borrow the fenced owner's execution domain. + let domain = cellar::sdk::ExecutionDomain::new(tokio::runtime::Handle::current()); + let result = domain + .run(async { + cellar::asyncrt::timeout(Duration::from_secs(20), async { + loop { + // This is a new read observation, never a command retry. A fenced + // route can close transport before its typed refusal arrives. + // Keep builder, decoding and arbitrary application errors fatal. + let response = async { + let response = client + .get(format!("{url}/rust/read?target={target}")) + .send() + .await?; + let status = response.status(); + Ok::<_, reqwest::Error>((status, response.text().await?)) + } + .await; + let (status, text) = match response { + Ok(observation) => observation, + Err(error) if transient_transport(&error) => { + cellar::asyncrt::sleep(Duration::from_millis(20)).await; + continue; + } + Err(error) => return Err(error.into()), + }; + if status.is_success() { + return Ok::<_, anyhow::Error>(serde_json::from_str(&text)?); + } + ensure!( transient(status, &text), "recovery read failed outside a transient RPC admission refusal: {status}: {text}" ); - tokio::time::sleep(Duration::from_millis(20)).await; - } - }) - .await - .context("proof recovery did not reopen read admission")? + cellar::asyncrt::sleep(Duration::from_millis(20)).await; + } + }) + .await + .context("proof recovery did not reopen read admission")? + }) + .await; + domain.shutdown().await; + result? } #[cfg(test)] diff --git a/tools/rpc/host/tests/proof_recovery.rs b/tools/rpc/host/tests/proof_recovery.rs index 54a9c980..a118b36c 100644 --- a/tools/rpc/host/tests/proof_recovery.rs +++ b/tools/rpc/host/tests/proof_recovery.rs @@ -10,9 +10,11 @@ use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; #[tokio::test] async fn recovery_reads_retry_disconnect_and_typed_refusal_inside_one_deadline() -> Result<()> { + let domain = cellar::sdk::ExecutionDomain::new(tokio::runtime::Handle::current()); + let result = domain.run(async { let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; let url = format!("http://{}", listener.local_addr()?); - let server = tokio::spawn(async move { + let server = cellar::asyncrt::spawn(async move { for index in 0..3 { let (mut socket, _) = listener.accept().await?; let mut request = Vec::new(); @@ -43,7 +45,7 @@ async fn recovery_reads_retry_disconnect_and_typed_refusal_inside_one_deadline() Ok::<_, anyhow::Error>(()) }); let client = reqwest::Client::new(); - let observed = tokio::time::timeout( + let observed = cellar::asyncrt::timeout( Duration::from_secs(1), recovery::read(&client, &url, "python"), ) @@ -55,5 +57,8 @@ async fn recovery_reads_retry_disconnect_and_typed_refusal_inside_one_deadline() let observed = observed??; ensure!(observed == serde_json::json!({"value":"7"})); served??; - Ok(()) + Ok::<_, anyhow::Error>(()) + }).await; + domain.shutdown().await; + result? } From cedf192ac9871b8d78b583672a1a29bd4e4692dc Mon Sep 17 00:00:00 2001 From: forhappy Date: Sun, 1 Nov 2026 17:22:20 -0700 Subject: [PATCH 02/44] fix: drain queued commit before membership fence --- crates/cellar/durability/raft/README.md | 11 ++-- crates/cellar/durability/raft/replica.rs | 34 +++++----- .../durability/raft/replica_delivery_tests.rs | 66 +++++++++++++++++-- 3 files changed, 86 insertions(+), 25 deletions(-) diff --git a/crates/cellar/durability/raft/README.md b/crates/cellar/durability/raft/README.md index c1662fb4..acf421b0 100644 --- a/crates/cellar/durability/raft/README.md +++ b/crates/cellar/durability/raft/README.md @@ -883,11 +883,12 @@ must retain or report immediate queue refusals. Dropping a delivery does not refund queued/in-flight work. Peer and node usage counters expose held bytes and items; permits release on completion, refusal, close and domain cancellation. -Before applying a committed configuration, the replica drains ownership of its -already-emitted protocol sends so membership revocation cannot invalidate their -delivery. Later ordinary Ready work waits behind this finite fence; an unstable -snapshot retains priority. Send completion only releases ownership: configuration -application still requires its own durable write. +Before applying a committed configuration, the replica includes one already-pending +ordinary Ready within existing admission limits, then drains ownership of its +protocol sends so membership revocation cannot invalidate their delivery. Later +ordinary Ready work waits behind this finite fence; an unstable snapshot retains +priority. Send completion only releases ownership: configuration application still +requires its own durable write. Receiver frame admission validates the current member and bounded message before queueing it, without waiting for that group's Ready or image installation. Raft diff --git a/crates/cellar/durability/raft/replica.rs b/crates/cellar/durability/raft/replica.rs index eab75ad9..405f3dce 100644 --- a/crates/cellar/durability/raft/replica.rs +++ b/crates/cellar/durability/raft/replica.rs @@ -108,6 +108,7 @@ pub(crate) struct Replica { pub write: Option<(WriteTicket, GroupWrite)>, outbound: VecDeque, deliveries: Vec>, + configuration_fence: Option, apply: VecDeque, apply_ticket: Option, finished_apply: Option, @@ -173,6 +174,7 @@ impl Replica { write: None, outbound: VecDeque::new(), deliveries: Vec::new(), + configuration_fence: None, apply: VecDeque::new(), apply_ticket: None, finished_apply: None, @@ -935,6 +937,7 @@ impl Replica { if let Some(entry) = self.apply.pop_front() { self.apply_bytes -= entry_bytes(&entry); } + self.configuration_fence = None; self.apply_ticket = None; self.phase = Phase::Idle; Ok(true) @@ -964,27 +967,26 @@ impl Replica { self.ready_bytes = 0; return Ok(true); } - // Drain the finite set of already-emitted sends before applying - // a committed configuration. Collecting another ordinary Ready - // here lets later heartbeats extend that fence indefinitely. - // An uninstalled snapshot retains its existing priority. - let configuration_pending = self + // Include one already-pending Ready in the send fence before + // membership revocation, then freeze that finite boundary. + // Later heartbeats cannot extend it. Snapshot priority and + // ordinary Ready admission limits remain unchanged. + let configuration_index = self .apply .front() - .is_some_and(|entry| entry.get_entry_type() != EntryType::EntryNormal) - && self.node.raft.raft_log.unstable_snapshot().is_none(); - if configuration_pending { - self.deliveries - .retain(|delivery| delivery.strong_count() != 0); - if !self.deliveries.is_empty() { - return Ok(false); - } - } - if !configuration_pending + .filter(|entry| entry.get_entry_type() != EntryType::EntryNormal) + .filter(|_| self.node.raft.raft_log.unstable_snapshot().is_none()) + .map(|entry| entry.index); + let fenced = configuration_index.is_some() + && self.configuration_fence == configuration_index; + if !fenced && self.node.has_ready() && ((can_collect && self.apply_bytes < options.max_group_apply_bytes) || self.node.raft.raft_log.unstable_snapshot().is_some()) { + if configuration_index.is_some() { + self.configuration_fence = configuration_index; + } let ready = self.node.ready(); self.ready_bytes = ready_bytes(&ready); if self.ready_bytes > options.max_ready_bytes { @@ -1018,6 +1020,7 @@ impl Replica { return Ok(false); }; if entry.get_entry_type() != EntryType::EntryNormal { + self.configuration_fence = Some(entry.index); self.deliveries .retain(|delivery| delivery.strong_count() != 0); if !self.deliveries.is_empty() { @@ -1138,6 +1141,7 @@ impl Replica { (!ready.snapshot().is_empty()).then_some(ready.snapshot().get_metadata().index); if snapshot_index.is_some() { self.apply.clear(); + self.configuration_fence = None; self.apply_bytes = 0; self.apply_ticket = None; self.finished_apply = None; diff --git a/crates/cellar/durability/raft/replica_delivery_tests.rs b/crates/cellar/durability/raft/replica_delivery_tests.rs index 1791b126..688e4794 100644 --- a/crates/cellar/durability/raft/replica_delivery_tests.rs +++ b/crates/cellar/durability/raft/replica_delivery_tests.rs @@ -9,15 +9,27 @@ use cellar_logic::consensus::*; #[tokio::test] async fn membership_apply_waits_for_prior_message_delivery() -> anyhow::Result<()> { - membership_case(false).await + membership_case(ReadyCase::None).await } #[tokio::test] async fn membership_delivery_fence_is_not_extended_by_later_protocol_ready() -> anyhow::Result<()> { - membership_case(true).await + membership_case(ReadyCase::LaterHeartbeat).await } -async fn membership_case(later_ready: bool) -> anyhow::Result<()> { +#[tokio::test] +async fn membership_apply_drains_an_already_queued_commit_notice() -> anyhow::Result<()> { + membership_case(ReadyCase::QueuedCommit).await +} + +#[derive(Clone, Copy)] +enum ReadyCase { + None, + LaterHeartbeat, + QueuedCommit, +} + +async fn membership_case(ready_case: ReadyCase) -> anyhow::Result<()> { let root = tempfile::tempdir()?; let domain = ExecutionDomain::new(tokio::runtime::Handle::current()); let log = domain @@ -75,7 +87,11 @@ async fn membership_case(later_ready: bool) -> anyhow::Result<()> { from: 1, to: 2, term: 1, - commit: 2, + commit: if matches!(ready_case, ReadyCase::QueuedCommit) { + 1 + } else { + 2 + }, ..Default::default() }; message.set_msg_type(MessageType::MsgAppend); @@ -87,7 +103,47 @@ async fn membership_case(later_ready: bool) -> anyhow::Result<()> { let event = outbox.events.pop_front().unwrap(); let transport_owner = event._permit.clone(); drop(event); - if later_ready { + let transport_owner = if matches!(ready_case, ReadyCase::QueuedCommit) { + drop(transport_owner); + let mut committed = Message { + from: 1, + to: 2, + term: 1, + commit: 2, + ..Default::default() + }; + committed.set_msg_type(MessageType::MsgAppend); + replica.node.raft.msgs.push(committed); + let mut owned = None; + for _ in 0..16 { + assert!( + replica.write.is_none(), + "configuration skipped a queued commit notice" + ); + assert!(replica.drive(&mut operations, &mut outbox, &options, true)?); + if let Some(event) = outbox.events.pop_front() { + let RuntimeEventKind::Message { message, .. } = &event.kind else { + panic!("queued commit notice was not emitted") + }; + assert_eq!((message.to, message.commit), (2, 2)); + owned = Some(event._permit.clone()); + break; + } + } + let owned = owned.expect("queued commit notice did not drain"); + for _ in 0..16 { + if replica.idle() && replica.ready_bytes == 0 { + break; + } + assert!(replica.write.is_none()); + assert!(replica.drive(&mut operations, &mut outbox, &options, true)?); + } + owned + } else { + transport_owner + }; + assert!(!replica.drive(&mut operations, &mut outbox, &options, true)?); + if matches!(ready_case, ReadyCase::LaterHeartbeat) { let mut heartbeat = Message { from: 1, to: 3, From 934922a97dac3e6e7a6f5143c75f06e1a414e25e Mon Sep 17 00:00:00 2001 From: forhappy Date: Sun, 1 Nov 2026 17:22:20 -0700 Subject: [PATCH 03/44] test: establish pending route before cancellation --- crates/cellar/node/event_wait_tests.rs | 69 +++++++++++++++----------- 1 file changed, 41 insertions(+), 28 deletions(-) diff --git a/crates/cellar/node/event_wait_tests.rs b/crates/cellar/node/event_wait_tests.rs index 40a27889..6f8af92c 100644 --- a/crates/cellar/node/event_wait_tests.rs +++ b/crates/cellar/node/event_wait_tests.rs @@ -366,41 +366,54 @@ async fn cancellation_of_an_unpolled_local_wait_releases_the_actual_activity() - async fn real_dispatch_route_timeout_cancels_the_pending_actor_request() -> Result<()> { use crate::application::event_stream::{Authority, Binding}; let domain = crate::asyncrt::ExecutionDomain::new(tokio::runtime::Handle::current()); - let result = domain.run(async { - let (app, mut messages) = app()?; + let result = domain.run(crate::asyncrt::timeout(std::time::Duration::from_secs(2), async { let binding: Binding = serde_json::from_value(serde_json::json!({ "environment": "EVENTS", "resource": {"tenant": "t", "application": "a", "id": "a".repeat(32), "name": "e", "family": "e"}, "principal": "test", "grant": "grant", "read": true }))?; - let authority = Authority::resolution(&binding)?; - let (reply, result) = oneshot::channel(); - let call = crate::host_channels::RpcCallReq { - reservation: None, - parent_cancellation: None, - signal: None, - deadline_ms: None, - authority: Some(authority.into()), request_id: None, cancel: None, - scope: crate::application::event_stream::cell_scope(&binding.resource)?, - name: None, method: "__streamResolve".into(), - args: crate::engine_api::RpcData::Json("[]".into()), reply, - }; - let dispatch = super::super::dispatch_rpc_call(app, call); - let actor = async { - let crate::actor::Message::Request { request, reply, .. } = messages.recv().await.unwrap() + loop { + let (app, mut messages) = app()?; + let authority = Authority::resolution(&binding)?; + let (reply, result) = oneshot::channel(); + let call = crate::host_channels::RpcCallReq { + reservation: None, + parent_cancellation: None, + signal: None, + deadline_ms: None, + authority: Some(authority.into()), request_id: None, cancel: None, + scope: crate::application::event_stream::cell_scope(&binding.resource)?, + name: None, method: "__streamResolve".into(), + args: crate::engine_api::RpcData::Json("[]".into()), reply, + }; + let mut dispatch = Box::pin(super::super::dispatch_rpc_call(app, call)); + // The host can exhaust this attempt's unchanged 10 ms budget before + // its first routing poll. Verify that refusal submitted nothing, then + // establish the pending-route state within the original outer bound. + if dispatch.as_mut().now_or_never().is_some() { + let error = result.await?.err().expect("RPC unexpectedly succeeded"); + assert_eq!(code(&error), ErrorCode::Unavailable); + untagged(&error); + assert!(messages.try_recv().is_err(), "completed setup submitted actor work"); + crate::asyncrt::sleep(std::time::Duration::from_millis(1)).await; + continue; + } + let crate::actor::Message::Request { request, reply, .. } = messages.try_recv()? else { panic!("routing request missing") }; - let cancelled = messages.recv().await.unwrap(); - assert!(matches!(cancelled, crate::actor::Message::CancelRoute { request: observed } if observed == request)); - drop(reply); - }; - crate::asyncrt::timeout(std::time::Duration::from_secs(2), futures_util::future::join(dispatch, actor)).await?; - let error = result.await?.err().expect("RPC unexpectedly succeeded"); - assert_eq!(code(&error), ErrorCode::Unavailable); - untagged(&error); - Ok::<_, anyhow::Error>(()) - }).await?; + let actor = async { + let cancelled = messages.recv().await.unwrap(); + assert!(matches!(cancelled, crate::actor::Message::CancelRoute { request: observed } if observed == request)); + drop(reply); + }; + futures_util::future::join(dispatch, actor).await; + let error = result.await?.err().expect("RPC unexpectedly succeeded"); + assert_eq!(code(&error), ErrorCode::Unavailable); + untagged(&error); + return Ok::<_, anyhow::Error>(()); + } + })).await?; domain.shutdown().await; - result + result? } #[tokio::test] From 7824ece074a2fe5f03bab41bdf7a2a20315938d0 Mon Sep 17 00:00:00 2001 From: forhappy Date: Sun, 1 Nov 2026 17:25:11 -0700 Subject: [PATCH 04/44] test: verify fleet confirmation fault is reached --- .../tests/support/raft_movement_aborted.rs | 32 +++++---- .../tests/support/raft_movement_completion.rs | 72 +++++++++++++++---- 2 files changed, 76 insertions(+), 28 deletions(-) diff --git a/crates/cellar/tests/support/raft_movement_aborted.rs b/crates/cellar/tests/support/raft_movement_aborted.rs index 2b21a06b..b151971d 100644 --- a/crates/cellar/tests/support/raft_movement_aborted.rs +++ b/crates/cellar/tests/support/raft_movement_aborted.rs @@ -105,21 +105,25 @@ async fn fleet_aborted_movement_retains_seeded_history_and_recovers_receipt() -> } async fn retained_case() -> Result<()> { - let s = Box::pin(Setup::with_capacity(true, 2)).await?; - let started = start(&s, true).await?; - abort(&s, &started).await?; - warm_owners(&s, &started).await?; - write_read(&s.f, &started.cells, 2, 5, 12).await?; - let (cut, request) = completion(&s, &started.movement, 230).await?; - s.f.store - .fail_fleet_confirmation - .store(true, Ordering::Release); - ensure!(s - .f - .catalog - .settle_aborted_movement(&cut, &request, &s.target, TIMEOUT) + let s = Box::pin(Setup::with_capacity(true, 2)) .await - .is_err()); + .context("retained setup")?; + let started = start(&s, true).await.context("seed retained target")?; + abort(&s, &started).await.context("abort seeded target")?; + warm_owners(&s, &started) + .await + .context("warm aborted physical owners")?; + write_read(&s.f, &started.cells, 2, 5, 12) + .await + .context("SQL after abort")?; + let (cut, request) = completion(&s, &started.movement, 230).await?; + lose_confirmation(&s, || async { + s.f.catalog + .settle_aborted_movement(&cut, &request, &s.target, TIMEOUT) + .await + .map(|_| ()) + }) + .await?; ensure!(s.f.catalog.read(TIMEOUT).await?.revision() == cut.revision() + 1); let catalog = Fixture::open_catalog(&s.f.domain, s.f.store.clone())?; let current = catalog.read(TIMEOUT).await?; diff --git a/crates/cellar/tests/support/raft_movement_completion.rs b/crates/cellar/tests/support/raft_movement_completion.rs index f0c20064..5ebf848d 100644 --- a/crates/cellar/tests/support/raft_movement_completion.rs +++ b/crates/cellar/tests/support/raft_movement_completion.rs @@ -108,7 +108,8 @@ async fn replace(s: &Setup, from_s3: bool) -> Result { cellar::asyncrt::sleep(Duration::from_millis(10)).await; } })) - .await???; + .await? + .context("warm terminal physical copies")??; Ok(Replaced { cells, target, @@ -133,6 +134,44 @@ async fn completion( )) } +async fn lose_confirmation(s: &Setup, mut operation: F) -> Result<()> +where + F: FnMut() -> Fut, + Fut: std::future::Future>, +{ + s.f.store + .fail_fleet_confirmation + .store(true, Ordering::Release); + s.f.domain + .run(cellar::asyncrt::timeout(TIMEOUT, async { + loop { + // Both callers retain their exact idempotent request. Stop at + // the injected post-CAS loss; an earlier physical-admission + // refusal does not establish that this fault was exercised. + let result = Box::pin(operation()).await; + if !s.f.store.fail_fleet_confirmation.load(Ordering::Acquire) { + ensure!( + result.is_err(), + "injected confirmation loss returned success" + ); + return Ok::<_, anyhow::Error>(()); + } + match result { + Err( + FleetCatalogError::InventoryIncomplete + | FleetCatalogError::Unconfirmed + | FleetCatalogError::Overloaded, + ) => (), + other => anyhow::bail!("confirmation fault was not reached: {other:?}"), + } + cellar::asyncrt::sleep(Duration::from_millis(10)).await; + } + })) + .await? + .context("reach injected fleet confirmation loss")??; + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn fleet_movement_completion_reuses_slot_preserves_floor() -> Result<()> { let s = Box::pin(Setup::with_capacity(true, 2)).await?; @@ -228,26 +267,31 @@ async fn fleet_movement_completion_reuses_slot_preserves_floor() -> Result<()> { #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn fleet_movement_completion_lost_reply_survives_target_shutdown() -> Result<()> { - let s = Box::pin(Setup::new(true)).await?; - let r = replace(&s, false).await?; - let (cut, request) = completion(&s, &r.request, 230).await?; - s.f.store - .fail_fleet_confirmation - .store(true, Ordering::Release); - ensure!(s - .f - .catalog - .complete_movement(&cut, &request, &s.target, TIMEOUT) + let s = Box::pin(Setup::new(true)) .await - .is_err()); + .context("lost-reply setup")?; + let r = replace(&s, false).await.context("replace physical owner")?; + let (cut, request) = completion(&s, &r.request, 230) + .await + .context("completion request")?; + lose_confirmation(&s, || async { + s.f.catalog + .complete_movement(&cut, &request, &s.target, TIMEOUT) + .await + .map(|_| ()) + }) + .await?; let root = s.f.catalog.read(TIMEOUT).await?; ensure!(root.revision() == cut.revision() + 1); - joined(s.target_node).await?; + joined(s.target_node) + .await + .context("join completed target")?; let catalog = Fixture::open_catalog(&s.f.domain, s.f.store.clone())?; let current = catalog.read(TIMEOUT).await?; let receipt = catalog .complete_movement(¤t, &request, &s.nodes[0].host, TIMEOUT) - .await?; + .await + .context("recover completion receipt")?; ensure!(receipt.applied_revision() == cut.revision() + 1); let mut changed = request; changed.attestation.used.memory_bytes += 1; From 4fec0776a90e759288e6a77ba0a005e8ba9fc057 Mon Sep 17 00:00:00 2001 From: forhappy Date: Sun, 1 Nov 2026 17:25:11 -0700 Subject: [PATCH 05/44] test: poll admitted Raft progress within bounds --- .../tests/support/raft_outbox_native.rs | 71 ++++++++++++++----- 1 file changed, 54 insertions(+), 17 deletions(-) diff --git a/crates/cellar/tests/support/raft_outbox_native.rs b/crates/cellar/tests/support/raft_outbox_native.rs index 165e5c5b..1a47caac 100644 --- a/crates/cellar/tests/support/raft_outbox_native.rs +++ b/crates/cellar/tests/support/raft_outbox_native.rs @@ -260,18 +260,46 @@ impl Cluster { } async fn read(&self, index: usize, minimum: Option) -> Result { - Ok(self - .cell(index)? - .query( - self.definitions[index].get.clone(), - vec![], - QueryOptions { - consistency: minimum.map_or(QueryConsistency::Stale, QueryConsistency::AtLeast), - timeout: Duration::from_millis(1000), - ..Default::default() - }, - ) - .await?) + let deadline = tokio::time::Instant::now() + Duration::from_millis(1000); + Ok(tokio::time::timeout_at(deadline, async { + loop { + let result = self + .cell(index)? + .query( + self.definitions[index].get.clone(), + vec![], + QueryOptions { + consistency: minimum + .clone() + .map_or(QueryConsistency::Stale, QueryConsistency::AtLeast), + timeout: deadline + .saturating_duration_since(tokio::time::Instant::now()), + ..Default::default() + }, + ) + .await; + if !matches!(result, Err(CommandFailure::Overloaded)) { + return Ok::<_, anyhow::Error>(result?); + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await??) + } + + async fn status(&self, index: usize) -> Result { + // Status shares admission with the scheduler's own observation. Keep + // its original SDK bound while waiting for a fresh successful sample. + Ok(tokio::time::timeout(Duration::from_secs(5), async { + loop { + let result = self.cell(index)?.status().await; + if !matches!(result, Err(CommandFailure::Overloaded)) { + return Ok::<_, anyhow::Error>(result?); + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await??) } async fn settled(&self, sequence: u64) -> Result { @@ -314,7 +342,7 @@ impl Cluster { let leader = self.leader().await?; tokio::time::timeout(Duration::from_secs(10), async { loop { - let status = self.cell(leader)?.status().await?; + let status = self.status(leader).await?; if status.command_ready && status.archived >= status.applied.index { return Ok::<_, anyhow::Error>(()); } @@ -626,7 +654,7 @@ pub async fn leadership_race() -> Result<()> { loop { let read = f.destination.read(target, None).await?; if i64::from_be_bytes(read.result.as_slice().try_into()?) == 9 { - let status = f.destination.cell(target)?.status().await?; + let status = f.destination.status(target).await?; if status.archived < read.version.position.index { return Ok::<_, anyhow::Error>(()); } @@ -681,8 +709,13 @@ pub async fn replicated_progress() -> Result<()> { ensure!(f.source.store.archive_unavailable.load(Ordering::SeqCst)); f.source .read(leader, Some(settled.version_token.clone())) - .await?; - let status = f.source.cell(leader)?.status().await?; + .await + .context("source settled minimum read")?; + let status = f + .source + .status(leader) + .await + .context("source terminal progress status")?; ensure!( status.applied.index >= settled.version.position.index && status.archived < settled.version.position.index, @@ -693,7 +726,11 @@ pub async fn replicated_progress() -> Result<()> { ); // Source cleanup committed while its promised S3 completion response is // unavailable. Stop the producer and recover progress on another voter. - let report = f.source.stop(leader).await?; + let report = f + .source + .stop(leader) + .await + .context("stop producer after settled progress")?; ensure!( !report.unfinished.is_empty(), "missing ambiguous S3 completion in close report" From 06bf1818f96721275711d3f93867a43e3c2c0c08 Mon Sep 17 00:00:00 2001 From: forhappy Date: Sun, 1 Nov 2026 17:33:34 -0700 Subject: [PATCH 06/44] ci: isolate serial native qualification --- .github/workflows/native-sdk.yml | 87 ++++++++++++++++++++------------ 1 file changed, 56 insertions(+), 31 deletions(-) diff --git a/.github/workflows/native-sdk.yml b/.github/workflows/native-sdk.yml index 711d7c0e..6d6304e1 100644 --- a/.github/workflows/native-sdk.yml +++ b/.github/workflows/native-sdk.yml @@ -152,8 +152,8 @@ jobs: if: github.event_name != 'workflow_dispatch' || inputs.mode == 'qualification' name: Rust framework and SDK (${{ matrix.python }}) runs-on: ubuntu-22.04 - # ABI/package qualification plus serial fleet tests and restart examples - # exceed 90 minutes on shared runners; individual test bounds remain intact. + # ABI/package qualification and examples retain their existing ceiling. + # Native campaigns run independently with all per-test bounds unchanged. timeout-minutes: 120 strategy: fail-fast: false @@ -331,28 +331,10 @@ jobs: cargo run -p cellar --example python-package-qualification --locked -- \ "$RUNNER_TEMP/python-package" "$RUNNER_TEMP" "$RUNNER_TEMP/python-package-v2" \ --supervisor "$PWD/target/debug/cellar" - - name: Test and lint - env: - # Failed native fixtures retain synthetic recovery observations here. - TMPDIR: ${{ runner.temp }}/native-failure-fixtures - CELLAR_FS_FAULT_REPORT: ${{ runner.temp }}/filesystem-proof-fault.json + - name: Doctests and Clippy run: | - mkdir -p "$TMPDIR" - # Fleet fixtures own concurrent runtimes and short authority leases. - # Run independent cases serially to keep their resource envelope stable. - # The opt-in RPC qualifier owns every guest runtime, including V8. - # Keep this native/Python job independent of JavaScript engines. - cargo test --workspace --exclude cellar-python --exclude cellar-javascript --exclude cellar-rpc-qualification --all-features --lib --tests --locked -- --test-threads=1 - cargo test -p cellar --bin cellar-perf-cells --bin cellar-perf-load --locked -- --test-threads=1 cargo test --workspace --exclude cellar-javascript --exclude cellar-rpc-qualification --all-features --doc --locked cargo clippy --workspace --exclude cellar-javascript --exclude cellar-rpc-qualification --all-targets --all-features --locked -- -D warnings - - name: Preserve filesystem proof and recovery ledger - uses: actions/upload-artifact@v6 - if: always() - with: - name: filesystem-proof-fault-${{ matrix.python }}-${{ github.run_id }}-${{ github.run_attempt }} - path: ${{ runner.temp }}/filesystem-proof-fault.json - if-no-files-found: ignore - name: Qualify dense filesystem writes and cold recovery env: CELLAR_FS_CAPACITY_MIB: "10" @@ -396,6 +378,59 @@ jobs: python3 examples/platform_smoke.py --cellar target/debug/cellar \ --embedded target/debug/cellar-example-embedded --with-containers + - uses: actions/upload-artifact@v6 + if: always() + with: + name: python-churn-${{ matrix.python }}-${{ github.run_id }}-${{ github.run_attempt }} + path: | + ${{ runner.temp }}/python-churn/**/*.json + ${{ runner.temp }}/python-churn/**/*.log + ${{ runner.temp }}/python-churn/report.md + if-no-files-found: ignore + retention-days: 7 + + native_tests: + if: github.event_name != 'workflow_dispatch' || inputs.mode == 'qualification' + name: Native framework tests (${{ matrix.python }}) + runs-on: ubuntu-22.04 + # Separate the serial native campaign from ABI/package qualification; + # preserve the original job ceiling and every existing test command. + timeout-minutes: 120 + strategy: + fail-fast: false + matrix: + python: ['3.14', '3.14t'] + steps: + - uses: actions/checkout@v6 + - name: Install Rust + run: | + rustup toolchain install "$RUST_VERSION" --profile minimal + rustup default "$RUST_VERSION" + - uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.python }} + - name: Select embedded Python + run: echo "PYO3_PYTHON=$(command -v python)" >> "$GITHUB_ENV" + - name: Native framework tests + env: + # Failed native fixtures retain synthetic recovery observations here. + TMPDIR: ${{ runner.temp }}/native-failure-fixtures + CELLAR_FS_FAULT_REPORT: ${{ runner.temp }}/filesystem-proof-fault.json + run: | + mkdir -p "$TMPDIR" + # Fleet fixtures own concurrent runtimes and short authority leases. + # Run independent cases serially to keep their resource envelope stable. + # The opt-in RPC qualifier owns every guest runtime, including V8. + # Keep this native/Python job independent of JavaScript engines. + cargo test --workspace --exclude cellar-python --exclude cellar-javascript --exclude cellar-rpc-qualification --all-features --lib --tests --locked -- --test-threads=1 + cargo test -p cellar --bin cellar-perf-cells --bin cellar-perf-load --locked -- --test-threads=1 + - name: Preserve filesystem proof and recovery ledger + uses: actions/upload-artifact@v6 + if: always() + with: + name: filesystem-proof-fault-${{ matrix.python }}-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/filesystem-proof-fault.json + if-no-files-found: ignore - name: Preserve native failure observations uses: actions/upload-artifact@v6 if: always() @@ -417,16 +452,6 @@ jobs: if-no-files-found: ignore retention-days: 7 - - uses: actions/upload-artifact@v6 - if: always() - with: - name: python-churn-${{ matrix.python }}-${{ github.run_id }}-${{ github.run_attempt }} - path: | - ${{ runner.temp }}/python-churn/**/*.json - ${{ runner.temp }}/python-churn/**/*.log - ${{ runner.temp }}/python-churn/report.md - if-no-files-found: ignore - retention-days: 7 python_load: if: github.event_name == 'workflow_dispatch' && inputs.mode == 'python-load' From 8d87c0e519a68ca17c3b90d9b742019040f2955e Mon Sep 17 00:00:00 2001 From: forhappy Date: Sun, 1 Nov 2026 17:33:34 -0700 Subject: [PATCH 07/44] test: prioritize completed operator CLI results --- .../raft_operator_maintenance_native.rs | 66 +++++++++++++++---- .../tests/support/raft_operator_native.rs | 5 +- .../support/raft_operator_restart_native.rs | 43 +++++++++--- 3 files changed, 94 insertions(+), 20 deletions(-) diff --git a/crates/cellar/tests/support/raft_operator_maintenance_native.rs b/crates/cellar/tests/support/raft_operator_maintenance_native.rs index 027eca77..eb1ca7bd 100644 --- a/crates/cellar/tests/support/raft_operator_maintenance_native.rs +++ b/crates/cellar/tests/support/raft_operator_maintenance_native.rs @@ -94,20 +94,64 @@ async fn routed(f: &Scenario, future: F) -> Result { let catalog = Fixture::open_catalog(&f.s.f.domain, f.s.f.store.clone())?; f.s.f .domain - .run(cellar::asyncrt::timeout(TIMEOUT, async { - // A peer-probe retry must not prevent polling a completed CLI result. - // Both branches share this original deadline; no task is detached. - cellar::asyncrt::select! { - value = &mut future => Ok(value), - result = refresh_routes(f, &catalog) => { - result?; - Err(anyhow::anyhow!("route refresher stopped")) - }, - } - })) + .run(cellar::asyncrt::timeout( + TIMEOUT, + cli_with_refresh(&mut future, refresh_routes(f, &catalog)), + )) .await?? } +async fn cli_with_refresh>>( + future: F, + refresh: R, +) -> Result { + cellar::asyncrt::select_biased! { + "a completed CLI result wins a simultaneous auxiliary route failure"; + value = future => Ok(value), + result = refresh => { + result?; + Err(anyhow::anyhow!("route refresher stopped")) + }, + } +} + +#[tokio::test] +async fn completed_cli_result_wins_a_ready_route_refresh_failure() -> Result<()> { + let domain = ExecutionDomain::new(tokio::runtime::Handle::current()); + let result = domain + .run(async { + for _ in 0..16 { + let value = cli_with_refresh(std::future::ready(37), async { + anyhow::bail!("route probe refused") + }) + .await?; + ensure!(value == 37); + } + Ok::<_, anyhow::Error>(()) + }) + .await; + domain.shutdown().await; + result? +} + +#[tokio::test] +async fn pending_cli_does_not_hide_a_route_refresh_failure() -> Result<()> { + let domain = ExecutionDomain::new(tokio::runtime::Handle::current()); + let result = domain + .run(async { + let error = cli_with_refresh(std::future::pending::<()>(), async { + anyhow::bail!("route probe refused") + }) + .await + .unwrap_err(); + ensure!(error.to_string() == "route probe refused"); + Ok::<_, anyhow::Error>(()) + }) + .await; + domain.shutdown().await; + result? +} + async fn view(server: &Server, operation: &str) -> Result { let args = vec![ "fleet".into(), diff --git a/crates/cellar/tests/support/raft_operator_native.rs b/crates/cellar/tests/support/raft_operator_native.rs index 47f1051c..ffd2707b 100644 --- a/crates/cellar/tests/support/raft_operator_native.rs +++ b/crates/cellar/tests/support/raft_operator_native.rs @@ -507,7 +507,10 @@ async fn cancel() -> Result<()> { let mut waiting = op_args("wait", &begin.operation_id); waiting.extend(["--timeout-seconds".into(), "1".into()]); let (exit, timed_out) = server.cli(&waiting, "reader").await?; - ensure!(exit == 6 && timed_out["error"]["code"] == "wait_timeout"); + ensure!( + exit == 6 && timed_out["error"]["code"] == "wait_timeout", + "CLI wait result {exit}: {timed_out}" + ); ensure!(f.s.f.catalog.read(TIMEOUT).await?.revision() == before.revision()); let current = f.s.f diff --git a/crates/cellar/tests/support/raft_operator_restart_native.rs b/crates/cellar/tests/support/raft_operator_restart_native.rs index 02f8ce07..9bdfe0e4 100644 --- a/crates/cellar/tests/support/raft_operator_restart_native.rs +++ b/crates/cellar/tests/support/raft_operator_restart_native.rs @@ -279,11 +279,24 @@ async fn resume_primary( service: FleetRestartService, host: RaftCells, ) -> Result<()> { - let target_id = host.observe_node(TIMEOUT).await?.node.identity.node; + let target_id = host + .observe_node(TIMEOUT) + .await + .context("observe returned node identity")? + .node + .identity + .node; let node = hex::encode(target_id.bytes()); let reader = Fixture::open_catalog(&f.s.f.domain, f.s.f.store.clone())?; - let mut attestation = attestation(&f.s.f, &f.spare, &host).await?; - let mut profile = host.observe_node(TIMEOUT).await?.node.profile; + let mut attestation = attestation(&f.s.f, &f.spare, &host) + .await + .context("read returned node attestation")?; + let mut profile = host + .observe_node(TIMEOUT) + .await + .context("observe returned node profile")? + .node + .profile; profile.draining = false; attestation.profile_digest = fleet_capacity_profile_digest(&profile)?; // A new intended profile uses a new capacity revision. Reusing the old @@ -327,7 +340,10 @@ async fn resume_primary( f.spare.clone(), None, )?; - controller.reconcile(TIMEOUT).await?; + controller + .reconcile(TIMEOUT) + .await + .context("initialize manual resume controller")?; let routes = Fixture::open_catalog(&f.s.f.domain, f.s.f.store.clone())?; f.s.f .domain @@ -335,7 +351,9 @@ async fn resume_primary( TIMEOUT, refresh_routes_once(f, &routes), )) - .await???; + .await? + .context("initial resume route refresh deadline")? + .context("initial resume route refresh")?; let before = reader.read(TIMEOUT).await?.revision(); let rejected = controller.reconcile(TIMEOUT).await; let error = rejected @@ -363,13 +381,22 @@ async fn resume_primary( .run(cellar::asyncrt::timeout(TIMEOUT, async { let mut refreshed = None; loop { - let revision = routes.read(TIMEOUT).await?.revision(); + let revision = routes + .read(TIMEOUT) + .await + .context("read resume proof Root")? + .revision(); if refreshed != Some(revision) { - refreshed = Some(refresh_routes_once(f, &routes).await?); + refreshed = Some( + refresh_routes_once(f, &routes) + .await + .context("refresh resume proof routes")?, + ); } let progress = controller .reconcile(TIMEOUT) - .await? + .await + .context("advance manual resume controller")? .context("owned resume operation")?; { let mut seen = seen.lock().map_err(|_| anyhow::anyhow!("progress mutex"))?; From 63f6b3800c95c3e7bba05425a46759237d70578d Mon Sep 17 00:00:00 2001 From: forhappy Date: Sun, 1 Nov 2026 17:42:22 -0700 Subject: [PATCH 08/44] docs: describe independent native CI campaign --- docs/19-testing-qualification.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/19-testing-qualification.md b/docs/19-testing-qualification.md index 6cca3ea6..26d061fa 100644 --- a/docs/19-testing-qualification.md +++ b/docs/19-testing-qualification.md @@ -32,6 +32,11 @@ Run independent test cases serially, as CI does. Node fixtures own several concurrent runtimes and short leases; unrelated fixtures competing for the runner can expire those leases. Concurrency inside each fixture stays enabled. +SDK CI runs the serial native campaign independently from Python ABI/package +qualification in both Python threading modes. Each campaign retains the original +commands, assertions and job ceiling; individual request and test deadlines stay +unchanged. + The allocator startup test uses a fresh process with allocator environment overrides removed. Linux must start background purging; macOS must leave that unsupported build default disabled. Forcing `background_thread:true` on macOS From 6780a1c7c7944ee6c53175dbb742f17f0943a543 Mon Sep 17 00:00:00 2001 From: forhappy Date: Sun, 1 Nov 2026 17:58:15 -0700 Subject: [PATCH 09/44] ci: retain required SDK qualification gates --- .github/workflows/native-sdk.yml | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/.github/workflows/native-sdk.yml b/.github/workflows/native-sdk.yml index 6d6304e1..ffc9873f 100644 --- a/.github/workflows/native-sdk.yml +++ b/.github/workflows/native-sdk.yml @@ -150,7 +150,7 @@ jobs: measure: ${{ inputs.fleet_measure }} check: if: github.event_name != 'workflow_dispatch' || inputs.mode == 'qualification' - name: Rust framework and SDK (${{ matrix.python }}) + name: Python ABI and SDK (${{ matrix.python }}) runs-on: ubuntu-22.04 # ABI/package qualification and examples retain their existing ceiling. # Native campaigns run independently with all per-test bounds unchanged. @@ -453,6 +453,27 @@ jobs: retention-days: 7 + qualification: + if: always() && (github.event_name != 'workflow_dispatch' || inputs.mode == 'qualification') + needs: [check, native_tests] + # Preserve the existing required contexts across the campaign split. A + # failed, cancelled or skipped campaign must fail the original gate. + name: Rust framework and SDK (${{ matrix.python }}) + runs-on: ubuntu-22.04 + timeout-minutes: 5 + strategy: + fail-fast: false + matrix: + python: ['3.14', '3.14t'] + steps: + - name: Require both qualification campaigns + env: + SDK_RESULT: ${{ needs.check.result }} + NATIVE_RESULT: ${{ needs.native_tests.result }} + run: | + test "$SDK_RESULT" = success + test "$NATIVE_RESULT" = success + python_load: if: github.event_name == 'workflow_dispatch' && inputs.mode == 'python-load' name: Optimized Python load (${{ matrix.python }}) From ddf54884fa925e84ef6184b5461cdf440a531d84 Mon Sep 17 00:00:00 2001 From: forhappy Date: Sun, 1 Nov 2026 18:25:07 -0700 Subject: [PATCH 10/44] ci: optimize assertion-enabled native tests Debug codec work exhausted fixed protocol windows. Keep assertions and overflow checks enabled at opt-level 1. --- .github/workflows/native-sdk.yml | 6 ++++++ docs/19-testing-qualification.md | 5 ++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/native-sdk.yml b/.github/workflows/native-sdk.yml index ffc9873f..3cc2abec 100644 --- a/.github/workflows/native-sdk.yml +++ b/.github/workflows/native-sdk.yml @@ -393,6 +393,12 @@ jobs: if: github.event_name != 'workflow_dispatch' || inputs.mode == 'qualification' name: Native framework tests (${{ matrix.python }}) runs-on: ubuntu-22.04 + env: + # Exercise native protocol deadlines with optimized code, retaining all + # debug assertions and integer overflow checks in the test artifact. + CARGO_PROFILE_TEST_OPT_LEVEL: "1" + CARGO_PROFILE_TEST_DEBUG_ASSERTIONS: "true" + CARGO_PROFILE_TEST_OVERFLOW_CHECKS: "true" # Separate the serial native campaign from ABI/package qualification; # preserve the original job ceiling and every existing test command. timeout-minutes: 120 diff --git a/docs/19-testing-qualification.md b/docs/19-testing-qualification.md index 26d061fa..be4a63bf 100644 --- a/docs/19-testing-qualification.md +++ b/docs/19-testing-qualification.md @@ -35,7 +35,10 @@ runner can expire those leases. Concurrency inside each fixture stays enabled. SDK CI runs the serial native campaign independently from Python ABI/package qualification in both Python threading modes. Each campaign retains the original commands, assertions and job ceiling; individual request and test deadlines stay -unchanged. +unchanged. Native CI sets `CARGO_PROFILE_TEST_OPT_LEVEL=1`, +`CARGO_PROFILE_TEST_DEBUG_ASSERTIONS=true` and +`CARGO_PROFILE_TEST_OVERFLOW_CHECKS=true`. Use these same settings when +reproducing its bounded native campaign locally. The allocator startup test uses a fresh process with allocator environment overrides removed. Linux must start background purging; macOS must leave that From e0aec01e48f2d75333b1a214f0ee903fea67b849 Mon Sep 17 00:00:00 2001 From: forhappy Date: Sun, 1 Nov 2026 18:25:07 -0700 Subject: [PATCH 11/44] test: count confirmed movement turns --- crates/cellar/tests/support/raft_movement_execution.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/cellar/tests/support/raft_movement_execution.rs b/crates/cellar/tests/support/raft_movement_execution.rs index 900c81dd..ef437a54 100644 --- a/crates/cellar/tests/support/raft_movement_execution.rs +++ b/crates/cellar/tests/support/raft_movement_execution.rs @@ -62,7 +62,10 @@ async fn finish( .run(cellar::asyncrt::timeout(TIMEOUT * 4, async { let mut phases = Vec::new(); let mut last_error = None; - for _ in 0..64 { + // Refused observations do not confirm a movement turn. Retain the + // 64 confirmed-turn limit and the original cohort deadline while + // waiting for bounded physical/transport work to finish. + while phases.len() < 64 { let catalog = Fixture::open_catalog(&s.f.domain, s.f.store.clone())?; let cut = match catalog.read(TIMEOUT).await { Ok(cut) => cut, From ceb95f61ecff1f846a05f622144fea1d6acec86d Mon Sep 17 00:00:00 2001 From: forhappy Date: Sun, 1 Nov 2026 18:32:14 -0700 Subject: [PATCH 12/44] test: report owned maintenance timeout phase --- .../tests/support/raft_operator_maintenance_native.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/crates/cellar/tests/support/raft_operator_maintenance_native.rs b/crates/cellar/tests/support/raft_operator_maintenance_native.rs index eb1ca7bd..87f88803 100644 --- a/crates/cellar/tests/support/raft_operator_maintenance_native.rs +++ b/crates/cellar/tests/support/raft_operator_maintenance_native.rs @@ -275,7 +275,9 @@ async fn owned() -> Result<()> { .await?; let (node, mut command, request) = prepare(&f, &server).await?; command.extend(["--wait".into(), "--timeout-seconds".into(), "5".into()]); - let (exit, value) = routed(&f, server.cli(&command, "maintainer")).await??; + let (exit, value) = routed(&f, server.cli(&command, "maintainer")) + .await + .with_context(|| format!("owned prepare wait; service {:?}", service.status()))??; ensure!(exit == 0, "maintenance prepare wait {exit}: {value}"); let ready: RaftFleetMaintenanceStatus = serde_json::from_value(value)?; ready.validate()?; @@ -298,7 +300,9 @@ async fn owned() -> Result<()> { "--timeout-seconds".into(), "5".into(), ]); - let (exit, value) = routed(&f, server.cli(&command, "canceller")).await??; + let (exit, value) = routed(&f, server.cli(&command, "canceller")) + .await + .with_context(|| format!("owned cancellation wait; service {:?}", service.status()))??; ensure!(exit == 0, "maintenance cancellation wait {exit}: {value}"); let status: RaftFleetMaintenanceStatus = serde_json::from_value(value)?; status.validate()?; From 2ed1936f0db39430d6f4cfda90fa728d462cad31 Mon Sep 17 00:00:00 2001 From: forhappy Date: Sun, 1 Nov 2026 18:38:09 -0700 Subject: [PATCH 13/44] test: simplify bounded observation results --- crates/cellar/tests/support/raft_outbox_native.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/cellar/tests/support/raft_outbox_native.rs b/crates/cellar/tests/support/raft_outbox_native.rs index 1a47caac..4f745968 100644 --- a/crates/cellar/tests/support/raft_outbox_native.rs +++ b/crates/cellar/tests/support/raft_outbox_native.rs @@ -261,7 +261,7 @@ impl Cluster { async fn read(&self, index: usize, minimum: Option) -> Result { let deadline = tokio::time::Instant::now() + Duration::from_millis(1000); - Ok(tokio::time::timeout_at(deadline, async { + tokio::time::timeout_at(deadline, async { loop { let result = self .cell(index)? @@ -284,13 +284,13 @@ impl Cluster { tokio::time::sleep(Duration::from_millis(10)).await; } }) - .await??) + .await? } async fn status(&self, index: usize) -> Result { // Status shares admission with the scheduler's own observation. Keep // its original SDK bound while waiting for a fresh successful sample. - Ok(tokio::time::timeout(Duration::from_secs(5), async { + tokio::time::timeout(Duration::from_secs(5), async { loop { let result = self.cell(index)?.status().await; if !matches!(result, Err(CommandFailure::Overloaded)) { @@ -299,7 +299,7 @@ impl Cluster { tokio::time::sleep(Duration::from_millis(10)).await; } }) - .await??) + .await? } async fn settled(&self, sequence: u64) -> Result { From 4b937d993db12f4a5370d98e345e21176cdc252e Mon Sep 17 00:00:00 2001 From: forhappy Date: Sun, 1 Nov 2026 18:41:11 -0700 Subject: [PATCH 14/44] ci: propagate captured validation failures --- .github/workflows/native-sdk.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/native-sdk.yml b/.github/workflows/native-sdk.yml index 3cc2abec..59414f28 100644 --- a/.github/workflows/native-sdk.yml +++ b/.github/workflows/native-sdk.yml @@ -104,6 +104,7 @@ jobs: git rev-parse HEAD > "$RUNNER_TEMP/filesystem-release/source.txt" - name: Build the release CLI and filesystem application run: | + set -o pipefail cargo build --release --locked -p cellar --bin cellar --bin cellar-native-examples \ 2>&1 | tee "$RUNNER_TEMP/filesystem-release/build.log" python3 - <<'PY' @@ -123,6 +124,7 @@ jobs: PY - name: Exercise the release filesystem application and restart run: | + set -o pipefail python3 examples/focused_smoke.py --cellar target/release/cellar \ --example filesystem --profile release \ 2>&1 | tee "$RUNNER_TEMP/filesystem-release/smoke.log" @@ -341,6 +343,7 @@ jobs: CELLAR_FS_CAPACITY_REPORT: ${{ runner.temp }}/filesystem-dense-capacity.json RUST_LOG: 'warn,cellar::actor::effects=info,store[node_lease_store]=debug' run: | + set -o pipefail cargo test --locked -p cellar --all-features --test native_node \ filesystem_dense_capacity_cold_recovery -- --ignored --nocapture \ 2>&1 | tee "$RUNNER_TEMP/filesystem-dense-capacity.log" From 3dd4da906d91fbc7ed2392261e1ce4da13e3df59 Mon Sep 17 00:00:00 2001 From: forhappy Date: Sun, 1 Nov 2026 19:45:46 -0700 Subject: [PATCH 15/44] test: isolate profile node authority writes Reuse the canonical durable fixture namespace so node lease CAS does not queue behind application data writers. Preserve the lease, handler, audit, and shutdown contracts. --- tools/rpc/host/peer_faults/mod.rs | 2 ++ tools/rpc/host/peer_faults/storage.rs | 2 +- tools/rpc/host/profile/matrix.rs | 8 ++++++-- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/tools/rpc/host/peer_faults/mod.rs b/tools/rpc/host/peer_faults/mod.rs index f433c261..3bf66e90 100644 --- a/tools/rpc/host/peer_faults/mod.rs +++ b/tools/rpc/host/peer_faults/mod.rs @@ -9,3 +9,5 @@ mod relay; mod storage; pub use matrix::run; + +pub(crate) use storage::open as durable_fixture_bucket; diff --git a/tools/rpc/host/peer_faults/storage.rs b/tools/rpc/host/peer_faults/storage.rs index 17e9b783..13d769c0 100644 --- a/tools/rpc/host/peer_faults/storage.rs +++ b/tools/rpc/host/peer_faults/storage.rs @@ -13,7 +13,7 @@ use std::{ sync::Arc, }; -pub(super) fn open(root: &std::path::Path, names: &[&str]) -> anyhow::Result { +pub(crate) fn open(root: &std::path::Path, names: &[&str]) -> anyhow::Result { let bucket = Bucket::open_dev(&root.join("fleet.sqlite"))?; let mut nodes = BTreeMap::new(); for name in names { diff --git a/tools/rpc/host/profile/matrix.rs b/tools/rpc/host/profile/matrix.rs index b211fb65..b044ec94 100644 --- a/tools/rpc/host/profile/matrix.rs +++ b/tools/rpc/host/profile/matrix.rs @@ -2,7 +2,6 @@ //! Every vector traverses real generated clients, native cells and peer routing. use super::*; -use cellar::bucket::Bucket; use serde_json::{json, Value}; async fn matrix( @@ -85,7 +84,12 @@ pub(super) async fn run( vectors: Arc>, ) -> Result { let directory = tempfile::tempdir()?; - let bucket = Bucket::open_dev(&directory.path().join("fleet.sqlite"))?; + // Node-key CAS stays durable and visible through the shared namespace, but + // cannot queue behind application-data writers and consume the lease TTL. + let bucket = crate::peer_faults::durable_fixture_bucket( + directory.path(), + &["profile-owner", "profile-source"], + )?; let client = reqwest::Client::builder() .no_proxy() .timeout(Duration::from_secs(20)) From 951cad9417cc910fe531519acc2238456fab3549 Mon Sep 17 00:00:00 2001 From: forhappy Date: Sun, 1 Nov 2026 19:45:54 -0700 Subject: [PATCH 16/44] test: report terminal observation failures Retain the last physical copies and topology service state on failure. Keep exact completion predicates and the original observation deadlines. --- .../tests/support/raft_movement_completion.rs | 37 +++++++++++++++++-- .../support/raft_operator_lifecycle_native.rs | 3 +- 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/crates/cellar/tests/support/raft_movement_completion.rs b/crates/cellar/tests/support/raft_movement_completion.rs index 5ebf848d..e41441f7 100644 --- a/crates/cellar/tests/support/raft_movement_completion.rs +++ b/crates/cellar/tests/support/raft_movement_completion.rs @@ -84,23 +84,47 @@ async fn replace(s: &Setup, from_s3: bool) -> Result { .await?; // Warm only actual physical observations. The terminal API must independently // obtain its own fresh quorum and complete physical probes afterward. + let seen = std::sync::Mutex::new((None, Vec::new())); s.f.domain .run(cellar::asyncrt::timeout(TIMEOUT, async { loop { let cut = s.f.catalog.read(TIMEOUT).await?; let mut complete = true; + seen.lock() + .map_err(|_| anyhow::anyhow!("physical observations mutex"))? + .1 + .clear(); for node in &s.nodes { + seen.lock() + .map_err(|_| anyhow::anyhow!("physical observations mutex"))? + .0 = Some(node.spec.node); let observed = node .host .inspect_peer_physical_inventory(node.spec.node, cut.revision(), TIMEOUT) .await?; - complete &= observed.view.seed_workers == 0 - && observed + let history = if observed.view.seed_workers == 0 { + observed .inventory .as_ref() .unwrap() .history(&request.resources.group)? - .is_some_and(|h| h.completed_sequence() == 1); + } else { + None + }; + let ready = observed.view.seed_workers == 0 + && history.is_some_and(|h| h.completed_sequence() == 1); + complete &= ready; + { + let mut seen = seen + .lock() + .map_err(|_| anyhow::anyhow!("physical observations mutex"))?; + seen.0 = None; + seen.1.push(( + node.spec.node, + observed.view.seed_workers, + history.map(|h| (h.completed_sequence(), h.in_configuration())), + )); + } } if complete { return Ok::<_, anyhow::Error>(()); @@ -109,7 +133,12 @@ async fn replace(s: &Setup, from_s3: bool) -> Result { } })) .await? - .context("warm terminal physical copies")??; + .with_context(|| { + format!( + "warm terminal physical copies; observations {:?}", + seen.lock() + ) + })??; Ok(Replaced { cells, target, diff --git a/crates/cellar/tests/support/raft_operator_lifecycle_native.rs b/crates/cellar/tests/support/raft_operator_lifecycle_native.rs index c55e0e8f..d1bd4a86 100644 --- a/crates/cellar/tests/support/raft_operator_lifecycle_native.rs +++ b/crates/cellar/tests/support/raft_operator_lifecycle_native.rs @@ -99,7 +99,8 @@ async fn redistribute(f: &mut Scenario, admin: &Server, node: &str) -> Result<() let (exit, value) = admin.cli(&args, "reader").await?; ensure!( exit == 0 && value["converged"] == true, - "redistribution {exit}: {value}" + "redistribution {exit}: {value}; service {:?}", + service.status() ); service.shutdown().await; completed_sql(f, 164).await?; From 85d5ec063ad7bc66f87692211e22e111935773f1 Mon Sep 17 00:00:00 2001 From: forhappy Date: Sun, 1 Nov 2026 20:53:17 -0700 Subject: [PATCH 17/44] fix: announce config before member revocation A paused append stream can miss the final commit broadcast. Drain prior sends, announce the committed index and term under durable leader authority, and drain that finite notification set before revoking membership. Preserve receiver log matching and physical closure proofs. --- crates/cellar/durability/raft/replica.rs | 104 ++++++++++++++-- .../durability/raft/replica_delivery_tests.rs | 116 +++++++++++++++++- 2 files changed, 208 insertions(+), 12 deletions(-) diff --git a/crates/cellar/durability/raft/replica.rs b/crates/cellar/durability/raft/replica.rs index 405f3dce..8b7978a4 100644 --- a/crates/cellar/durability/raft/replica.rs +++ b/crates/cellar/durability/raft/replica.rs @@ -101,6 +101,11 @@ enum Phase { }, } +struct ConfigurationFence { + index: u64, + announced: bool, +} + pub(crate) struct Replica { pub log: GroupLog, node: RawNode, @@ -108,7 +113,7 @@ pub(crate) struct Replica { pub write: Option<(WriteTicket, GroupWrite)>, outbound: VecDeque, deliveries: Vec>, - configuration_fence: Option, + configuration_fence: Option, apply: VecDeque, apply_ticket: Option, finished_apply: Option, @@ -694,6 +699,59 @@ impl Replica { Ok(()) } + fn announce_configuration(&mut self, index: u64) -> Result<(), RuntimeError> { + if self + .configuration_fence + .as_ref() + .is_some_and(|fence| fence.index == index && fence.announced) + { + return Ok(()); + } + self.configuration_fence = Some(ConfigurationFence { + index, + announced: true, + }); + if self.node.raft.state != StateRole::Leader { + return Ok(()); + } + let hard = self.log.initial_state()?.hard_state; + // A concurrent term change cannot advertise an unpersisted promise. + if hard.term != self.node.raft.term || hard.vote != self.node.raft.vote { + return Ok(()); + } + let entry = self + .apply + .front() + .filter(|entry| entry.index == index) + .ok_or_else(|| RuntimeError::Protocol("missing configuration apply".into()))?; + // A paused append stream can miss Raft's commit broadcast. Once this + // configuration removes that peer, ordinary routing cannot reach it. + // Announce this already committed entry once while the old binding is + // still valid. An empty Append requires the receiver to match its exact + // index/term; transport acceptance is never a durability proof. A lagging + // or unavailable copy still fails the catalog's physical closure check. + for (&member, _) in self.node.raft.prs().iter() { + if member == self.node.raft.id { + continue; + } + let mut message = Message { + from: self.node.raft.id, + to: member, + term: self.node.raft.term, + index, + log_term: entry.term, + commit: index, + ..Default::default() + }; + message.set_msg_type(MessageType::MsgAppend); + self.outbound.push_back(RuntimeEventKind::Message { + replica: self.log.identity().clone(), + message, + }); + } + Ok(()) + } + pub fn drive( &mut self, operations: &mut Operations, @@ -977,15 +1035,20 @@ impl Replica { .filter(|entry| entry.get_entry_type() != EntryType::EntryNormal) .filter(|_| self.node.raft.raft_log.unstable_snapshot().is_none()) .map(|entry| entry.index); - let fenced = configuration_index.is_some() - && self.configuration_fence == configuration_index; + let fenced = self + .configuration_fence + .as_ref() + .is_some_and(|fence| configuration_index == Some(fence.index)); if !fenced && self.node.has_ready() && ((can_collect && self.apply_bytes < options.max_group_apply_bytes) || self.node.raft.raft_log.unstable_snapshot().is_some()) { - if configuration_index.is_some() { - self.configuration_fence = configuration_index; + if let Some(index) = configuration_index { + self.configuration_fence = Some(ConfigurationFence { + index, + announced: false, + }); } let ready = self.node.ready(); self.ready_bytes = ready_bytes(&ready); @@ -1016,17 +1079,38 @@ impl Replica { }; return Ok(true); } - let Some(entry) = self.apply.front() else { - return Ok(false); - }; - if entry.get_entry_type() != EntryType::EntryNormal { - self.configuration_fence = Some(entry.index); + if let Some(index) = self + .apply + .front() + .filter(|entry| entry.get_entry_type() != EntryType::EntryNormal) + .map(|entry| entry.index) + { + if self + .configuration_fence + .as_ref() + .is_none_or(|fence| fence.index != index) + { + self.configuration_fence = Some(ConfigurationFence { + index, + announced: false, + }); + } self.deliveries .retain(|delivery| delivery.strong_count() != 0); if !self.deliveries.is_empty() { return Ok(false); } + // The notification must not overtake an earlier Append on + // another transport stream. Its own sends then join the + // same finite fence before membership is revoked. + self.announce_configuration(index)?; + if !self.outbound.is_empty() { + return Ok(true); + } } + let Some(entry) = self.apply.front() else { + return Ok(false); + }; let position = entry_position(entry); if entry.get_entry_type() == EntryType::EntryNormal && !entry.data.is_empty() { if let Some(completed) = self.finished_apply.take() { diff --git a/crates/cellar/durability/raft/replica_delivery_tests.rs b/crates/cellar/durability/raft/replica_delivery_tests.rs index 688e4794..54bdf271 100644 --- a/crates/cellar/durability/raft/replica_delivery_tests.rs +++ b/crates/cellar/durability/raft/replica_delivery_tests.rs @@ -22,11 +22,17 @@ async fn membership_apply_drains_an_already_queued_commit_notice() -> anyhow::Re membership_case(ReadyCase::QueuedCommit).await } +#[tokio::test] +async fn membership_apply_announces_commit_to_a_paused_peer() -> anyhow::Result<()> { + membership_case(ReadyCase::PausedPeer).await +} + #[derive(Clone, Copy)] enum ReadyCase { None, LaterHeartbeat, QueuedCommit, + PausedPeer, } async fn membership_case(ready_case: ReadyCase) -> anyhow::Result<()> { @@ -138,9 +144,76 @@ async fn membership_case(ready_case: ReadyCase) -> anyhow::Result<()> { assert!(replica.write.is_none()); assert!(replica.drive(&mut operations, &mut outbox, &options, true)?); } - owned + vec![owned] + } else if matches!(ready_case, ReadyCase::PausedPeer) { + replica.node.raft.state = StateRole::Leader; + replica.node.raft.mut_prs().get_mut(2).unwrap().pause(); + replica.node.raft.send_append(2); + assert!(replica.node.raft.msgs.is_empty()); + for _ in 0..16 { + if replica.configuration_fence.is_some() + && replica.idle() + && replica.ready_bytes == 0 + { + break; + } + assert!(replica.drive(&mut operations, &mut outbox, &options, true)?); + assert!(outbox.events.is_empty()); + assert!(replica.write.is_none()); + } + assert!(!replica.drive(&mut operations, &mut outbox, &options, true)?); + assert!( + replica.outbound.is_empty(), + "commit notice overtook prior send" + ); + // A new in-memory promise cannot escape before persistence. + let hard = replica.log.initial_state()?.hard_state; + for (term, vote) in [(hard.term + 1, hard.vote), (hard.term, 1)] { + replica.node.raft.term = term; + replica.node.raft.vote = vote; + replica.announce_configuration(2)?; + assert!(replica.outbound.is_empty()); + replica.configuration_fence.as_mut().unwrap().announced = false; + } + replica.node.raft.term = hard.term; + replica.node.raft.vote = hard.vote; + drop(transport_owner); + let mut owners = Vec::new(); + let mut peers = std::collections::BTreeSet::new(); + for _ in 0..16 { + assert!(replica.write.is_none()); + assert!(replica.drive(&mut operations, &mut outbox, &options, true)?); + if let Some(event) = outbox.events.pop_front() { + let RuntimeEventKind::Message { message, .. } = &event.kind else { + panic!("configuration commit was not announced") + }; + assert_eq!(message.get_msg_type(), MessageType::MsgAppend); + assert_eq!( + ( + message.from, + message.term, + message.index, + message.log_term, + message.commit + ), + (1, 1, 2, 1, 2) + ); + assert!(message.entries.is_empty()); + if message.to == 2 { + check_notice_matching(message)?; + } + assert!(peers.insert(message.to)); + owners.push(event._permit.clone()); + } + if owners.len() == 2 && replica.idle() && replica.ready_bytes == 0 { + break; + } + } + assert_eq!(peers, [2, 3].into_iter().collect()); + assert!(replica.node.raft.prs().get(2).unwrap().is_paused()); + owners } else { - transport_owner + vec![transport_owner] }; assert!(!replica.drive(&mut operations, &mut outbox, &options, true)?); if matches!(ready_case, ReadyCase::LaterHeartbeat) { @@ -180,3 +253,42 @@ async fn membership_case(ready_case: ReadyCase) -> anyhow::Result<()> { domain.shutdown().await; Ok(()) } + +fn check_notice_matching(notice: &Message) -> anyhow::Result<()> { + for (present, matching) in [(true, true), (true, false), (false, true)] { + let store = ::raft::storage::MemStorage::new_with_conf_state((vec![1, 2, 3], vec![])); + { + let mut state = store.wl(); + let entries = (1..=if present { 2 } else { 1 }) + .map(|index| Entry { + index, + term: 1, + ..Default::default() + }) + .collect::>(); + state.append(&entries)?; + state.mut_hard_state().term = 1; + state.mut_hard_state().commit = 1; + } + let mut follower = RawNode::new( + &Config { + id: 2, + applied: 1, + ..Default::default() + }, + store, + &slog::Logger::root(slog::Discard, slog::o!()), + )?; + let mut message = notice.clone(); + if !matching { + message.log_term += 1; + } + follower.step(message)?; + assert_eq!( + follower.raft.raft_log.committed, + if present && matching { 2 } else { 1 }, + "a commit notification must match the actual receiver log" + ); + } + Ok(()) +} From 73fa2eef03b63cc37a7d6b7f9d44b186f1a4b73c Mon Sep 17 00:00:00 2001 From: forhappy Date: Sun, 1 Nov 2026 21:11:35 -0700 Subject: [PATCH 18/44] test: isolate worker lifecycle node writes Use the canonical FULL-synchronous namespace for both live worker cases and recovery. Keep ordinary reads, CAS and listing coherent while separating node authority from application-data writers. Preserve all shutdown and receipt assertions. --- tools/rpc/host/worker/matrix.rs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tools/rpc/host/worker/matrix.rs b/tools/rpc/host/worker/matrix.rs index db1abff7..cb2ce06a 100644 --- a/tools/rpc/host/worker/matrix.rs +++ b/tools/rpc/host/worker/matrix.rs @@ -3,10 +3,7 @@ use super::{application::Fixture, backend::Backend, trace::Trace}; use crate::{application::Composite, crash::process::wait_groups_gone, fleet}; use anyhow::{ensure, Context as _, Result}; -use cellar::{ - bucket::Bucket, - sdk::{NodeOutcome, ShutdownMode}, -}; +use cellar::sdk::{NodeOutcome, ShutdownMode}; use cellar_javascript::{BridgeUsage, JavaScriptApplication, Options}; use serde_json::{json, Value}; use std::{ @@ -214,7 +211,7 @@ async fn live_case( trace: trace.clone(), provider_class: backend.class(), }); - let bucket = Bucket::open_dev(&path.join("fleet.sqlite"))?; + let bucket = crate::peer_faults::durable_fixture_bucket(path, &[OWNER, SOURCE])?; let client = reqwest::Client::builder() .timeout(Duration::from_secs(20)) .build()?; @@ -456,7 +453,7 @@ async fn recovered( trace, provider_class: backend.class(), }); - let bucket = Bucket::open_dev(&path.join("fleet.sqlite"))?; + let bucket = crate::peer_faults::durable_fixture_bucket(path, &[OWNER, SOURCE])?; let (node, url) = fleet::start(&path.join("owner"), fixture, OWNER, bucket).await?; let client = reqwest::Client::builder() .timeout(Duration::from_secs(20)) From 0ac7e51121033f6a6f99859137fab409176b3563 Mon Sep 17 00:00:00 2001 From: forhappy Date: Sun, 1 Nov 2026 21:13:39 -0700 Subject: [PATCH 19/44] test: identify departure refusal stage Keep evidence, service startup, CLI, SQL and finalizing-route failures distinguishable without changing observation deadlines or success predicates. --- .../support/raft_operator_lifecycle_native.rs | 35 ++++++++++++++----- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/crates/cellar/tests/support/raft_operator_lifecycle_native.rs b/crates/cellar/tests/support/raft_operator_lifecycle_native.rs index d1bd4a86..f7d95c51 100644 --- a/crates/cellar/tests/support/raft_operator_lifecycle_native.rs +++ b/crates/cellar/tests/support/raft_operator_lifecycle_native.rs @@ -76,13 +76,19 @@ async fn fleet_operator_departure_redistributes_verifies_joins_and_removes() -> } async fn redistribute(f: &mut Scenario, admin: &Server, node: &str) -> Result<()> { - let mut begin = request(f, 164).await?; + let mut begin = request(f, 164) + .await + .context("departure redistribution evidence")?; begin.keep.retain(|n| n.as_str() != node); let (exit, value) = admin .cli(&admin.begin_args(&begin, "departure-keep.json")?, "owner") .await?; ensure!(exit == 0, "redistribution admission {exit}: {value}"); - let evidence = Arc::new(Inputs::new(f).await?); + let evidence = Arc::new( + Inputs::new(f) + .await + .context("departure redistribution service inputs")?, + ); let service = f.s.f .catalog @@ -93,17 +99,23 @@ async fn redistribute(f: &mut Scenario, admin: &Server, node: &str) -> Result<() options(), FleetTopologyCollectionOptions::default(), ) - .await?; + .await + .context("departure redistribution service start")?; let mut args = op_args("wait", &begin.operation_id); args.extend(["--timeout-seconds".into(), "40".into()]); - let (exit, value) = admin.cli(&args, "reader").await?; + let (exit, value) = admin + .cli(&args, "reader") + .await + .context("departure redistribution CLI wait")?; ensure!( exit == 0 && value["converged"] == true, "redistribution {exit}: {value}; service {:?}", service.status() ); service.shutdown().await; - completed_sql(f, 164).await?; + completed_sql(f, 164) + .await + .context("departure redistribution SQL receipts")?; ensure!(f .live .iter() @@ -113,8 +125,10 @@ async fn redistribute(f: &mut Scenario, admin: &Server, node: &str) -> Result<() } async fn departure() -> Result<()> { - let mut f = Box::pin(Scenario::new(true)).await?; - let mut admin = server(&f).await?; + let mut f = Box::pin(Scenario::new(true)) + .await + .context("departure scenario")?; + let mut admin = server(&f).await.context("departure admin server")?; let node = hex::encode([4; 16]); let revision = f.s.f.catalog.read(TIMEOUT).await?.revision(); let refused = operation(&admin, 165, revision); @@ -130,7 +144,9 @@ async fn departure() -> Result<()> { ); ensure!(f.s.f.catalog.read(TIMEOUT).await?.revision() == revision); - redistribute(&mut f, &admin, &node).await?; + redistribute(&mut f, &admin, &node) + .await + .context("departure redistribution")?; let exclusion = operation(&admin, 167, f.s.f.catalog.read(TIMEOUT).await?.revision()); let args = admin.departure_args(&node, "evacuate", &exclusion); @@ -254,7 +270,8 @@ async fn departure() -> Result<()> { // authenticated discovery path before the exact stop RPC. f.s.f .remote(&f.s.nodes[1].host, NodeId::new([4; 16])?, revision) - .await?; + .await + .context("departure finalizing process metadata")?; let (exit, value) = admin.cli(&stop_args, "stopper").await?; ensure!( (exit == 0 && accepted(value.clone())?.phase == RaftFleetEvacuationPhase::StopRequested) From 30c88c1625e3c740da69991f5ad00397ad6be74c Mon Sep 17 00:00:00 2001 From: forhappy Date: Sun, 1 Nov 2026 22:37:00 -0700 Subject: [PATCH 20/44] test: isolate Python lifecycle node writes --- tools/rpc/host/worker/python/case.rs | 10 ++++------ tools/rpc/host/worker/python/recovery.rs | 6 +++--- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/tools/rpc/host/worker/python/case.rs b/tools/rpc/host/worker/python/case.rs index f30a1d2d..be0f1209 100644 --- a/tools/rpc/host/worker/python/case.rs +++ b/tools/rpc/host/worker/python/case.rs @@ -8,10 +8,7 @@ use super::super::{ use super::{probes, recovery, Case, Trigger, DELTA, OWNER, SOURCE}; use crate::{application::Composite, fleet}; use anyhow::{ensure, Context as _, Result}; -use cellar::{ - bucket::Bucket, - sdk::{NodeOutcome, ShutdownMode}, -}; +use cellar::sdk::{NodeOutcome, ShutdownMode}; use cellar_python::PythonApplication; use serde_json::{json, Value}; use std::{ @@ -34,7 +31,7 @@ pub(super) async fn run( trace: trace.clone(), provider_class: "PythonCounter", }); - let bucket = Bucket::open_dev(&root.join("fleet.sqlite"))?; + let bucket = crate::peer_faults::durable_fixture_bucket(root, &[OWNER, SOURCE])?; let client = reqwest::Client::builder() .timeout(Duration::from_secs(20)) .build()?; @@ -111,7 +108,8 @@ pub(super) async fn run( } else { let stopping = owner.take().context("Python owner already stopped")?.shutdown(ShutdownMode::Preserve); let (stopped, returned) = tokio::join!(stopping, tokio::time::timeout(Duration::from_secs(12), &mut call)); - let NodeOutcome::Stopped(proof) = stopped? else { anyhow::bail!("Python owner failed during shutdown"); }; + let outcome = stopped?; + let NodeOutcome::Stopped(proof) = outcome else { anyhow::bail!("Python owner failed during shutdown: {outcome:?}; case={spec:?}"); }; ensure!(proof.drain_timed_out && !proof.application_work_drained && !proof.cells_drained && !proof.clean_reload_prepared && proof.pending_application_events > 0 && proof.pending_dispatches > 0 && proof.log_quiesced && proof.local_durability_closed, "Python forced shutdown proof differs: {proof:?}"); diff --git a/tools/rpc/host/worker/python/recovery.rs b/tools/rpc/host/worker/python/recovery.rs index 1ef967e0..150270d5 100644 --- a/tools/rpc/host/worker/python/recovery.rs +++ b/tools/rpc/host/worker/python/recovery.rs @@ -1,10 +1,10 @@ // Copyright 2026 Cellar Project Authors. Licensed under Apache-2.0. use super::super::{application::Fixture, matrix::request, trace::Trace}; -use super::{DELTA, OWNER}; +use super::{DELTA, OWNER, SOURCE}; use crate::{application::Composite, fleet}; use anyhow::{ensure, Context as _, Result}; -use cellar::{bucket::Bucket, sdk::ShutdownMode}; +use cellar::sdk::ShutdownMode; use cellar_python::PythonApplication; use serde_json::{json, Value}; use std::{path::Path, sync::Arc}; @@ -24,7 +24,7 @@ pub(super) async fn run( trace: trace.clone(), provider_class: "PythonCounter", }); - let bucket = Bucket::open_dev(&root.join("fleet.sqlite"))?; + let bucket = crate::peer_faults::durable_fixture_bucket(root, &[OWNER, SOURCE])?; let (node, url) = fleet::start(&root.join("owner"), app, OWNER, bucket).await?; let result = async { let before = before.to_string(); From 001a1613d59dad596ff650fd13df85a29de840e3 Mon Sep 17 00:00:00 2001 From: forhappy Date: Sun, 1 Nov 2026 22:40:26 -0700 Subject: [PATCH 21/44] test: await coherent restart observations --- .../tests/support/raft_initial_native.rs | 5 +- .../support/raft_restart_resume_native.rs | 70 ++++++++++++++----- .../support/raft_restart_return_native.rs | 68 ++++++++++++++---- .../tests/support/raft_restart_stop_native.rs | 11 ++- 4 files changed, 121 insertions(+), 33 deletions(-) diff --git a/crates/cellar/tests/support/raft_initial_native.rs b/crates/cellar/tests/support/raft_initial_native.rs index 47cd0c0f..0b72f44a 100644 --- a/crates/cellar/tests/support/raft_initial_native.rs +++ b/crates/cellar/tests/support/raft_initial_native.rs @@ -278,7 +278,7 @@ async fn activate(cells: &[RaftCellHandle], auth: &FleetGroupAuthorization) -> R } async fn primary(f: &Fixture, cells: &[RaftCellHandle]) -> Result { - Ok(f.domain + f.domain .run(cellar::asyncrt::timeout(TIMEOUT, async { loop { for (i, cell) in cells.iter().enumerate() { @@ -289,7 +289,8 @@ async fn primary(f: &Fixture, cells: &[RaftCellHandle]) -> Result { cellar::asyncrt::sleep(Duration::from_millis(10)).await; } })) - .await??) + .await? + .context("command-ready primary deadline") } async fn write_read( diff --git a/crates/cellar/tests/support/raft_restart_resume_native.rs b/crates/cellar/tests/support/raft_restart_resume_native.rs index 92a1fa51..b5354b98 100644 --- a/crates/cellar/tests/support/raft_restart_resume_native.rs +++ b/crates/cellar/tests/support/raft_restart_resume_native.rs @@ -59,17 +59,56 @@ async fn prepare_resume( request: &FleetRestartResumeRequest, ) -> Result { trust(s).await?; - let root = s.f.catalog.read(TIMEOUT).await?; - Ok(s.f - .catalog - .prepare_restart_resume( - &root, - request, - std::slice::from_ref(&s.namespace), - &s.spare, - TIMEOUT, - ) - .await?) + // Returning engines register asynchronously. Construct the verification + // from a fresh, complete inventory within the existing call budget. + s.f.domain + .run(cellar::asyncrt::timeout(TIMEOUT, async { + loop { + let root = s.f.catalog.read(TIMEOUT).await?; + match s + .f + .catalog + .prepare_restart_resume( + &root, + request, + std::slice::from_ref(&s.namespace), + &s.spare, + TIMEOUT, + ) + .await + { + Ok(walk) => return Ok(walk), + Err(FleetCatalogError::InventoryIncomplete) => { + cellar::asyncrt::sleep(Duration::from_millis(10)).await; + } + Err(error) => return Err(error.into()), + } + } + })) + .await? + .context("restart resume inventory preparation deadline")? +} + +async fn coherent_inspection( + s: &Setup, + cell: &RaftCellHandle, +) -> Result { + // inspect() itself has a five-second budget. A mixed runtime/image sample + // is refused with Overloaded; keep that same bound across fresh samples. + s.f.domain + .run(cellar::asyncrt::timeout(TIMEOUT, async { + loop { + match cell.inspect().await { + Ok(view) => return Ok(view), + Err(CommandFailure::Overloaded) => { + cellar::asyncrt::sleep(Duration::from_millis(10)).await; + } + Err(error) => return Err(error.into()), + } + } + })) + .await? + .context("coherent restart inspection deadline")? } async fn consume_resume(s: &Setup, walk: &mut FleetRestartResumeVerification) -> Result { @@ -392,8 +431,7 @@ async fn fleet_restart_resume_rechecks_and_hands_off_regained_leadership() -> Re drop(warm); let source = primary(&s.f, &s.cells).await?; ensure!(source != p); - let state = s.cells[source] - .inspect() + let state = coherent_inspection(&s, &s.cells[source]) .await? .member .context("surviving leader")?; @@ -409,8 +447,7 @@ async fn fleet_restart_resume_rechecks_and_hands_off_regained_leadership() -> Re .await?; s.f.domain .run(cellar::asyncrt::timeout(TIMEOUT, async { - while !s.cells[p] - .inspect() + while !coherent_inspection(&s, &s.cells[p]) .await? .member .is_some_and(|m| m.is_leader) @@ -433,8 +470,7 @@ async fn fleet_restart_resume_rechecks_and_hands_off_regained_leadership() -> Re .await?; pending(&s, p, false).await?; ensure!( - !s.cells[p] - .inspect() + !coherent_inspection(&s, &s.cells[p]) .await? .member .context("returned member")? diff --git a/crates/cellar/tests/support/raft_restart_return_native.rs b/crates/cellar/tests/support/raft_restart_return_native.rs index 27f647ba..f6e692fc 100644 --- a/crates/cellar/tests/support/raft_restart_return_native.rs +++ b/crates/cellar/tests/support/raft_restart_return_native.rs @@ -7,7 +7,7 @@ mod resume; use cellar::application::consensus::verification::FleetRestartReturnRequest; async fn offline(s: &mut Setup, p: usize) -> Result<(Running, FleetRestartReturnRequest)> { - let (gate, request, root) = consent(s, p).await?; + let (gate, request, root) = consent(s, p).await.context("restart return stop consent")?; s.f.catalog .admit_restart_stop(&root, &request, TIMEOUT) .await?; @@ -27,7 +27,9 @@ async fn offline(s: &mut Setup, p: usize) -> Result<(Running, FleetRestartReturn result.as_ref().err() ); let mut target = s.nodes.remove(p); - joined(&mut target.node, &s.f).await?; + joined(&mut target.node, &s.f) + .await + .context("restart return old process join")?; let root = s.f.catalog.read(TIMEOUT).await?; let mut identity = request_identity(180, root.revision(), gate.identity.operation); let offline = FleetRestartOfflineRequest { @@ -82,14 +84,20 @@ async fn configuration( } async fn returned(primary_target: bool, lose_confirmations: bool) -> Result<()> { - let mut s = Box::pin(Setup::new()).await?; - let leader = primary(&s.f, &s.cells).await?; + let mut s = Box::pin(Setup::new()) + .await + .context("restart return setup")?; + let leader = primary(&s.f, &s.cells) + .await + .context("restart return initial primary")?; let p = if primary_target { leader } else { (leader + 1) % 3 }; - let (mut target, request) = offline(&mut s, p).await?; + let (mut target, request) = offline(&mut s, p) + .await + .context("restart return offline predecessor")?; let old = target.host.observe_node(TIMEOUT).await; ensure!(old.is_err()); let root = s.f.catalog.read(TIMEOUT).await?; @@ -124,7 +132,9 @@ async fn returned(primary_target: bool, lose_confirmations: bool) -> Result<()> .filter(|(i, _)| *i != p) .map(|(_, c)| c.clone()) .collect::>(); - write_read(&s.f, &survivors, 2, 5, 12).await?; + write_read(&s.f, &survivors, 2, 5, 12) + .await + .context("restart return survivor write before startup")?; let node = public_node::node_named( target.root.path(), s.f.definition.application.clone(), @@ -210,16 +220,24 @@ async fn returned(primary_target: bool, lose_confirmations: bool) -> Result<()> ); let revision = root.revision(); for survivor in &s.nodes { - s.f.remote(&survivor.host, request.node, revision).await?; - s.f.remote(&host, survivor.spec.node, revision).await?; + s.f.remote(&survivor.host, request.node, revision) + .await + .context("restart return install returned peer")?; + s.f.remote(&host, survivor.spec.node, revision) + .await + .context("restart return install survivor peer")?; } - write_read(&s.f, &survivors, 3, 2, 14).await?; + write_read(&s.f, &survivors, 3, 2, 14) + .await + .context("restart return survivor write after startup")?; let wanted = survivors[0] .inspect() - .await? + .await + .context("restart return survivor progress")? .member .context("survivor progress")? .applied; + let seen = std::sync::Mutex::new((0u64, None)); s.f.domain .run(cellar::asyncrt::timeout(TIMEOUT, async { loop { @@ -231,6 +249,25 @@ async fn returned(primary_target: bool, lose_confirmations: bool) -> Result<()> Err(CommandFailure::Overloaded) => None, Err(error) => return Err(error.into()), }; + { + let mut seen = seen + .lock() + .map_err(|_| anyhow::anyhow!("restart return observation mutex"))?; + if let Some(view) = observed.as_ref() { + seen.1 = Some(view.member.as_ref().map(|m| { + ( + m.applied, + m.committed, + m.membership_revision, + m.term, + m.leader, + m.is_leader, + ) + })); + } else { + seen.0 += 1; + } + } if observed .and_then(|view| view.member) .is_some_and(|m| m.applied.index > wanted.index || m.applied == wanted) @@ -240,7 +277,14 @@ async fn returned(primary_target: bool, lose_confirmations: bool) -> Result<()> cellar::asyncrt::sleep(Duration::from_millis(10)).await; } })) - .await???; + .await? + .with_context(|| { + format!( + "restart return catch-up: wanted={wanted:?}, seen={:?}", + seen.lock() + ) + })? + .context("restart return inspect sample")?; ensure!(host.observe_node(TIMEOUT).await?.node.profile.draining); ensure!(reopened .begin_evacuation( @@ -255,7 +299,7 @@ async fn returned(primary_target: bool, lose_confirmations: bool) -> Result<()> target.node = node; target.host = host; s.nodes.push(target); - s.close().await + s.close().await.context("restart return joined cleanup") } #[tokio::test(flavor = "multi_thread", worker_threads = 4)] diff --git a/crates/cellar/tests/support/raft_restart_stop_native.rs b/crates/cellar/tests/support/raft_restart_stop_native.rs index 61c134a6..e7a59675 100644 --- a/crates/cellar/tests/support/raft_restart_stop_native.rs +++ b/crates/cellar/tests/support/raft_restart_stop_native.rs @@ -58,10 +58,17 @@ async fn trust(s: &Setup) -> Result<()> { } async fn joined(node: &mut cellar::sdk::Node, f: &Fixture) -> Result<()> { - let outcome = f + let completed = f .domain .run(cellar::asyncrt::timeout(TIMEOUT, node.finished())) - .await???; + .await?; + let outcome = completed.with_context(|| { + format!( + "restart stopped process join: node={}, closed={}", + node.info().node, + node.is_closed() + ) + })??; let NodeOutcome::Stopped(report) = outcome else { anyhow::bail!("maintenance did not autonomously stop"); }; From eed299e2ef84cbd9e2e4092f40df0a348512b239 Mon Sep 17 00:00:00 2001 From: forhappy Date: Mon, 2 Nov 2026 00:13:22 -0700 Subject: [PATCH 22/44] test: require complete physical proof observations --- .../tests/support/raft_catalog_fault.rs | 3 ++ .../support/raft_operator_lifecycle_native.rs | 43 ++++++++++++++++++- .../raft_restart_operator_cancel_native.rs | 11 ++++- 3 files changed, 54 insertions(+), 3 deletions(-) diff --git a/crates/cellar/tests/support/raft_catalog_fault.rs b/crates/cellar/tests/support/raft_catalog_fault.rs index 2db8a6c7..fa78a7f8 100644 --- a/crates/cellar/tests/support/raft_catalog_fault.rs +++ b/crates/cellar/tests/support/raft_catalog_fault.rs @@ -144,6 +144,7 @@ pub struct Store { pub payload_size_delta: AtomicUsize, pub payload_chunks: std::sync::Arc, pub refuse_archive_payload_reads: AtomicBool, + pub refused_archive_payload_reads: AtomicUsize, pub hold_next_payload: AtomicBool, pub payload_hang: AtomicBool, pub payload_waiting: AtomicBool, @@ -722,6 +723,8 @@ impl ObjectStore for Store { } if self.refuse_archive_payload_reads.load(Ordering::Acquire) { + self.refused_archive_payload_reads + .fetch_add(1, Ordering::AcqRel); return Err(failure()); } } diff --git a/crates/cellar/tests/support/raft_operator_lifecycle_native.rs b/crates/cellar/tests/support/raft_operator_lifecycle_native.rs index f7d95c51..5719ecfa 100644 --- a/crates/cellar/tests/support/raft_operator_lifecycle_native.rs +++ b/crates/cellar/tests/support/raft_operator_lifecycle_native.rs @@ -180,12 +180,53 @@ async fn departure() -> Result<()> { let args = admin.departure_args(&node, "prepare-removal", &preparation); let (exit, value) = admin.cli(&args, "evacuator").await?; ensure!(exit == 4 && value["error"]["code"] == "forbidden"); + let refused_before = + f.s.f + .store + .refused_archive_payload_reads + .load(Ordering::Acquire); f.s.f .store .refuse_archive_payload_reads .store(true, Ordering::Release); - let (exit, value) = admin.cli(&args, "remover").await?; + // A read-only quorum blocker can precede replay. It is not the injected + // failure and must not count as coverage. Reconcile the exact request only + // while its validated progress proves no history or authority was admitted. + let (exit, value) = f + .s + .f + .domain + .run(cellar::asyncrt::timeout(TIMEOUT, async { + loop { + let (exit, value) = admin.cli(&args, "remover").await?; + if exit != 0 { + return Ok::<_, anyhow::Error>((exit, value)); + } + let progress = accepted(value.clone())?; + ensure!( + progress.phase == RaftFleetEvacuationPhase::VerifyingBeforeStop + && progress.checked_histories == 0 + && progress.blocker.as_ref().is_some_and(|blocker| { + blocker.stage == RaftFleetVerificationStage::HistoryQuorum + }) + && progress.applied_revision.is_none() + && progress.fence_id.is_none(), + "replay fault allowed progress or authority: {value}" + ); + ensure!(f.s.f.catalog.read(TIMEOUT).await?.revision() == status.observed_revision); + cellar::asyncrt::sleep(Duration::from_millis(10)).await; + } + })) + .await???; ensure!(exit != 0, "physical replay failure admitted fence: {value}"); + ensure!( + f.s.f + .store + .refused_archive_payload_reads + .load(Ordering::Acquire) + > refused_before, + "archive replay fault was not reached: {value}" + ); f.s.f .store .refuse_archive_payload_reads diff --git a/crates/cellar/tests/support/raft_restart_operator_cancel_native.rs b/crates/cellar/tests/support/raft_restart_operator_cancel_native.rs index ed149555..b0c97ab0 100644 --- a/crates/cellar/tests/support/raft_restart_operator_cancel_native.rs +++ b/crates/cellar/tests/support/raft_restart_operator_cancel_native.rs @@ -34,14 +34,21 @@ pub(super) async fn ready( s.f.domain .run(cellar::asyncrt::timeout(TIMEOUT, async { loop { - routes(s).await?; + routes(s).await.context("operator readiness discovery")?; match controller.reconcile(TIMEOUT).await { Ok(Some(progress)) => { if let Some(ready) = progress.ready { return Ok::<_, anyhow::Error>(ready); } } - Err(FleetCatalogError::Unconfirmed | FleetCatalogError::Overloaded) + // The driver owns durable child identities across turns. + // Incomplete physical registration is not readiness; keep + // reconciling that same operation within this call's bound. + Err( + FleetCatalogError::Unconfirmed + | FleetCatalogError::Overloaded + | FleetCatalogError::InventoryIncomplete, + ) | Err(FleetCatalogError::Observation( InspectionError::Unavailable | InspectionError::Overloaded, )) => (), From e7104c5e0b73bcf9179ef8d140706baff3a12bd8 Mon Sep 17 00:00:00 2001 From: forhappy Date: Mon, 2 Nov 2026 00:13:22 -0700 Subject: [PATCH 23/44] ci: optimize bounded Node delivery tests --- .github/workflows/javascript-sdk.yml | 5 +++++ docs/19-testing-qualification.md | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/.github/workflows/javascript-sdk.yml b/.github/workflows/javascript-sdk.yml index 29bfe566..1f63f27c 100644 --- a/.github/workflows/javascript-sdk.yml +++ b/.github/workflows/javascript-sdk.yml @@ -242,6 +242,11 @@ jobs: if: matrix.profile == 'packages' env: RUST_LOG: timing=debug,cellar::runtime=debug,store=debug + # Keep the existing handler deadlines and all debug/overflow checks. + # Optimize this repeated cold-restart artifact, as in native SDK CI. + CARGO_PROFILE_TEST_OPT_LEVEL: '1' + CARGO_PROFILE_TEST_DEBUG_ASSERTIONS: 'true' + CARGO_PROFILE_TEST_OVERFLOW_CHECKS: 'true' run: | cargo build --locked -p cellar-javascript --no-default-features --features node --bin cellar-node-example mkdir -p .cellar diff --git a/docs/19-testing-qualification.md b/docs/19-testing-qualification.md index be4a63bf..f211eee3 100644 --- a/docs/19-testing-qualification.md +++ b/docs/19-testing-qualification.md @@ -40,6 +40,10 @@ unchanged. Native CI sets `CARGO_PROFILE_TEST_OPT_LEVEL=1`, `CARGO_PROFILE_TEST_OVERFLOW_CHECKS=true`. Use these same settings when reproducing its bounded native campaign locally. +The packaged Node delivery restart campaign uses the same test-profile settings +for its repeated cold-start fixture. Its 250 ms handler budget and all 100 fresh +store/process cases remain unchanged. + The allocator startup test uses a fresh process with allocator environment overrides removed. Linux must start background purging; macOS must leave that unsupported build default disabled. Forcing `background_thread:true` on macOS From f09e51b1151b7694fdd33f33518ff0b122341b28 Mon Sep 17 00:00:00 2001 From: forhappy Date: Mon, 2 Nov 2026 01:39:42 -0700 Subject: [PATCH 24/44] test: isolate dense recovery lease writes --- crates/cellar/tests/support/node_filesystem_capacity.rs | 5 ++++- docs/19-testing-qualification.md | 5 +++++ .../storage.rs => qualification/durable_bucket.rs} | 0 tools/rpc/host/peer_faults/mod.rs | 1 + tools/rpc/host/tests/peer_storage.rs | 2 +- 5 files changed, 11 insertions(+), 2 deletions(-) rename tools/{rpc/host/peer_faults/storage.rs => qualification/durable_bucket.rs} (100%) diff --git a/crates/cellar/tests/support/node_filesystem_capacity.rs b/crates/cellar/tests/support/node_filesystem_capacity.rs index ac068781..de8e339a 100644 --- a/crates/cellar/tests/support/node_filesystem_capacity.rs +++ b/crates/cellar/tests/support/node_filesystem_capacity.rs @@ -12,6 +12,9 @@ use std::{ time::{Duration, Instant}, }; +#[path = "../../../../tools/qualification/durable_bucket.rs"] +mod durable_bucket; + const BLOCK: usize = 1 << 20; const PATCH: usize = 4096; @@ -336,7 +339,7 @@ async fn profile(mib: usize) -> Result { let directory = tempfile::tempdir()?; let management = ExecutionDomain::new(tokio::runtime::Handle::current()); let bucket = management - .run(async { cellar::bucket::Bucket::open_dev(&directory.path().join("bucket.sqlite")) }) + .run(async { durable_bucket::open(directory.path(), &["dense-owner", "dense-successor"]) }) .await??; let core = Executor::new("filesystem-dense-core")?; let http = reqwest::Client::builder() diff --git a/docs/19-testing-qualification.md b/docs/19-testing-qualification.md index f211eee3..13b53fd6 100644 --- a/docs/19-testing-qualification.md +++ b/docs/19-testing-qualification.md @@ -44,6 +44,11 @@ The packaged Node delivery restart campaign uses the same test-profile settings for its repeated cold-start fixture. Its 250 ms handler budget and all 100 fresh store/process cases remain unchanged. +The [shared durable fixture bucket](../tools/qualification/durable_bucket.rs) +keeps node-record CAS independent of application-data writers using FULL-sync +SQLite stores. Ordinary reads, CAS, listings and recovery use one coherent +namespace, including the dense filesystem cold-recovery fixture. + The allocator startup test uses a fresh process with allocator environment overrides removed. Linux must start background purging; macOS must leave that unsupported build default disabled. Forcing `background_thread:true` on macOS diff --git a/tools/rpc/host/peer_faults/storage.rs b/tools/qualification/durable_bucket.rs similarity index 100% rename from tools/rpc/host/peer_faults/storage.rs rename to tools/qualification/durable_bucket.rs diff --git a/tools/rpc/host/peer_faults/mod.rs b/tools/rpc/host/peer_faults/mod.rs index 3bf66e90..e2526bd7 100644 --- a/tools/rpc/host/peer_faults/mod.rs +++ b/tools/rpc/host/peer_faults/mod.rs @@ -6,6 +6,7 @@ mod framing; mod matrix; mod receiver; mod relay; +#[path = "../../../qualification/durable_bucket.rs"] mod storage; pub use matrix::run; diff --git a/tools/rpc/host/tests/peer_storage.rs b/tools/rpc/host/tests/peer_storage.rs index 271a5c3b..c721d50d 100644 --- a/tools/rpc/host/tests/peer_storage.rs +++ b/tools/rpc/host/tests/peer_storage.rs @@ -2,7 +2,7 @@ //! Node authority remains durable and visible through every logical client. #![allow(clippy::disallowed_methods)] -#[path = "../peer_faults/storage.rs"] +#[path = "../../../qualification/durable_bucket.rs"] mod storage; use anyhow::{ensure, Result}; From 4fbfadfaf4244a10d8e4ffe61400830938fc2aaf Mon Sep 17 00:00:00 2001 From: forhappy Date: Mon, 2 Nov 2026 03:33:23 -0700 Subject: [PATCH 25/44] test: stabilize native state observations Start the dormant repair participant, route the same permit at the highest observed term, and await complete restart proofs within existing bounds. Match the two archive-readiness gauges in the exact metrics inventory. Qualify changes to the shared durable fixture source. --- .github/workflows/javascript-sdk.yml | 1 + .github/workflows/typed-rpc.yml | 1 + .../application/consensus/resource_metrics.rs | 2 +- crates/cellar/durability/raft/README.md | 2 +- .../cellar/tests/native_raft_fleet_permits.rs | 97 ++++++++++++++++--- .../tests/support/raft_metrics_native.rs | 13 ++- .../support/raft_restart_operator_native.rs | 9 +- .../support/raft_restart_resume_native.rs | 77 ++++++++++----- 8 files changed, 160 insertions(+), 42 deletions(-) diff --git a/.github/workflows/javascript-sdk.yml b/.github/workflows/javascript-sdk.yml index 1f63f27c..6388ff43 100644 --- a/.github/workflows/javascript-sdk.yml +++ b/.github/workflows/javascript-sdk.yml @@ -17,6 +17,7 @@ on: - 'tools/javascript/**' - 'tools/sandbox/**' - 'tools/rpc/**' + - 'tools/qualification/**' - 'tools/cellar-rpc-codegen/**' - 'proto/**' - 'docs/design/named-typed-rpc/*.proto' diff --git a/.github/workflows/typed-rpc.yml b/.github/workflows/typed-rpc.yml index 028df205..dd9a703d 100644 --- a/.github/workflows/typed-rpc.yml +++ b/.github/workflows/typed-rpc.yml @@ -7,6 +7,7 @@ on: - 'crates/**' - 'packages/cellar-sdk/**' - 'tools/rpc/**' + - 'tools/qualification/**' - 'tools/javascript/**' - 'tools/cellar-rpc-codegen/**' - 'tools/python/**' diff --git a/crates/cellar/application/consensus/resource_metrics.rs b/crates/cellar/application/consensus/resource_metrics.rs index 3946bedc..e4336803 100644 --- a/crates/cellar/application/consensus/resource_metrics.rs +++ b/crates/cellar/application/consensus/resource_metrics.rs @@ -55,7 +55,7 @@ fn budget(text: &mut Text, prefix: &str, labels: &str, usage: RaftBudgetUsage) { } impl RaftResourceUsage { - /// Bounded label vocabularies, with 111 samples when SQL, image, group and + /// Bounded label vocabularies, with 113 samples when SQL, image, group and /// transport sources are live, plus 13 for a configured controller. /// A live extent inventory adds 13 samples. /// Unavailable sources expose only their availability gauge, diff --git a/crates/cellar/durability/raft/README.md b/crates/cellar/durability/raft/README.md index acf421b0..a52b15c9 100644 --- a/crates/cellar/durability/raft/README.md +++ b/crates/cellar/durability/raft/README.md @@ -188,7 +188,7 @@ I/O or group enumeration. The internal generation-bound, authenticated `/peer/raft/metrics` endpoint exports these counters, actual SQL work reservations and incoming/outgoing transport and snapshot/metadata banks, without identity labels or callback execution. Live SQL, image, group and transport sources use -111 fixed samples. A configured controller and a live extent inventory each +113 fixed samples. A configured controller and a live extent inventory each add 13 samples. Unavailable sources omit their values rather than synthesizing zero. Weak transport and inventory observers cannot retain physical resources after joined shutdown. The [operations contract](../../../../docs/design/multi-raft/operations-design.md) diff --git a/crates/cellar/tests/native_raft_fleet_permits.rs b/crates/cellar/tests/native_raft_fleet_permits.rs index 5b45712c..d2183b7d 100644 --- a/crates/cellar/tests/native_raft_fleet_permits.rs +++ b/crates/cellar/tests/native_raft_fleet_permits.rs @@ -62,14 +62,34 @@ struct Fixture { } async fn leader(domain: &ExecutionDomain, cells: &[RaftCellHandle]) -> Result { + Ok(leader_status(domain, cells).await?.0) +} + +// Local readiness can briefly outlive isolation. Route at the highest +// observed term; the guarded operation still has to establish its proof. +async fn leader_status( + domain: &ExecutionDomain, + cells: &[RaftCellHandle], +) -> Result<(usize, CellStatus)> { Ok(domain .run(cellar::asyncrt::timeout(TIMEOUT, async { loop { + let mut highest_term = 0; + let mut ready = None; for (index, cell) in cells.iter().enumerate() { - if cell.status().await.is_ok_and(|status| status.command_ready) { - return index; + if let Ok(status) = cell.status().await { + if status.term.get() > highest_term { + highest_term = status.term.get(); + ready = None; + } + if status.term.get() == highest_term && status.command_ready { + ready = Some((index, status)); + } } } + if let Some(ready) = ready { + return ready; + } cellar::asyncrt::sleep(Duration::from_millis(10)).await; } })) @@ -319,14 +339,19 @@ impl Fixture { authorization: &FleetPermitAuthorization, step: RepairStep, ) -> Result { - self.domain + let mut last_status = None; + let mut last_refusal = None; + let result = self + .domain .run(cellar::asyncrt::timeout(TIMEOUT, async { loop { - let expected = self.handles[self.primary] - .status() - .await? - .membership_revision; - match self.handles[self.primary] + // A repair survives leader loss. Route the same permit + // and idempotent step to a freshly observed leader; the + // initial fixture leader is not permanent authority. + let (primary, status) = leader_status(&self.domain, &self.handles).await?; + let expected = status.membership_revision; + last_status = Some((primary, status)); + match self.handles[primary] .advance_managed_repair( authorization.clone(), step, @@ -336,15 +361,23 @@ impl Fixture { .await { Ok(status) => return Ok(status), - Err(CommandFailure::Unavailable | CommandFailure::Overloaded) => { + Err( + error @ (CommandFailure::Unavailable + | CommandFailure::Overloaded + | CommandFailure::NotLeader { .. }), + ) => { + last_refusal = Some(error); cellar::asyncrt::sleep(Duration::from_millis(10)).await } Err(error) => return Err(error.into()), } } })) - .await? - .context("managed repair step timed out")? + .await?; + result.map_err(anyhow::Error::from).and_then(|status| status) + .with_context(|| format!( + "managed repair {step:?}: last leader status {last_status:?}, refusal {last_refusal:?}", + )) } fn activation(&self, from_s3: bool) -> Result { @@ -456,7 +489,10 @@ impl Fixture { #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn fleet_permit_delayed_after_cancel() -> Result<()> { - let f = Fixture::new().await?; + let mut f = Fixture::new().await?; + // The reserved replacement receives protocol traffic once the plan starts. + // Start its real dormant process; activation still requires normal seeding. + f.standby().await?; let plan = f.plan(40).await?; let cached = f.reserve(1, 20, &plan).await?; let status = f.handles[f.primary] @@ -507,6 +543,43 @@ async fn fleet_permit_delayed_after_cancel() -> Result<()> { f.close().await } +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn fleet_permit_repair_step_follows_new_quorum_leader() -> Result<()> { + let mut f = Fixture::new().await?; + f.standby().await?; + let plan = f.plan(40).await?; + let authorization = f.reserve(1, 20, &plan).await?; + let repair = f.handles[f.primary] + .begin_managed_repair(authorization.clone(), plan, TIMEOUT) + .await?; + ensure!(repair.phase == RepairPhase::Seeding); + let (source, status) = leader_status(&f.domain, &f.handles).await?; + ensure!(source == f.primary); + // Optional transfers correctly refuse an active repair. Exercise actual + // leader loss instead, retaining the same permit and committed plan. + f.cluster.isolate_consensus(source).await?; + let survivors = f + .handles + .iter() + .enumerate() + .filter(|(index, _)| *index != source) + .map(|(_, cell)| cell.clone()) + .collect::>(); + let (_, successor) = leader_status(&f.domain, &survivors).await?; + ensure!(successor.term > status.term); + let advanced = f.step(&authorization, RepairStep::AbortLearner).await; + if advanced.is_err() { + for (index, cell) in f.handles.iter().enumerate() { + eprintln!("stalled repair node {index}: {:?}", cell.status().await); + } + } + f.cluster.heal(); + let closed = f.close().await; + let advanced = advanced.with_context(|| format!("joined repair fixture: {closed:?}"))?; + ensure!(advanced.phase == RepairPhase::Aborted); + closed +} + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn fleet_permit_crash_recovery() -> Result<()> { let mut f = Fixture::new().await?; diff --git a/crates/cellar/tests/support/raft_metrics_native.rs b/crates/cellar/tests/support/raft_metrics_native.rs index b3f27352..c8d5bd38 100644 --- a/crates/cellar/tests/support/raft_metrics_native.rs +++ b/crates/cellar/tests/support/raft_metrics_native.rs @@ -123,7 +123,18 @@ fn samples(text: &str) -> Result> { "duplicate or malformed sample" ); } - ensure!(identities.len() == 111); + // Archive readiness added two fixed group gauges. Require their exact + // identities as well as the total; neither may replace an older series. + ensure!(identities.len() == 113); + for name in [ + "cellar_raft_groups_archive_current", + "cellar_raft_groups_archive_unconfirmed", + ] { + ensure!( + identities.contains(name), + "missing archive readiness gauge: {name}" + ); + } ensure!(text.contains("cellar_raft_extents_sample_available 0\n")); for identity in &identities { ensure!( diff --git a/crates/cellar/tests/support/raft_restart_operator_native.rs b/crates/cellar/tests/support/raft_restart_operator_native.rs index 117dcc04..681bc1aa 100644 --- a/crates/cellar/tests/support/raft_restart_operator_native.rs +++ b/crates/cellar/tests/support/raft_restart_operator_native.rs @@ -327,7 +327,14 @@ async fn preparation(recreate: bool, concurrent: bool, churn: bool) -> Result<() } Err(FleetCatalogError::Conflict | FleetCatalogError::PayloadConflict) if concurrent => {} - Err(FleetCatalogError::Unconfirmed | FleetCatalogError::Overloaded) + // Recreated controllers must obtain a complete live + // inventory before returning the same durable intent's + // Ready proof. Refusal is not preparation progress. + Err( + FleetCatalogError::Unconfirmed + | FleetCatalogError::Overloaded + | FleetCatalogError::InventoryIncomplete, + ) | Err(FleetCatalogError::Observation( InspectionError::Unavailable | InspectionError::Overloaded, )) => {} diff --git a/crates/cellar/tests/support/raft_restart_resume_native.rs b/crates/cellar/tests/support/raft_restart_resume_native.rs index b5354b98..32ac4a7f 100644 --- a/crates/cellar/tests/support/raft_restart_resume_native.rs +++ b/crates/cellar/tests/support/raft_restart_resume_native.rs @@ -58,30 +58,35 @@ async fn prepare_resume( s: &Setup, request: &FleetRestartResumeRequest, ) -> Result { - trust(s).await?; + trust(s).await.context("restart resume peer discovery")?; // Returning engines register asynchronously. Construct the verification // from a fresh, complete inventory within the existing call budget. s.f.domain .run(cellar::asyncrt::timeout(TIMEOUT, async { loop { - let root = s.f.catalog.read(TIMEOUT).await?; - match s - .f - .catalog - .prepare_restart_resume( - &root, - request, - std::slice::from_ref(&s.namespace), - &s.spare, - TIMEOUT, - ) - .await - { + let observed = async { + let root = s.f.catalog.read(TIMEOUT).await?; + s.f.catalog + .prepare_restart_resume( + &root, + request, + std::slice::from_ref(&s.namespace), + &s.spare, + TIMEOUT, + ) + .await + } + .await; + match observed { Ok(walk) => return Ok(walk), - Err(FleetCatalogError::InventoryIncomplete) => { + Err(FleetCatalogError::InventoryIncomplete | FleetCatalogError::Overloaded) => { cellar::asyncrt::sleep(Duration::from_millis(10)).await; } - Err(error) => return Err(error.into()), + Err(error) => { + return Err( + anyhow::Error::from(error).context("restart resume proof construction") + ); + } } } })) @@ -425,9 +430,15 @@ async fn fleet_restart_resume_root_drift_requires_fresh_same_scope_consent() -> #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn fleet_restart_resume_rechecks_and_hands_off_regained_leadership() -> Result<()> { - let (s, p, request) = Box::pin(setup(false)).await?; - let mut warm = prepare_resume(&s, &request).await?; - consume_resume(&s, &mut warm).await?; + let (s, p, request) = Box::pin(setup(false)) + .await + .context("regained leader setup")?; + let mut warm = prepare_resume(&s, &request) + .await + .context("initial resume proof")?; + consume_resume(&s, &mut warm) + .await + .context("initial resume histories")?; drop(warm); let source = primary(&s.f, &s.cells).await?; ensure!(source != p); @@ -457,18 +468,32 @@ async fn fleet_restart_resume_rechecks_and_hands_off_regained_leadership() -> Re Ok::<_, anyhow::Error>(()) })) .await???; - pending(&s, p, true).await?; - let mut walk = prepare_resume(&s, &request).await?; - ensure!(consume_resume(&s, &mut walk).await? > 0); + pending(&s, p, true) + .await + .context("regained leader paused state")?; + let mut walk = prepare_resume(&s, &request) + .await + .context("regained leader fresh proof")?; + ensure!( + consume_resume(&s, &mut walk) + .await + .context("regained leader handoff proof")? + > 0 + ); let admission = walk .admit_resume(&request, TIMEOUT) .await .context("publish refreshed resume consent")?; - trust(&s).await?; + trust(&s) + .await + .context("regained leader resume discovery")?; s.f.catalog .resume_restart_maintenance(&admission, &s.spare, TIMEOUT) - .await?; - pending(&s, p, false).await?; + .await + .context("guarded regained leader resume")?; + pending(&s, p, false) + .await + .context("regained leader resumed state")?; ensure!( !coherent_inspection(&s, &s.cells[p]) .await? @@ -476,7 +501,7 @@ async fn fleet_restart_resume_rechecks_and_hands_off_regained_leadership() -> Re .context("returned member")? .is_leader ); - s.close().await + s.close().await.context("regained leader joined cleanup") } #[tokio::test(flavor = "multi_thread", worker_threads = 4)] From b4761defc9f0e7dc223513dbe2dbd2b4b271d960 Mon Sep 17 00:00:00 2001 From: forhappy Date: Mon, 2 Nov 2026 04:31:01 -0700 Subject: [PATCH 26/44] test: reconcile exact movement completion proof A queued seed worker can start after the warm inventory. Keep the same\ncompletion identity and require its genuine receipt within the existing\nfive-second readiness bound. Preserve independent API probes and floors. --- .../tests/support/raft_movement_isolation.rs | 106 +++++++++++------- 1 file changed, 66 insertions(+), 40 deletions(-) diff --git a/crates/cellar/tests/support/raft_movement_isolation.rs b/crates/cellar/tests/support/raft_movement_isolation.rs index e255624c..1da2c6f2 100644 --- a/crates/cellar/tests/support/raft_movement_isolation.rs +++ b/crates/cellar/tests/support/raft_movement_isolation.rs @@ -10,7 +10,11 @@ async fn isolate( node: NodeId, boot: FleetBootBinding, ) -> Result { - let cut = s.f.catalog.read(TIMEOUT).await?; + let cut = + s.f.catalog + .read(TIMEOUT) + .await + .context("read exclusion Root")?; s.f.catalog .begin_evacuation( &cut, @@ -19,7 +23,8 @@ async fn isolate( std::slice::from_ref(&s.namespace), TIMEOUT, ) - .await?; + .await + .context("admit node exclusion before isolation")?; let cut = s.f.catalog.read(TIMEOUT).await?; let request = FleetNodeIsolationRequest { identity: Fixture::request(221, cut.revision()), @@ -48,11 +53,15 @@ async fn isolate( .await .is_err()); let reopened = Fixture::open_catalog(&s.f.domain, s.f.store.clone())?; - let root = reopened.read(TIMEOUT).await?; + let root = reopened + .read(TIMEOUT) + .await + .context("read Root after lost isolation confirmation")?; ensure!(root.revision() == cut.revision() + 1); let receipt = reopened .attest_node_isolation(&root, &request, TIMEOUT) - .await?; + .await + .context("reconcile exact node isolation receipt")?; ensure!( receipt.boot() == &request.attestation.boot && receipt.applied_revision() == root.revision() @@ -145,7 +154,9 @@ async fn refresh(s: &Setup) -> Result<()> { async fn fleet_movement_isolated_obsolete_repairs_three_histories() -> Result<()> { // Three actual moves exceed the fleet's two-active-move book; each must // settle while the obsolete node stays offline and its engine is retained. - let mut s = Box::pin(Setup::with_histories(true, 3, 3)).await?; + let mut s = Box::pin(Setup::with_histories(true, 3, 3)) + .await + .context("set up three isolated histories")?; let mut groups = Vec::new(); for proposal in &s.proposals { let cut = s.f.catalog.read(TIMEOUT).await?; @@ -187,7 +198,9 @@ async fn fleet_movement_isolated_obsolete_repairs_three_histories() -> Result<() .await .is_err()); joined(stopped.node).await?; - let evidence = isolate(&s, stopped.spec.node, boot).await?; + let evidence = isolate(&s, stopped.spec.node, boot) + .await + .context("attest joined obsolete boot")?; // Even a saved startup capability cannot restart the retained engine. let retry_node = public_node::node_named( stopped.root.path(), @@ -230,7 +243,8 @@ async fn fleet_movement_isolated_obsolete_repairs_three_histories() -> Result<() std::slice::from_ref(&s.namespace), TIMEOUT, ) - .await?; + .await + .with_context(|| format!("reserve isolated movement {index}"))?; let permit = accepted.authorization(); let target = s .target @@ -275,46 +289,58 @@ async fn fleet_movement_isolated_obsolete_repairs_three_histories() -> Result<() ); cells.push(target); write_read(&s.f, &cells, 2, 5, 12).await?; - s.f.domain - .run(cellar::asyncrt::timeout(TIMEOUT, async { - loop { - let cut = s.f.catalog.read(TIMEOUT).await?; - let mut complete = true; - for node in &s.nodes { - let observed = node - .host - .inspect_peer_physical_inventory( - node.spec.node, - cut.revision(), - TIMEOUT, - ) - .await?; - complete &= observed.view.seed_workers == 0 - && observed - .inventory - .as_ref() - .unwrap() - .history(scope)? - .is_some_and(|h| h.completed_sequence() == 1); - } - if complete { - return Ok::<_, anyhow::Error>(()); - } - cellar::asyncrt::sleep(Duration::from_millis(10)).await; - } - })) - .await???; let (cut, completion) = completion(&s, &request, 231 + index as u8 * 2).await?; let done = - s.f.catalog - .complete_movement(&cut, &completion, &s.target, TIMEOUT) - .await?; + s.f.domain + .run(cellar::asyncrt::timeout(TIMEOUT, async { + loop { + let observed_cut = s.f.catalog.read(TIMEOUT).await?; + let mut complete = true; + for node in &s.nodes { + let observed = node + .host + .inspect_peer_physical_inventory( + node.spec.node, + observed_cut.revision(), + TIMEOUT, + ) + .await?; + complete &= observed.view.seed_workers == 0 + && observed + .inventory + .as_ref() + .unwrap() + .history(scope)? + .is_some_and(|h| h.completed_sequence() == 1); + } + if complete { + // A queued seed worker can start between the warm + // inventory and the API's independent physical probe. + // Reconcile this exact idempotent request within the + // original readiness bound; only a real receipt ends it. + match s + .f + .catalog + .complete_movement(&cut, &completion, &s.target, TIMEOUT) + .await + { + Ok(done) => return Ok(done), + Err(FleetCatalogError::Unconfirmed) => {} + Err(error) => return Err(anyhow::Error::from(error)), + } + } + cellar::asyncrt::sleep(Duration::from_millis(10)).await; + } + })) + .await? + .with_context(|| format!("complete isolated movement {index}"))??; ensure!(done.isolated_sources() == [stopped.spec.node.bytes()]); let root = s.f.catalog.read(TIMEOUT).await?; let replay = s.f.catalog .complete_movement(&root, &completion, &s.target, TIMEOUT) - .await?; + .await + .with_context(|| format!("reconcile isolated movement {index}"))?; ensure!( replay.applied_revision() == done.applied_revision() && replay.isolated_sources() == done.isolated_sources() From aa35745b27172d9702e61b1ed48744e7e5705f38 Mon Sep 17 00:00:00 2001 From: forhappy Date: Mon, 2 Nov 2026 04:56:26 -0700 Subject: [PATCH 27/44] test: retain handoff on incomplete inventory --- .../cellar/tests/support/raft_restart_retained_runtime.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/cellar/tests/support/raft_restart_retained_runtime.rs b/crates/cellar/tests/support/raft_restart_retained_runtime.rs index fdf5597e..18ccfc01 100644 --- a/crates/cellar/tests/support/raft_restart_retained_runtime.rs +++ b/crates/cellar/tests/support/raft_restart_retained_runtime.rs @@ -171,7 +171,11 @@ async fn verified(dynamic: bool, owned: bool) -> Result<()> { Ok(Some(cellar::application::consensus::verification::FleetRestartRuntimeProgress::HandoffRequested { from, to, .. })) => { ensure!(dynamic && from == r.request.plan.replacement.member && from != to); } - Err(FleetCatalogError::Unconfirmed | FleetCatalogError::Overloaded) + // Handoff can overlap a fresh physical inventory cut. Keep + // the pending history until an actual Verified turn; an + // incomplete cut is neither progress nor a matched vote. + Err(FleetCatalogError::Unconfirmed | FleetCatalogError::Overloaded + | FleetCatalogError::InventoryIncomplete) | Err(FleetCatalogError::Observation(cellar::application::consensus::InspectionError::Unavailable | cellar::application::consensus::InspectionError::Overloaded)) => { cellar::asyncrt::sleep(Duration::from_millis(10)).await; From f322a0225683f83873576fe374f7fa28dd78ad8b Mon Sep 17 00:00:00 2001 From: forhappy Date: Mon, 2 Nov 2026 05:53:50 -0700 Subject: [PATCH 28/44] test: isolate proof fixture lease writers --- tools/rpc/README.md | 3 +++ tools/rpc/host/proof/mod.rs | 7 +++++-- tools/rpc/host/proof/wait.rs | 5 ++++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/tools/rpc/README.md b/tools/rpc/README.md index bc34135a..c5ba342b 100644 --- a/tools/rpc/README.md +++ b/tools/rpc/README.md @@ -1801,6 +1801,9 @@ The suite runs all 16 caller/provider pairs at four boundaries across three authenticated nodes, reusing the native SDK's real log-transport and object-store fault gates. Distinct acknowledged warmup commands establish both replica members before any fault is armed. Lease renewal and reads remain independent of data proof. +Proof and proof-wait fixtures use the shared canonical FULL-synchronous bucket: +each node key has its own SQLite writer, while application data shares the +ordinary durable data store. Lease CAS and ordinary reads see the same records. | Point | Original handler at the barrier | Coordinator action | | --- | --- | --- | diff --git a/tools/rpc/host/proof/mod.rs b/tools/rpc/host/proof/mod.rs index 925ac75d..2396b435 100644 --- a/tools/rpc/host/proof/mod.rs +++ b/tools/rpc/host/proof/mod.rs @@ -41,7 +41,7 @@ mod faults; mod fault_tests; use anyhow::{ensure, Context as _, Result}; -use cellar::{bucket::Bucket, sdk::*}; +use cellar::sdk::*; use faults::{Gate, Peer, Release, Store}; use futures_util::FutureExt as _; use serde_json::{json, Value}; @@ -230,7 +230,10 @@ async fn case( // captured evidence no longer needs live cursors into this bounded log. app.audit.reset_after_node_drain(); let directory = tempfile::tempdir()?; - let bucket = Bucket::open_dev(&directory.path().join("fleet.sqlite"))?; + let bucket = crate::peer_faults::durable_fixture_bucket( + directory.path(), + &["proof-owner", "proof-ingress", "proof-replica"], + )?; let controls = Controls::default(); let (mut owner, owner_url) = fleet::start_controlled( &directory.path().join("owner"), diff --git a/tools/rpc/host/proof/wait.rs b/tools/rpc/host/proof/wait.rs index dc6a42f2..82eb15c5 100644 --- a/tools/rpc/host/proof/wait.rs +++ b/tools/rpc/host/proof/wait.rs @@ -91,7 +91,10 @@ async fn case_inner(app: Arc, client: &reqwest::Client, case: Case) - lease_trace.mark("proof-wait case start"); app.audit.reset_after_node_drain(); let directory = tempfile::tempdir()?; - let bucket = Bucket::open_dev(&directory.path().join("fleet.sqlite"))?; + let bucket = crate::peer_faults::durable_fixture_bucket( + directory.path(), + &["proof-owner", "proof-ingress", "proof-replica"], + )?; let controls = Controls::default(); let (owner, owner_url) = fleet::start_controlled( &directory.path().join("owner"), From d0c58ae7d7765cefb6e364b0e635959fc883e331 Mon Sep 17 00:00:00 2001 From: forhappy Date: Mon, 2 Nov 2026 07:30:23 -0700 Subject: [PATCH 29/44] test: start empty learner protocol standby --- crates/cellar/tests/support/raft_repair_native.rs | 11 ++++++++++- crates/cellar/tests/support/raft_standby_native.rs | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/crates/cellar/tests/support/raft_repair_native.rs b/crates/cellar/tests/support/raft_repair_native.rs index 507e6203..6a1928f8 100644 --- a/crates/cellar/tests/support/raft_repair_native.rs +++ b/crates/cellar/tests/support/raft_repair_native.rs @@ -453,7 +453,11 @@ impl Fixture { } pub(super) async fn learner_before_promotion() -> Result<()> { - let f = Fixture::new(false).await.context("start learner fixture")?; + let mut f = Fixture::new(false).await.context("start learner fixture")?; + // The advertised peer must own a real protocol listener even while its + // learner remains unactivated. Otherwise unrelated sends wait for transport + // timeouts while the test is trying to observe a committed abort. + let learner = standby::registered(&mut f).await?; f.fail_obsolete().await.context("stop obsolete voter")?; let started = f.begin().await?; let current = f.handles[f.primary].status().await?; @@ -484,6 +488,11 @@ pub(super) async fn learner_before_promotion() -> Result<()> { matches!(result, Err(CommandFailure::Unavailable)), "empty learner promoted: {result:?}" ); + ensure!(matches!( + learner.status().await, + Err(CommandFailure::UnsupportedCapability) + )); + assert_eq!(f.calls[3].load(Ordering::SeqCst), 0); assert_eq!( f.handles[f.primary].repair_status().await?.unwrap().phase, RepairPhase::Seeding diff --git a/crates/cellar/tests/support/raft_standby_native.rs b/crates/cellar/tests/support/raft_standby_native.rs index af7085c2..9ecbc1e7 100644 --- a/crates/cellar/tests/support/raft_standby_native.rs +++ b/crates/cellar/tests/support/raft_standby_native.rs @@ -58,7 +58,7 @@ pub(in super::super) async fn registered_standby_refuses_quorumless_s3_activatio f.cluster.close().await } -async fn registered(f: &mut Fixture) -> Result { +pub(super) async fn registered(f: &mut Fixture) -> Result { let service = f .cluster .join_standby( From 37f49ced63b31d379c9f47628e3db1df19242985 Mon Sep 17 00:00:00 2001 From: forhappy Date: Mon, 2 Nov 2026 07:48:41 -0700 Subject: [PATCH 30/44] test: start stale-plan peer and install rustfmt --- .github/workflows/native-sdk.yml | 2 +- crates/cellar/tests/support/raft_repair_native.rs | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/native-sdk.yml b/.github/workflows/native-sdk.yml index 59414f28..72214ee0 100644 --- a/.github/workflows/native-sdk.yml +++ b/.github/workflows/native-sdk.yml @@ -413,7 +413,7 @@ jobs: - uses: actions/checkout@v6 - name: Install Rust run: | - rustup toolchain install "$RUST_VERSION" --profile minimal + rustup toolchain install "$RUST_VERSION" --profile minimal --component rustfmt rustup default "$RUST_VERSION" - uses: actions/setup-python@v6 with: diff --git a/crates/cellar/tests/support/raft_repair_native.rs b/crates/cellar/tests/support/raft_repair_native.rs index 6a1928f8..a673b3f7 100644 --- a/crates/cellar/tests/support/raft_repair_native.rs +++ b/crates/cellar/tests/support/raft_repair_native.rs @@ -520,7 +520,8 @@ pub(super) async fn learner_before_promotion() -> Result<()> { } pub(super) async fn stale_plan_idempotence() -> Result<()> { - let f = Fixture::new(false).await?; + let mut f = Fixture::new(false).await?; + let _learner = standby::registered(&mut f).await?; f.fail_obsolete().await?; // Admission starts synchronously even while its returned future is unpolled. let lost = f.handles[f.primary].begin_repair(f.plan.clone(), Duration::from_secs(5)); From c953102582cb76b36f32a7acafaad43dcd7466f4 Mon Sep 17 00:00:00 2001 From: forhappy Date: Mon, 2 Nov 2026 07:48:41 -0700 Subject: [PATCH 31/44] test: isolate crash fixture authority writers --- tools/rpc/README.md | 5 +++++ tools/rpc/host/crash/mod.rs | 29 ++++++++++++++++++++++++++--- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/tools/rpc/README.md b/tools/rpc/README.md index c5ba342b..ebeae2f3 100644 --- a/tools/rpc/README.md +++ b/tools/rpc/README.md @@ -1666,6 +1666,11 @@ The surviving caller runs in a separate process with a distinct node directory and the same development bucket. Peer calls use the production signed transport with fleet durability enabled. +Both processes open the same `sqlite-full-per-node-key` namespace described +above. The owner's and successor's authority records each use their canonical +FULL-synchronous file, including after the owner is killed. Cell state, receipt +recovery and the one-second lease retain their normal durability requirements. + ```text before_handler admit RPC -> hold before business method -> SIGKILL owner diff --git a/tools/rpc/host/crash/mod.rs b/tools/rpc/host/crash/mod.rs index 8bbd675d..cfbd70ab 100644 --- a/tools/rpc/host/crash/mod.rs +++ b/tools/rpc/host/crash/mod.rs @@ -12,7 +12,12 @@ pub use gate::{Counter, Gate, Point}; use anyhow::{ensure, Context as _, Result}; use cellar::{bucket::Bucket, sdk::ShutdownMode}; use serde_json::{json, Value}; -use std::{collections::BTreeSet, path::PathBuf, sync::Arc, time::Duration}; +use std::{ + collections::BTreeSet, + path::{Path, PathBuf}, + sync::Arc, + time::Duration, +}; use super::{application::Composite, fleet}; @@ -23,13 +28,31 @@ pub struct Server { pub ready: PathBuf, } +fn durable_bucket(database: &Path) -> Result { + // The coordinator and the killed owner must reopen the same canonical + // namespace. Independent authority writes retain FULL sync without waiting + // behind the other node's application-data transaction. + ensure!( + database + .file_name() + .is_some_and(|name| name == "fleet.sqlite"), + "crash fixture requires its canonical fleet database" + ); + crate::peer_faults::durable_fixture_bucket( + database + .parent() + .context("crash fixture bucket directory")?, + &["crash-owner", "crash-next"], + ) +} + pub async fn serve(app: Arc, server: Server) -> Result { // Only the disposable crash-owner process installs this observer. Native // applications and the qualification coordinator keep their SQLite hooks. sqlite::install()?; app.crash .evidence_file(server.ready.with_file_name("barrier.json")); - let bucket = Bucket::open_dev(&server.bucket)?; + let bucket = durable_bucket(&server.bucket)?; let controls = app.durability.clone(); let (mut node, url) = fleet::start_controlled(&server.data, app, &server.name, bucket, Some(controls)).await?; @@ -98,7 +121,7 @@ async fn case( class: &'static str, ) -> Result { let directory = tempfile::tempdir()?; - let bucket = Bucket::open_dev(&directory.path().join("fleet.sqlite"))?; + let bucket = durable_bucket(&directory.path().join("fleet.sqlite"))?; let (mut owner, owner_url) = process::Owner::start(directory.path(), "crash-owner").await?; let worker = get(client, &owner_url, "/node/process").await?; owner.worker( From ff92f8fda2b2b1c940febe0bc2a06b12387794a0 Mon Sep 17 00:00:00 2001 From: forhappy Date: Mon, 2 Nov 2026 07:57:14 -0700 Subject: [PATCH 32/44] ci: drain build writes before Node restart tests On the same x64 runner, 100 fresh cases had restarted-write p99 142.579 ms and max 178.144 ms. Two drained cohorts (200 cases) retained p95 near 19 ms while reducing p99 to 20.498 ms and max to 21.267 ms. All cases passed. Prebuild the unchanged test artifact and sync its filesystem before the existing 100-case campaign. Preserve FULL sync, all assertions, and every timeout. Kernel writeback tuning added no gain and is excluded. --- .github/workflows/javascript-sdk.yml | 4 ++++ docs/19-testing-qualification.md | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/javascript-sdk.yml b/.github/workflows/javascript-sdk.yml index 6388ff43..97f9313a 100644 --- a/.github/workflows/javascript-sdk.yml +++ b/.github/workflows/javascript-sdk.yml @@ -260,6 +260,10 @@ jobs: .cellar/node-delivery/launch.sh --help export CELLAR_TEST_NODE_ENTRY="$PWD/.cellar/node-delivery/app/dist/app.js" export CELLAR_TEST_NODE_EXECUTABLE="$PWD/.cellar/node-delivery/bin/node" + cargo test --locked -p cellar-javascript --no-default-features --features node --test node_runtime --no-run + # Finish build writeback before measuring fresh FULL-sync fixtures. + # Keep their real fsyncs and the existing handler deadline unchanged. + sync -f "${CARGO_TARGET_DIR:-$PWD/target}" # Fresh stores and Node processes in every case. Stop at the first # failure; these repetitions never retry a failed request or test. for iteration in {1..100}; do diff --git a/docs/19-testing-qualification.md b/docs/19-testing-qualification.md index 13b53fd6..3c4611e0 100644 --- a/docs/19-testing-qualification.md +++ b/docs/19-testing-qualification.md @@ -42,7 +42,9 @@ reproducing its bounded native campaign locally. The packaged Node delivery restart campaign uses the same test-profile settings for its repeated cold-start fixture. Its 250 ms handler budget and all 100 fresh -store/process cases remain unchanged. +store/process cases remain unchanged. CI builds the test artifact before the +campaign and flushes completed build output to keep compiler writeback out of +the fixture's disk queue. Every fixture still performs its normal FULL syncs. The [shared durable fixture bucket](../tools/qualification/durable_bucket.rs) keeps node-record CAS independent of application-data writers using FULL-sync From 81ab324f50ea9a8dcae4336412c170b1c2a3b5a9 Mon Sep 17 00:00:00 2001 From: forhappy Date: Mon, 2 Nov 2026 08:23:43 -0700 Subject: [PATCH 33/44] ci: drain build writes before live RPC fixtures On x64, fresh lease-commit p95 was 1.101 ms with a 421.319 ms maximum. Three drained 128-case cohorts passed with p95 0.836-0.906 ms and maximum 1.845-2.212 ms. ARM reproduced a fresh 692.225 ms commit and failure; all three drained cohorts passed. Build the existing full-feature qualifier and flush completed compiler output before live campaigns. Retain every request/lease deadline, FULL sync, original command, and cleanup assertion. Kernel tuning did not consistently improve on drainage and is excluded. --- .github/workflows/typed-rpc.yml | 6 ++++++ docs/19-testing-qualification.md | 2 ++ 2 files changed, 8 insertions(+) diff --git a/.github/workflows/typed-rpc.yml b/.github/workflows/typed-rpc.yml index dd9a703d..132ad812 100644 --- a/.github/workflows/typed-rpc.yml +++ b/.github/workflows/typed-rpc.yml @@ -89,6 +89,12 @@ jobs: python3 -m unittest discover -s tools/perf -p test_python_fleet.py cargo clippy --locked -p cellar-rpc-qualification --all-targets --features full -- -D warnings cargo test --locked -p cellar-rpc-qualification --features full -- --test-threads=1 + - name: Finish build writeback before live qualification + run: | + cargo build --locked -p cellar-rpc-qualification --features full + # Preserve real FULL syncs and the one-second authority lease while + # keeping completed compiler writes out of the fixture's disk queue. + sync -f "$CARGO_TARGET_DIR" - name: Live portable Protobuf profile if: matrix.campaign == 'focused' run: | diff --git a/docs/19-testing-qualification.md b/docs/19-testing-qualification.md index 3c4611e0..a802c2db 100644 --- a/docs/19-testing-qualification.md +++ b/docs/19-testing-qualification.md @@ -45,6 +45,8 @@ for its repeated cold-start fixture. Its 250 ms handler budget and all 100 fresh store/process cases remain unchanged. CI builds the test artifact before the campaign and flushes completed build output to keep compiler writeback out of the fixture's disk queue. Every fixture still performs its normal FULL syncs. +The live RPC campaigns use the same build/writeback separation without changing +their one-second node leases, per-request deadlines or runtime cleanup checks. The [shared durable fixture bucket](../tools/qualification/durable_bucket.rs) keeps node-record CAS independent of application-data writers using FULL-sync From 2fd6fb2c4d38ad4abf15366c355c5f58c9c9af4b Mon Sep 17 00:00:00 2001 From: forhappy Date: Mon, 2 Nov 2026 08:45:15 -0700 Subject: [PATCH 34/44] ci: bound RPC runner background writeback A full FT matrix still encountered simultaneous 1.08-1.14 second COMMIT stalls on three independent authority files after the build flush; nodes correctly fenced. Use the previously measured 4/16 MiB dirty limits and 1s/500ms expiry/writeback policy on the disposable x64 RPC runner. Paired x64 cohorts reduced median commit p95 from 873 to 765 us (12.4%); the bounded full matrix passed with max commit 4.528 ms. Restore the original policy in an always-run step. Keep FULL sync, filesystem barriers, every lease/request deadline, matrix case and cleanup assertion unchanged. This is runner isolation, not production storage tuning. --- .github/workflows/typed-rpc.yml | 26 ++++++++++++++++++++++++-- docs/19-testing-qualification.md | 5 +++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/.github/workflows/typed-rpc.yml b/.github/workflows/typed-rpc.yml index 132ad812..13eace9f 100644 --- a/.github/workflows/typed-rpc.yml +++ b/.github/workflows/typed-rpc.yml @@ -92,8 +92,23 @@ jobs: - name: Finish build writeback before live qualification run: | cargo build --locked -p cellar-rpc-qualification --features full - # Preserve real FULL syncs and the one-second authority lease while - # keeping completed compiler writes out of the fixture's disk queue. + # Bound ongoing dirty writeback on this disposable x64 runner. A + # one-time flush alone still allowed >1s commits in the full matrix. + # FULL sync, barriers and the one-second authority lease stay enabled. + snapshot="$RUNNER_TEMP/cellar-rpc-writeback.conf" + : > "$snapshot" + for setting in dirty_background dirty; do + bytes=$(sysctl -n "vm.${setting}_bytes") + if [ "$bytes" -eq 0 ]; then + key="vm.${setting}_ratio" + else + key="vm.${setting}_bytes" + fi + printf '%s=%s\n' "$key" "$(sysctl -n "$key")" >> "$snapshot" + done + sysctl vm.dirty_expire_centisecs vm.dirty_writeback_centisecs >> "$snapshot" + sudo sysctl -w vm.dirty_background_bytes=4194304 vm.dirty_bytes=16777216 \ + vm.dirty_expire_centisecs=100 vm.dirty_writeback_centisecs=50 sync -f "$CARGO_TARGET_DIR" - name: Live portable Protobuf profile if: matrix.campaign == 'focused' @@ -142,6 +157,13 @@ jobs: python3 tools/rpc/check_python_mode.py --expected "$CELLAR_RPC_PYTHON_MODE" .cellar/rpc/all.json python3 tools/rpc/check_provenance.py .cellar/rpc/all.json python3 tools/rpc/check_provider_limits.py .cellar/rpc/all.json + - name: Restore runner writeback policy + if: always() + run: | + snapshot="$RUNNER_TEMP/cellar-rpc-writeback.conf" + if [ -f "$snapshot" ]; then + sudo sysctl -p "$snapshot" + fi - name: Retain lifecycle evidence if: always() uses: actions/upload-artifact@v6 diff --git a/docs/19-testing-qualification.md b/docs/19-testing-qualification.md index a802c2db..1e674a55 100644 --- a/docs/19-testing-qualification.md +++ b/docs/19-testing-qualification.md @@ -47,6 +47,11 @@ campaign and flushes completed build output to keep compiler writeback out of the fixture's disk queue. Every fixture still performs its normal FULL syncs. The live RPC campaigns use the same build/writeback separation without changing their one-second node leases, per-request deadlines or runtime cleanup checks. +Their disposable x64 Linux runners also bound ongoing dirty writeback to 4 MiB +in the background and 16 MiB overall, with one-second expiry and 500 ms writeback +intervals. This limits interference between buffered writes and FULL-sync +commits; it does not disable syncs or filesystem barriers. An always-run cleanup +step restores the runner's original writeback policy. The [shared durable fixture bucket](../tools/qualification/durable_bucket.rs) keeps node-record CAS independent of application-data writers using FULL-sync From 0425b3df2b578105dc00b992101070517de161b7 Mon Sep 17 00:00:00 2001 From: forhappy Date: Mon, 2 Nov 2026 12:15:22 -0700 Subject: [PATCH 35/44] fix(raft): yield unclaimed snapshot offers An ignored voter offer could hold the only source slot for a learner's entire deadline. Atomically retire unclaimed offers while protecting active transfers and retaining physical I/O reservations. Select partial-body faults from actual bulk bytes, verify incompressible SQL state after recovery, and match the native TCP_NODELAY policy so the proxy cannot add delayed-ACK stalls. Keep all deadlines and correctness assertions. --- crates/cellar/durability/raft/README.md | 7 ++ crates/cellar/durability/raft/host_seed.rs | 9 +- .../durability/raft/snapshot_codec_tests.rs | 90 ++++++++++++++++++- .../durability/raft/snapshot_transport.rs | 69 +++++++++++--- crates/cellar/tests/native_raft_snapshot.rs | 30 +++++-- crates/cellar/tests/support/raft_cluster.rs | 59 ++++++++---- crates/cellar/tests/support/raft_node.rs | 36 +++++++- 7 files changed, 252 insertions(+), 48 deletions(-) diff --git a/crates/cellar/durability/raft/README.md b/crates/cellar/durability/raft/README.md index a52b15c9..d2ce011a 100644 --- a/crates/cellar/durability/raft/README.md +++ b/crates/cellar/durability/raft/README.md @@ -1410,6 +1410,13 @@ supersedes the earlier source lease to that learner, whose outstanding stream retains its pins until bounded I/O exits. Identity/authentication failures remain refusals. +The single source slot distinguishes an offered snapshot from an authenticated +pull that has started. An authorized fresh seed request may retire an unclaimed +offer to another voter, which might have ignored an obsolete announcement. +Retiring an offer and claiming its first pull are mutually exclusive atomic +transitions. Active transfers to other peers stay protected, and every retained +physical owner keeps its source reservation through cancellation and I/O exit. + `snapshot_seed.rs` validates the original birth, exact learner binding, included configuration and barrier before sealing the physical image. The engine creates the mapping, snapshot, current configuration, bindings and applied marker in one diff --git a/crates/cellar/durability/raft/host_seed.rs b/crates/cellar/durability/raft/host_seed.rs index 779ce01d..5c32ca60 100644 --- a/crates/cellar/durability/raft/host_seed.rs +++ b/crates/cellar/durability/raft/host_seed.rs @@ -59,13 +59,14 @@ impl Driver { reply(&mut request, Err(CommandFailure::Unavailable)); return; } - // A fresh authenticated request supersedes an earlier transfer to the - // same physical learner. Old stream owners retain their pins until - // their bounded I/O exits; no unverified progress escapes either one. + // A valid seed request may release an unclaimed snapshot offer to an + // unrelated voter. Otherwise an ignored offer can hold the sole source + // slot for the seed's entire deadline. Active transfers to other peers + // remain protected; this learner's earlier transfer is superseded. if self .snapshot_source .as_ref() - .is_some_and(|source| source.targets(&request.scope, target.member)) + .is_some_and(|source| source.yield_for_seed(&request.scope, target.member)) { if let Some(source) = self.snapshot_source.take() { source.cancel(); diff --git a/crates/cellar/durability/raft/snapshot_codec_tests.rs b/crates/cellar/durability/raft/snapshot_codec_tests.rs index 516c6af7..4dd09b60 100644 --- a/crates/cellar/durability/raft/snapshot_codec_tests.rs +++ b/crates/cellar/durability/raft/snapshot_codec_tests.rs @@ -106,8 +106,7 @@ async fn prepared_owner_shutdown_joins_even_if_the_stop_waiter_was_never_polled( fixture.close().await } -#[tokio::test] -async fn obsolete_snapshot_routes_cancel_sources_without_releasing_live_owners() -> Result<()> { +async fn source_fixture() -> Result<(Fixture, GroupRoute)> { use cellar_logic::consensus::{ ClusterId, DiskLineage, GroupId, IncarnationId, NamespaceId, ProcessGeneration, }; @@ -156,7 +155,7 @@ async fn obsolete_snapshot_routes_cancel_sources_without_releasing_live_owners() .persist(cellar_logic::consensus::OperationId::new(1), vec![write]) .await?; - let mut route = GroupRoute { + let route = GroupRoute { scope: scope.clone(), schema_digest: [9; 32], revision: 1, @@ -186,6 +185,91 @@ async fn obsolete_snapshot_routes_cancel_sources_without_releasing_live_owners() ], }; fixture.transport.install_group(route.clone())?; + Ok((fixture, route)) +} + +fn source_offer(fixture: &Fixture, route: &GroupRoute) -> Result { + let group = fixture + .transport + .handle + .0 + .runtime + .log() + .group(&route.scope)?; + let package = + SnapshotPackage::capture_empty(&group, route.schema_digest, SnapshotLimits::default())?; + Ok(fixture.transport.serve_snapshot( + &route.scope, + 2, + package, + fixture.transport.reserve_snapshot_source()?, + )?) +} + +#[tokio::test] +async fn unclaimed_offer_yields_to_seed_without_releasing_retained_owners() -> Result<()> { + let (fixture, route) = source_fixture().await?; + let state = fixture.transport.handle.0.clone(); + let source = source_offer(&fixture, &route)?; + let delayed_pull = source.0.clone(); + ensure!(matches!( + fixture.transport.reserve_snapshot_source(), + Err(TransportError::Overloaded) + )); + // A voter may ignore an obsolete snapshot announcement. Its unclaimed + // offer must not occupy the only source slot for a learner's full deadline. + ensure!(source.yield_for_seed(&route.scope, MemberId::new(3)?)); + ensure!(source.finished(0)); + ensure!( + !delayed_pull.begin_pull(), + "a yielded offer admitted a pull" + ); + drop(source); + ensure!(state.snapshot_source_budget.usage().1 == 1); + ensure!(matches!( + fixture.transport.reserve_snapshot_source(), + Err(TransportError::Overloaded) + )); + drop(delayed_pull); + ensure!(state.snapshot_source_budget.usage() == (0, 0)); + let next = source_offer(&fixture, &route)?; + ensure!(!next.finished(0)); + next.cancel(); + ensure!(state.snapshot_source_budget.usage() == (0, 0)); + fixture.close().await +} + +#[tokio::test] +async fn claimed_offer_protects_other_peers_and_retains_cancelled_io_charge() -> Result<()> { + let (fixture, route) = source_fixture().await?; + let state = fixture.transport.handle.0.clone(); + let source = source_offer(&fixture, &route)?; + let active = source.0.clone(); + ensure!(active.begin_pull()); + ensure!(!source.yield_for_seed(&route.scope, MemberId::new(3)?)); + ensure!(!source.finished(0)); + ensure!(matches!( + fixture.transport.reserve_snapshot_source(), + Err(TransportError::Overloaded) + )); + // A fresh request for this same physical learner still supersedes its + // earlier transfer. The old physical owner keeps its reservation. + ensure!(source.yield_for_seed(&route.scope, MemberId::new(2)?)); + source.cancel(); + ensure!(active.is_finished()); + ensure!(!active.begin_pull()); + ensure!(state.snapshot_source_budget.usage().1 == 1); + drop(active); + ensure!(state.snapshot_source_budget.usage() == (0, 0)); + fixture.close().await +} + +#[tokio::test] +async fn obsolete_snapshot_routes_cancel_sources_without_releasing_live_owners() -> Result<()> { + let (fixture, mut route) = source_fixture().await?; + let state = fixture.transport.handle.0.clone(); + let scope = route.scope.clone(); + let group = state.runtime.log().group(&scope)?; for remove in [false, true] { let package = SnapshotPackage::capture_empty(&group, [9; 32], SnapshotLimits::default())?; let lease = fixture.transport.serve_snapshot( diff --git a/crates/cellar/durability/raft/snapshot_transport.rs b/crates/cellar/durability/raft/snapshot_transport.rs index 8fdfa15a..79a6a327 100644 --- a/crates/cellar/durability/raft/snapshot_transport.rs +++ b/crates/cellar/durability/raft/snapshot_transport.rs @@ -8,7 +8,7 @@ use crate::raft::completion::ApplyTicket; use crate::raft::snapshot::{PreparedSnapshot, SnapshotDownload, SnapshotLimits, SnapshotPackage}; use cellar_logic::consensus::MemberId; use sha2::{Digest, Sha256}; -use std::sync::{atomic::AtomicBool, Weak}; +use std::sync::{atomic::AtomicU8, Weak}; #[path = "snapshot_receive.rs"] mod receive; @@ -102,6 +102,10 @@ impl SourceReservation { } } +const SOURCE_OFFERED: u8 = 0; +const SOURCE_STREAMING: u8 = 1; +const SOURCE_FINISHED: u8 = 2; + pub(super) struct Source { id: u64, route: Arc, @@ -109,10 +113,32 @@ pub(super) struct Source { package: SnapshotPackage, digest: [u8; 32], expires: u64, - finished: AtomicBool, + phase: AtomicU8, _reservation: SourceReservation, } +impl Source { + fn finish(&self) { + self.phase.store(SOURCE_FINISHED, Ordering::Release); + } + + fn is_finished(&self) -> bool { + self.phase.load(Ordering::Acquire) == SOURCE_FINISHED + } + + fn begin_pull(&self) -> bool { + matches!( + self.phase.compare_exchange( + SOURCE_OFFERED, + SOURCE_STREAMING, + Ordering::AcqRel, + Ordering::Acquire, + ), + Ok(_) | Err(SOURCE_STREAMING) + ) + } +} + pub(crate) struct SnapshotSource(Arc); struct ActiveSource { @@ -122,17 +148,31 @@ struct ActiveSource { impl Drop for ActiveSource { fn drop(&mut self) { - self.source.finished.store(true, Ordering::Release); + self.source.finish(); } } impl SnapshotSource { - pub(crate) fn targets(&self, scope: &GroupScope, member: MemberId) -> bool { - self.0.peer.replica.scope == *scope && self.0.peer.replica.member == member + /// Yield an unused offer or supersede this learner's previous transfer. + /// Claiming a pull and yielding an offer are mutually exclusive. All Arc + /// owners still retain the physical reservation after cancellation. + pub(crate) fn yield_for_seed(&self, scope: &GroupScope, member: MemberId) -> bool { + if self.0.peer.replica.scope == *scope && self.0.peer.replica.member == member { + return true; + } + matches!( + self.0.phase.compare_exchange( + SOURCE_OFFERED, + SOURCE_FINISHED, + Ordering::AcqRel, + Ordering::Acquire, + ), + Ok(_) | Err(SOURCE_FINISHED) + ) } pub(crate) fn cancel(self) { - self.0.finished.store(true, Ordering::Release); + self.0.finish(); } pub fn id(&self) -> u64 { @@ -140,7 +180,7 @@ impl SnapshotSource { } pub fn finished(&self, now: u64) -> bool { - self.0.finished.load(Ordering::Acquire) || now >= self.0.expires + self.0.is_finished() || now >= self.0.expires } pub fn matches( @@ -276,7 +316,7 @@ impl RaftTransport { .mono_ms() .map_err(|_| TransportError::Closed)? .saturating_add(state.options.snapshot_timeout_ms), - finished: AtomicBool::new(false), + phase: AtomicU8::new(SOURCE_OFFERED), _reservation: reservation, }); // The host owns the body and expiry. The registry cannot extend either @@ -288,7 +328,7 @@ impl RaftTransport { // Route replacement may race source construction. Publishing before // rechecking closes the gap with install/remove's invalidation pass. state.invalidate_snapshot_routes(); - if source.finished.load(Ordering::Acquire) { + if source.is_finished() { return Err(TransportError::StalePeer); } Ok(SnapshotSource(source)) @@ -481,7 +521,7 @@ impl State { .and_then(|source| source.as_ref().and_then(Weak::upgrade)); if let Some(source) = source { if !self.current_route(&source.peer.replica.scope, &source.route, &source.peer) { - source.finished.store(true, Ordering::Release); + source.finish(); } } } @@ -493,7 +533,7 @@ impl State { .and_then(Weak::upgrade) .filter(|source| source.id == id) { - source.finished.store(true, Ordering::Release); + source.finish(); } } } @@ -592,7 +632,7 @@ impl State { if selected .as_ref() .is_some_and(|old| !Arc::ptr_eq(&old.source, &lease)) - || lease.finished.load(Ordering::Acquire) + || lease.is_finished() || (frame::decimal(&request.lease)? != 0 && frame::decimal(&request.lease)? != lease.id) || lease.peer.replica != recipient @@ -615,6 +655,9 @@ impl State { { return Err(TransportError::Identity); } + if !lease.begin_pull() { + return Err(TransportError::StalePeer); + } transfer.get_or_insert(request.transfer.clone()); selected.get_or_insert_with(|| ActiveSource { source: lease.clone(), @@ -662,7 +705,7 @@ impl State { .await?; offset += body.len(); if offset == lease.package.encoded_length() { - lease.finished.store(true, Ordering::Release); + lease.finish(); return Ok(()); } } diff --git a/crates/cellar/tests/native_raft_snapshot.rs b/crates/cellar/tests/native_raft_snapshot.rs index db2695fc..46e61105 100644 --- a/crates/cellar/tests/native_raft_snapshot.rs +++ b/crates/cellar/tests/native_raft_snapshot.rs @@ -751,7 +751,13 @@ async fn native_seed_drill_store( let calls: Vec<_> = (0..4).map(|_| Arc::new(AtomicUsize::new(0))).collect(); let definitions = calls .iter() - .map(|calls| public_node::counter(calls.clone(), gate.clone())) + .map(|calls| { + if interrupt { + public_node::counter_with_seed_payload(calls.clone(), gate.clone()) + } else { + public_node::counter(calls.clone(), gate.clone()) + } + }) .collect::>>()?; let mut namespace = native_snapshot_config(); namespace.schema_digest = definitions[0].schema; @@ -2068,14 +2074,16 @@ async fn native_snapshot_drill( // Exercise independent-group progress on the same node whose // snapshot download is held, rather than depending on election luck. let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + let mut placement_observed = vec![String::new(); healthy.len()]; + let mut transfer_observed = String::new(); tokio::time::timeout_at(deadline, async { loop { - if healthy[follower] + let target_status = healthy[follower] .status() .await - .context("recovering node healthy-group leadership status")? - .command_ready - { + .context("recovering node healthy-group leadership status")?; + placement_observed[follower] = format!("{target_status:?}"); + if target_status.command_ready { return Ok::<_, anyhow::Error>(()); } for (node, handle) in healthy.iter().enumerate() { @@ -2088,8 +2096,9 @@ async fn native_snapshot_drill( let status = handle.status().await.with_context(|| { format!("healthy-group transfer source status on node {node}") })?; + placement_observed[node] = format!("{status:?}"); if status.command_ready { - match handle + let result = handle .transfer_leadership( LeadershipTransfer { target: MemberId::new(follower as u64 + 1)?, @@ -2098,8 +2107,9 @@ async fn native_snapshot_drill( }, deadline.saturating_duration_since(tokio::time::Instant::now()), ) - .await - { + .await; + transfer_observed = format!("node {node}: {result:?}"); + match result { Ok(()) | Err( CommandFailure::Overloaded @@ -2117,7 +2127,9 @@ async fn native_snapshot_drill( } }) .await - .context("place healthy leader on snapshot recovery node")??; + .with_context(|| format!( + "place healthy leader on snapshot recovery node {follower}: statuses {placement_observed:?}, last transfer {transfer_observed}" + ))??; } let mut observed = [None, None, None]; let healthy_primary = tokio::time::timeout(Duration::from_secs(10), async { diff --git a/crates/cellar/tests/support/raft_cluster.rs b/crates/cellar/tests/support/raft_cluster.rs index 52526f67..9a3885e9 100644 --- a/crates/cellar/tests/support/raft_cluster.rs +++ b/crates/cellar/tests/support/raft_cluster.rs @@ -29,7 +29,7 @@ struct Fault { stopped: bool, hold_response: bool, hold_after: Option, - hold_connection: Option, + hold_once: bool, } #[derive(Clone, Copy, PartialEq, Eq)] @@ -45,6 +45,7 @@ struct Proxy { active: Arc, attempts: Arc, held_response_bytes: Arc, + held_connection: Arc, task: Option>>, } @@ -57,6 +58,7 @@ impl Drop for HeldResponses { self.0.send_modify(|fault| { fault.hold_response = false; fault.hold_after = None; + fault.hold_once = false; }); } } @@ -75,6 +77,7 @@ async fn forward_responses( mut writer: impl AsyncWrite + Unpin, mut faults: watch::Receiver, held: Arc, + selected: Arc, connection: usize, ) -> std::io::Result<()> { let mut buffer = [0; 16 * 1024]; @@ -88,7 +91,8 @@ async fn forward_responses( let mut counted = false; while offset < count { let mut fault = *faults.borrow_and_update(); - if fault.hold_connection.is_some_and(|held| held != connection) { + let chosen = selected.load(Ordering::SeqCst); + if fault.hold_once && chosen != 0 && chosen != connection { fault.hold_response = false; fault.hold_after = None; } @@ -100,6 +104,13 @@ async fn forward_responses( .map_or(count - offset, |after| after.saturating_sub(forwarded)) }; if allowance == 0 { + if fault.hold_once + && selected + .compare_exchange(0, connection, Ordering::SeqCst, Ordering::SeqCst) + .is_err_and(|chosen| chosen != connection) + { + continue; + } if !counted { held.fetch_add(count - offset, Ordering::SeqCst); counted = true; @@ -133,7 +144,9 @@ impl Proxy { let active = Arc::new(AtomicUsize::new(0)); let attempts = Arc::new(AtomicUsize::new(0)); let held_response_bytes = Arc::new(AtomicUsize::new(0)); + let held_connection = Arc::new(AtomicUsize::new(0)); let held = held_response_bytes.clone(); + let selected = held_connection.clone(); let counted = active.clone(); let attempted = attempts.clone(); let task = tokio::spawn(async move { @@ -146,10 +159,14 @@ impl Proxy { done = connections.join_next(), if !connections.is_empty() => { if let Some(done) = done { done?; } } accepted = listener.accept(), if connections.len() < 8 => { let (mut source, _) = accepted?; + // Match the native transport: fault injection must not + // add Nagle/delayed-ACK stalls to small snapshot chunks. + source.set_nodelay(true)?; let connection=attempted.fetch_add(1, Ordering::SeqCst)+1; if changes.borrow().blocked { continue; } let cut = changes.clone(); let held = held.clone(); + let selected = selected.clone(); let active = counted.clone(); active.fetch_add(1, Ordering::SeqCst); let guard = Active(active); connections.spawn(async move { @@ -162,14 +179,15 @@ impl Proxy { cellar::asyncrt::select_biased! { "test fault transitions close sockets before more bytes are forwarded"; _ = connection_cut(cut.clone()) => (), _ = async { - if let Ok(target) = TcpStream::connect(destination).await { - let (mut from_source, to_source) = source.split(); - let (from_target, mut to_target) = target.into_split(); - cellar::asyncrt::select! { - _ = tokio::io::copy(&mut from_source, &mut to_target) => (), - _ = forward_responses(from_target, to_source, cut, held, connection) => (), - } + let target = TcpStream::connect(destination).await?; + target.set_nodelay(true)?; + let (mut from_source, to_source) = source.split(); + let (from_target, mut to_target) = target.into_split(); + cellar::asyncrt::select! { + _ = tokio::io::copy(&mut from_source, &mut to_target) => (), + _ = forward_responses(from_target, to_source, cut, held, selected, connection) => (), } + Ok::<_, std::io::Error>(()) } => (), } }); @@ -190,6 +208,7 @@ impl Proxy { active, attempts, held_response_bytes, + held_connection, task: Some(task), }) } @@ -1029,9 +1048,15 @@ impl Cluster { .find(|(s, t, lane, _)| *s == source && *t == target && *lane == Link::Data) .map(|(_, _, _, p)| p) .ok_or_else(|| anyhow::anyhow!("seed data link missing"))?; + proxy.held_connection.store(0, Ordering::SeqCst); + proxy.held_response_bytes.store(0, Ordering::SeqCst); proxy.state.send_modify(|fault| { - fault.hold_connection = Some(2); - fault.hold_after = Some(4096); + // Grant metadata is bounded to 64 KiB plus its header and TLS. + // The seed fixture carries a verified 256-KiB SQL payload, so a + // response beyond 128 KiB is a real partial bulk body. Select it + // once regardless of how many authorization connections preceded it. + fault.hold_once = true; + fault.hold_after = Some(128 * 1024); }); Ok(proxy.held_response_bytes.clone()) } @@ -1044,7 +1069,10 @@ impl Cluster { .find(|(s, t, lane, _)| *s == source && *t == target && *lane == Link::Data) .map(|(_, _, _, proxy)| proxy) .ok_or_else(|| anyhow::anyhow!("data link not found"))?; - proxy.state.send_modify(|fault| fault.hold_response = true); + proxy.state.send_modify(|fault| { + fault.hold_response = true; + fault.hold_once = false; + }); Ok(HeldResponses(proxy.state.clone())) } @@ -1061,9 +1089,10 @@ impl Cluster { .find(|(s, t, lane, _)| *s == source && *t == target && *lane == Link::Data) .map(|(_, _, _, proxy)| proxy) .ok_or_else(|| anyhow::anyhow!("data link not found"))?; - proxy - .state - .send_modify(|fault| fault.hold_after = Some(after)); + proxy.state.send_modify(|fault| { + fault.hold_after = Some(after); + fault.hold_once = false; + }); Ok(HeldResponses(proxy.state.clone())) } diff --git a/crates/cellar/tests/support/raft_node.rs b/crates/cellar/tests/support/raft_node.rs index b2a6aa85..6fa8ada7 100644 --- a/crates/cellar/tests/support/raft_node.rs +++ b/crates/cellar/tests/support/raft_node.rs @@ -430,7 +430,7 @@ pub fn counter_with_blobs( Ok(()) }; if blobs { - counter_observed_with_blobs(observe, gate, true) + counter_definition(observe, gate, true, None) } else { counter_observed(observe, gate) } @@ -441,29 +441,57 @@ pub fn counter_observed( observe: impl Fn(&GroupScope) -> Result<()> + Send + Sync + 'static, gate: Arc, ) -> Result { - counter_observed_with_blobs(observe, gate, false) + counter_definition(observe, gate, false, None) } -fn counter_observed_with_blobs( +/// A bulk seed must exceed the bounded grant metadata, even after compression. +#[allow(dead_code)] // Shared support; only snapshot tests need the bulk payload. +pub fn counter_with_seed_payload( + calls: Arc, + gate: Arc, +) -> Result { + use sha2::Digest as _; + let mut bytes = Vec::with_capacity(256 * 1024); + for block in 0_u64..8192 { + bytes.extend_from_slice(&sha2::Sha256::digest(block.to_be_bytes())); + } + let observe = move |_: &GroupScope| { + calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + Ok(()) + }; + counter_definition(observe, gate, false, Some(bytes.into())) +} + +fn counter_definition( observe: impl Fn(&GroupScope) -> Result<()> + Send + Sync + 'static, gate: Arc, blobs: bool, + padding: Option>, ) -> Result { let mut class = RaftClass::new("Counter", 1, [21; 32])?; if blobs { class = class.immutable_blobs(RaftBlobLimits::default())?; } + let expected_padding = padding.clone(); let class=class .command("Increment",[22;32],move |context,input|{ observe(context.identity())?; let delta=i64::from_be_bytes(input.try_into()?); + if let Some(padding)=&padding { + context.sql().execute_batch("CREATE TABLE IF NOT EXISTS seed_payload(data BLOB NOT NULL)")?; + context.sql().execute("INSERT INTO seed_payload SELECT ?1 WHERE NOT EXISTS(SELECT 1 FROM seed_payload)",[padding.as_ref()])?; + } context.sql().execute_batch("CREATE TABLE IF NOT EXISTS counter(value INTEGER); INSERT INTO counter SELECT 0 WHERE NOT EXISTS(SELECT 1 FROM counter)")?; context.sql().execute("UPDATE counter SET value=value+?1",[delta])?; if context.command().sequence()==2 {gate.wait()?;} let value:i64=context.sql().query_row("SELECT value FROM counter",[],|row|row.get(0))?; Ok(value.to_be_bytes().to_vec()) })? - .query("Get",[23;32],|context,_|{ + .query("Get",[23;32],move |context,_|{ + if let Some(expected)=&expected_padding { + let actual:Vec=context.query_row("SELECT data FROM seed_payload",[],|row|row.get(0))?; + anyhow::ensure!(actual.as_slice()==expected.as_ref(),"seed payload changed"); + } let value:i64=context.query_row("SELECT value FROM counter",[],|row|row.get(0))?; Ok(value.to_be_bytes().to_vec()) })?; From 3e99a02a746e17aa12d20b474961258cc1dd0cc1 Mon Sep 17 00:00:00 2001 From: forhappy Date: Mon, 2 Nov 2026 14:32:10 -0700 Subject: [PATCH 36/44] test: isolate event fixture authority writers Shared data transactions delayed FULL-sync lease CAS past the one-second authority bound. Reuse the canonical fixture namespace so authority keys keep independent durable writers and coherent ordinary reads. The outage wraps the entire namespace before either owner starts. Keep metrics, fencing, pre-replay audit and closure assertions intact. Expose the exact expired-readiness stop error without changing its accepted outcomes. --- crates/cellar/tests/native_node.rs | 2 ++ crates/cellar/tests/support/event_owner_outage.rs | 7 ++++++- crates/cellar/tests/support/node_events.rs | 5 +++-- crates/cellar/tests/support/node_filesystem_capacity.rs | 5 +---- crates/cellar/tests/support/raft_restart_stop_native.rs | 4 +++- 5 files changed, 15 insertions(+), 8 deletions(-) diff --git a/crates/cellar/tests/native_node.rs b/crates/cellar/tests/native_node.rs index 3a741a29..d27eaf29 100644 --- a/crates/cellar/tests/native_node.rs +++ b/crates/cellar/tests/native_node.rs @@ -2,6 +2,8 @@ //! Full public node bootstrap on caller-owned executors and infrastructure. #![allow(clippy::disallowed_methods)] +#[path = "../../../tools/qualification/durable_bucket.rs"] +mod durable_bucket; #[allow(dead_code)] #[path = "support/html.rs"] mod fixture; diff --git a/crates/cellar/tests/support/event_owner_outage.rs b/crates/cellar/tests/support/event_owner_outage.rs index 843c4a63..0bb5ebf5 100644 --- a/crates/cellar/tests/support/event_owner_outage.rs +++ b/crates/cellar/tests/support/event_owner_outage.rs @@ -45,7 +45,12 @@ async fn subscription(client: &reqwest::Client, node: &Node) -> Result { async fn storage_isolated_event_owner_expires_and_recovers_before_replay() -> Result<()> { let directory = tempfile::tempdir()?; let old_core = Executor::new("isolated-event-core")?; - let bucket = cellar::bucket::Bucket::open_dev(&directory.path().join("fleet.sqlite"))?; + // Wrap the entire coherent namespace: isolation must refuse authority as + // well as data, while healthy successor CAS has its own FULL-sync writer. + let bucket = super::super::durable_bucket::open( + directory.path(), + &["isolated-event-owner", "successor-event-owner"], + )?; let (isolated, outage) = Outage::wrap(&bucket)?; let (mut config, services) = inputs( "isolated-event-owner", diff --git a/crates/cellar/tests/support/node_events.rs b/crates/cellar/tests/support/node_events.rs index f40a0905..4d3b5f7b 100644 --- a/crates/cellar/tests/support/node_events.rs +++ b/crates/cellar/tests/support/node_events.rs @@ -2428,7 +2428,7 @@ fn metric(snapshot: &cellar::sdk::EventMetricsSnapshot, name: &str) -> u64 { async fn event_metrics_export_tracks_real_work_and_keeps_hosts_isolated() -> Result<()> { let directory = tempfile::tempdir()?; let core = Executor::new("event-metrics-core")?; - let bucket = cellar::bucket::Bucket::open_dev(&directory.path().join("fleet.sqlite"))?; + let bucket = super::durable_bucket::open(directory.path(), &["event-metrics-owner"])?; let (config, services) = inputs( "event-metrics-owner", &directory.path().join("owner"), @@ -2439,8 +2439,9 @@ async fn event_metrics_export_tracks_real_work_and_keeps_hosts_isolated() -> Res .await?; let owner = Node::start(config, services, graph(1, false), applications("owner")?).await?; super::ready(&owner).await?; + let sibling_directory = tempfile::tempdir_in(directory.path())?; let sibling_bucket = - cellar::bucket::Bucket::open_dev(&directory.path().join("sibling-fleet.sqlite"))?; + super::durable_bucket::open(sibling_directory.path(), &["event-metrics-sibling"])?; let (config, services) = inputs( "event-metrics-sibling", &directory.path().join("sibling"), diff --git a/crates/cellar/tests/support/node_filesystem_capacity.rs b/crates/cellar/tests/support/node_filesystem_capacity.rs index de8e339a..8a6bbd68 100644 --- a/crates/cellar/tests/support/node_filesystem_capacity.rs +++ b/crates/cellar/tests/support/node_filesystem_capacity.rs @@ -1,7 +1,7 @@ // Copyright 2026 Cellar Project Authors. Licensed under Apache-2.0. //! Opt-in dense payload qualification through public SDK turns and bucket proof. -use super::{inputs, ready, response, Executor}; +use super::{durable_bucket, inputs, ready, response, Executor}; use anyhow::{ensure, Context as _, Result}; use cellar::sdk::{filesystem::*, *}; use serde_json::{json, Value}; @@ -12,9 +12,6 @@ use std::{ time::{Duration, Instant}, }; -#[path = "../../../../tools/qualification/durable_bucket.rs"] -mod durable_bucket; - const BLOCK: usize = 1 << 20; const PATCH: usize = 4096; diff --git a/crates/cellar/tests/support/raft_restart_stop_native.rs b/crates/cellar/tests/support/raft_restart_stop_native.rs index e7a59675..45e4761e 100644 --- a/crates/cellar/tests/support/raft_restart_stop_native.rs +++ b/crates/cellar/tests/support/raft_restart_stop_native.rs @@ -266,7 +266,9 @@ async fn fleet_restart_stop_renews_expired_readiness_against_current_sql() -> Re || matches!( result, Err(FleetCatalogError::Unconfirmed | FleetCatalogError::Closed) - ) + ), + "expired-readiness stop refusal: {:?}", + result.as_ref().err().map(ToString::to_string) ); let mut target = s.nodes.remove(p); joined(&mut target.node, &s.f).await?; From 41449d2a03f716e364c8bef661867e276714cb1d Mon Sep 17 00:00:00 2001 From: forhappy Date: Mon, 2 Nov 2026 16:53:36 -0700 Subject: [PATCH 37/44] test: reconcile recovery refusals Bound read-only proof reset and exact-request stop reconciliation by their existing deadlines. Preserve mutation attempt and closure checks. Prebuild dense qualification with optimized code, debug assertions, and overflow checks; drain build writeback before live proof. --- .github/workflows/native-sdk.yml | 7 ++++ crates/cellar-python/tests/bridge.py | 11 +++++- .../cellar-python/tests/support/durability.rs | 34 ++++++++++++++--- .../support/raft_restart_return_native.rs | 5 +-- .../tests/support/raft_restart_stop_native.rs | 38 +++++++++++++++---- 5 files changed, 76 insertions(+), 19 deletions(-) diff --git a/.github/workflows/native-sdk.yml b/.github/workflows/native-sdk.yml index 72214ee0..aae6cc4f 100644 --- a/.github/workflows/native-sdk.yml +++ b/.github/workflows/native-sdk.yml @@ -339,11 +339,18 @@ jobs: cargo clippy --workspace --exclude cellar-javascript --exclude cellar-rpc-qualification --all-targets --all-features --locked -- -D warnings - name: Qualify dense filesystem writes and cold recovery env: + # Match the native campaign while retaining its correctness checks. + CARGO_PROFILE_TEST_OPT_LEVEL: "1" + CARGO_PROFILE_TEST_DEBUG_ASSERTIONS: "true" + CARGO_PROFILE_TEST_OVERFLOW_CHECKS: "true" CELLAR_FS_CAPACITY_MIB: "10" CELLAR_FS_CAPACITY_REPORT: ${{ runner.temp }}/filesystem-dense-capacity.json RUST_LOG: 'warn,cellar::actor::effects=info,store[node_lease_store]=debug' run: | set -o pipefail + cargo test --locked -p cellar --all-features --test native_node --no-run + # Complete build writeback before the live durability qualification. + sync -f "${CARGO_TARGET_DIR:-$PWD/target}" cargo test --locked -p cellar --all-features --test native_node \ filesystem_dense_capacity_cold_recovery -- --ignored --nocapture \ 2>&1 | tee "$RUNNER_TEMP/filesystem-dense-capacity.log" diff --git a/crates/cellar-python/tests/bridge.py b/crates/cellar-python/tests/bridge.py index 69090077..d30d2018 100644 --- a/crates/cellar-python/tests/bridge.py +++ b/crates/cellar-python/tests/bridge.py @@ -319,7 +319,16 @@ async def wrong_context(): if request.path == "/library": # A real native extension (_hashlib) plus pure Python library imports. return Response.json(hashlib.sha256(b"cellar").hexdigest()) - return await ctx.namespace("COUNTERS").get_by_name("one").fetch(request) + cell = ctx.namespace("COUNTERS").get_by_name("one") + if request.path == "/proof/state": + try: + return await cell.fetch(request) + except Ambiguous as error: + # Only this fixed, read-only recovery route can be reconciled. A + # failed/reset callee never establishes a mutation's rollback. + assert error.code == "Ambiguous" and error.outcome == "unknown" + return Response.json({"code": error.code, "outcome": error.outcome}, status=503) + return await cell.fetch(request) class Counter: diff --git a/crates/cellar-python/tests/support/durability.rs b/crates/cellar-python/tests/support/durability.rs index ab31e1b2..287a8b88 100644 --- a/crates/cellar-python/tests/support/durability.rs +++ b/crates/cellar-python/tests/support/durability.rs @@ -16,12 +16,23 @@ async fn restored_state(client: &reqwest::Client, url: &str) -> Result { if status.is_success() { return Ok::<_, anyhow::Error>(serde_json::from_str(&body)?); } - ensure!( - status.is_server_error() - && body.contains("cellar output-gate durability proof failed for ") - && body.contains("ltx durability timed out for "), - "restored proof read failed outside the pending proof reset: {status}: {body}" - ); + if status == reqwest::StatusCode::SERVICE_UNAVAILABLE { + // Python preserves the native refusal on this read-only route. + // It is an unknown outcome, never rollback or an acknowledged + // mutation. Reject unrelated errors instead of retrying a 500. + ensure!( + serde_json::from_str::(&body)? + == json!({"code":"Ambiguous","outcome":"unknown"}), + "recovery read lost its native refusal: {body}" + ); + } else { + ensure!( + status.is_server_error() + && body.contains("cellar output-gate durability proof failed for ") + && body.contains("ltx durability timed out for "), + "restored proof read failed outside the pending proof reset: {status}: {body}" + ); + } tokio::time::sleep(Duration::from_millis(20)).await; } }) @@ -198,6 +209,17 @@ pub(super) fn child() -> Result<()> { "an unproven mutation handler was blindly replayed: {attempts}" ); let state = restored_state(&client, &url).await?; + let restored_attempts: Value = client + .get(format!("{url}/proof/attempts")) + .send() + .await? + .error_for_status()? + .json() + .await?; + ensure!( + restored_attempts == attempts, + "recovery read replayed a mutation: {restored_attempts}" + ); ensure!( state["sync"] == 1 && state["output"] == 1, "restored proof access lost previously acknowledged state: {state}" diff --git a/crates/cellar/tests/support/raft_restart_return_native.rs b/crates/cellar/tests/support/raft_restart_return_native.rs index f6e692fc..59e5fd45 100644 --- a/crates/cellar/tests/support/raft_restart_return_native.rs +++ b/crates/cellar/tests/support/raft_restart_return_native.rs @@ -13,10 +13,7 @@ async fn offline(s: &mut Setup, p: usize) -> Result<(Running, FleetRestartReturn .await?; trust(s).await?; let cut = s.f.catalog.read(TIMEOUT).await?; - let result = - s.f.catalog - .stop_restart_maintenance(&cut, &request, &s.spare, TIMEOUT) - .await; + let result = submit_stop(s, &s.f.catalog, &cut, &request).await?; ensure!( result.is_ok() || matches!( diff --git a/crates/cellar/tests/support/raft_restart_stop_native.rs b/crates/cellar/tests/support/raft_restart_stop_native.rs index 45e4761e..a881dd4d 100644 --- a/crates/cellar/tests/support/raft_restart_stop_native.rs +++ b/crates/cellar/tests/support/raft_restart_stop_native.rs @@ -6,7 +6,7 @@ use super::*; mod rejoin; use cellar::application::consensus::verification::{ FleetChange, FleetMutation, FleetRestartOfflineRequest, FleetRestartReadyRequest, - FleetRestartStopRequest, FleetSnapshot, FleetTable, + FleetRestartStopAdmission, FleetRestartStopRequest, FleetSnapshot, FleetTable, }; async fn consent( @@ -77,6 +77,33 @@ async fn joined(node: &mut cellar::sdk::Node, f: &Fixture) -> Result<()> { Ok(()) } +// Overload is a retryable observation, not proof of stop or closure. Reuse the +// exact admitted request within the original stop-call budget; callers still +// validate the final verdict and join the actual process below. +async fn submit_stop( + s: &Setup, + catalog: &FleetCatalog, + snapshot: &FleetSnapshot, + request: &FleetRestartStopRequest, +) -> Result> { + s.f.domain + .run(cellar::asyncrt::timeout(TIMEOUT, async { + loop { + match catalog + .stop_restart_maintenance(snapshot, request, &s.spare, TIMEOUT) + .await + { + Err(FleetCatalogError::Overloaded) => { + cellar::asyncrt::sleep(Duration::from_millis(10)).await; + } + result => return result, + } + } + })) + .await? + .context("maintenance stop submission deadline") +} + async fn stop_and_confirm(primary_target: bool, lose_replies: bool) -> Result<()> { let mut s = Box::pin(Setup::new()).await?; let leader = primary(&s.f, &s.cells).await?; @@ -123,9 +150,7 @@ async fn stop_and_confirm(primary_target: bool, lose_replies: bool) -> Result<() )); ensure!(catalog.read(TIMEOUT).await?.revision() == cut.revision()); trust(&s).await?; - let outcome = catalog - .stop_restart_maintenance(&cut, &request, &s.spare, TIMEOUT) - .await; + let outcome = submit_stop(&s, &catalog, &cut, &request).await?; ensure!( outcome.is_ok() || matches!( @@ -257,10 +282,7 @@ async fn fleet_restart_stop_renews_expired_readiness_against_current_sql() -> Re s.f.domain .run(async { cellar::asyncrt::sleep(Duration::from_millis(5001)).await }) .await?; - let result = - s.f.catalog - .stop_restart_maintenance(&root, &request, &s.spare, TIMEOUT) - .await; + let result = submit_stop(&s, &s.f.catalog, &root, &request).await?; ensure!( result.is_ok() || matches!( From 25ff1616aa6c3e1ae8df0b36743ca92166c60113 Mon Sep 17 00:00:00 2001 From: forhappy Date: Mon, 2 Nov 2026 19:16:09 -0700 Subject: [PATCH 38/44] fix: refuse shutdown work after domain closure Return cancellation when forced closure refuses listener or drain-token admission. Keep listeners owned and joined; never fabricate graceful proof. The old closed-domain panic reproduces. Candidate boundary, 968 library, 109 native-node and 200 unchanged management receipt cases pass, together with strict Clippy and repository checks. --- crates/cellar/node/driver.rs | 43 +++++++++---- crates/cellar/node/mod.rs | 14 ++-- crates/cellar/node/shutdown_acceptor_tests.rs | 64 +++++++++++++++++++ 3 files changed, 106 insertions(+), 15 deletions(-) create mode 100644 crates/cellar/node/shutdown_acceptor_tests.rs diff --git a/crates/cellar/node/driver.rs b/crates/cellar/node/driver.rs index 0a3d5792..ce44de35 100644 --- a/crates/cellar/node/driver.rs +++ b/crates/cellar/node/driver.rs @@ -568,37 +568,55 @@ where shutdown_accept_failure_test_active, "listener failure armed", ); + // Forced closure can win during this poll. Refused shutdown work means + // cancellation and must never produce a graceful completion proof. #[cfg(all(test, cellar_internal_tests))] { if let Some(listener) = listener { - spawn_shutdown_acceptor( + if spawn_shutdown_acceptor( listener, HttpSurface::Public, shutdown_connection_tx.clone(), shutdown_accept_failure_test_active, - ); + ) + .is_err() + { + return Ok(NodeOutcome::Cancelled); + } } - spawn_shutdown_acceptor( + if spawn_shutdown_acceptor( internal_listener, HttpSurface::Internal, shutdown_connection_tx, false, - ); + ) + .is_err() + { + return Ok(NodeOutcome::Cancelled); + } } #[cfg(not(all(test, cellar_internal_tests)))] { if let Some(listener) = listener { - spawn_shutdown_acceptor( + if spawn_shutdown_acceptor( listener, HttpSurface::Public, shutdown_connection_tx.clone(), - ); + ) + .is_err() + { + return Ok(NodeOutcome::Cancelled); + } } - spawn_shutdown_acceptor( + if spawn_shutdown_acceptor( internal_listener, HttpSurface::Internal, shutdown_connection_tx, - ); + ) + .is_err() + { + return Ok(NodeOutcome::Cancelled); + } } let mut drain_token_hold: Option = None; let (admitted_work_drained, ingress_drained_before_cut) = { @@ -617,7 +635,7 @@ where let baseline_ownership = ready_ownership.clone(); let wait = std::time::Duration::from_millis(drain_token_wait_ms) .min(handoff_deadline.saturating_duration_since(tokio::time::Instant::now())); - crate::asyncrt::spawn(async move { + let task = crate::asyncrt::try_spawn(async move { let now_ms = crate::ownership_store::now_ms(); let restoration_baseline = match baseline_ownership { Some(ownership) => match ownership.read_capacity_peers().await { @@ -662,8 +680,11 @@ where ) .await; *slot.lock().expect("drain token slot") = Some(outcome); - }) - .detach(); + }); + match task { + Ok(task) => task.detach(), + Err(_) => return Ok(NodeOutcome::Cancelled), + } } _ => { *drain_token.lock().expect("drain token slot") = diff --git a/crates/cellar/node/mod.rs b/crates/cellar/node/mod.rs index acb7d5ca..1a2834ec 100644 --- a/crates/cellar/node/mod.rs +++ b/crates/cellar/node/mod.rs @@ -72,6 +72,9 @@ pub(crate) mod operations; mod raft_metrics; mod raft_startup; mod rpc_deadline; +#[cfg(test)] +#[path = "shutdown_acceptor_tests.rs"] +mod shutdown_acceptor_tests; mod source_operator; mod state_sampling; mod stream_operator; @@ -3841,11 +3844,13 @@ fn spawn_shutdown_acceptor( surface: HttpSurface, connections: mpsc::Sender, #[cfg(all(test, cellar_internal_tests))] mut fail_next_for_test: bool, -) { +) -> Result<(), crate::asyncrt::DomainClosed> { // Shutdown selects finish whenever a timer, request, or actor message // wins. Keep each listener's accept and retry waits in one task so those // unrelated events cannot cancel a wait or reset its retry delay. - crate::asyncrt::spawn(async move { + // Forced closure can race this supervisor poll. Refuse task admission and + // drop the captured listener instead of panicking or detaching unowned work. + let task = crate::asyncrt::try_spawn(async move { let mut retry_delay = SHUTDOWN_ACCEPT_RETRY_INITIAL; let mut recovering = false; #[cfg(all(test, cellar_internal_tests))] @@ -3922,8 +3927,9 @@ fn spawn_shutdown_acceptor( } } } - }) - .detach(); + })?; + task.detach(); + Ok(()) } fn serve_http_connection( diff --git a/crates/cellar/node/shutdown_acceptor_tests.rs b/crates/cellar/node/shutdown_acceptor_tests.rs new file mode 100644 index 00000000..834058d5 --- /dev/null +++ b/crates/cellar/node/shutdown_acceptor_tests.rs @@ -0,0 +1,64 @@ +// Copyright 2026 Cellar Project Authors. Licensed under Apache-2.0. + +//! Forced closure can arrive during the supervisor's current shutdown poll. +#![allow(clippy::disallowed_methods)] +use super::*; +use futures_util::FutureExt as _; + +#[tokio::test] +async fn closed_shutdown_acceptor_releases_its_listener_without_panicking() { + let domain = Arc::new(crate::asyncrt::ExecutionDomain::new( + tokio::runtime::Handle::current(), + )); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let (connections, mut received) = mpsc::channel(1); + let closing = domain.clone(); + let outcome = std::panic::AssertUnwindSafe(domain.run(async move { + // No await separates closure from admission. This reproduces closure + // racing a supervisor poll which already passed its cancellation check. + closing.close(); + #[cfg(not(cellar_internal_tests))] + let _ = spawn_shutdown_acceptor(listener, HttpSurface::Internal, connections); + #[cfg(cellar_internal_tests)] + let _ = spawn_shutdown_acceptor(listener, HttpSurface::Internal, connections, false); + assert!(matches!( + received.try_recv(), + Err(mpsc::error::TryRecvError::Disconnected) + )); + let rebound = std::net::TcpListener::bind(address).unwrap(); + drop(rebound); + })) + .catch_unwind() + .await; + domain.shutdown().await; + assert!(outcome.is_ok(), "shutdown admission panicked after closure"); + assert!(domain.is_closed()); + assert_eq!(tokio::spawn(async { 7 }).await.unwrap(), 7); +} + +#[tokio::test] +async fn open_shutdown_acceptor_delivers_its_owned_connection_and_joins() { + let domain = Arc::new(crate::asyncrt::ExecutionDomain::new( + tokio::runtime::Handle::current(), + )); + let outcome = domain + .run(async { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let (connections, mut received) = mpsc::channel(1); + #[cfg(not(cellar_internal_tests))] + let _ = spawn_shutdown_acceptor(listener, HttpSurface::Internal, connections); + #[cfg(cellar_internal_tests)] + let _ = spawn_shutdown_acceptor(listener, HttpSurface::Internal, connections, false); + let client = tokio::net::TcpStream::connect(address).await.unwrap(); + let connection = received.recv().await.unwrap(); + assert_eq!(connection.surface.name(), "internal"); + assert!(connection.received.send(()).is_ok()); + drop(connection.stream); + drop(client); + }) + .await; + domain.shutdown().await; + outcome.unwrap(); +} From 15563b31fc9f76923726807009c4a9b12bc64de3 Mon Sep 17 00:00:00 2001 From: forhappy Date: Mon, 2 Nov 2026 19:16:21 -0700 Subject: [PATCH 39/44] test: isolate saturation node-key writers Reuse the coherent FULL-sync namespace so node lease CAS writes cannot queue behind application-data SQLite transactions. Keep held-result, cleanup, lease and resource assertions unchanged. Both full RPC ABI matrices pass. The same 144 saturation cases complete 43% faster with regular Python and 20% faster with free-threaded Python across the retained comparisons; this is qualification campaign throughput. --- tools/rpc/host/saturation/mod.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tools/rpc/host/saturation/mod.rs b/tools/rpc/host/saturation/mod.rs index 766ddc2f..3e8f65bb 100644 --- a/tools/rpc/host/saturation/mod.rs +++ b/tools/rpc/host/saturation/mod.rs @@ -5,7 +5,7 @@ mod gate; mod matrix; use anyhow::{ensure, Context as _, Result}; -use cellar::{bucket::Bucket, sdk::*}; +use cellar::sdk::*; use serde_json::{json, Value}; use std::{sync::Arc, time::Duration}; @@ -118,7 +118,10 @@ async fn request(client: &reqwest::Client, base: &str, path: &str) -> Result, remote: bool, limit: Limit, side: Side) -> Result { app.audit.reset_after_node_drain(); let directory = tempfile::tempdir()?; - let bucket = Bucket::open_dev(&directory.path().join("fleet.sqlite"))?; + // Saturate RPC resources without serializing independent node authority + // behind application-object writes. Every logical client still observes + // the same canonical FULL-synchronous records. + let bucket = crate::peer_faults::durable_fixture_bucket(directory.path(), &[SOURCE, OWNER])?; let client = reqwest::Client::builder() .timeout(Duration::from_secs(20)) .build()?; From e2715ad695b6e1ad6dc666fbc4b3d57661cf0402 Mon Sep 17 00:00:00 2001 From: forhappy Date: Mon, 2 Nov 2026 19:16:21 -0700 Subject: [PATCH 40/44] ci: drain build writes before Axum qualification Prebuild the no-default-features artifact and finish build writeback before the original one-second lease checks. Keep test concurrency, assertions, feature commands and timeout budgets unchanged. Ten alternating full-package runs pass; sampled lease-write commit p95 falls from 7.25 ms to 3.91 ms. All-features tests and strict Clippy pass. --- .github/workflows/native-sdk.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/native-sdk.yml b/.github/workflows/native-sdk.yml index aae6cc4f..a2dca52d 100644 --- a/.github/workflows/native-sdk.yml +++ b/.github/workflows/native-sdk.yml @@ -365,6 +365,9 @@ jobs: if-no-files-found: ignore - name: Test Axum feature modes and generated contract run: | + cargo test -p cellar-axum --no-default-features --locked --no-run + # Keep compiler writeback outside the live one-second lease checks. + sync -f "${CARGO_TARGET_DIR:-$PWD/target}" cargo test -p cellar-axum --no-default-features --locked cargo check -p cellar-axum-counter --no-default-features --bins --locked cargo run -p cellar-axum-counter --bin export-openapi --locked -- "$RUNNER_TEMP/counter-openapi.json" From 935bb7233c6ea33ff1dc61fa16e23e7305d05d7a Mon Sep 17 00:00:00 2001 From: forhappy Date: Mon, 2 Nov 2026 19:25:22 -0700 Subject: [PATCH 41/44] test: identify isolated repair refusal phases Report setup, activation, capacity, observation, repair and join context without changing requests, retries, assertions or timeout budgets. The original Overloaded refusal did not reproduce in the instrumented 292-case fleet suite or 100 fresh minority/majority repair cases. Retain phase context so the complete SDK campaign can identify a recurrence. --- .../support/raft_isolated_repair_service.rs | 86 ++++++++++++++----- 1 file changed, 66 insertions(+), 20 deletions(-) diff --git a/crates/cellar/tests/support/raft_isolated_repair_service.rs b/crates/cellar/tests/support/raft_isolated_repair_service.rs index efdb19f3..fa6b9b74 100644 --- a/crates/cellar/tests/support/raft_isolated_repair_service.rs +++ b/crates/cellar/tests/support/raft_isolated_repair_service.rs @@ -60,8 +60,13 @@ impl RepairFleet { } async fn with_admission(extra: bool, isolated: Option, histories: u8) -> Result { - let mut s = Box::pin(Setup::with_histories(true, 6, histories)).await?; - let (spare_root, spare_node, spare) = s.f.peer_host(8).await?; + let mut s = Box::pin(Setup::with_histories(true, 6, histories)) + .await + .context("construct isolated-repair resource fixture")?; + let (spare_root, spare_node, spare) = + s.f.peer_host(8) + .await + .context("start isolated-repair spare host")?; if extra { let mut proposal = s.proposals[0].clone(); proposal.scope.cell = "Counter:independent-majority".into(); @@ -133,11 +138,19 @@ impl RepairFleet { }; cells.push( host.register_managed_history(authorization.clone(), TIMEOUT) - .await?, + .await + .with_context(|| { + format!("register isolated-repair voter for {}", proposal.scope.cell) + })?, ); nodes.push(member.node); } - activate(&cells, &authorization).await?; + activate(&cells, &authorization).await.with_context(|| { + format!( + "activate isolated-repair voters for {}", + proposal.scope.cell + ) + })?; write_read(&s.f, &cells, 1, 7, 7).await.with_context(|| { format!( "initial SQL for isolated-repair history {}", @@ -147,7 +160,9 @@ impl RepairFleet { live.push(nodes.into_iter().zip(cells).collect::>()); } for (index, host) in s.nodes.iter().map(|n| &n.host).chain([&spare]).enumerate() { - let evidence = crate::hosts::capacity_tests::attestation(&s.f, &s.target, host).await?; + let evidence = crate::hosts::capacity_tests::attestation(&s.f, &s.target, host) + .await + .with_context(|| format!("attest isolated-repair host {index}"))?; let cut = s.f.catalog.read(TIMEOUT).await?; s.f.catalog .admit_node_capacity( @@ -161,7 +176,8 @@ impl RepairFleet { std::slice::from_ref(&s.namespace), TIMEOUT, ) - .await?; + .await + .with_context(|| format!("admit isolated-repair host capacity {index}"))?; } discover(&s.f, &hosts).await?; // Exercise a failed leader, rather than only a failed follower. @@ -178,7 +194,8 @@ impl RepairFleet { }, TIMEOUT, ) - .await?; + .await + .context("transfer initial isolated-repair leader")?; } s.f.domain .run(cellar::asyncrt::timeout(TIMEOUT, async { @@ -193,7 +210,8 @@ impl RepairFleet { let boot = old .host .observe_node(TIMEOUT) - .await? + .await + .context("observe failed isolated-repair leader")? .fleet_boot .context("failed leader boot")?; let old_engine = old.node.info().data_dir.join("raft"); @@ -245,7 +263,12 @@ impl RepairFleet { .chain([&self.s.target, &self.spare]) { hosts.push(( - host.observe_node(TIMEOUT).await?.node.identity.node, + host.observe_node(TIMEOUT) + .await + .context("observe isolated-repair inventory host")? + .node + .identity + .node, host.clone(), )); } @@ -291,7 +314,13 @@ impl RepairFleet { .map(|(_, c)| c.clone()) .collect::>(); for host in [&self.s.target, &self.spare] { - let node = host.observe_node(TIMEOUT).await?.node.identity.node; + let node = host + .observe_node(TIMEOUT) + .await + .context("observe post-repair host identity")? + .node + .identity + .node; if self.live[index].iter().any(|(n, _)| *n == node) { continue; } @@ -310,7 +339,8 @@ impl RepairFleet { for cell in &cells { let proof = cell .observe_fleet_membership(authorization.clone(), TIMEOUT) - .await?; + .await + .with_context(|| format!("observe repaired membership for {}", scope.cell))?; ensure!( proof.configuration().voters().len() == 3 && proof.configuration().outgoing_voters().is_empty() @@ -604,12 +634,19 @@ async fn fleet_isolated_repair_service_skips_minority_and_repairs_independent_ma } async fn minority() -> Result<()> { - let mut fleet = Box::pin(RepairFleet::new(true)).await?; + let mut fleet = Box::pin(RepairFleet::new(true)) + .await + .context("construct minority isolated-repair fleet")?; let second = fleet.s.nodes.remove(0); ensure!(second.spec.node == NodeId::new([5; 16])?); let second_engine = second.node.info().data_dir.join("raft"); - joined(second.node).await?; - let evidence = fleet.evidence().await?; + joined(second.node) + .await + .context("join second failed minority voter")?; + let evidence = fleet + .evidence() + .await + .context("collect minority isolated-repair evidence")?; let hosts = fleet .s .nodes @@ -617,7 +654,9 @@ async fn minority() -> Result<()> { .map(|n| &n.host) .chain([&fleet.s.target, &fleet.spare]) .collect::>(); - discover(&fleet.s.f, &hosts).await?; + discover(&fleet.s.f, &hosts) + .await + .context("discover surviving minority-repair hosts")?; let service = fleet .s .f @@ -629,7 +668,8 @@ async fn minority() -> Result<()> { ProposalId::new([100; 16])?, options(), ) - .await?; + .await + .context("start minority isolated-repair service")?; fleet .wait(&service, |s| { s.settled_movements == 1 @@ -638,7 +678,8 @@ async fn minority() -> Result<()> { r.inventory_exhausted && r.blocked_histories == 3 && r.histories_checked == 4 }) }) - .await?; + .await + .context("settle independent-majority repair")?; service.shutdown().await; let cut = fleet.s.f.catalog.read(TIMEOUT).await?; for proposal in &fleet.s.proposals[..3] { @@ -655,9 +696,14 @@ async fn minority() -> Result<()> { .await? .is_none()); } - fleet.verify(3).await?; - retained_engine(&fleet.s.f, second_engine).await?; - fleet.close().await + fleet + .verify(3) + .await + .context("verify independently repaired majority")?; + retained_engine(&fleet.s.f, second_engine) + .await + .context("verify retained minority engine")?; + fleet.close().await.context("join minority-repair fleet") } async fn retained_engine(f: &Fixture, path: std::path::PathBuf) -> Result<()> { From bd4bac23e6b31b4987900c3486279fdc15630907 Mon Sep 17 00:00:00 2001 From: forhappy Date: Mon, 2 Nov 2026 22:03:38 -0700 Subject: [PATCH 42/44] test: identify isolated removal refusal phases --- .../tests/support/raft_isolated_removal.rs | 73 +++++++++++++------ 1 file changed, 52 insertions(+), 21 deletions(-) diff --git a/crates/cellar/tests/support/raft_isolated_removal.rs b/crates/cellar/tests/support/raft_isolated_removal.rs index 03ebebe8..353efce6 100644 --- a/crates/cellar/tests/support/raft_isolated_removal.rs +++ b/crates/cellar/tests/support/raft_isolated_removal.rs @@ -33,9 +33,15 @@ struct Repaired { impl Repaired { async fn new() -> Result { - let mut s = Box::pin(Started::new()).await?; - let (namespace, _) = initial_birth(&s.f).await?; - write_read(&s.f, &s.cells, 1, 7, 7).await?; + let mut s = Box::pin(Started::new()) + .await + .context("start isolated removal fixture")?; + let (namespace, _) = initial_birth(&s.f) + .await + .context("initialize isolated removal history")?; + write_read(&s.f, &s.cells, 1, 7, 7) + .await + .context("verify state before isolated removal")?; let p = primary(&s.f, &s.cells).await?; let obsolete = (p + 1) % 3; let evidence = attestation(&s.f, &s.nodes[p].host, &s.observer).await?; @@ -52,7 +58,8 @@ impl Repaired { std::slice::from_ref(&namespace), TIMEOUT, ) - .await?; + .await + .context("admit capacity before isolated replacement")?; let old = s.nodes.remove(obsolete); let boot = old .host @@ -60,7 +67,9 @@ impl Repaired { .await? .fleet_boot .context("old boot")?; - joined(old.node).await?; + joined(old.node) + .await + .context("join obsolete owner before isolation")?; let cut = s.f.catalog.read(TIMEOUT).await?; s.f.catalog .begin_evacuation( @@ -70,7 +79,8 @@ impl Repaired { std::slice::from_ref(&namespace), TIMEOUT, ) - .await?; + .await + .context("begin evacuation of joined obsolete owner")?; let cut = s.f.catalog.read(TIMEOUT).await?; s.f.catalog .attest_node_isolation( @@ -85,7 +95,8 @@ impl Repaired { }, TIMEOUT, ) - .await?; + .await + .context("attest isolation of joined obsolete owner")?; let mut active = s .cells .iter() @@ -141,16 +152,19 @@ impl Repaired { std::slice::from_ref(&namespace), TIMEOUT, ) - .await?; + .await + .context("reserve isolated replacement movement")?; let permit = admitted.authorization(); let target = s .observer .register_managed_history(s.authorization.clone(), TIMEOUT) - .await?; + .await + .context("register isolated replacement history")?; discover(&s.f, &[&s.nodes[0].host, &s.nodes[1].host, &s.observer]).await?; active[p] .begin_managed_repair(permit.clone(), movement.plan.clone(), TIMEOUT) - .await?; + .await + .context("begin isolated replacement repair")?; let source_member = s .proposal .voters @@ -169,7 +183,8 @@ impl Repaired { timeout: TIMEOUT, }, ) - .await?; + .await + .context("activate isolated replacement from archive")?; ensure!( super::super::super::retirement::step( &s.f, @@ -177,7 +192,8 @@ impl Repaired { &permit, RepairStep::PromoteAndRemove ) - .await? + .await + .context("promote isolated replacement and remove obsolete voter")? .phase == RepairPhase::Joint ); @@ -188,12 +204,15 @@ impl Repaired { &permit, RepairStep::FinishJoint ) - .await? + .await + .context("finish isolated replacement joint configuration")? .phase == RepairPhase::Complete ); active.push(target); - write_read(&s.f, &active, 2, 5, 12).await?; + write_read(&s.f, &active, 2, 5, 12) + .await + .context("verify state after isolated replacement")?; s.f.domain .run(cellar::asyncrt::timeout(TIMEOUT, async { loop { @@ -236,7 +255,8 @@ impl Repaired { &s.observer, TIMEOUT, ) - .await?; + .await + .context("complete isolated replacement movement")?; ensure!(complete.isolated_sources() == [old.spec.node.bytes()]); let peers = [ s.nodes[0].spec.node, @@ -270,11 +290,17 @@ impl Repaired { std::slice::from_ref(&self.namespace), TIMEOUT, ) - .await?; + .await + .context("prepare isolated removal verification")?; let fixed = proof .freeze_live(&self.s.observer, self.peers, TIMEOUT) - .await?; - let reference = self.s.reference(&fixed).await?; + .await + .context("freeze isolated removal survivor cut")?; + let reference = self + .s + .reference(&fixed) + .await + .context("capture isolated removal archive reference")?; proof .verify_live( &self.s.observer, @@ -282,13 +308,17 @@ impl Repaired { archive(&self.s, &reference), TIMEOUT, ) - .await?; + .await + .context("verify isolated removal archive replay and survivors")?; Ok(proof) } async fn finalizing(&self) -> Result { let mut proof = self.verification(&self.s.f.catalog).await?; - let candidate = proof.publish_candidate(TIMEOUT).await?; + let candidate = proof + .publish_candidate(TIMEOUT) + .await + .context("publish isolated removal candidate")?; let cut = self.s.f.catalog.read(TIMEOUT).await?; let intent = request(230, cut.revision()); ensure!(candidate @@ -330,7 +360,8 @@ impl Repaired { ensure!(root.revision() == cut.revision() + 1); let fence = candidate .begin_isolated_finalization(&root, &intent, ProposalId::new(ISOLATION)?, &[], TIMEOUT) - .await?; + .await + .context("reconcile isolated finalization after lost confirmation")?; ensure!(fence.isolation_request() == Some(ISOLATION)); ensure!(fence.stop_request(ProposalId::new([238; 16])?).is_err()); ensure!(fence From 47b50d4836eafccc17848f52081bc0da62e5b76f Mon Sep 17 00:00:00 2001 From: forhappy Date: Mon, 2 Nov 2026 22:05:02 -0700 Subject: [PATCH 43/44] fix: defer snapshots until prior publication Retain one runtime event with its admission until the preceding durable snapshot finishes SQL publication. Earlier overlaps and a second deferred event still close admission; read version and image proof gates remain unchanged. --- crates/cellar/durability/raft/host.rs | 6 +- .../cellar/durability/raft/host_snapshot.rs | 48 ++++++-- .../durability/raft/host_snapshot_tests.rs | 114 ++++++++++++++++++ docs/design/multi-raft/snapshot-audit.md | 2 + 4 files changed, 157 insertions(+), 13 deletions(-) create mode 100644 crates/cellar/durability/raft/host_snapshot_tests.rs diff --git a/crates/cellar/durability/raft/host.rs b/crates/cellar/durability/raft/host.rs index f676aad4..6fece202 100644 --- a/crates/cellar/durability/raft/host.rs +++ b/crates/cellar/durability/raft/host.rs @@ -3189,11 +3189,7 @@ impl Driver { tracing::error!(group = ?replica.scope.group, ?error, "Raft runtime failed"); self.fail(&replica.scope); } - RuntimeEventKind::Snapshot { - ticket, - snapshot, - source, - } => self.receive_snapshot(ticket.clone(), snapshot.clone(), *source), + RuntimeEventKind::Snapshot { .. } => self.receive_snapshot(event), RuntimeEventKind::SnapshotInstalled { ticket, .. } => self.snapshot_installed(ticket), RuntimeEventKind::SnapshotAborted { ticket, .. } => self.snapshot_aborted(ticket), RuntimeEventKind::Checkpointed { ticket } => { diff --git a/crates/cellar/durability/raft/host_snapshot.rs b/crates/cellar/durability/raft/host_snapshot.rs index ed26e826..d1c5bde5 100644 --- a/crates/cellar/durability/raft/host_snapshot.rs +++ b/crates/cellar/durability/raft/host_snapshot.rs @@ -12,6 +12,10 @@ pub(super) struct SourceRequest { reservation: SourceReservation, } +#[cfg(test)] +#[path = "host_snapshot_tests.rs"] +mod tests; + #[derive(PartialEq, Eq)] enum Phase { Fetching, @@ -32,9 +36,18 @@ pub(super) struct Incoming { received: Option, completion: Option, retry_at: u64, + next: Option, } impl Incoming { + fn defer(&mut self, event: RuntimeEvent) -> bool { + if !matches!(self.phase, Phase::Publish | Phase::Publishing) || self.next.is_some() { + return false; + } + self.next = Some(event); + true + } + pub(super) fn allows_apply(&self) -> bool { matches!(self.phase, Phase::NeedAbort | Phase::Aborting) } @@ -175,17 +188,32 @@ impl Driver { } } - pub(super) fn receive_snapshot( - &mut self, - ticket: ApplyTicket, - snapshot: ::raft::prelude::Snapshot, - source: u64, - ) { + pub(super) fn receive_snapshot(&mut self, event: RuntimeEvent) { + let (ticket, snapshot, source) = match &event.kind { + RuntimeEventKind::Snapshot { + ticket, + snapshot, + source, + } => (ticket.clone(), snapshot.clone(), *source), + _ => return, + }; let scope = ticket.replica().scope.clone(); let Some(cell) = self.cells.get_mut(&scope) else { return; }; - if cell.incoming.is_some() || cell.group.identity() != ticket.replica() { + if cell.group.identity() != ticket.replica() { + self.fail(&scope); + return; + } + if let Some(incoming) = cell.incoming.as_mut() { + // The runtime may accept its next snapshot after durable install, + // before the SQL owner has published the preceding image. Retain + // one exact event, including its admission, until publication. + // That runtime cannot emit another snapshot until this ticket is + // completed or aborted, so a second deferred event is a violation. + if incoming.defer(event) { + return; + } self.fail(&scope); return; } @@ -200,6 +228,7 @@ impl Driver { received: None, completion: None, retry_at: 0, + next: None, }); self.refuse(&scope, CommandFailure::Unavailable); if !abort { @@ -513,12 +542,15 @@ impl Driver { .. }) if valid => { cell.image = view; - cell.incoming.take(); + let next = cell.incoming.take().and_then(|incoming| incoming.next); if let Ok(Some(progress)) = progress(&cell.group, cell.image.as_ref(), cell.archived) { let _ = cell.core.observe_progress(progress); } + if let Some(event) = next { + self.on_event(event); + } } _ => self.fail(&scope), } diff --git a/crates/cellar/durability/raft/host_snapshot_tests.rs b/crates/cellar/durability/raft/host_snapshot_tests.rs new file mode 100644 index 00000000..1789fb14 --- /dev/null +++ b/crates/cellar/durability/raft/host_snapshot_tests.rs @@ -0,0 +1,114 @@ +// Copyright 2026 Cellar Project Authors. Licensed under Apache-2.0. + +use super::*; + +fn ticket() -> ApplyTicket { + ApplyTicket { + owner: Arc::new(()), + engine_owner: Arc::new(()), + replica: ReplicaIdentity { + scope: GroupScope { + cluster: ClusterId::new([1; 16]).unwrap(), + namespace: NamespaceId::new([2; 16]).unwrap(), + cell: "Counter:snapshot-order".into(), + group: GroupId::new([3; 16]).unwrap(), + incarnation: IncarnationId::new([4; 16]).unwrap(), + }, + node: NodeId::new([5; 16]).unwrap(), + member: MemberId::new(1).unwrap(), + disk: DiskLineage::new([6; 16]).unwrap(), + process: ProcessGeneration::new([7; 16]).unwrap(), + }, + operation: OperationId::new(1), + position: LogPosition { + index: LogIndex::new(4), + term: Term::new(2), + }, + digest: [8; 32], + } +} + +fn incoming(phase: Phase) -> Incoming { + Incoming { + ticket: ticket(), + phase, + received: None, + completion: None, + retry_at: 0, + next: None, + } +} + +fn event(budget: &Arc, ticket: ApplyTicket, source: u64) -> RuntimeEvent { + let mut snapshot = ::raft::prelude::Snapshot::default(); + snapshot.mut_metadata().index = ticket.position().index.get(); + snapshot.mut_metadata().term = ticket.position().term.get(); + RuntimeEvent { + kind: RuntimeEventKind::Snapshot { + ticket, + snapshot, + source, + }, + _permit: Arc::new(budget.acquire(4096).unwrap()), + } +} + +#[test] +fn next_snapshot_retains_exact_ticket_and_admission_until_publication_owner_releases_it() { + for phase in [Phase::Publish, Phase::Publishing] { + let budget = Budget::new(8192, 2); + let mut incoming = incoming(phase); + let mut next = incoming.ticket.clone(); + next.operation = OperationId::new(2); + next.position.index = LogIndex::new(5); + next.digest = [9; 32]; + assert!(incoming.defer(event(&budget, next.clone(), 3))); + assert_eq!(budget.usage(), (4096, 1)); + + // Refusing another event must neither overwrite the admitted ticket + // nor refund its retained event bytes prematurely. + let mut later = next.clone(); + later.operation = OperationId::new(3); + later.position.index = LogIndex::new(6); + assert!(!incoming.defer(event(&budget, later, 2))); + assert_eq!(budget.usage(), (4096, 1)); + let deferred = incoming.next.as_ref().unwrap(); + let RuntimeEventKind::Snapshot { + ticket, + snapshot, + source, + } = &deferred.kind + else { + panic!("deferred snapshot changed kind") + }; + assert_eq!(ticket, &next); + assert_eq!(snapshot.get_metadata().index, 5); + assert_eq!(*source, 3); + + let deferred = incoming.next.take().unwrap(); + drop(incoming); + assert_eq!(budget.usage(), (4096, 1)); + drop(deferred); + assert_eq!(budget.usage(), (0, 0)); + } +} + +#[test] +fn overlapping_snapshot_before_durable_install_remains_refused() { + for phase in [ + Phase::Fetching, + Phase::Downloaded, + Phase::Preparing, + Phase::Prepared, + Phase::Authorizing, + Phase::Installing, + Phase::NeedAbort, + Phase::Aborting, + ] { + let budget = Budget::new(4096, 1); + let mut incoming = incoming(phase); + assert!(!incoming.defer(event(&budget, ticket(), 3))); + assert!(incoming.next.is_none()); + assert_eq!(budget.usage(), (0, 0)); + } +} diff --git a/docs/design/multi-raft/snapshot-audit.md b/docs/design/multi-raft/snapshot-audit.md index 5f8b14c2..8a887fa6 100644 --- a/docs/design/multi-raft/snapshot-audit.md +++ b/docs/design/multi-raft/snapshot-audit.md @@ -121,6 +121,8 @@ The [cancellation record](evidence/MR-10-abort-progress.json), [peer-admission r - The existing runtime persists the snapshot, applied marker, configuration and required HardState before emitting `SnapshotInstalled` or releasing dependent messages. - Physical completion also supplies an opaque verified reference bound to the exact ticket/digest/image; mechanical fixtures explicitly use no such capability. - `PreparedSnapshot` checks the durable engine snapshot and exact applied image before publication; sealing bytes or merely submitting a completion does not make state queryable. +- After `SnapshotInstalled`, the runtime can accept another snapshot while the SQL owner is still publishing the preceding image. The host retains at most one next snapshot event, including its original runtime admission, until that publication completes. It starts no second download or installation before releasing the preceding publication owner. +- An overlap before durable installation, a changed replica identity or a second deferred event still closes admission. Reads remain blocked by the incoming snapshot, and every installation and publication retains its exact ticket and image checks. - `ImageApplier` separately retains the durable snapshot generation and the latest applied generation. - Recovery validates/adopts both before retiring other inventory. From 300df5ceb64d70f796d811f7c565e1232bcb1954 Mon Sep 17 00:00:00 2001 From: forhappy Date: Tue, 3 Nov 2026 02:53:14 -0700 Subject: [PATCH 44/44] fix: preserve completed pool reclamation A concurrent inspection of a freed worker mutex could revoke the observed drain proof and make generation retirement miss its maintenance tick. Publish reclamation after destruction; retain active-work and slot-reuse checks and the original retirement deadlines. --- .../application/service/persistent_tests.rs | 53 ++++++++++++++++--- crates/cellar/runtime/workers/native/tests.rs | 17 ++++++ crates/cellar/runtime/workers/pool.rs | 17 ++++-- 3 files changed, 77 insertions(+), 10 deletions(-) diff --git a/crates/cellar/application/service/persistent_tests.rs b/crates/cellar/application/service/persistent_tests.rs index 59a1bd8c..6aa115c5 100644 --- a/crates/cellar/application/service/persistent_tests.rs +++ b/crates/cellar/application/service/persistent_tests.rs @@ -241,7 +241,13 @@ async fn retained_routes_use_fresh_epoch_gates_without_pinning_the_installing_ev crate::host_service::dispatch_fetch(Some(&target), call).await; } }); - assert_eq!(install(&runtime).await?.body, b"captured"); + assert_eq!( + install(&runtime) + .await + .context("install retained route")? + .body, + b"captured" + ); let route = signals .route .lock() @@ -255,13 +261,15 @@ async fn retained_routes_use_fresh_epoch_gates_without_pinning_the_installing_ev crate::asyncrt::sleep(Duration::from_millis(1)).await; } }) - .await?; + .await + .context("installing event and IO did not release within 2s")?; for admitted in [true, false, true] { let live = Arc::new(AtomicBool::new(true)); let future = dispatch(&route, live); let call = crate::asyncrt::spawn(future); let gate = crate::asyncrt::timeout(Duration::from_secs(2), gates.recv()) - .await? + .await + .context("fresh route gate exceeded 2s")? .context("fresh gate missing")?; check_gate(&gate); let before = signals.first.load(Ordering::SeqCst); @@ -283,7 +291,8 @@ async fn retained_routes_use_fresh_epoch_gates_without_pinning_the_installing_ev } let call = crate::asyncrt::spawn(dispatch(&route, Arc::new(AtomicBool::new(true)))); let gate = crate::asyncrt::timeout(Duration::from_secs(2), gates.recv()) - .await? + .await + .context("cancellation gate exceeded 2s")? .context("cancellation gate missing")?; check_gate(&gate); call.abort(); @@ -299,7 +308,8 @@ async fn retained_routes_use_fresh_epoch_gates_without_pinning_the_installing_ev let live = Arc::new(AtomicBool::new(true)); let call = crate::asyncrt::spawn(dispatch(&route, live.clone())); let gate = crate::asyncrt::timeout(Duration::from_secs(2), gates.recv()) - .await? + .await + .context("retirement gate exceeded 2s")? .context("retirement gate missing")?; live.store(false, Ordering::Release); gate.reply.send(Ok(())).unwrap(); @@ -312,8 +322,25 @@ async fn retained_routes_use_fresh_epoch_gates_without_pinning_the_installing_ev other.shutdown().await; assert!(foreign.is_err()); assert!(gates.try_recv().is_err()); + let old_service = runtime + .generation() + .service("persistent-route") + .context("old service missing")?; + let inspecting = old_service + .isolates + .admit(false) + .map_err(|error| anyhow::anyhow!("{error}"))?; + let freed_slot = inspecting.slot().clone(); + drop(inspecting); runtime.adopt(generation(2, signals.clone())?); runtime.swap_out_cell("Root:retained", 7).await?; + assert!( + old_service.isolates.is_drained(), + "old stateless worker did not physically free before inspection" + ); + // Another pool maintenance pass may hold this already-empty mutex + // at the generation tick. Physical reclamation remains complete. + let inspecting_freed = freed_slot.hold_turn_for_test().await; // Generation maintenance runs every 30 s in production. Exercise the // real reap boundary rather than manufacturing a missing route id. crate::asyncrt::timeout(Duration::from_secs(35), async { @@ -325,10 +352,21 @@ async fn retained_routes_use_fresh_epoch_gates_without_pinning_the_installing_ev crate::asyncrt::sleep(Duration::from_millis(10)).await; } }) - .await?; + .await + .with_context(|| { + format!( + "generation 1 did not reap within 35s: {:?}", + runtime + .draining_generations() + .iter() + .map(|g| (g.id(), g.isolate_census())) + .collect::>() + ) + })?; let call = crate::asyncrt::spawn(dispatch(&route, Arc::new(AtomicBool::new(true)))); let gate = crate::asyncrt::timeout(Duration::from_secs(2), gates.recv()) - .await? + .await + .context("expired generation gate exceeded 2s")? .context("expired generation gate missing")?; check_gate(&gate); gate.reply.send(Ok(())).unwrap(); @@ -337,6 +375,7 @@ async fn retained_routes_use_fresh_epoch_gates_without_pinning_the_installing_ev "retired route selected the current deployment" ); assert_eq!(signals.second.load(Ordering::SeqCst), 0); + drop(inspecting_freed); forwarding.abort(); let _ = forwarding.await; Ok::<_, anyhow::Error>(()) diff --git a/crates/cellar/runtime/workers/native/tests.rs b/crates/cellar/runtime/workers/native/tests.rs index cfe30756..b105dde1 100644 --- a/crates/cellar/runtime/workers/native/tests.rs +++ b/crates/cellar/runtime/workers/native/tests.rs @@ -449,6 +449,23 @@ async fn check_shared_worker_pool() -> Result<()> { .map_err(anyhow::Error::msg)?; pool.reap(); assert!(pool.is_drained()); + // A concurrent maintenance pass can inspect the now-empty worker mutex. + // Its lock must not revoke the completed physical reclamation proof. + let inspecting_freed = slot.hold_turn_for_test().await; + assert!( + pool.is_drained(), + "freed worker mutex contention revoked the pool drain proof" + ); + drop(inspecting_freed); + let reused = pool.admit(false).map_err(|error| anyhow!("{error}"))?; + assert!(reused.slot().is_retiring()); + assert!( + !pool.is_drained(), + "a reused slot inherited its predecessor's drain proof" + ); + drop(reused); + pool.reap(); + assert!(pool.is_drained()); assert!(crate::pool::current_slot().is_none()); assert!(installed_context().is_none()); crate::host_tcp::tests::native_tls_application(|application| { diff --git a/crates/cellar/runtime/workers/pool.rs b/crates/cellar/runtime/workers/pool.rs index 398a2aea..d6ba7b8c 100644 --- a/crates/cellar/runtime/workers/pool.rs +++ b/crates/cellar/runtime/workers/pool.rs @@ -210,6 +210,10 @@ pub struct Slot { /// retiring slot is filtered out of every placement decision, so nothing /// can arrive afterwards. Entering one would be a bug in this module. worker: tokio::sync::Mutex>, + // Published only after the worker has actually been dropped. Slots are + // replaced rather than repopulated, so this physical reclamation proof is + // monotonic even while another maintenance pass holds the empty mutex. + reclaimed: AtomicBool, application_work: Arc, handler_budget: std::time::Duration, turn_scheduler: TurnScheduler, @@ -397,6 +401,7 @@ impl Slot { id: 0, heap_id: next_heap_id(), worker: tokio::sync::Mutex::new(Some(worker)), + reclaimed: AtomicBool::new(false), application_work: crate::asyncrt::services().application_work_registry(), handler_budget: crate::engine_api::handler_budget(), turn_scheduler: TurnScheduler::default(), @@ -432,6 +437,7 @@ impl Slot { id: 0, heap_id: HeapId::new(0), worker: tokio::sync::Mutex::new(None), + reclaimed: AtomicBool::new(true), handler_budget: crate::engine_api::handler_budget(), turn_scheduler: TurnScheduler::default(), queue_producers: Mutex::new(std::collections::HashMap::new()), @@ -448,7 +454,7 @@ impl Slot { }) } - #[cfg(cellar_internal_tests)] + #[cfg(any(test, cellar_internal_tests))] #[doc(hidden)] pub async fn hold_turn_for_test(&self) -> impl Drop + '_ { self.worker.lock().await @@ -736,6 +742,7 @@ impl Pool { id, heap_id, worker: tokio::sync::Mutex::new(Some(worker)), + reclaimed: AtomicBool::new(false), application_work: self.application_work.clone(), handler_budget: self.handler_budget, turn_scheduler: TurnScheduler::default(), @@ -939,7 +946,7 @@ impl Pool { .read() .expect("pool poisoned") .iter() - .all(|slot| slot.worker.try_lock().is_ok_and(|worker| worker.is_none())) + .all(|slot| slot.reclaimed.load(Ordering::Acquire)) } fn retire_one(&self) -> bool { @@ -970,7 +977,11 @@ impl Pool { if slot.worker_pins.load(Ordering::Acquire) != 0 { continue; } - if worker.take().is_some() { + let retired = worker.take(); + let was_live = retired.is_some(); + drop(retired); + slot.reclaimed.store(true, Ordering::Release); + if was_live { tracing::info!(isolate = %slot.heap_id, slot = slot.id, "isolate freed"); } }