diff --git a/.github/workflows/javascript-sdk.yml b/.github/workflows/javascript-sdk.yml index 29bfe5668..97f9313a3 100644 --- a/.github/workflows/javascript-sdk.yml +++ b/.github/workflows/javascript-sdk.yml @@ -17,6 +17,7 @@ on: - 'tools/javascript/**' - 'tools/sandbox/**' - 'tools/rpc/**' + - 'tools/qualification/**' - 'tools/cellar-rpc-codegen/**' - 'proto/**' - 'docs/design/named-typed-rpc/*.proto' @@ -242,6 +243,11 @@ jobs: if: matrix.profile == 'packages' env: RUST_LOG: timing=debug,cellar::runtime=debug,store=debug + # Keep the existing handler deadlines and all debug/overflow checks. + # Optimize this repeated cold-restart artifact, as in native SDK CI. + CARGO_PROFILE_TEST_OPT_LEVEL: '1' + CARGO_PROFILE_TEST_DEBUG_ASSERTIONS: 'true' + CARGO_PROFILE_TEST_OVERFLOW_CHECKS: 'true' run: | cargo build --locked -p cellar-javascript --no-default-features --features node --bin cellar-node-example mkdir -p .cellar @@ -254,6 +260,10 @@ jobs: .cellar/node-delivery/launch.sh --help export CELLAR_TEST_NODE_ENTRY="$PWD/.cellar/node-delivery/app/dist/app.js" export CELLAR_TEST_NODE_EXECUTABLE="$PWD/.cellar/node-delivery/bin/node" + cargo test --locked -p cellar-javascript --no-default-features --features node --test node_runtime --no-run + # Finish build writeback before measuring fresh FULL-sync fixtures. + # Keep their real fsyncs and the existing handler deadline unchanged. + sync -f "${CARGO_TARGET_DIR:-$PWD/target}" # Fresh stores and Node processes in every case. Stop at the first # failure; these repetitions never retry a failed request or test. for iteration in {1..100}; do diff --git a/.github/workflows/native-sdk.yml b/.github/workflows/native-sdk.yml index 711d7c0e9..a2dca52d6 100644 --- a/.github/workflows/native-sdk.yml +++ b/.github/workflows/native-sdk.yml @@ -104,6 +104,7 @@ jobs: git rev-parse HEAD > "$RUNNER_TEMP/filesystem-release/source.txt" - name: Build the release CLI and filesystem application run: | + set -o pipefail cargo build --release --locked -p cellar --bin cellar --bin cellar-native-examples \ 2>&1 | tee "$RUNNER_TEMP/filesystem-release/build.log" python3 - <<'PY' @@ -123,6 +124,7 @@ jobs: PY - name: Exercise the release filesystem application and restart run: | + set -o pipefail python3 examples/focused_smoke.py --cellar target/release/cellar \ --example filesystem --profile release \ 2>&1 | tee "$RUNNER_TEMP/filesystem-release/smoke.log" @@ -150,10 +152,10 @@ jobs: measure: ${{ inputs.fleet_measure }} check: if: github.event_name != 'workflow_dispatch' || inputs.mode == 'qualification' - name: Rust framework and SDK (${{ matrix.python }}) + name: Python ABI and SDK (${{ matrix.python }}) runs-on: ubuntu-22.04 - # ABI/package qualification plus serial fleet tests and restart examples - # exceed 90 minutes on shared runners; individual test bounds remain intact. + # ABI/package qualification and examples retain their existing ceiling. + # Native campaigns run independently with all per-test bounds unchanged. timeout-minutes: 120 strategy: fail-fast: false @@ -331,34 +333,24 @@ jobs: cargo run -p cellar --example python-package-qualification --locked -- \ "$RUNNER_TEMP/python-package" "$RUNNER_TEMP" "$RUNNER_TEMP/python-package-v2" \ --supervisor "$PWD/target/debug/cellar" - - name: Test and lint - env: - # Failed native fixtures retain synthetic recovery observations here. - TMPDIR: ${{ runner.temp }}/native-failure-fixtures - CELLAR_FS_FAULT_REPORT: ${{ runner.temp }}/filesystem-proof-fault.json + - name: Doctests and Clippy run: | - mkdir -p "$TMPDIR" - # Fleet fixtures own concurrent runtimes and short authority leases. - # Run independent cases serially to keep their resource envelope stable. - # The opt-in RPC qualifier owns every guest runtime, including V8. - # Keep this native/Python job independent of JavaScript engines. - cargo test --workspace --exclude cellar-python --exclude cellar-javascript --exclude cellar-rpc-qualification --all-features --lib --tests --locked -- --test-threads=1 - cargo test -p cellar --bin cellar-perf-cells --bin cellar-perf-load --locked -- --test-threads=1 cargo test --workspace --exclude cellar-javascript --exclude cellar-rpc-qualification --all-features --doc --locked cargo clippy --workspace --exclude cellar-javascript --exclude cellar-rpc-qualification --all-targets --all-features --locked -- -D warnings - - name: Preserve filesystem proof and recovery ledger - uses: actions/upload-artifact@v6 - if: always() - with: - name: filesystem-proof-fault-${{ matrix.python }}-${{ github.run_id }}-${{ github.run_attempt }} - path: ${{ runner.temp }}/filesystem-proof-fault.json - if-no-files-found: ignore - name: Qualify dense filesystem writes and cold recovery env: + # Match the native campaign while retaining its correctness checks. + CARGO_PROFILE_TEST_OPT_LEVEL: "1" + CARGO_PROFILE_TEST_DEBUG_ASSERTIONS: "true" + CARGO_PROFILE_TEST_OVERFLOW_CHECKS: "true" CELLAR_FS_CAPACITY_MIB: "10" CELLAR_FS_CAPACITY_REPORT: ${{ runner.temp }}/filesystem-dense-capacity.json RUST_LOG: 'warn,cellar::actor::effects=info,store[node_lease_store]=debug' run: | + set -o pipefail + cargo test --locked -p cellar --all-features --test native_node --no-run + # Complete build writeback before the live durability qualification. + sync -f "${CARGO_TARGET_DIR:-$PWD/target}" cargo test --locked -p cellar --all-features --test native_node \ filesystem_dense_capacity_cold_recovery -- --ignored --nocapture \ 2>&1 | tee "$RUNNER_TEMP/filesystem-dense-capacity.log" @@ -373,6 +365,9 @@ jobs: if-no-files-found: ignore - name: Test Axum feature modes and generated contract run: | + cargo test -p cellar-axum --no-default-features --locked --no-run + # Keep compiler writeback outside the live one-second lease checks. + sync -f "${CARGO_TARGET_DIR:-$PWD/target}" cargo test -p cellar-axum --no-default-features --locked cargo check -p cellar-axum-counter --no-default-features --bins --locked cargo run -p cellar-axum-counter --bin export-openapi --locked -- "$RUNNER_TEMP/counter-openapi.json" @@ -396,6 +391,65 @@ jobs: python3 examples/platform_smoke.py --cellar target/debug/cellar \ --embedded target/debug/cellar-example-embedded --with-containers + - uses: actions/upload-artifact@v6 + if: always() + with: + name: python-churn-${{ matrix.python }}-${{ github.run_id }}-${{ github.run_attempt }} + path: | + ${{ runner.temp }}/python-churn/**/*.json + ${{ runner.temp }}/python-churn/**/*.log + ${{ runner.temp }}/python-churn/report.md + if-no-files-found: ignore + retention-days: 7 + + native_tests: + if: github.event_name != 'workflow_dispatch' || inputs.mode == 'qualification' + name: Native framework tests (${{ matrix.python }}) + runs-on: ubuntu-22.04 + env: + # Exercise native protocol deadlines with optimized code, retaining all + # debug assertions and integer overflow checks in the test artifact. + CARGO_PROFILE_TEST_OPT_LEVEL: "1" + CARGO_PROFILE_TEST_DEBUG_ASSERTIONS: "true" + CARGO_PROFILE_TEST_OVERFLOW_CHECKS: "true" + # Separate the serial native campaign from ABI/package qualification; + # preserve the original job ceiling and every existing test command. + timeout-minutes: 120 + strategy: + fail-fast: false + matrix: + python: ['3.14', '3.14t'] + steps: + - uses: actions/checkout@v6 + - name: Install Rust + run: | + rustup toolchain install "$RUST_VERSION" --profile minimal --component rustfmt + rustup default "$RUST_VERSION" + - uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.python }} + - name: Select embedded Python + run: echo "PYO3_PYTHON=$(command -v python)" >> "$GITHUB_ENV" + - name: Native framework tests + env: + # Failed native fixtures retain synthetic recovery observations here. + TMPDIR: ${{ runner.temp }}/native-failure-fixtures + CELLAR_FS_FAULT_REPORT: ${{ runner.temp }}/filesystem-proof-fault.json + run: | + mkdir -p "$TMPDIR" + # Fleet fixtures own concurrent runtimes and short authority leases. + # Run independent cases serially to keep their resource envelope stable. + # The opt-in RPC qualifier owns every guest runtime, including V8. + # Keep this native/Python job independent of JavaScript engines. + cargo test --workspace --exclude cellar-python --exclude cellar-javascript --exclude cellar-rpc-qualification --all-features --lib --tests --locked -- --test-threads=1 + cargo test -p cellar --bin cellar-perf-cells --bin cellar-perf-load --locked -- --test-threads=1 + - name: Preserve filesystem proof and recovery ledger + uses: actions/upload-artifact@v6 + if: always() + with: + name: filesystem-proof-fault-${{ matrix.python }}-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/filesystem-proof-fault.json + if-no-files-found: ignore - name: Preserve native failure observations uses: actions/upload-artifact@v6 if: always() @@ -417,16 +471,27 @@ jobs: if-no-files-found: ignore retention-days: 7 - - uses: actions/upload-artifact@v6 - if: always() - with: - name: python-churn-${{ matrix.python }}-${{ github.run_id }}-${{ github.run_attempt }} - path: | - ${{ runner.temp }}/python-churn/**/*.json - ${{ runner.temp }}/python-churn/**/*.log - ${{ runner.temp }}/python-churn/report.md - if-no-files-found: ignore - retention-days: 7 + + qualification: + if: always() && (github.event_name != 'workflow_dispatch' || inputs.mode == 'qualification') + needs: [check, native_tests] + # Preserve the existing required contexts across the campaign split. A + # failed, cancelled or skipped campaign must fail the original gate. + name: Rust framework and SDK (${{ matrix.python }}) + runs-on: ubuntu-22.04 + timeout-minutes: 5 + strategy: + fail-fast: false + matrix: + python: ['3.14', '3.14t'] + steps: + - name: Require both qualification campaigns + env: + SDK_RESULT: ${{ needs.check.result }} + NATIVE_RESULT: ${{ needs.native_tests.result }} + run: | + test "$SDK_RESULT" = success + test "$NATIVE_RESULT" = success python_load: if: github.event_name == 'workflow_dispatch' && inputs.mode == 'python-load' diff --git a/.github/workflows/typed-rpc.yml b/.github/workflows/typed-rpc.yml index 028df2053..13eace9f6 100644 --- a/.github/workflows/typed-rpc.yml +++ b/.github/workflows/typed-rpc.yml @@ -7,6 +7,7 @@ on: - 'crates/**' - 'packages/cellar-sdk/**' - 'tools/rpc/**' + - 'tools/qualification/**' - 'tools/javascript/**' - 'tools/cellar-rpc-codegen/**' - 'tools/python/**' @@ -88,6 +89,27 @@ jobs: python3 -m unittest discover -s tools/perf -p test_python_fleet.py cargo clippy --locked -p cellar-rpc-qualification --all-targets --features full -- -D warnings cargo test --locked -p cellar-rpc-qualification --features full -- --test-threads=1 + - name: Finish build writeback before live qualification + run: | + cargo build --locked -p cellar-rpc-qualification --features full + # Bound ongoing dirty writeback on this disposable x64 runner. A + # one-time flush alone still allowed >1s commits in the full matrix. + # FULL sync, barriers and the one-second authority lease stay enabled. + snapshot="$RUNNER_TEMP/cellar-rpc-writeback.conf" + : > "$snapshot" + for setting in dirty_background dirty; do + bytes=$(sysctl -n "vm.${setting}_bytes") + if [ "$bytes" -eq 0 ]; then + key="vm.${setting}_ratio" + else + key="vm.${setting}_bytes" + fi + printf '%s=%s\n' "$key" "$(sysctl -n "$key")" >> "$snapshot" + done + sysctl vm.dirty_expire_centisecs vm.dirty_writeback_centisecs >> "$snapshot" + sudo sysctl -w vm.dirty_background_bytes=4194304 vm.dirty_bytes=16777216 \ + vm.dirty_expire_centisecs=100 vm.dirty_writeback_centisecs=50 + sync -f "$CARGO_TARGET_DIR" - name: Live portable Protobuf profile if: matrix.campaign == 'focused' run: | @@ -135,6 +157,13 @@ jobs: python3 tools/rpc/check_python_mode.py --expected "$CELLAR_RPC_PYTHON_MODE" .cellar/rpc/all.json python3 tools/rpc/check_provenance.py .cellar/rpc/all.json python3 tools/rpc/check_provider_limits.py .cellar/rpc/all.json + - name: Restore runner writeback policy + if: always() + run: | + snapshot="$RUNNER_TEMP/cellar-rpc-writeback.conf" + if [ -f "$snapshot" ]; then + sudo sysctl -p "$snapshot" + fi - name: Retain lifecycle evidence if: always() uses: actions/upload-artifact@v6 diff --git a/crates/cellar-python/tests/bridge.py b/crates/cellar-python/tests/bridge.py index 69090077e..d30d20185 100644 --- a/crates/cellar-python/tests/bridge.py +++ b/crates/cellar-python/tests/bridge.py @@ -319,7 +319,16 @@ async def wrong_context(): if request.path == "/library": # A real native extension (_hashlib) plus pure Python library imports. return Response.json(hashlib.sha256(b"cellar").hexdigest()) - return await ctx.namespace("COUNTERS").get_by_name("one").fetch(request) + cell = ctx.namespace("COUNTERS").get_by_name("one") + if request.path == "/proof/state": + try: + return await cell.fetch(request) + except Ambiguous as error: + # Only this fixed, read-only recovery route can be reconciled. A + # failed/reset callee never establishes a mutation's rollback. + assert error.code == "Ambiguous" and error.outcome == "unknown" + return Response.json({"code": error.code, "outcome": error.outcome}, status=503) + return await cell.fetch(request) class Counter: diff --git a/crates/cellar-python/tests/support/durability.rs b/crates/cellar-python/tests/support/durability.rs index ab31e1b21..287a8b881 100644 --- a/crates/cellar-python/tests/support/durability.rs +++ b/crates/cellar-python/tests/support/durability.rs @@ -16,12 +16,23 @@ async fn restored_state(client: &reqwest::Client, url: &str) -> Result { if status.is_success() { return Ok::<_, anyhow::Error>(serde_json::from_str(&body)?); } - ensure!( - status.is_server_error() - && body.contains("cellar output-gate durability proof failed for ") - && body.contains("ltx durability timed out for "), - "restored proof read failed outside the pending proof reset: {status}: {body}" - ); + if status == reqwest::StatusCode::SERVICE_UNAVAILABLE { + // Python preserves the native refusal on this read-only route. + // It is an unknown outcome, never rollback or an acknowledged + // mutation. Reject unrelated errors instead of retrying a 500. + ensure!( + serde_json::from_str::(&body)? + == json!({"code":"Ambiguous","outcome":"unknown"}), + "recovery read lost its native refusal: {body}" + ); + } else { + ensure!( + status.is_server_error() + && body.contains("cellar output-gate durability proof failed for ") + && body.contains("ltx durability timed out for "), + "restored proof read failed outside the pending proof reset: {status}: {body}" + ); + } tokio::time::sleep(Duration::from_millis(20)).await; } }) @@ -198,6 +209,17 @@ pub(super) fn child() -> Result<()> { "an unproven mutation handler was blindly replayed: {attempts}" ); let state = restored_state(&client, &url).await?; + let restored_attempts: Value = client + .get(format!("{url}/proof/attempts")) + .send() + .await? + .error_for_status()? + .json() + .await?; + ensure!( + restored_attempts == attempts, + "recovery read replayed a mutation: {restored_attempts}" + ); ensure!( state["sync"] == 1 && state["output"] == 1, "restored proof access lost previously acknowledged state: {state}" diff --git a/crates/cellar/application/consensus/resource_metrics.rs b/crates/cellar/application/consensus/resource_metrics.rs index 3946bedcd..e4336803a 100644 --- a/crates/cellar/application/consensus/resource_metrics.rs +++ b/crates/cellar/application/consensus/resource_metrics.rs @@ -55,7 +55,7 @@ fn budget(text: &mut Text, prefix: &str, labels: &str, usage: RaftBudgetUsage) { } impl RaftResourceUsage { - /// Bounded label vocabularies, with 111 samples when SQL, image, group and + /// Bounded label vocabularies, with 113 samples when SQL, image, group and /// transport sources are live, plus 13 for a configured controller. /// A live extent inventory adds 13 samples. /// Unavailable sources expose only their availability gauge, diff --git a/crates/cellar/application/service/persistent_tests.rs b/crates/cellar/application/service/persistent_tests.rs index 59a1bd8c6..6aa115c53 100644 --- a/crates/cellar/application/service/persistent_tests.rs +++ b/crates/cellar/application/service/persistent_tests.rs @@ -241,7 +241,13 @@ async fn retained_routes_use_fresh_epoch_gates_without_pinning_the_installing_ev crate::host_service::dispatch_fetch(Some(&target), call).await; } }); - assert_eq!(install(&runtime).await?.body, b"captured"); + assert_eq!( + install(&runtime) + .await + .context("install retained route")? + .body, + b"captured" + ); let route = signals .route .lock() @@ -255,13 +261,15 @@ async fn retained_routes_use_fresh_epoch_gates_without_pinning_the_installing_ev crate::asyncrt::sleep(Duration::from_millis(1)).await; } }) - .await?; + .await + .context("installing event and IO did not release within 2s")?; for admitted in [true, false, true] { let live = Arc::new(AtomicBool::new(true)); let future = dispatch(&route, live); let call = crate::asyncrt::spawn(future); let gate = crate::asyncrt::timeout(Duration::from_secs(2), gates.recv()) - .await? + .await + .context("fresh route gate exceeded 2s")? .context("fresh gate missing")?; check_gate(&gate); let before = signals.first.load(Ordering::SeqCst); @@ -283,7 +291,8 @@ async fn retained_routes_use_fresh_epoch_gates_without_pinning_the_installing_ev } let call = crate::asyncrt::spawn(dispatch(&route, Arc::new(AtomicBool::new(true)))); let gate = crate::asyncrt::timeout(Duration::from_secs(2), gates.recv()) - .await? + .await + .context("cancellation gate exceeded 2s")? .context("cancellation gate missing")?; check_gate(&gate); call.abort(); @@ -299,7 +308,8 @@ async fn retained_routes_use_fresh_epoch_gates_without_pinning_the_installing_ev let live = Arc::new(AtomicBool::new(true)); let call = crate::asyncrt::spawn(dispatch(&route, live.clone())); let gate = crate::asyncrt::timeout(Duration::from_secs(2), gates.recv()) - .await? + .await + .context("retirement gate exceeded 2s")? .context("retirement gate missing")?; live.store(false, Ordering::Release); gate.reply.send(Ok(())).unwrap(); @@ -312,8 +322,25 @@ async fn retained_routes_use_fresh_epoch_gates_without_pinning_the_installing_ev other.shutdown().await; assert!(foreign.is_err()); assert!(gates.try_recv().is_err()); + let old_service = runtime + .generation() + .service("persistent-route") + .context("old service missing")?; + let inspecting = old_service + .isolates + .admit(false) + .map_err(|error| anyhow::anyhow!("{error}"))?; + let freed_slot = inspecting.slot().clone(); + drop(inspecting); runtime.adopt(generation(2, signals.clone())?); runtime.swap_out_cell("Root:retained", 7).await?; + assert!( + old_service.isolates.is_drained(), + "old stateless worker did not physically free before inspection" + ); + // Another pool maintenance pass may hold this already-empty mutex + // at the generation tick. Physical reclamation remains complete. + let inspecting_freed = freed_slot.hold_turn_for_test().await; // Generation maintenance runs every 30 s in production. Exercise the // real reap boundary rather than manufacturing a missing route id. crate::asyncrt::timeout(Duration::from_secs(35), async { @@ -325,10 +352,21 @@ async fn retained_routes_use_fresh_epoch_gates_without_pinning_the_installing_ev crate::asyncrt::sleep(Duration::from_millis(10)).await; } }) - .await?; + .await + .with_context(|| { + format!( + "generation 1 did not reap within 35s: {:?}", + runtime + .draining_generations() + .iter() + .map(|g| (g.id(), g.isolate_census())) + .collect::>() + ) + })?; let call = crate::asyncrt::spawn(dispatch(&route, Arc::new(AtomicBool::new(true)))); let gate = crate::asyncrt::timeout(Duration::from_secs(2), gates.recv()) - .await? + .await + .context("expired generation gate exceeded 2s")? .context("expired generation gate missing")?; check_gate(&gate); gate.reply.send(Ok(())).unwrap(); @@ -337,6 +375,7 @@ async fn retained_routes_use_fresh_epoch_gates_without_pinning_the_installing_ev "retired route selected the current deployment" ); assert_eq!(signals.second.load(Ordering::SeqCst), 0); + drop(inspecting_freed); forwarding.abort(); let _ = forwarding.await; Ok::<_, anyhow::Error>(()) diff --git a/crates/cellar/durability/raft/README.md b/crates/cellar/durability/raft/README.md index c1662fb4f..d2ce011ad 100644 --- a/crates/cellar/durability/raft/README.md +++ b/crates/cellar/durability/raft/README.md @@ -188,7 +188,7 @@ I/O or group enumeration. The internal generation-bound, authenticated `/peer/raft/metrics` endpoint exports these counters, actual SQL work reservations and incoming/outgoing transport and snapshot/metadata banks, without identity labels or callback execution. Live SQL, image, group and transport sources use -111 fixed samples. A configured controller and a live extent inventory each +113 fixed samples. A configured controller and a live extent inventory each add 13 samples. Unavailable sources omit their values rather than synthesizing zero. Weak transport and inventory observers cannot retain physical resources after joined shutdown. The [operations contract](../../../../docs/design/multi-raft/operations-design.md) @@ -883,11 +883,12 @@ must retain or report immediate queue refusals. Dropping a delivery does not refund queued/in-flight work. Peer and node usage counters expose held bytes and items; permits release on completion, refusal, close and domain cancellation. -Before applying a committed configuration, the replica drains ownership of its -already-emitted protocol sends so membership revocation cannot invalidate their -delivery. Later ordinary Ready work waits behind this finite fence; an unstable -snapshot retains priority. Send completion only releases ownership: configuration -application still requires its own durable write. +Before applying a committed configuration, the replica includes one already-pending +ordinary Ready within existing admission limits, then drains ownership of its +protocol sends so membership revocation cannot invalidate their delivery. Later +ordinary Ready work waits behind this finite fence; an unstable snapshot retains +priority. Send completion only releases ownership: configuration application still +requires its own durable write. Receiver frame admission validates the current member and bounded message before queueing it, without waiting for that group's Ready or image installation. Raft @@ -1409,6 +1410,13 @@ supersedes the earlier source lease to that learner, whose outstanding stream retains its pins until bounded I/O exits. Identity/authentication failures remain refusals. +The single source slot distinguishes an offered snapshot from an authenticated +pull that has started. An authorized fresh seed request may retire an unclaimed +offer to another voter, which might have ignored an obsolete announcement. +Retiring an offer and claiming its first pull are mutually exclusive atomic +transitions. Active transfers to other peers stay protected, and every retained +physical owner keeps its source reservation through cancellation and I/O exit. + `snapshot_seed.rs` validates the original birth, exact learner binding, included configuration and barrier before sealing the physical image. The engine creates the mapping, snapshot, current configuration, bindings and applied marker in one diff --git a/crates/cellar/durability/raft/host.rs b/crates/cellar/durability/raft/host.rs index f676aad40..6fece2022 100644 --- a/crates/cellar/durability/raft/host.rs +++ b/crates/cellar/durability/raft/host.rs @@ -3189,11 +3189,7 @@ impl Driver { tracing::error!(group = ?replica.scope.group, ?error, "Raft runtime failed"); self.fail(&replica.scope); } - RuntimeEventKind::Snapshot { - ticket, - snapshot, - source, - } => self.receive_snapshot(ticket.clone(), snapshot.clone(), *source), + RuntimeEventKind::Snapshot { .. } => self.receive_snapshot(event), RuntimeEventKind::SnapshotInstalled { ticket, .. } => self.snapshot_installed(ticket), RuntimeEventKind::SnapshotAborted { ticket, .. } => self.snapshot_aborted(ticket), RuntimeEventKind::Checkpointed { ticket } => { diff --git a/crates/cellar/durability/raft/host_seed.rs b/crates/cellar/durability/raft/host_seed.rs index 779ce01d4..5c32ca603 100644 --- a/crates/cellar/durability/raft/host_seed.rs +++ b/crates/cellar/durability/raft/host_seed.rs @@ -59,13 +59,14 @@ impl Driver { reply(&mut request, Err(CommandFailure::Unavailable)); return; } - // A fresh authenticated request supersedes an earlier transfer to the - // same physical learner. Old stream owners retain their pins until - // their bounded I/O exits; no unverified progress escapes either one. + // A valid seed request may release an unclaimed snapshot offer to an + // unrelated voter. Otherwise an ignored offer can hold the sole source + // slot for the seed's entire deadline. Active transfers to other peers + // remain protected; this learner's earlier transfer is superseded. if self .snapshot_source .as_ref() - .is_some_and(|source| source.targets(&request.scope, target.member)) + .is_some_and(|source| source.yield_for_seed(&request.scope, target.member)) { if let Some(source) = self.snapshot_source.take() { source.cancel(); diff --git a/crates/cellar/durability/raft/host_snapshot.rs b/crates/cellar/durability/raft/host_snapshot.rs index ed26e8267..d1c5bde5e 100644 --- a/crates/cellar/durability/raft/host_snapshot.rs +++ b/crates/cellar/durability/raft/host_snapshot.rs @@ -12,6 +12,10 @@ pub(super) struct SourceRequest { reservation: SourceReservation, } +#[cfg(test)] +#[path = "host_snapshot_tests.rs"] +mod tests; + #[derive(PartialEq, Eq)] enum Phase { Fetching, @@ -32,9 +36,18 @@ pub(super) struct Incoming { received: Option, completion: Option, retry_at: u64, + next: Option, } impl Incoming { + fn defer(&mut self, event: RuntimeEvent) -> bool { + if !matches!(self.phase, Phase::Publish | Phase::Publishing) || self.next.is_some() { + return false; + } + self.next = Some(event); + true + } + pub(super) fn allows_apply(&self) -> bool { matches!(self.phase, Phase::NeedAbort | Phase::Aborting) } @@ -175,17 +188,32 @@ impl Driver { } } - pub(super) fn receive_snapshot( - &mut self, - ticket: ApplyTicket, - snapshot: ::raft::prelude::Snapshot, - source: u64, - ) { + pub(super) fn receive_snapshot(&mut self, event: RuntimeEvent) { + let (ticket, snapshot, source) = match &event.kind { + RuntimeEventKind::Snapshot { + ticket, + snapshot, + source, + } => (ticket.clone(), snapshot.clone(), *source), + _ => return, + }; let scope = ticket.replica().scope.clone(); let Some(cell) = self.cells.get_mut(&scope) else { return; }; - if cell.incoming.is_some() || cell.group.identity() != ticket.replica() { + if cell.group.identity() != ticket.replica() { + self.fail(&scope); + return; + } + if let Some(incoming) = cell.incoming.as_mut() { + // The runtime may accept its next snapshot after durable install, + // before the SQL owner has published the preceding image. Retain + // one exact event, including its admission, until publication. + // That runtime cannot emit another snapshot until this ticket is + // completed or aborted, so a second deferred event is a violation. + if incoming.defer(event) { + return; + } self.fail(&scope); return; } @@ -200,6 +228,7 @@ impl Driver { received: None, completion: None, retry_at: 0, + next: None, }); self.refuse(&scope, CommandFailure::Unavailable); if !abort { @@ -513,12 +542,15 @@ impl Driver { .. }) if valid => { cell.image = view; - cell.incoming.take(); + let next = cell.incoming.take().and_then(|incoming| incoming.next); if let Ok(Some(progress)) = progress(&cell.group, cell.image.as_ref(), cell.archived) { let _ = cell.core.observe_progress(progress); } + if let Some(event) = next { + self.on_event(event); + } } _ => self.fail(&scope), } diff --git a/crates/cellar/durability/raft/host_snapshot_tests.rs b/crates/cellar/durability/raft/host_snapshot_tests.rs new file mode 100644 index 000000000..1789fb144 --- /dev/null +++ b/crates/cellar/durability/raft/host_snapshot_tests.rs @@ -0,0 +1,114 @@ +// Copyright 2026 Cellar Project Authors. Licensed under Apache-2.0. + +use super::*; + +fn ticket() -> ApplyTicket { + ApplyTicket { + owner: Arc::new(()), + engine_owner: Arc::new(()), + replica: ReplicaIdentity { + scope: GroupScope { + cluster: ClusterId::new([1; 16]).unwrap(), + namespace: NamespaceId::new([2; 16]).unwrap(), + cell: "Counter:snapshot-order".into(), + group: GroupId::new([3; 16]).unwrap(), + incarnation: IncarnationId::new([4; 16]).unwrap(), + }, + node: NodeId::new([5; 16]).unwrap(), + member: MemberId::new(1).unwrap(), + disk: DiskLineage::new([6; 16]).unwrap(), + process: ProcessGeneration::new([7; 16]).unwrap(), + }, + operation: OperationId::new(1), + position: LogPosition { + index: LogIndex::new(4), + term: Term::new(2), + }, + digest: [8; 32], + } +} + +fn incoming(phase: Phase) -> Incoming { + Incoming { + ticket: ticket(), + phase, + received: None, + completion: None, + retry_at: 0, + next: None, + } +} + +fn event(budget: &Arc, ticket: ApplyTicket, source: u64) -> RuntimeEvent { + let mut snapshot = ::raft::prelude::Snapshot::default(); + snapshot.mut_metadata().index = ticket.position().index.get(); + snapshot.mut_metadata().term = ticket.position().term.get(); + RuntimeEvent { + kind: RuntimeEventKind::Snapshot { + ticket, + snapshot, + source, + }, + _permit: Arc::new(budget.acquire(4096).unwrap()), + } +} + +#[test] +fn next_snapshot_retains_exact_ticket_and_admission_until_publication_owner_releases_it() { + for phase in [Phase::Publish, Phase::Publishing] { + let budget = Budget::new(8192, 2); + let mut incoming = incoming(phase); + let mut next = incoming.ticket.clone(); + next.operation = OperationId::new(2); + next.position.index = LogIndex::new(5); + next.digest = [9; 32]; + assert!(incoming.defer(event(&budget, next.clone(), 3))); + assert_eq!(budget.usage(), (4096, 1)); + + // Refusing another event must neither overwrite the admitted ticket + // nor refund its retained event bytes prematurely. + let mut later = next.clone(); + later.operation = OperationId::new(3); + later.position.index = LogIndex::new(6); + assert!(!incoming.defer(event(&budget, later, 2))); + assert_eq!(budget.usage(), (4096, 1)); + let deferred = incoming.next.as_ref().unwrap(); + let RuntimeEventKind::Snapshot { + ticket, + snapshot, + source, + } = &deferred.kind + else { + panic!("deferred snapshot changed kind") + }; + assert_eq!(ticket, &next); + assert_eq!(snapshot.get_metadata().index, 5); + assert_eq!(*source, 3); + + let deferred = incoming.next.take().unwrap(); + drop(incoming); + assert_eq!(budget.usage(), (4096, 1)); + drop(deferred); + assert_eq!(budget.usage(), (0, 0)); + } +} + +#[test] +fn overlapping_snapshot_before_durable_install_remains_refused() { + for phase in [ + Phase::Fetching, + Phase::Downloaded, + Phase::Preparing, + Phase::Prepared, + Phase::Authorizing, + Phase::Installing, + Phase::NeedAbort, + Phase::Aborting, + ] { + let budget = Budget::new(4096, 1); + let mut incoming = incoming(phase); + assert!(!incoming.defer(event(&budget, ticket(), 3))); + assert!(incoming.next.is_none()); + assert_eq!(budget.usage(), (0, 0)); + } +} diff --git a/crates/cellar/durability/raft/replica.rs b/crates/cellar/durability/raft/replica.rs index eab75ad9b..8b7978a44 100644 --- a/crates/cellar/durability/raft/replica.rs +++ b/crates/cellar/durability/raft/replica.rs @@ -101,6 +101,11 @@ enum Phase { }, } +struct ConfigurationFence { + index: u64, + announced: bool, +} + pub(crate) struct Replica { pub log: GroupLog, node: RawNode, @@ -108,6 +113,7 @@ pub(crate) struct Replica { pub write: Option<(WriteTicket, GroupWrite)>, outbound: VecDeque, deliveries: Vec>, + configuration_fence: Option, apply: VecDeque, apply_ticket: Option, finished_apply: Option, @@ -173,6 +179,7 @@ impl Replica { write: None, outbound: VecDeque::new(), deliveries: Vec::new(), + configuration_fence: None, apply: VecDeque::new(), apply_ticket: None, finished_apply: None, @@ -692,6 +699,59 @@ impl Replica { Ok(()) } + fn announce_configuration(&mut self, index: u64) -> Result<(), RuntimeError> { + if self + .configuration_fence + .as_ref() + .is_some_and(|fence| fence.index == index && fence.announced) + { + return Ok(()); + } + self.configuration_fence = Some(ConfigurationFence { + index, + announced: true, + }); + if self.node.raft.state != StateRole::Leader { + return Ok(()); + } + let hard = self.log.initial_state()?.hard_state; + // A concurrent term change cannot advertise an unpersisted promise. + if hard.term != self.node.raft.term || hard.vote != self.node.raft.vote { + return Ok(()); + } + let entry = self + .apply + .front() + .filter(|entry| entry.index == index) + .ok_or_else(|| RuntimeError::Protocol("missing configuration apply".into()))?; + // A paused append stream can miss Raft's commit broadcast. Once this + // configuration removes that peer, ordinary routing cannot reach it. + // Announce this already committed entry once while the old binding is + // still valid. An empty Append requires the receiver to match its exact + // index/term; transport acceptance is never a durability proof. A lagging + // or unavailable copy still fails the catalog's physical closure check. + for (&member, _) in self.node.raft.prs().iter() { + if member == self.node.raft.id { + continue; + } + let mut message = Message { + from: self.node.raft.id, + to: member, + term: self.node.raft.term, + index, + log_term: entry.term, + commit: index, + ..Default::default() + }; + message.set_msg_type(MessageType::MsgAppend); + self.outbound.push_back(RuntimeEventKind::Message { + replica: self.log.identity().clone(), + message, + }); + } + Ok(()) + } + pub fn drive( &mut self, operations: &mut Operations, @@ -935,6 +995,7 @@ impl Replica { if let Some(entry) = self.apply.pop_front() { self.apply_bytes -= entry_bytes(&entry); } + self.configuration_fence = None; self.apply_ticket = None; self.phase = Phase::Idle; Ok(true) @@ -964,27 +1025,31 @@ impl Replica { self.ready_bytes = 0; return Ok(true); } - // Drain the finite set of already-emitted sends before applying - // a committed configuration. Collecting another ordinary Ready - // here lets later heartbeats extend that fence indefinitely. - // An uninstalled snapshot retains its existing priority. - let configuration_pending = self + // Include one already-pending Ready in the send fence before + // membership revocation, then freeze that finite boundary. + // Later heartbeats cannot extend it. Snapshot priority and + // ordinary Ready admission limits remain unchanged. + let configuration_index = self .apply .front() - .is_some_and(|entry| entry.get_entry_type() != EntryType::EntryNormal) - && self.node.raft.raft_log.unstable_snapshot().is_none(); - if configuration_pending { - self.deliveries - .retain(|delivery| delivery.strong_count() != 0); - if !self.deliveries.is_empty() { - return Ok(false); - } - } - if !configuration_pending + .filter(|entry| entry.get_entry_type() != EntryType::EntryNormal) + .filter(|_| self.node.raft.raft_log.unstable_snapshot().is_none()) + .map(|entry| entry.index); + let fenced = self + .configuration_fence + .as_ref() + .is_some_and(|fence| configuration_index == Some(fence.index)); + if !fenced && self.node.has_ready() && ((can_collect && self.apply_bytes < options.max_group_apply_bytes) || self.node.raft.raft_log.unstable_snapshot().is_some()) { + if let Some(index) = configuration_index { + self.configuration_fence = Some(ConfigurationFence { + index, + announced: false, + }); + } let ready = self.node.ready(); self.ready_bytes = ready_bytes(&ready); if self.ready_bytes > options.max_ready_bytes { @@ -1014,16 +1079,38 @@ impl Replica { }; return Ok(true); } - let Some(entry) = self.apply.front() else { - return Ok(false); - }; - if entry.get_entry_type() != EntryType::EntryNormal { + if let Some(index) = self + .apply + .front() + .filter(|entry| entry.get_entry_type() != EntryType::EntryNormal) + .map(|entry| entry.index) + { + if self + .configuration_fence + .as_ref() + .is_none_or(|fence| fence.index != index) + { + self.configuration_fence = Some(ConfigurationFence { + index, + announced: false, + }); + } self.deliveries .retain(|delivery| delivery.strong_count() != 0); if !self.deliveries.is_empty() { return Ok(false); } + // The notification must not overtake an earlier Append on + // another transport stream. Its own sends then join the + // same finite fence before membership is revoked. + self.announce_configuration(index)?; + if !self.outbound.is_empty() { + return Ok(true); + } } + let Some(entry) = self.apply.front() else { + return Ok(false); + }; let position = entry_position(entry); if entry.get_entry_type() == EntryType::EntryNormal && !entry.data.is_empty() { if let Some(completed) = self.finished_apply.take() { @@ -1138,6 +1225,7 @@ impl Replica { (!ready.snapshot().is_empty()).then_some(ready.snapshot().get_metadata().index); if snapshot_index.is_some() { self.apply.clear(); + self.configuration_fence = None; self.apply_bytes = 0; self.apply_ticket = None; self.finished_apply = None; diff --git a/crates/cellar/durability/raft/replica_delivery_tests.rs b/crates/cellar/durability/raft/replica_delivery_tests.rs index 1791b1267..54bdf271d 100644 --- a/crates/cellar/durability/raft/replica_delivery_tests.rs +++ b/crates/cellar/durability/raft/replica_delivery_tests.rs @@ -9,15 +9,33 @@ use cellar_logic::consensus::*; #[tokio::test] async fn membership_apply_waits_for_prior_message_delivery() -> anyhow::Result<()> { - membership_case(false).await + membership_case(ReadyCase::None).await } #[tokio::test] async fn membership_delivery_fence_is_not_extended_by_later_protocol_ready() -> anyhow::Result<()> { - membership_case(true).await + membership_case(ReadyCase::LaterHeartbeat).await } -async fn membership_case(later_ready: bool) -> anyhow::Result<()> { +#[tokio::test] +async fn membership_apply_drains_an_already_queued_commit_notice() -> anyhow::Result<()> { + membership_case(ReadyCase::QueuedCommit).await +} + +#[tokio::test] +async fn membership_apply_announces_commit_to_a_paused_peer() -> anyhow::Result<()> { + membership_case(ReadyCase::PausedPeer).await +} + +#[derive(Clone, Copy)] +enum ReadyCase { + None, + LaterHeartbeat, + QueuedCommit, + PausedPeer, +} + +async fn membership_case(ready_case: ReadyCase) -> anyhow::Result<()> { let root = tempfile::tempdir()?; let domain = ExecutionDomain::new(tokio::runtime::Handle::current()); let log = domain @@ -75,7 +93,11 @@ async fn membership_case(later_ready: bool) -> anyhow::Result<()> { from: 1, to: 2, term: 1, - commit: 2, + commit: if matches!(ready_case, ReadyCase::QueuedCommit) { + 1 + } else { + 2 + }, ..Default::default() }; message.set_msg_type(MessageType::MsgAppend); @@ -87,7 +109,114 @@ async fn membership_case(later_ready: bool) -> anyhow::Result<()> { let event = outbox.events.pop_front().unwrap(); let transport_owner = event._permit.clone(); drop(event); - if later_ready { + let transport_owner = if matches!(ready_case, ReadyCase::QueuedCommit) { + drop(transport_owner); + let mut committed = Message { + from: 1, + to: 2, + term: 1, + commit: 2, + ..Default::default() + }; + committed.set_msg_type(MessageType::MsgAppend); + replica.node.raft.msgs.push(committed); + let mut owned = None; + for _ in 0..16 { + assert!( + replica.write.is_none(), + "configuration skipped a queued commit notice" + ); + assert!(replica.drive(&mut operations, &mut outbox, &options, true)?); + if let Some(event) = outbox.events.pop_front() { + let RuntimeEventKind::Message { message, .. } = &event.kind else { + panic!("queued commit notice was not emitted") + }; + assert_eq!((message.to, message.commit), (2, 2)); + owned = Some(event._permit.clone()); + break; + } + } + let owned = owned.expect("queued commit notice did not drain"); + for _ in 0..16 { + if replica.idle() && replica.ready_bytes == 0 { + break; + } + assert!(replica.write.is_none()); + assert!(replica.drive(&mut operations, &mut outbox, &options, true)?); + } + vec![owned] + } else if matches!(ready_case, ReadyCase::PausedPeer) { + replica.node.raft.state = StateRole::Leader; + replica.node.raft.mut_prs().get_mut(2).unwrap().pause(); + replica.node.raft.send_append(2); + assert!(replica.node.raft.msgs.is_empty()); + for _ in 0..16 { + if replica.configuration_fence.is_some() + && replica.idle() + && replica.ready_bytes == 0 + { + break; + } + assert!(replica.drive(&mut operations, &mut outbox, &options, true)?); + assert!(outbox.events.is_empty()); + assert!(replica.write.is_none()); + } + assert!(!replica.drive(&mut operations, &mut outbox, &options, true)?); + assert!( + replica.outbound.is_empty(), + "commit notice overtook prior send" + ); + // A new in-memory promise cannot escape before persistence. + let hard = replica.log.initial_state()?.hard_state; + for (term, vote) in [(hard.term + 1, hard.vote), (hard.term, 1)] { + replica.node.raft.term = term; + replica.node.raft.vote = vote; + replica.announce_configuration(2)?; + assert!(replica.outbound.is_empty()); + replica.configuration_fence.as_mut().unwrap().announced = false; + } + replica.node.raft.term = hard.term; + replica.node.raft.vote = hard.vote; + drop(transport_owner); + let mut owners = Vec::new(); + let mut peers = std::collections::BTreeSet::new(); + for _ in 0..16 { + assert!(replica.write.is_none()); + assert!(replica.drive(&mut operations, &mut outbox, &options, true)?); + if let Some(event) = outbox.events.pop_front() { + let RuntimeEventKind::Message { message, .. } = &event.kind else { + panic!("configuration commit was not announced") + }; + assert_eq!(message.get_msg_type(), MessageType::MsgAppend); + assert_eq!( + ( + message.from, + message.term, + message.index, + message.log_term, + message.commit + ), + (1, 1, 2, 1, 2) + ); + assert!(message.entries.is_empty()); + if message.to == 2 { + check_notice_matching(message)?; + } + assert!(peers.insert(message.to)); + owners.push(event._permit.clone()); + } + if owners.len() == 2 && replica.idle() && replica.ready_bytes == 0 { + break; + } + } + assert_eq!(peers, [2, 3].into_iter().collect()); + assert!(replica.node.raft.prs().get(2).unwrap().is_paused()); + owners + } else { + vec![transport_owner] + }; + assert!(!replica.drive(&mut operations, &mut outbox, &options, true)?); + if matches!(ready_case, ReadyCase::LaterHeartbeat) { let mut heartbeat = Message { from: 1, to: 3, @@ -124,3 +253,42 @@ async fn membership_case(later_ready: bool) -> anyhow::Result<()> { domain.shutdown().await; Ok(()) } + +fn check_notice_matching(notice: &Message) -> anyhow::Result<()> { + for (present, matching) in [(true, true), (true, false), (false, true)] { + let store = ::raft::storage::MemStorage::new_with_conf_state((vec![1, 2, 3], vec![])); + { + let mut state = store.wl(); + let entries = (1..=if present { 2 } else { 1 }) + .map(|index| Entry { + index, + term: 1, + ..Default::default() + }) + .collect::>(); + state.append(&entries)?; + state.mut_hard_state().term = 1; + state.mut_hard_state().commit = 1; + } + let mut follower = RawNode::new( + &Config { + id: 2, + applied: 1, + ..Default::default() + }, + store, + &slog::Logger::root(slog::Discard, slog::o!()), + )?; + let mut message = notice.clone(); + if !matching { + message.log_term += 1; + } + follower.step(message)?; + assert_eq!( + follower.raft.raft_log.committed, + if present && matching { 2 } else { 1 }, + "a commit notification must match the actual receiver log" + ); + } + Ok(()) +} diff --git a/crates/cellar/durability/raft/snapshot_codec_tests.rs b/crates/cellar/durability/raft/snapshot_codec_tests.rs index 516c6af76..4dd09b60c 100644 --- a/crates/cellar/durability/raft/snapshot_codec_tests.rs +++ b/crates/cellar/durability/raft/snapshot_codec_tests.rs @@ -106,8 +106,7 @@ async fn prepared_owner_shutdown_joins_even_if_the_stop_waiter_was_never_polled( fixture.close().await } -#[tokio::test] -async fn obsolete_snapshot_routes_cancel_sources_without_releasing_live_owners() -> Result<()> { +async fn source_fixture() -> Result<(Fixture, GroupRoute)> { use cellar_logic::consensus::{ ClusterId, DiskLineage, GroupId, IncarnationId, NamespaceId, ProcessGeneration, }; @@ -156,7 +155,7 @@ async fn obsolete_snapshot_routes_cancel_sources_without_releasing_live_owners() .persist(cellar_logic::consensus::OperationId::new(1), vec![write]) .await?; - let mut route = GroupRoute { + let route = GroupRoute { scope: scope.clone(), schema_digest: [9; 32], revision: 1, @@ -186,6 +185,91 @@ async fn obsolete_snapshot_routes_cancel_sources_without_releasing_live_owners() ], }; fixture.transport.install_group(route.clone())?; + Ok((fixture, route)) +} + +fn source_offer(fixture: &Fixture, route: &GroupRoute) -> Result { + let group = fixture + .transport + .handle + .0 + .runtime + .log() + .group(&route.scope)?; + let package = + SnapshotPackage::capture_empty(&group, route.schema_digest, SnapshotLimits::default())?; + Ok(fixture.transport.serve_snapshot( + &route.scope, + 2, + package, + fixture.transport.reserve_snapshot_source()?, + )?) +} + +#[tokio::test] +async fn unclaimed_offer_yields_to_seed_without_releasing_retained_owners() -> Result<()> { + let (fixture, route) = source_fixture().await?; + let state = fixture.transport.handle.0.clone(); + let source = source_offer(&fixture, &route)?; + let delayed_pull = source.0.clone(); + ensure!(matches!( + fixture.transport.reserve_snapshot_source(), + Err(TransportError::Overloaded) + )); + // A voter may ignore an obsolete snapshot announcement. Its unclaimed + // offer must not occupy the only source slot for a learner's full deadline. + ensure!(source.yield_for_seed(&route.scope, MemberId::new(3)?)); + ensure!(source.finished(0)); + ensure!( + !delayed_pull.begin_pull(), + "a yielded offer admitted a pull" + ); + drop(source); + ensure!(state.snapshot_source_budget.usage().1 == 1); + ensure!(matches!( + fixture.transport.reserve_snapshot_source(), + Err(TransportError::Overloaded) + )); + drop(delayed_pull); + ensure!(state.snapshot_source_budget.usage() == (0, 0)); + let next = source_offer(&fixture, &route)?; + ensure!(!next.finished(0)); + next.cancel(); + ensure!(state.snapshot_source_budget.usage() == (0, 0)); + fixture.close().await +} + +#[tokio::test] +async fn claimed_offer_protects_other_peers_and_retains_cancelled_io_charge() -> Result<()> { + let (fixture, route) = source_fixture().await?; + let state = fixture.transport.handle.0.clone(); + let source = source_offer(&fixture, &route)?; + let active = source.0.clone(); + ensure!(active.begin_pull()); + ensure!(!source.yield_for_seed(&route.scope, MemberId::new(3)?)); + ensure!(!source.finished(0)); + ensure!(matches!( + fixture.transport.reserve_snapshot_source(), + Err(TransportError::Overloaded) + )); + // A fresh request for this same physical learner still supersedes its + // earlier transfer. The old physical owner keeps its reservation. + ensure!(source.yield_for_seed(&route.scope, MemberId::new(2)?)); + source.cancel(); + ensure!(active.is_finished()); + ensure!(!active.begin_pull()); + ensure!(state.snapshot_source_budget.usage().1 == 1); + drop(active); + ensure!(state.snapshot_source_budget.usage() == (0, 0)); + fixture.close().await +} + +#[tokio::test] +async fn obsolete_snapshot_routes_cancel_sources_without_releasing_live_owners() -> Result<()> { + let (fixture, mut route) = source_fixture().await?; + let state = fixture.transport.handle.0.clone(); + let scope = route.scope.clone(); + let group = state.runtime.log().group(&scope)?; for remove in [false, true] { let package = SnapshotPackage::capture_empty(&group, [9; 32], SnapshotLimits::default())?; let lease = fixture.transport.serve_snapshot( diff --git a/crates/cellar/durability/raft/snapshot_transport.rs b/crates/cellar/durability/raft/snapshot_transport.rs index 8fdfa15a0..79a6a3272 100644 --- a/crates/cellar/durability/raft/snapshot_transport.rs +++ b/crates/cellar/durability/raft/snapshot_transport.rs @@ -8,7 +8,7 @@ use crate::raft::completion::ApplyTicket; use crate::raft::snapshot::{PreparedSnapshot, SnapshotDownload, SnapshotLimits, SnapshotPackage}; use cellar_logic::consensus::MemberId; use sha2::{Digest, Sha256}; -use std::sync::{atomic::AtomicBool, Weak}; +use std::sync::{atomic::AtomicU8, Weak}; #[path = "snapshot_receive.rs"] mod receive; @@ -102,6 +102,10 @@ impl SourceReservation { } } +const SOURCE_OFFERED: u8 = 0; +const SOURCE_STREAMING: u8 = 1; +const SOURCE_FINISHED: u8 = 2; + pub(super) struct Source { id: u64, route: Arc, @@ -109,10 +113,32 @@ pub(super) struct Source { package: SnapshotPackage, digest: [u8; 32], expires: u64, - finished: AtomicBool, + phase: AtomicU8, _reservation: SourceReservation, } +impl Source { + fn finish(&self) { + self.phase.store(SOURCE_FINISHED, Ordering::Release); + } + + fn is_finished(&self) -> bool { + self.phase.load(Ordering::Acquire) == SOURCE_FINISHED + } + + fn begin_pull(&self) -> bool { + matches!( + self.phase.compare_exchange( + SOURCE_OFFERED, + SOURCE_STREAMING, + Ordering::AcqRel, + Ordering::Acquire, + ), + Ok(_) | Err(SOURCE_STREAMING) + ) + } +} + pub(crate) struct SnapshotSource(Arc); struct ActiveSource { @@ -122,17 +148,31 @@ struct ActiveSource { impl Drop for ActiveSource { fn drop(&mut self) { - self.source.finished.store(true, Ordering::Release); + self.source.finish(); } } impl SnapshotSource { - pub(crate) fn targets(&self, scope: &GroupScope, member: MemberId) -> bool { - self.0.peer.replica.scope == *scope && self.0.peer.replica.member == member + /// Yield an unused offer or supersede this learner's previous transfer. + /// Claiming a pull and yielding an offer are mutually exclusive. All Arc + /// owners still retain the physical reservation after cancellation. + pub(crate) fn yield_for_seed(&self, scope: &GroupScope, member: MemberId) -> bool { + if self.0.peer.replica.scope == *scope && self.0.peer.replica.member == member { + return true; + } + matches!( + self.0.phase.compare_exchange( + SOURCE_OFFERED, + SOURCE_FINISHED, + Ordering::AcqRel, + Ordering::Acquire, + ), + Ok(_) | Err(SOURCE_FINISHED) + ) } pub(crate) fn cancel(self) { - self.0.finished.store(true, Ordering::Release); + self.0.finish(); } pub fn id(&self) -> u64 { @@ -140,7 +180,7 @@ impl SnapshotSource { } pub fn finished(&self, now: u64) -> bool { - self.0.finished.load(Ordering::Acquire) || now >= self.0.expires + self.0.is_finished() || now >= self.0.expires } pub fn matches( @@ -276,7 +316,7 @@ impl RaftTransport { .mono_ms() .map_err(|_| TransportError::Closed)? .saturating_add(state.options.snapshot_timeout_ms), - finished: AtomicBool::new(false), + phase: AtomicU8::new(SOURCE_OFFERED), _reservation: reservation, }); // The host owns the body and expiry. The registry cannot extend either @@ -288,7 +328,7 @@ impl RaftTransport { // Route replacement may race source construction. Publishing before // rechecking closes the gap with install/remove's invalidation pass. state.invalidate_snapshot_routes(); - if source.finished.load(Ordering::Acquire) { + if source.is_finished() { return Err(TransportError::StalePeer); } Ok(SnapshotSource(source)) @@ -481,7 +521,7 @@ impl State { .and_then(|source| source.as_ref().and_then(Weak::upgrade)); if let Some(source) = source { if !self.current_route(&source.peer.replica.scope, &source.route, &source.peer) { - source.finished.store(true, Ordering::Release); + source.finish(); } } } @@ -493,7 +533,7 @@ impl State { .and_then(Weak::upgrade) .filter(|source| source.id == id) { - source.finished.store(true, Ordering::Release); + source.finish(); } } } @@ -592,7 +632,7 @@ impl State { if selected .as_ref() .is_some_and(|old| !Arc::ptr_eq(&old.source, &lease)) - || lease.finished.load(Ordering::Acquire) + || lease.is_finished() || (frame::decimal(&request.lease)? != 0 && frame::decimal(&request.lease)? != lease.id) || lease.peer.replica != recipient @@ -615,6 +655,9 @@ impl State { { return Err(TransportError::Identity); } + if !lease.begin_pull() { + return Err(TransportError::StalePeer); + } transfer.get_or_insert(request.transfer.clone()); selected.get_or_insert_with(|| ActiveSource { source: lease.clone(), @@ -662,7 +705,7 @@ impl State { .await?; offset += body.len(); if offset == lease.package.encoded_length() { - lease.finished.store(true, Ordering::Release); + lease.finish(); return Ok(()); } } diff --git a/crates/cellar/node/driver.rs b/crates/cellar/node/driver.rs index 0a3d5792b..ce44de35d 100644 --- a/crates/cellar/node/driver.rs +++ b/crates/cellar/node/driver.rs @@ -568,37 +568,55 @@ where shutdown_accept_failure_test_active, "listener failure armed", ); + // Forced closure can win during this poll. Refused shutdown work means + // cancellation and must never produce a graceful completion proof. #[cfg(all(test, cellar_internal_tests))] { if let Some(listener) = listener { - spawn_shutdown_acceptor( + if spawn_shutdown_acceptor( listener, HttpSurface::Public, shutdown_connection_tx.clone(), shutdown_accept_failure_test_active, - ); + ) + .is_err() + { + return Ok(NodeOutcome::Cancelled); + } } - spawn_shutdown_acceptor( + if spawn_shutdown_acceptor( internal_listener, HttpSurface::Internal, shutdown_connection_tx, false, - ); + ) + .is_err() + { + return Ok(NodeOutcome::Cancelled); + } } #[cfg(not(all(test, cellar_internal_tests)))] { if let Some(listener) = listener { - spawn_shutdown_acceptor( + if spawn_shutdown_acceptor( listener, HttpSurface::Public, shutdown_connection_tx.clone(), - ); + ) + .is_err() + { + return Ok(NodeOutcome::Cancelled); + } } - spawn_shutdown_acceptor( + if spawn_shutdown_acceptor( internal_listener, HttpSurface::Internal, shutdown_connection_tx, - ); + ) + .is_err() + { + return Ok(NodeOutcome::Cancelled); + } } let mut drain_token_hold: Option = None; let (admitted_work_drained, ingress_drained_before_cut) = { @@ -617,7 +635,7 @@ where let baseline_ownership = ready_ownership.clone(); let wait = std::time::Duration::from_millis(drain_token_wait_ms) .min(handoff_deadline.saturating_duration_since(tokio::time::Instant::now())); - crate::asyncrt::spawn(async move { + let task = crate::asyncrt::try_spawn(async move { let now_ms = crate::ownership_store::now_ms(); let restoration_baseline = match baseline_ownership { Some(ownership) => match ownership.read_capacity_peers().await { @@ -662,8 +680,11 @@ where ) .await; *slot.lock().expect("drain token slot") = Some(outcome); - }) - .detach(); + }); + match task { + Ok(task) => task.detach(), + Err(_) => return Ok(NodeOutcome::Cancelled), + } } _ => { *drain_token.lock().expect("drain token slot") = diff --git a/crates/cellar/node/event_wait_tests.rs b/crates/cellar/node/event_wait_tests.rs index 40a278896..6f8af92cb 100644 --- a/crates/cellar/node/event_wait_tests.rs +++ b/crates/cellar/node/event_wait_tests.rs @@ -366,41 +366,54 @@ async fn cancellation_of_an_unpolled_local_wait_releases_the_actual_activity() - async fn real_dispatch_route_timeout_cancels_the_pending_actor_request() -> Result<()> { use crate::application::event_stream::{Authority, Binding}; let domain = crate::asyncrt::ExecutionDomain::new(tokio::runtime::Handle::current()); - let result = domain.run(async { - let (app, mut messages) = app()?; + let result = domain.run(crate::asyncrt::timeout(std::time::Duration::from_secs(2), async { let binding: Binding = serde_json::from_value(serde_json::json!({ "environment": "EVENTS", "resource": {"tenant": "t", "application": "a", "id": "a".repeat(32), "name": "e", "family": "e"}, "principal": "test", "grant": "grant", "read": true }))?; - let authority = Authority::resolution(&binding)?; - let (reply, result) = oneshot::channel(); - let call = crate::host_channels::RpcCallReq { - reservation: None, - parent_cancellation: None, - signal: None, - deadline_ms: None, - authority: Some(authority.into()), request_id: None, cancel: None, - scope: crate::application::event_stream::cell_scope(&binding.resource)?, - name: None, method: "__streamResolve".into(), - args: crate::engine_api::RpcData::Json("[]".into()), reply, - }; - let dispatch = super::super::dispatch_rpc_call(app, call); - let actor = async { - let crate::actor::Message::Request { request, reply, .. } = messages.recv().await.unwrap() + loop { + let (app, mut messages) = app()?; + let authority = Authority::resolution(&binding)?; + let (reply, result) = oneshot::channel(); + let call = crate::host_channels::RpcCallReq { + reservation: None, + parent_cancellation: None, + signal: None, + deadline_ms: None, + authority: Some(authority.into()), request_id: None, cancel: None, + scope: crate::application::event_stream::cell_scope(&binding.resource)?, + name: None, method: "__streamResolve".into(), + args: crate::engine_api::RpcData::Json("[]".into()), reply, + }; + let mut dispatch = Box::pin(super::super::dispatch_rpc_call(app, call)); + // The host can exhaust this attempt's unchanged 10 ms budget before + // its first routing poll. Verify that refusal submitted nothing, then + // establish the pending-route state within the original outer bound. + if dispatch.as_mut().now_or_never().is_some() { + let error = result.await?.err().expect("RPC unexpectedly succeeded"); + assert_eq!(code(&error), ErrorCode::Unavailable); + untagged(&error); + assert!(messages.try_recv().is_err(), "completed setup submitted actor work"); + crate::asyncrt::sleep(std::time::Duration::from_millis(1)).await; + continue; + } + let crate::actor::Message::Request { request, reply, .. } = messages.try_recv()? else { panic!("routing request missing") }; - let cancelled = messages.recv().await.unwrap(); - assert!(matches!(cancelled, crate::actor::Message::CancelRoute { request: observed } if observed == request)); - drop(reply); - }; - crate::asyncrt::timeout(std::time::Duration::from_secs(2), futures_util::future::join(dispatch, actor)).await?; - let error = result.await?.err().expect("RPC unexpectedly succeeded"); - assert_eq!(code(&error), ErrorCode::Unavailable); - untagged(&error); - Ok::<_, anyhow::Error>(()) - }).await?; + let actor = async { + let cancelled = messages.recv().await.unwrap(); + assert!(matches!(cancelled, crate::actor::Message::CancelRoute { request: observed } if observed == request)); + drop(reply); + }; + futures_util::future::join(dispatch, actor).await; + let error = result.await?.err().expect("RPC unexpectedly succeeded"); + assert_eq!(code(&error), ErrorCode::Unavailable); + untagged(&error); + return Ok::<_, anyhow::Error>(()); + } + })).await?; domain.shutdown().await; - result + result? } #[tokio::test] diff --git a/crates/cellar/node/mod.rs b/crates/cellar/node/mod.rs index acb7d5ca6..1a2834ec3 100644 --- a/crates/cellar/node/mod.rs +++ b/crates/cellar/node/mod.rs @@ -72,6 +72,9 @@ pub(crate) mod operations; mod raft_metrics; mod raft_startup; mod rpc_deadline; +#[cfg(test)] +#[path = "shutdown_acceptor_tests.rs"] +mod shutdown_acceptor_tests; mod source_operator; mod state_sampling; mod stream_operator; @@ -3841,11 +3844,13 @@ fn spawn_shutdown_acceptor( surface: HttpSurface, connections: mpsc::Sender, #[cfg(all(test, cellar_internal_tests))] mut fail_next_for_test: bool, -) { +) -> Result<(), crate::asyncrt::DomainClosed> { // Shutdown selects finish whenever a timer, request, or actor message // wins. Keep each listener's accept and retry waits in one task so those // unrelated events cannot cancel a wait or reset its retry delay. - crate::asyncrt::spawn(async move { + // Forced closure can race this supervisor poll. Refuse task admission and + // drop the captured listener instead of panicking or detaching unowned work. + let task = crate::asyncrt::try_spawn(async move { let mut retry_delay = SHUTDOWN_ACCEPT_RETRY_INITIAL; let mut recovering = false; #[cfg(all(test, cellar_internal_tests))] @@ -3922,8 +3927,9 @@ fn spawn_shutdown_acceptor( } } } - }) - .detach(); + })?; + task.detach(); + Ok(()) } fn serve_http_connection( diff --git a/crates/cellar/node/shutdown_acceptor_tests.rs b/crates/cellar/node/shutdown_acceptor_tests.rs new file mode 100644 index 000000000..834058d59 --- /dev/null +++ b/crates/cellar/node/shutdown_acceptor_tests.rs @@ -0,0 +1,64 @@ +// Copyright 2026 Cellar Project Authors. Licensed under Apache-2.0. + +//! Forced closure can arrive during the supervisor's current shutdown poll. +#![allow(clippy::disallowed_methods)] +use super::*; +use futures_util::FutureExt as _; + +#[tokio::test] +async fn closed_shutdown_acceptor_releases_its_listener_without_panicking() { + let domain = Arc::new(crate::asyncrt::ExecutionDomain::new( + tokio::runtime::Handle::current(), + )); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let (connections, mut received) = mpsc::channel(1); + let closing = domain.clone(); + let outcome = std::panic::AssertUnwindSafe(domain.run(async move { + // No await separates closure from admission. This reproduces closure + // racing a supervisor poll which already passed its cancellation check. + closing.close(); + #[cfg(not(cellar_internal_tests))] + let _ = spawn_shutdown_acceptor(listener, HttpSurface::Internal, connections); + #[cfg(cellar_internal_tests)] + let _ = spawn_shutdown_acceptor(listener, HttpSurface::Internal, connections, false); + assert!(matches!( + received.try_recv(), + Err(mpsc::error::TryRecvError::Disconnected) + )); + let rebound = std::net::TcpListener::bind(address).unwrap(); + drop(rebound); + })) + .catch_unwind() + .await; + domain.shutdown().await; + assert!(outcome.is_ok(), "shutdown admission panicked after closure"); + assert!(domain.is_closed()); + assert_eq!(tokio::spawn(async { 7 }).await.unwrap(), 7); +} + +#[tokio::test] +async fn open_shutdown_acceptor_delivers_its_owned_connection_and_joins() { + let domain = Arc::new(crate::asyncrt::ExecutionDomain::new( + tokio::runtime::Handle::current(), + )); + let outcome = domain + .run(async { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let (connections, mut received) = mpsc::channel(1); + #[cfg(not(cellar_internal_tests))] + let _ = spawn_shutdown_acceptor(listener, HttpSurface::Internal, connections); + #[cfg(cellar_internal_tests)] + let _ = spawn_shutdown_acceptor(listener, HttpSurface::Internal, connections, false); + let client = tokio::net::TcpStream::connect(address).await.unwrap(); + let connection = received.recv().await.unwrap(); + assert_eq!(connection.surface.name(), "internal"); + assert!(connection.received.send(()).is_ok()); + drop(connection.stream); + drop(client); + }) + .await; + domain.shutdown().await; + outcome.unwrap(); +} diff --git a/crates/cellar/runtime/workers/native/tests.rs b/crates/cellar/runtime/workers/native/tests.rs index cfe307563..b105dde12 100644 --- a/crates/cellar/runtime/workers/native/tests.rs +++ b/crates/cellar/runtime/workers/native/tests.rs @@ -449,6 +449,23 @@ async fn check_shared_worker_pool() -> Result<()> { .map_err(anyhow::Error::msg)?; pool.reap(); assert!(pool.is_drained()); + // A concurrent maintenance pass can inspect the now-empty worker mutex. + // Its lock must not revoke the completed physical reclamation proof. + let inspecting_freed = slot.hold_turn_for_test().await; + assert!( + pool.is_drained(), + "freed worker mutex contention revoked the pool drain proof" + ); + drop(inspecting_freed); + let reused = pool.admit(false).map_err(|error| anyhow!("{error}"))?; + assert!(reused.slot().is_retiring()); + assert!( + !pool.is_drained(), + "a reused slot inherited its predecessor's drain proof" + ); + drop(reused); + pool.reap(); + assert!(pool.is_drained()); assert!(crate::pool::current_slot().is_none()); assert!(installed_context().is_none()); crate::host_tcp::tests::native_tls_application(|application| { diff --git a/crates/cellar/runtime/workers/pool.rs b/crates/cellar/runtime/workers/pool.rs index 398a2aeaa..d6ba7b8cf 100644 --- a/crates/cellar/runtime/workers/pool.rs +++ b/crates/cellar/runtime/workers/pool.rs @@ -210,6 +210,10 @@ pub struct Slot { /// retiring slot is filtered out of every placement decision, so nothing /// can arrive afterwards. Entering one would be a bug in this module. worker: tokio::sync::Mutex>, + // Published only after the worker has actually been dropped. Slots are + // replaced rather than repopulated, so this physical reclamation proof is + // monotonic even while another maintenance pass holds the empty mutex. + reclaimed: AtomicBool, application_work: Arc, handler_budget: std::time::Duration, turn_scheduler: TurnScheduler, @@ -397,6 +401,7 @@ impl Slot { id: 0, heap_id: next_heap_id(), worker: tokio::sync::Mutex::new(Some(worker)), + reclaimed: AtomicBool::new(false), application_work: crate::asyncrt::services().application_work_registry(), handler_budget: crate::engine_api::handler_budget(), turn_scheduler: TurnScheduler::default(), @@ -432,6 +437,7 @@ impl Slot { id: 0, heap_id: HeapId::new(0), worker: tokio::sync::Mutex::new(None), + reclaimed: AtomicBool::new(true), handler_budget: crate::engine_api::handler_budget(), turn_scheduler: TurnScheduler::default(), queue_producers: Mutex::new(std::collections::HashMap::new()), @@ -448,7 +454,7 @@ impl Slot { }) } - #[cfg(cellar_internal_tests)] + #[cfg(any(test, cellar_internal_tests))] #[doc(hidden)] pub async fn hold_turn_for_test(&self) -> impl Drop + '_ { self.worker.lock().await @@ -736,6 +742,7 @@ impl Pool { id, heap_id, worker: tokio::sync::Mutex::new(Some(worker)), + reclaimed: AtomicBool::new(false), application_work: self.application_work.clone(), handler_budget: self.handler_budget, turn_scheduler: TurnScheduler::default(), @@ -939,7 +946,7 @@ impl Pool { .read() .expect("pool poisoned") .iter() - .all(|slot| slot.worker.try_lock().is_ok_and(|worker| worker.is_none())) + .all(|slot| slot.reclaimed.load(Ordering::Acquire)) } fn retire_one(&self) -> bool { @@ -970,7 +977,11 @@ impl Pool { if slot.worker_pins.load(Ordering::Acquire) != 0 { continue; } - if worker.take().is_some() { + let retired = worker.take(); + let was_live = retired.is_some(); + drop(retired); + slot.reclaimed.store(true, Ordering::Release); + if was_live { tracing::info!(isolate = %slot.heap_id, slot = slot.id, "isolate freed"); } } diff --git a/crates/cellar/tests/native_node.rs b/crates/cellar/tests/native_node.rs index 3a741a29e..d27eaf29b 100644 --- a/crates/cellar/tests/native_node.rs +++ b/crates/cellar/tests/native_node.rs @@ -2,6 +2,8 @@ //! Full public node bootstrap on caller-owned executors and infrastructure. #![allow(clippy::disallowed_methods)] +#[path = "../../../tools/qualification/durable_bucket.rs"] +mod durable_bucket; #[allow(dead_code)] #[path = "support/html.rs"] mod fixture; diff --git a/crates/cellar/tests/native_raft_fleet_permits.rs b/crates/cellar/tests/native_raft_fleet_permits.rs index 5b45712c9..d2183b7d1 100644 --- a/crates/cellar/tests/native_raft_fleet_permits.rs +++ b/crates/cellar/tests/native_raft_fleet_permits.rs @@ -62,14 +62,34 @@ struct Fixture { } async fn leader(domain: &ExecutionDomain, cells: &[RaftCellHandle]) -> Result { + Ok(leader_status(domain, cells).await?.0) +} + +// Local readiness can briefly outlive isolation. Route at the highest +// observed term; the guarded operation still has to establish its proof. +async fn leader_status( + domain: &ExecutionDomain, + cells: &[RaftCellHandle], +) -> Result<(usize, CellStatus)> { Ok(domain .run(cellar::asyncrt::timeout(TIMEOUT, async { loop { + let mut highest_term = 0; + let mut ready = None; for (index, cell) in cells.iter().enumerate() { - if cell.status().await.is_ok_and(|status| status.command_ready) { - return index; + if let Ok(status) = cell.status().await { + if status.term.get() > highest_term { + highest_term = status.term.get(); + ready = None; + } + if status.term.get() == highest_term && status.command_ready { + ready = Some((index, status)); + } } } + if let Some(ready) = ready { + return ready; + } cellar::asyncrt::sleep(Duration::from_millis(10)).await; } })) @@ -319,14 +339,19 @@ impl Fixture { authorization: &FleetPermitAuthorization, step: RepairStep, ) -> Result { - self.domain + let mut last_status = None; + let mut last_refusal = None; + let result = self + .domain .run(cellar::asyncrt::timeout(TIMEOUT, async { loop { - let expected = self.handles[self.primary] - .status() - .await? - .membership_revision; - match self.handles[self.primary] + // A repair survives leader loss. Route the same permit + // and idempotent step to a freshly observed leader; the + // initial fixture leader is not permanent authority. + let (primary, status) = leader_status(&self.domain, &self.handles).await?; + let expected = status.membership_revision; + last_status = Some((primary, status)); + match self.handles[primary] .advance_managed_repair( authorization.clone(), step, @@ -336,15 +361,23 @@ impl Fixture { .await { Ok(status) => return Ok(status), - Err(CommandFailure::Unavailable | CommandFailure::Overloaded) => { + Err( + error @ (CommandFailure::Unavailable + | CommandFailure::Overloaded + | CommandFailure::NotLeader { .. }), + ) => { + last_refusal = Some(error); cellar::asyncrt::sleep(Duration::from_millis(10)).await } Err(error) => return Err(error.into()), } } })) - .await? - .context("managed repair step timed out")? + .await?; + result.map_err(anyhow::Error::from).and_then(|status| status) + .with_context(|| format!( + "managed repair {step:?}: last leader status {last_status:?}, refusal {last_refusal:?}", + )) } fn activation(&self, from_s3: bool) -> Result { @@ -456,7 +489,10 @@ impl Fixture { #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn fleet_permit_delayed_after_cancel() -> Result<()> { - let f = Fixture::new().await?; + let mut f = Fixture::new().await?; + // The reserved replacement receives protocol traffic once the plan starts. + // Start its real dormant process; activation still requires normal seeding. + f.standby().await?; let plan = f.plan(40).await?; let cached = f.reserve(1, 20, &plan).await?; let status = f.handles[f.primary] @@ -507,6 +543,43 @@ async fn fleet_permit_delayed_after_cancel() -> Result<()> { f.close().await } +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn fleet_permit_repair_step_follows_new_quorum_leader() -> Result<()> { + let mut f = Fixture::new().await?; + f.standby().await?; + let plan = f.plan(40).await?; + let authorization = f.reserve(1, 20, &plan).await?; + let repair = f.handles[f.primary] + .begin_managed_repair(authorization.clone(), plan, TIMEOUT) + .await?; + ensure!(repair.phase == RepairPhase::Seeding); + let (source, status) = leader_status(&f.domain, &f.handles).await?; + ensure!(source == f.primary); + // Optional transfers correctly refuse an active repair. Exercise actual + // leader loss instead, retaining the same permit and committed plan. + f.cluster.isolate_consensus(source).await?; + let survivors = f + .handles + .iter() + .enumerate() + .filter(|(index, _)| *index != source) + .map(|(_, cell)| cell.clone()) + .collect::>(); + let (_, successor) = leader_status(&f.domain, &survivors).await?; + ensure!(successor.term > status.term); + let advanced = f.step(&authorization, RepairStep::AbortLearner).await; + if advanced.is_err() { + for (index, cell) in f.handles.iter().enumerate() { + eprintln!("stalled repair node {index}: {:?}", cell.status().await); + } + } + f.cluster.heal(); + let closed = f.close().await; + let advanced = advanced.with_context(|| format!("joined repair fixture: {closed:?}"))?; + ensure!(advanced.phase == RepairPhase::Aborted); + closed +} + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn fleet_permit_crash_recovery() -> Result<()> { let mut f = Fixture::new().await?; diff --git a/crates/cellar/tests/native_raft_snapshot.rs b/crates/cellar/tests/native_raft_snapshot.rs index db2695fcd..46e611055 100644 --- a/crates/cellar/tests/native_raft_snapshot.rs +++ b/crates/cellar/tests/native_raft_snapshot.rs @@ -751,7 +751,13 @@ async fn native_seed_drill_store( let calls: Vec<_> = (0..4).map(|_| Arc::new(AtomicUsize::new(0))).collect(); let definitions = calls .iter() - .map(|calls| public_node::counter(calls.clone(), gate.clone())) + .map(|calls| { + if interrupt { + public_node::counter_with_seed_payload(calls.clone(), gate.clone()) + } else { + public_node::counter(calls.clone(), gate.clone()) + } + }) .collect::>>()?; let mut namespace = native_snapshot_config(); namespace.schema_digest = definitions[0].schema; @@ -2068,14 +2074,16 @@ async fn native_snapshot_drill( // Exercise independent-group progress on the same node whose // snapshot download is held, rather than depending on election luck. let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + let mut placement_observed = vec![String::new(); healthy.len()]; + let mut transfer_observed = String::new(); tokio::time::timeout_at(deadline, async { loop { - if healthy[follower] + let target_status = healthy[follower] .status() .await - .context("recovering node healthy-group leadership status")? - .command_ready - { + .context("recovering node healthy-group leadership status")?; + placement_observed[follower] = format!("{target_status:?}"); + if target_status.command_ready { return Ok::<_, anyhow::Error>(()); } for (node, handle) in healthy.iter().enumerate() { @@ -2088,8 +2096,9 @@ async fn native_snapshot_drill( let status = handle.status().await.with_context(|| { format!("healthy-group transfer source status on node {node}") })?; + placement_observed[node] = format!("{status:?}"); if status.command_ready { - match handle + let result = handle .transfer_leadership( LeadershipTransfer { target: MemberId::new(follower as u64 + 1)?, @@ -2098,8 +2107,9 @@ async fn native_snapshot_drill( }, deadline.saturating_duration_since(tokio::time::Instant::now()), ) - .await - { + .await; + transfer_observed = format!("node {node}: {result:?}"); + match result { Ok(()) | Err( CommandFailure::Overloaded @@ -2117,7 +2127,9 @@ async fn native_snapshot_drill( } }) .await - .context("place healthy leader on snapshot recovery node")??; + .with_context(|| format!( + "place healthy leader on snapshot recovery node {follower}: statuses {placement_observed:?}, last transfer {transfer_observed}" + ))??; } let mut observed = [None, None, None]; let healthy_primary = tokio::time::timeout(Duration::from_secs(10), async { diff --git a/crates/cellar/tests/support/event_owner_outage.rs b/crates/cellar/tests/support/event_owner_outage.rs index 843c4a63f..0bb5ebf5d 100644 --- a/crates/cellar/tests/support/event_owner_outage.rs +++ b/crates/cellar/tests/support/event_owner_outage.rs @@ -45,7 +45,12 @@ async fn subscription(client: &reqwest::Client, node: &Node) -> Result { async fn storage_isolated_event_owner_expires_and_recovers_before_replay() -> Result<()> { let directory = tempfile::tempdir()?; let old_core = Executor::new("isolated-event-core")?; - let bucket = cellar::bucket::Bucket::open_dev(&directory.path().join("fleet.sqlite"))?; + // Wrap the entire coherent namespace: isolation must refuse authority as + // well as data, while healthy successor CAS has its own FULL-sync writer. + let bucket = super::super::durable_bucket::open( + directory.path(), + &["isolated-event-owner", "successor-event-owner"], + )?; let (isolated, outage) = Outage::wrap(&bucket)?; let (mut config, services) = inputs( "isolated-event-owner", diff --git a/crates/cellar/tests/support/node_events.rs b/crates/cellar/tests/support/node_events.rs index f40a09055..4d3b5f7b9 100644 --- a/crates/cellar/tests/support/node_events.rs +++ b/crates/cellar/tests/support/node_events.rs @@ -2428,7 +2428,7 @@ fn metric(snapshot: &cellar::sdk::EventMetricsSnapshot, name: &str) -> u64 { async fn event_metrics_export_tracks_real_work_and_keeps_hosts_isolated() -> Result<()> { let directory = tempfile::tempdir()?; let core = Executor::new("event-metrics-core")?; - let bucket = cellar::bucket::Bucket::open_dev(&directory.path().join("fleet.sqlite"))?; + let bucket = super::durable_bucket::open(directory.path(), &["event-metrics-owner"])?; let (config, services) = inputs( "event-metrics-owner", &directory.path().join("owner"), @@ -2439,8 +2439,9 @@ async fn event_metrics_export_tracks_real_work_and_keeps_hosts_isolated() -> Res .await?; let owner = Node::start(config, services, graph(1, false), applications("owner")?).await?; super::ready(&owner).await?; + let sibling_directory = tempfile::tempdir_in(directory.path())?; let sibling_bucket = - cellar::bucket::Bucket::open_dev(&directory.path().join("sibling-fleet.sqlite"))?; + super::durable_bucket::open(sibling_directory.path(), &["event-metrics-sibling"])?; let (config, services) = inputs( "event-metrics-sibling", &directory.path().join("sibling"), diff --git a/crates/cellar/tests/support/node_filesystem_capacity.rs b/crates/cellar/tests/support/node_filesystem_capacity.rs index ac0687817..8a6bbd68a 100644 --- a/crates/cellar/tests/support/node_filesystem_capacity.rs +++ b/crates/cellar/tests/support/node_filesystem_capacity.rs @@ -1,7 +1,7 @@ // Copyright 2026 Cellar Project Authors. Licensed under Apache-2.0. //! Opt-in dense payload qualification through public SDK turns and bucket proof. -use super::{inputs, ready, response, Executor}; +use super::{durable_bucket, inputs, ready, response, Executor}; use anyhow::{ensure, Context as _, Result}; use cellar::sdk::{filesystem::*, *}; use serde_json::{json, Value}; @@ -336,7 +336,7 @@ async fn profile(mib: usize) -> Result { let directory = tempfile::tempdir()?; let management = ExecutionDomain::new(tokio::runtime::Handle::current()); let bucket = management - .run(async { cellar::bucket::Bucket::open_dev(&directory.path().join("bucket.sqlite")) }) + .run(async { durable_bucket::open(directory.path(), &["dense-owner", "dense-successor"]) }) .await??; let core = Executor::new("filesystem-dense-core")?; let http = reqwest::Client::builder() diff --git a/crates/cellar/tests/support/raft_catalog_fault.rs b/crates/cellar/tests/support/raft_catalog_fault.rs index 2db8a6c74..fa78a7f86 100644 --- a/crates/cellar/tests/support/raft_catalog_fault.rs +++ b/crates/cellar/tests/support/raft_catalog_fault.rs @@ -144,6 +144,7 @@ pub struct Store { pub payload_size_delta: AtomicUsize, pub payload_chunks: std::sync::Arc, pub refuse_archive_payload_reads: AtomicBool, + pub refused_archive_payload_reads: AtomicUsize, pub hold_next_payload: AtomicBool, pub payload_hang: AtomicBool, pub payload_waiting: AtomicBool, @@ -722,6 +723,8 @@ impl ObjectStore for Store { } if self.refuse_archive_payload_reads.load(Ordering::Acquire) { + self.refused_archive_payload_reads + .fetch_add(1, Ordering::AcqRel); return Err(failure()); } } diff --git a/crates/cellar/tests/support/raft_cluster.rs b/crates/cellar/tests/support/raft_cluster.rs index 52526f674..9a3885e90 100644 --- a/crates/cellar/tests/support/raft_cluster.rs +++ b/crates/cellar/tests/support/raft_cluster.rs @@ -29,7 +29,7 @@ struct Fault { stopped: bool, hold_response: bool, hold_after: Option, - hold_connection: Option, + hold_once: bool, } #[derive(Clone, Copy, PartialEq, Eq)] @@ -45,6 +45,7 @@ struct Proxy { active: Arc, attempts: Arc, held_response_bytes: Arc, + held_connection: Arc, task: Option>>, } @@ -57,6 +58,7 @@ impl Drop for HeldResponses { self.0.send_modify(|fault| { fault.hold_response = false; fault.hold_after = None; + fault.hold_once = false; }); } } @@ -75,6 +77,7 @@ async fn forward_responses( mut writer: impl AsyncWrite + Unpin, mut faults: watch::Receiver, held: Arc, + selected: Arc, connection: usize, ) -> std::io::Result<()> { let mut buffer = [0; 16 * 1024]; @@ -88,7 +91,8 @@ async fn forward_responses( let mut counted = false; while offset < count { let mut fault = *faults.borrow_and_update(); - if fault.hold_connection.is_some_and(|held| held != connection) { + let chosen = selected.load(Ordering::SeqCst); + if fault.hold_once && chosen != 0 && chosen != connection { fault.hold_response = false; fault.hold_after = None; } @@ -100,6 +104,13 @@ async fn forward_responses( .map_or(count - offset, |after| after.saturating_sub(forwarded)) }; if allowance == 0 { + if fault.hold_once + && selected + .compare_exchange(0, connection, Ordering::SeqCst, Ordering::SeqCst) + .is_err_and(|chosen| chosen != connection) + { + continue; + } if !counted { held.fetch_add(count - offset, Ordering::SeqCst); counted = true; @@ -133,7 +144,9 @@ impl Proxy { let active = Arc::new(AtomicUsize::new(0)); let attempts = Arc::new(AtomicUsize::new(0)); let held_response_bytes = Arc::new(AtomicUsize::new(0)); + let held_connection = Arc::new(AtomicUsize::new(0)); let held = held_response_bytes.clone(); + let selected = held_connection.clone(); let counted = active.clone(); let attempted = attempts.clone(); let task = tokio::spawn(async move { @@ -146,10 +159,14 @@ impl Proxy { done = connections.join_next(), if !connections.is_empty() => { if let Some(done) = done { done?; } } accepted = listener.accept(), if connections.len() < 8 => { let (mut source, _) = accepted?; + // Match the native transport: fault injection must not + // add Nagle/delayed-ACK stalls to small snapshot chunks. + source.set_nodelay(true)?; let connection=attempted.fetch_add(1, Ordering::SeqCst)+1; if changes.borrow().blocked { continue; } let cut = changes.clone(); let held = held.clone(); + let selected = selected.clone(); let active = counted.clone(); active.fetch_add(1, Ordering::SeqCst); let guard = Active(active); connections.spawn(async move { @@ -162,14 +179,15 @@ impl Proxy { cellar::asyncrt::select_biased! { "test fault transitions close sockets before more bytes are forwarded"; _ = connection_cut(cut.clone()) => (), _ = async { - if let Ok(target) = TcpStream::connect(destination).await { - let (mut from_source, to_source) = source.split(); - let (from_target, mut to_target) = target.into_split(); - cellar::asyncrt::select! { - _ = tokio::io::copy(&mut from_source, &mut to_target) => (), - _ = forward_responses(from_target, to_source, cut, held, connection) => (), - } + let target = TcpStream::connect(destination).await?; + target.set_nodelay(true)?; + let (mut from_source, to_source) = source.split(); + let (from_target, mut to_target) = target.into_split(); + cellar::asyncrt::select! { + _ = tokio::io::copy(&mut from_source, &mut to_target) => (), + _ = forward_responses(from_target, to_source, cut, held, selected, connection) => (), } + Ok::<_, std::io::Error>(()) } => (), } }); @@ -190,6 +208,7 @@ impl Proxy { active, attempts, held_response_bytes, + held_connection, task: Some(task), }) } @@ -1029,9 +1048,15 @@ impl Cluster { .find(|(s, t, lane, _)| *s == source && *t == target && *lane == Link::Data) .map(|(_, _, _, p)| p) .ok_or_else(|| anyhow::anyhow!("seed data link missing"))?; + proxy.held_connection.store(0, Ordering::SeqCst); + proxy.held_response_bytes.store(0, Ordering::SeqCst); proxy.state.send_modify(|fault| { - fault.hold_connection = Some(2); - fault.hold_after = Some(4096); + // Grant metadata is bounded to 64 KiB plus its header and TLS. + // The seed fixture carries a verified 256-KiB SQL payload, so a + // response beyond 128 KiB is a real partial bulk body. Select it + // once regardless of how many authorization connections preceded it. + fault.hold_once = true; + fault.hold_after = Some(128 * 1024); }); Ok(proxy.held_response_bytes.clone()) } @@ -1044,7 +1069,10 @@ impl Cluster { .find(|(s, t, lane, _)| *s == source && *t == target && *lane == Link::Data) .map(|(_, _, _, proxy)| proxy) .ok_or_else(|| anyhow::anyhow!("data link not found"))?; - proxy.state.send_modify(|fault| fault.hold_response = true); + proxy.state.send_modify(|fault| { + fault.hold_response = true; + fault.hold_once = false; + }); Ok(HeldResponses(proxy.state.clone())) } @@ -1061,9 +1089,10 @@ impl Cluster { .find(|(s, t, lane, _)| *s == source && *t == target && *lane == Link::Data) .map(|(_, _, _, proxy)| proxy) .ok_or_else(|| anyhow::anyhow!("data link not found"))?; - proxy - .state - .send_modify(|fault| fault.hold_after = Some(after)); + proxy.state.send_modify(|fault| { + fault.hold_after = Some(after); + fault.hold_once = false; + }); Ok(HeldResponses(proxy.state.clone())) } diff --git a/crates/cellar/tests/support/raft_initial_native.rs b/crates/cellar/tests/support/raft_initial_native.rs index 47cd0c0f9..0b72f44ad 100644 --- a/crates/cellar/tests/support/raft_initial_native.rs +++ b/crates/cellar/tests/support/raft_initial_native.rs @@ -278,7 +278,7 @@ async fn activate(cells: &[RaftCellHandle], auth: &FleetGroupAuthorization) -> R } async fn primary(f: &Fixture, cells: &[RaftCellHandle]) -> Result { - Ok(f.domain + f.domain .run(cellar::asyncrt::timeout(TIMEOUT, async { loop { for (i, cell) in cells.iter().enumerate() { @@ -289,7 +289,8 @@ async fn primary(f: &Fixture, cells: &[RaftCellHandle]) -> Result { cellar::asyncrt::sleep(Duration::from_millis(10)).await; } })) - .await??) + .await? + .context("command-ready primary deadline") } async fn write_read( diff --git a/crates/cellar/tests/support/raft_isolated_removal.rs b/crates/cellar/tests/support/raft_isolated_removal.rs index 03ebebe8d..353efce61 100644 --- a/crates/cellar/tests/support/raft_isolated_removal.rs +++ b/crates/cellar/tests/support/raft_isolated_removal.rs @@ -33,9 +33,15 @@ struct Repaired { impl Repaired { async fn new() -> Result { - let mut s = Box::pin(Started::new()).await?; - let (namespace, _) = initial_birth(&s.f).await?; - write_read(&s.f, &s.cells, 1, 7, 7).await?; + let mut s = Box::pin(Started::new()) + .await + .context("start isolated removal fixture")?; + let (namespace, _) = initial_birth(&s.f) + .await + .context("initialize isolated removal history")?; + write_read(&s.f, &s.cells, 1, 7, 7) + .await + .context("verify state before isolated removal")?; let p = primary(&s.f, &s.cells).await?; let obsolete = (p + 1) % 3; let evidence = attestation(&s.f, &s.nodes[p].host, &s.observer).await?; @@ -52,7 +58,8 @@ impl Repaired { std::slice::from_ref(&namespace), TIMEOUT, ) - .await?; + .await + .context("admit capacity before isolated replacement")?; let old = s.nodes.remove(obsolete); let boot = old .host @@ -60,7 +67,9 @@ impl Repaired { .await? .fleet_boot .context("old boot")?; - joined(old.node).await?; + joined(old.node) + .await + .context("join obsolete owner before isolation")?; let cut = s.f.catalog.read(TIMEOUT).await?; s.f.catalog .begin_evacuation( @@ -70,7 +79,8 @@ impl Repaired { std::slice::from_ref(&namespace), TIMEOUT, ) - .await?; + .await + .context("begin evacuation of joined obsolete owner")?; let cut = s.f.catalog.read(TIMEOUT).await?; s.f.catalog .attest_node_isolation( @@ -85,7 +95,8 @@ impl Repaired { }, TIMEOUT, ) - .await?; + .await + .context("attest isolation of joined obsolete owner")?; let mut active = s .cells .iter() @@ -141,16 +152,19 @@ impl Repaired { std::slice::from_ref(&namespace), TIMEOUT, ) - .await?; + .await + .context("reserve isolated replacement movement")?; let permit = admitted.authorization(); let target = s .observer .register_managed_history(s.authorization.clone(), TIMEOUT) - .await?; + .await + .context("register isolated replacement history")?; discover(&s.f, &[&s.nodes[0].host, &s.nodes[1].host, &s.observer]).await?; active[p] .begin_managed_repair(permit.clone(), movement.plan.clone(), TIMEOUT) - .await?; + .await + .context("begin isolated replacement repair")?; let source_member = s .proposal .voters @@ -169,7 +183,8 @@ impl Repaired { timeout: TIMEOUT, }, ) - .await?; + .await + .context("activate isolated replacement from archive")?; ensure!( super::super::super::retirement::step( &s.f, @@ -177,7 +192,8 @@ impl Repaired { &permit, RepairStep::PromoteAndRemove ) - .await? + .await + .context("promote isolated replacement and remove obsolete voter")? .phase == RepairPhase::Joint ); @@ -188,12 +204,15 @@ impl Repaired { &permit, RepairStep::FinishJoint ) - .await? + .await + .context("finish isolated replacement joint configuration")? .phase == RepairPhase::Complete ); active.push(target); - write_read(&s.f, &active, 2, 5, 12).await?; + write_read(&s.f, &active, 2, 5, 12) + .await + .context("verify state after isolated replacement")?; s.f.domain .run(cellar::asyncrt::timeout(TIMEOUT, async { loop { @@ -236,7 +255,8 @@ impl Repaired { &s.observer, TIMEOUT, ) - .await?; + .await + .context("complete isolated replacement movement")?; ensure!(complete.isolated_sources() == [old.spec.node.bytes()]); let peers = [ s.nodes[0].spec.node, @@ -270,11 +290,17 @@ impl Repaired { std::slice::from_ref(&self.namespace), TIMEOUT, ) - .await?; + .await + .context("prepare isolated removal verification")?; let fixed = proof .freeze_live(&self.s.observer, self.peers, TIMEOUT) - .await?; - let reference = self.s.reference(&fixed).await?; + .await + .context("freeze isolated removal survivor cut")?; + let reference = self + .s + .reference(&fixed) + .await + .context("capture isolated removal archive reference")?; proof .verify_live( &self.s.observer, @@ -282,13 +308,17 @@ impl Repaired { archive(&self.s, &reference), TIMEOUT, ) - .await?; + .await + .context("verify isolated removal archive replay and survivors")?; Ok(proof) } async fn finalizing(&self) -> Result { let mut proof = self.verification(&self.s.f.catalog).await?; - let candidate = proof.publish_candidate(TIMEOUT).await?; + let candidate = proof + .publish_candidate(TIMEOUT) + .await + .context("publish isolated removal candidate")?; let cut = self.s.f.catalog.read(TIMEOUT).await?; let intent = request(230, cut.revision()); ensure!(candidate @@ -330,7 +360,8 @@ impl Repaired { ensure!(root.revision() == cut.revision() + 1); let fence = candidate .begin_isolated_finalization(&root, &intent, ProposalId::new(ISOLATION)?, &[], TIMEOUT) - .await?; + .await + .context("reconcile isolated finalization after lost confirmation")?; ensure!(fence.isolation_request() == Some(ISOLATION)); ensure!(fence.stop_request(ProposalId::new([238; 16])?).is_err()); ensure!(fence diff --git a/crates/cellar/tests/support/raft_isolated_repair_service.rs b/crates/cellar/tests/support/raft_isolated_repair_service.rs index efdb19f38..fa6b9b744 100644 --- a/crates/cellar/tests/support/raft_isolated_repair_service.rs +++ b/crates/cellar/tests/support/raft_isolated_repair_service.rs @@ -60,8 +60,13 @@ impl RepairFleet { } async fn with_admission(extra: bool, isolated: Option, histories: u8) -> Result { - let mut s = Box::pin(Setup::with_histories(true, 6, histories)).await?; - let (spare_root, spare_node, spare) = s.f.peer_host(8).await?; + let mut s = Box::pin(Setup::with_histories(true, 6, histories)) + .await + .context("construct isolated-repair resource fixture")?; + let (spare_root, spare_node, spare) = + s.f.peer_host(8) + .await + .context("start isolated-repair spare host")?; if extra { let mut proposal = s.proposals[0].clone(); proposal.scope.cell = "Counter:independent-majority".into(); @@ -133,11 +138,19 @@ impl RepairFleet { }; cells.push( host.register_managed_history(authorization.clone(), TIMEOUT) - .await?, + .await + .with_context(|| { + format!("register isolated-repair voter for {}", proposal.scope.cell) + })?, ); nodes.push(member.node); } - activate(&cells, &authorization).await?; + activate(&cells, &authorization).await.with_context(|| { + format!( + "activate isolated-repair voters for {}", + proposal.scope.cell + ) + })?; write_read(&s.f, &cells, 1, 7, 7).await.with_context(|| { format!( "initial SQL for isolated-repair history {}", @@ -147,7 +160,9 @@ impl RepairFleet { live.push(nodes.into_iter().zip(cells).collect::>()); } for (index, host) in s.nodes.iter().map(|n| &n.host).chain([&spare]).enumerate() { - let evidence = crate::hosts::capacity_tests::attestation(&s.f, &s.target, host).await?; + let evidence = crate::hosts::capacity_tests::attestation(&s.f, &s.target, host) + .await + .with_context(|| format!("attest isolated-repair host {index}"))?; let cut = s.f.catalog.read(TIMEOUT).await?; s.f.catalog .admit_node_capacity( @@ -161,7 +176,8 @@ impl RepairFleet { std::slice::from_ref(&s.namespace), TIMEOUT, ) - .await?; + .await + .with_context(|| format!("admit isolated-repair host capacity {index}"))?; } discover(&s.f, &hosts).await?; // Exercise a failed leader, rather than only a failed follower. @@ -178,7 +194,8 @@ impl RepairFleet { }, TIMEOUT, ) - .await?; + .await + .context("transfer initial isolated-repair leader")?; } s.f.domain .run(cellar::asyncrt::timeout(TIMEOUT, async { @@ -193,7 +210,8 @@ impl RepairFleet { let boot = old .host .observe_node(TIMEOUT) - .await? + .await + .context("observe failed isolated-repair leader")? .fleet_boot .context("failed leader boot")?; let old_engine = old.node.info().data_dir.join("raft"); @@ -245,7 +263,12 @@ impl RepairFleet { .chain([&self.s.target, &self.spare]) { hosts.push(( - host.observe_node(TIMEOUT).await?.node.identity.node, + host.observe_node(TIMEOUT) + .await + .context("observe isolated-repair inventory host")? + .node + .identity + .node, host.clone(), )); } @@ -291,7 +314,13 @@ impl RepairFleet { .map(|(_, c)| c.clone()) .collect::>(); for host in [&self.s.target, &self.spare] { - let node = host.observe_node(TIMEOUT).await?.node.identity.node; + let node = host + .observe_node(TIMEOUT) + .await + .context("observe post-repair host identity")? + .node + .identity + .node; if self.live[index].iter().any(|(n, _)| *n == node) { continue; } @@ -310,7 +339,8 @@ impl RepairFleet { for cell in &cells { let proof = cell .observe_fleet_membership(authorization.clone(), TIMEOUT) - .await?; + .await + .with_context(|| format!("observe repaired membership for {}", scope.cell))?; ensure!( proof.configuration().voters().len() == 3 && proof.configuration().outgoing_voters().is_empty() @@ -604,12 +634,19 @@ async fn fleet_isolated_repair_service_skips_minority_and_repairs_independent_ma } async fn minority() -> Result<()> { - let mut fleet = Box::pin(RepairFleet::new(true)).await?; + let mut fleet = Box::pin(RepairFleet::new(true)) + .await + .context("construct minority isolated-repair fleet")?; let second = fleet.s.nodes.remove(0); ensure!(second.spec.node == NodeId::new([5; 16])?); let second_engine = second.node.info().data_dir.join("raft"); - joined(second.node).await?; - let evidence = fleet.evidence().await?; + joined(second.node) + .await + .context("join second failed minority voter")?; + let evidence = fleet + .evidence() + .await + .context("collect minority isolated-repair evidence")?; let hosts = fleet .s .nodes @@ -617,7 +654,9 @@ async fn minority() -> Result<()> { .map(|n| &n.host) .chain([&fleet.s.target, &fleet.spare]) .collect::>(); - discover(&fleet.s.f, &hosts).await?; + discover(&fleet.s.f, &hosts) + .await + .context("discover surviving minority-repair hosts")?; let service = fleet .s .f @@ -629,7 +668,8 @@ async fn minority() -> Result<()> { ProposalId::new([100; 16])?, options(), ) - .await?; + .await + .context("start minority isolated-repair service")?; fleet .wait(&service, |s| { s.settled_movements == 1 @@ -638,7 +678,8 @@ async fn minority() -> Result<()> { r.inventory_exhausted && r.blocked_histories == 3 && r.histories_checked == 4 }) }) - .await?; + .await + .context("settle independent-majority repair")?; service.shutdown().await; let cut = fleet.s.f.catalog.read(TIMEOUT).await?; for proposal in &fleet.s.proposals[..3] { @@ -655,9 +696,14 @@ async fn minority() -> Result<()> { .await? .is_none()); } - fleet.verify(3).await?; - retained_engine(&fleet.s.f, second_engine).await?; - fleet.close().await + fleet + .verify(3) + .await + .context("verify independently repaired majority")?; + retained_engine(&fleet.s.f, second_engine) + .await + .context("verify retained minority engine")?; + fleet.close().await.context("join minority-repair fleet") } async fn retained_engine(f: &Fixture, path: std::path::PathBuf) -> Result<()> { diff --git a/crates/cellar/tests/support/raft_metrics_native.rs b/crates/cellar/tests/support/raft_metrics_native.rs index b3f273525..c8d5bd385 100644 --- a/crates/cellar/tests/support/raft_metrics_native.rs +++ b/crates/cellar/tests/support/raft_metrics_native.rs @@ -123,7 +123,18 @@ fn samples(text: &str) -> Result> { "duplicate or malformed sample" ); } - ensure!(identities.len() == 111); + // Archive readiness added two fixed group gauges. Require their exact + // identities as well as the total; neither may replace an older series. + ensure!(identities.len() == 113); + for name in [ + "cellar_raft_groups_archive_current", + "cellar_raft_groups_archive_unconfirmed", + ] { + ensure!( + identities.contains(name), + "missing archive readiness gauge: {name}" + ); + } ensure!(text.contains("cellar_raft_extents_sample_available 0\n")); for identity in &identities { ensure!( diff --git a/crates/cellar/tests/support/raft_movement_aborted.rs b/crates/cellar/tests/support/raft_movement_aborted.rs index 2b21a06be..b151971db 100644 --- a/crates/cellar/tests/support/raft_movement_aborted.rs +++ b/crates/cellar/tests/support/raft_movement_aborted.rs @@ -105,21 +105,25 @@ async fn fleet_aborted_movement_retains_seeded_history_and_recovers_receipt() -> } async fn retained_case() -> Result<()> { - let s = Box::pin(Setup::with_capacity(true, 2)).await?; - let started = start(&s, true).await?; - abort(&s, &started).await?; - warm_owners(&s, &started).await?; - write_read(&s.f, &started.cells, 2, 5, 12).await?; - let (cut, request) = completion(&s, &started.movement, 230).await?; - s.f.store - .fail_fleet_confirmation - .store(true, Ordering::Release); - ensure!(s - .f - .catalog - .settle_aborted_movement(&cut, &request, &s.target, TIMEOUT) + let s = Box::pin(Setup::with_capacity(true, 2)) .await - .is_err()); + .context("retained setup")?; + let started = start(&s, true).await.context("seed retained target")?; + abort(&s, &started).await.context("abort seeded target")?; + warm_owners(&s, &started) + .await + .context("warm aborted physical owners")?; + write_read(&s.f, &started.cells, 2, 5, 12) + .await + .context("SQL after abort")?; + let (cut, request) = completion(&s, &started.movement, 230).await?; + lose_confirmation(&s, || async { + s.f.catalog + .settle_aborted_movement(&cut, &request, &s.target, TIMEOUT) + .await + .map(|_| ()) + }) + .await?; ensure!(s.f.catalog.read(TIMEOUT).await?.revision() == cut.revision() + 1); let catalog = Fixture::open_catalog(&s.f.domain, s.f.store.clone())?; let current = catalog.read(TIMEOUT).await?; diff --git a/crates/cellar/tests/support/raft_movement_completion.rs b/crates/cellar/tests/support/raft_movement_completion.rs index f0c200649..e41441f72 100644 --- a/crates/cellar/tests/support/raft_movement_completion.rs +++ b/crates/cellar/tests/support/raft_movement_completion.rs @@ -84,23 +84,47 @@ async fn replace(s: &Setup, from_s3: bool) -> Result { .await?; // Warm only actual physical observations. The terminal API must independently // obtain its own fresh quorum and complete physical probes afterward. + let seen = std::sync::Mutex::new((None, Vec::new())); s.f.domain .run(cellar::asyncrt::timeout(TIMEOUT, async { loop { let cut = s.f.catalog.read(TIMEOUT).await?; let mut complete = true; + seen.lock() + .map_err(|_| anyhow::anyhow!("physical observations mutex"))? + .1 + .clear(); for node in &s.nodes { + seen.lock() + .map_err(|_| anyhow::anyhow!("physical observations mutex"))? + .0 = Some(node.spec.node); let observed = node .host .inspect_peer_physical_inventory(node.spec.node, cut.revision(), TIMEOUT) .await?; - complete &= observed.view.seed_workers == 0 - && observed + let history = if observed.view.seed_workers == 0 { + observed .inventory .as_ref() .unwrap() .history(&request.resources.group)? - .is_some_and(|h| h.completed_sequence() == 1); + } else { + None + }; + let ready = observed.view.seed_workers == 0 + && history.is_some_and(|h| h.completed_sequence() == 1); + complete &= ready; + { + let mut seen = seen + .lock() + .map_err(|_| anyhow::anyhow!("physical observations mutex"))?; + seen.0 = None; + seen.1.push(( + node.spec.node, + observed.view.seed_workers, + history.map(|h| (h.completed_sequence(), h.in_configuration())), + )); + } } if complete { return Ok::<_, anyhow::Error>(()); @@ -108,7 +132,13 @@ async fn replace(s: &Setup, from_s3: bool) -> Result { cellar::asyncrt::sleep(Duration::from_millis(10)).await; } })) - .await???; + .await? + .with_context(|| { + format!( + "warm terminal physical copies; observations {:?}", + seen.lock() + ) + })??; Ok(Replaced { cells, target, @@ -133,6 +163,44 @@ async fn completion( )) } +async fn lose_confirmation(s: &Setup, mut operation: F) -> Result<()> +where + F: FnMut() -> Fut, + Fut: std::future::Future>, +{ + s.f.store + .fail_fleet_confirmation + .store(true, Ordering::Release); + s.f.domain + .run(cellar::asyncrt::timeout(TIMEOUT, async { + loop { + // Both callers retain their exact idempotent request. Stop at + // the injected post-CAS loss; an earlier physical-admission + // refusal does not establish that this fault was exercised. + let result = Box::pin(operation()).await; + if !s.f.store.fail_fleet_confirmation.load(Ordering::Acquire) { + ensure!( + result.is_err(), + "injected confirmation loss returned success" + ); + return Ok::<_, anyhow::Error>(()); + } + match result { + Err( + FleetCatalogError::InventoryIncomplete + | FleetCatalogError::Unconfirmed + | FleetCatalogError::Overloaded, + ) => (), + other => anyhow::bail!("confirmation fault was not reached: {other:?}"), + } + cellar::asyncrt::sleep(Duration::from_millis(10)).await; + } + })) + .await? + .context("reach injected fleet confirmation loss")??; + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn fleet_movement_completion_reuses_slot_preserves_floor() -> Result<()> { let s = Box::pin(Setup::with_capacity(true, 2)).await?; @@ -228,26 +296,31 @@ async fn fleet_movement_completion_reuses_slot_preserves_floor() -> Result<()> { #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn fleet_movement_completion_lost_reply_survives_target_shutdown() -> Result<()> { - let s = Box::pin(Setup::new(true)).await?; - let r = replace(&s, false).await?; - let (cut, request) = completion(&s, &r.request, 230).await?; - s.f.store - .fail_fleet_confirmation - .store(true, Ordering::Release); - ensure!(s - .f - .catalog - .complete_movement(&cut, &request, &s.target, TIMEOUT) + let s = Box::pin(Setup::new(true)) .await - .is_err()); + .context("lost-reply setup")?; + let r = replace(&s, false).await.context("replace physical owner")?; + let (cut, request) = completion(&s, &r.request, 230) + .await + .context("completion request")?; + lose_confirmation(&s, || async { + s.f.catalog + .complete_movement(&cut, &request, &s.target, TIMEOUT) + .await + .map(|_| ()) + }) + .await?; let root = s.f.catalog.read(TIMEOUT).await?; ensure!(root.revision() == cut.revision() + 1); - joined(s.target_node).await?; + joined(s.target_node) + .await + .context("join completed target")?; let catalog = Fixture::open_catalog(&s.f.domain, s.f.store.clone())?; let current = catalog.read(TIMEOUT).await?; let receipt = catalog .complete_movement(¤t, &request, &s.nodes[0].host, TIMEOUT) - .await?; + .await + .context("recover completion receipt")?; ensure!(receipt.applied_revision() == cut.revision() + 1); let mut changed = request; changed.attestation.used.memory_bytes += 1; diff --git a/crates/cellar/tests/support/raft_movement_execution.rs b/crates/cellar/tests/support/raft_movement_execution.rs index 900c81dd4..ef437a546 100644 --- a/crates/cellar/tests/support/raft_movement_execution.rs +++ b/crates/cellar/tests/support/raft_movement_execution.rs @@ -62,7 +62,10 @@ async fn finish( .run(cellar::asyncrt::timeout(TIMEOUT * 4, async { let mut phases = Vec::new(); let mut last_error = None; - for _ in 0..64 { + // Refused observations do not confirm a movement turn. Retain the + // 64 confirmed-turn limit and the original cohort deadline while + // waiting for bounded physical/transport work to finish. + while phases.len() < 64 { let catalog = Fixture::open_catalog(&s.f.domain, s.f.store.clone())?; let cut = match catalog.read(TIMEOUT).await { Ok(cut) => cut, diff --git a/crates/cellar/tests/support/raft_movement_isolation.rs b/crates/cellar/tests/support/raft_movement_isolation.rs index e255624c0..1da2c6f2e 100644 --- a/crates/cellar/tests/support/raft_movement_isolation.rs +++ b/crates/cellar/tests/support/raft_movement_isolation.rs @@ -10,7 +10,11 @@ async fn isolate( node: NodeId, boot: FleetBootBinding, ) -> Result { - let cut = s.f.catalog.read(TIMEOUT).await?; + let cut = + s.f.catalog + .read(TIMEOUT) + .await + .context("read exclusion Root")?; s.f.catalog .begin_evacuation( &cut, @@ -19,7 +23,8 @@ async fn isolate( std::slice::from_ref(&s.namespace), TIMEOUT, ) - .await?; + .await + .context("admit node exclusion before isolation")?; let cut = s.f.catalog.read(TIMEOUT).await?; let request = FleetNodeIsolationRequest { identity: Fixture::request(221, cut.revision()), @@ -48,11 +53,15 @@ async fn isolate( .await .is_err()); let reopened = Fixture::open_catalog(&s.f.domain, s.f.store.clone())?; - let root = reopened.read(TIMEOUT).await?; + let root = reopened + .read(TIMEOUT) + .await + .context("read Root after lost isolation confirmation")?; ensure!(root.revision() == cut.revision() + 1); let receipt = reopened .attest_node_isolation(&root, &request, TIMEOUT) - .await?; + .await + .context("reconcile exact node isolation receipt")?; ensure!( receipt.boot() == &request.attestation.boot && receipt.applied_revision() == root.revision() @@ -145,7 +154,9 @@ async fn refresh(s: &Setup) -> Result<()> { async fn fleet_movement_isolated_obsolete_repairs_three_histories() -> Result<()> { // Three actual moves exceed the fleet's two-active-move book; each must // settle while the obsolete node stays offline and its engine is retained. - let mut s = Box::pin(Setup::with_histories(true, 3, 3)).await?; + let mut s = Box::pin(Setup::with_histories(true, 3, 3)) + .await + .context("set up three isolated histories")?; let mut groups = Vec::new(); for proposal in &s.proposals { let cut = s.f.catalog.read(TIMEOUT).await?; @@ -187,7 +198,9 @@ async fn fleet_movement_isolated_obsolete_repairs_three_histories() -> Result<() .await .is_err()); joined(stopped.node).await?; - let evidence = isolate(&s, stopped.spec.node, boot).await?; + let evidence = isolate(&s, stopped.spec.node, boot) + .await + .context("attest joined obsolete boot")?; // Even a saved startup capability cannot restart the retained engine. let retry_node = public_node::node_named( stopped.root.path(), @@ -230,7 +243,8 @@ async fn fleet_movement_isolated_obsolete_repairs_three_histories() -> Result<() std::slice::from_ref(&s.namespace), TIMEOUT, ) - .await?; + .await + .with_context(|| format!("reserve isolated movement {index}"))?; let permit = accepted.authorization(); let target = s .target @@ -275,46 +289,58 @@ async fn fleet_movement_isolated_obsolete_repairs_three_histories() -> Result<() ); cells.push(target); write_read(&s.f, &cells, 2, 5, 12).await?; - s.f.domain - .run(cellar::asyncrt::timeout(TIMEOUT, async { - loop { - let cut = s.f.catalog.read(TIMEOUT).await?; - let mut complete = true; - for node in &s.nodes { - let observed = node - .host - .inspect_peer_physical_inventory( - node.spec.node, - cut.revision(), - TIMEOUT, - ) - .await?; - complete &= observed.view.seed_workers == 0 - && observed - .inventory - .as_ref() - .unwrap() - .history(scope)? - .is_some_and(|h| h.completed_sequence() == 1); - } - if complete { - return Ok::<_, anyhow::Error>(()); - } - cellar::asyncrt::sleep(Duration::from_millis(10)).await; - } - })) - .await???; let (cut, completion) = completion(&s, &request, 231 + index as u8 * 2).await?; let done = - s.f.catalog - .complete_movement(&cut, &completion, &s.target, TIMEOUT) - .await?; + s.f.domain + .run(cellar::asyncrt::timeout(TIMEOUT, async { + loop { + let observed_cut = s.f.catalog.read(TIMEOUT).await?; + let mut complete = true; + for node in &s.nodes { + let observed = node + .host + .inspect_peer_physical_inventory( + node.spec.node, + observed_cut.revision(), + TIMEOUT, + ) + .await?; + complete &= observed.view.seed_workers == 0 + && observed + .inventory + .as_ref() + .unwrap() + .history(scope)? + .is_some_and(|h| h.completed_sequence() == 1); + } + if complete { + // A queued seed worker can start between the warm + // inventory and the API's independent physical probe. + // Reconcile this exact idempotent request within the + // original readiness bound; only a real receipt ends it. + match s + .f + .catalog + .complete_movement(&cut, &completion, &s.target, TIMEOUT) + .await + { + Ok(done) => return Ok(done), + Err(FleetCatalogError::Unconfirmed) => {} + Err(error) => return Err(anyhow::Error::from(error)), + } + } + cellar::asyncrt::sleep(Duration::from_millis(10)).await; + } + })) + .await? + .with_context(|| format!("complete isolated movement {index}"))??; ensure!(done.isolated_sources() == [stopped.spec.node.bytes()]); let root = s.f.catalog.read(TIMEOUT).await?; let replay = s.f.catalog .complete_movement(&root, &completion, &s.target, TIMEOUT) - .await?; + .await + .with_context(|| format!("reconcile isolated movement {index}"))?; ensure!( replay.applied_revision() == done.applied_revision() && replay.isolated_sources() == done.isolated_sources() diff --git a/crates/cellar/tests/support/raft_node.rs b/crates/cellar/tests/support/raft_node.rs index b2a6aa856..6fa8ada76 100644 --- a/crates/cellar/tests/support/raft_node.rs +++ b/crates/cellar/tests/support/raft_node.rs @@ -430,7 +430,7 @@ pub fn counter_with_blobs( Ok(()) }; if blobs { - counter_observed_with_blobs(observe, gate, true) + counter_definition(observe, gate, true, None) } else { counter_observed(observe, gate) } @@ -441,29 +441,57 @@ pub fn counter_observed( observe: impl Fn(&GroupScope) -> Result<()> + Send + Sync + 'static, gate: Arc, ) -> Result { - counter_observed_with_blobs(observe, gate, false) + counter_definition(observe, gate, false, None) } -fn counter_observed_with_blobs( +/// A bulk seed must exceed the bounded grant metadata, even after compression. +#[allow(dead_code)] // Shared support; only snapshot tests need the bulk payload. +pub fn counter_with_seed_payload( + calls: Arc, + gate: Arc, +) -> Result { + use sha2::Digest as _; + let mut bytes = Vec::with_capacity(256 * 1024); + for block in 0_u64..8192 { + bytes.extend_from_slice(&sha2::Sha256::digest(block.to_be_bytes())); + } + let observe = move |_: &GroupScope| { + calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + Ok(()) + }; + counter_definition(observe, gate, false, Some(bytes.into())) +} + +fn counter_definition( observe: impl Fn(&GroupScope) -> Result<()> + Send + Sync + 'static, gate: Arc, blobs: bool, + padding: Option>, ) -> Result { let mut class = RaftClass::new("Counter", 1, [21; 32])?; if blobs { class = class.immutable_blobs(RaftBlobLimits::default())?; } + let expected_padding = padding.clone(); let class=class .command("Increment",[22;32],move |context,input|{ observe(context.identity())?; let delta=i64::from_be_bytes(input.try_into()?); + if let Some(padding)=&padding { + context.sql().execute_batch("CREATE TABLE IF NOT EXISTS seed_payload(data BLOB NOT NULL)")?; + context.sql().execute("INSERT INTO seed_payload SELECT ?1 WHERE NOT EXISTS(SELECT 1 FROM seed_payload)",[padding.as_ref()])?; + } context.sql().execute_batch("CREATE TABLE IF NOT EXISTS counter(value INTEGER); INSERT INTO counter SELECT 0 WHERE NOT EXISTS(SELECT 1 FROM counter)")?; context.sql().execute("UPDATE counter SET value=value+?1",[delta])?; if context.command().sequence()==2 {gate.wait()?;} let value:i64=context.sql().query_row("SELECT value FROM counter",[],|row|row.get(0))?; Ok(value.to_be_bytes().to_vec()) })? - .query("Get",[23;32],|context,_|{ + .query("Get",[23;32],move |context,_|{ + if let Some(expected)=&expected_padding { + let actual:Vec=context.query_row("SELECT data FROM seed_payload",[],|row|row.get(0))?; + anyhow::ensure!(actual.as_slice()==expected.as_ref(),"seed payload changed"); + } let value:i64=context.query_row("SELECT value FROM counter",[],|row|row.get(0))?; Ok(value.to_be_bytes().to_vec()) })?; diff --git a/crates/cellar/tests/support/raft_operator_lifecycle_native.rs b/crates/cellar/tests/support/raft_operator_lifecycle_native.rs index c55e0e8fb..5719ecfaa 100644 --- a/crates/cellar/tests/support/raft_operator_lifecycle_native.rs +++ b/crates/cellar/tests/support/raft_operator_lifecycle_native.rs @@ -76,13 +76,19 @@ async fn fleet_operator_departure_redistributes_verifies_joins_and_removes() -> } async fn redistribute(f: &mut Scenario, admin: &Server, node: &str) -> Result<()> { - let mut begin = request(f, 164).await?; + let mut begin = request(f, 164) + .await + .context("departure redistribution evidence")?; begin.keep.retain(|n| n.as_str() != node); let (exit, value) = admin .cli(&admin.begin_args(&begin, "departure-keep.json")?, "owner") .await?; ensure!(exit == 0, "redistribution admission {exit}: {value}"); - let evidence = Arc::new(Inputs::new(f).await?); + let evidence = Arc::new( + Inputs::new(f) + .await + .context("departure redistribution service inputs")?, + ); let service = f.s.f .catalog @@ -93,16 +99,23 @@ async fn redistribute(f: &mut Scenario, admin: &Server, node: &str) -> Result<() options(), FleetTopologyCollectionOptions::default(), ) - .await?; + .await + .context("departure redistribution service start")?; let mut args = op_args("wait", &begin.operation_id); args.extend(["--timeout-seconds".into(), "40".into()]); - let (exit, value) = admin.cli(&args, "reader").await?; + let (exit, value) = admin + .cli(&args, "reader") + .await + .context("departure redistribution CLI wait")?; ensure!( exit == 0 && value["converged"] == true, - "redistribution {exit}: {value}" + "redistribution {exit}: {value}; service {:?}", + service.status() ); service.shutdown().await; - completed_sql(f, 164).await?; + completed_sql(f, 164) + .await + .context("departure redistribution SQL receipts")?; ensure!(f .live .iter() @@ -112,8 +125,10 @@ async fn redistribute(f: &mut Scenario, admin: &Server, node: &str) -> Result<() } async fn departure() -> Result<()> { - let mut f = Box::pin(Scenario::new(true)).await?; - let mut admin = server(&f).await?; + let mut f = Box::pin(Scenario::new(true)) + .await + .context("departure scenario")?; + let mut admin = server(&f).await.context("departure admin server")?; let node = hex::encode([4; 16]); let revision = f.s.f.catalog.read(TIMEOUT).await?.revision(); let refused = operation(&admin, 165, revision); @@ -129,7 +144,9 @@ async fn departure() -> Result<()> { ); ensure!(f.s.f.catalog.read(TIMEOUT).await?.revision() == revision); - redistribute(&mut f, &admin, &node).await?; + redistribute(&mut f, &admin, &node) + .await + .context("departure redistribution")?; let exclusion = operation(&admin, 167, f.s.f.catalog.read(TIMEOUT).await?.revision()); let args = admin.departure_args(&node, "evacuate", &exclusion); @@ -163,12 +180,53 @@ async fn departure() -> Result<()> { let args = admin.departure_args(&node, "prepare-removal", &preparation); let (exit, value) = admin.cli(&args, "evacuator").await?; ensure!(exit == 4 && value["error"]["code"] == "forbidden"); + let refused_before = + f.s.f + .store + .refused_archive_payload_reads + .load(Ordering::Acquire); f.s.f .store .refuse_archive_payload_reads .store(true, Ordering::Release); - let (exit, value) = admin.cli(&args, "remover").await?; + // A read-only quorum blocker can precede replay. It is not the injected + // failure and must not count as coverage. Reconcile the exact request only + // while its validated progress proves no history or authority was admitted. + let (exit, value) = f + .s + .f + .domain + .run(cellar::asyncrt::timeout(TIMEOUT, async { + loop { + let (exit, value) = admin.cli(&args, "remover").await?; + if exit != 0 { + return Ok::<_, anyhow::Error>((exit, value)); + } + let progress = accepted(value.clone())?; + ensure!( + progress.phase == RaftFleetEvacuationPhase::VerifyingBeforeStop + && progress.checked_histories == 0 + && progress.blocker.as_ref().is_some_and(|blocker| { + blocker.stage == RaftFleetVerificationStage::HistoryQuorum + }) + && progress.applied_revision.is_none() + && progress.fence_id.is_none(), + "replay fault allowed progress or authority: {value}" + ); + ensure!(f.s.f.catalog.read(TIMEOUT).await?.revision() == status.observed_revision); + cellar::asyncrt::sleep(Duration::from_millis(10)).await; + } + })) + .await???; ensure!(exit != 0, "physical replay failure admitted fence: {value}"); + ensure!( + f.s.f + .store + .refused_archive_payload_reads + .load(Ordering::Acquire) + > refused_before, + "archive replay fault was not reached: {value}" + ); f.s.f .store .refuse_archive_payload_reads @@ -253,7 +311,8 @@ async fn departure() -> Result<()> { // authenticated discovery path before the exact stop RPC. f.s.f .remote(&f.s.nodes[1].host, NodeId::new([4; 16])?, revision) - .await?; + .await + .context("departure finalizing process metadata")?; let (exit, value) = admin.cli(&stop_args, "stopper").await?; ensure!( (exit == 0 && accepted(value.clone())?.phase == RaftFleetEvacuationPhase::StopRequested) diff --git a/crates/cellar/tests/support/raft_operator_maintenance_native.rs b/crates/cellar/tests/support/raft_operator_maintenance_native.rs index 027eca77a..87f88803d 100644 --- a/crates/cellar/tests/support/raft_operator_maintenance_native.rs +++ b/crates/cellar/tests/support/raft_operator_maintenance_native.rs @@ -94,20 +94,64 @@ async fn routed(f: &Scenario, future: F) -> Result { let catalog = Fixture::open_catalog(&f.s.f.domain, f.s.f.store.clone())?; f.s.f .domain - .run(cellar::asyncrt::timeout(TIMEOUT, async { - // A peer-probe retry must not prevent polling a completed CLI result. - // Both branches share this original deadline; no task is detached. - cellar::asyncrt::select! { - value = &mut future => Ok(value), - result = refresh_routes(f, &catalog) => { - result?; - Err(anyhow::anyhow!("route refresher stopped")) - }, - } - })) + .run(cellar::asyncrt::timeout( + TIMEOUT, + cli_with_refresh(&mut future, refresh_routes(f, &catalog)), + )) .await?? } +async fn cli_with_refresh>>( + future: F, + refresh: R, +) -> Result { + cellar::asyncrt::select_biased! { + "a completed CLI result wins a simultaneous auxiliary route failure"; + value = future => Ok(value), + result = refresh => { + result?; + Err(anyhow::anyhow!("route refresher stopped")) + }, + } +} + +#[tokio::test] +async fn completed_cli_result_wins_a_ready_route_refresh_failure() -> Result<()> { + let domain = ExecutionDomain::new(tokio::runtime::Handle::current()); + let result = domain + .run(async { + for _ in 0..16 { + let value = cli_with_refresh(std::future::ready(37), async { + anyhow::bail!("route probe refused") + }) + .await?; + ensure!(value == 37); + } + Ok::<_, anyhow::Error>(()) + }) + .await; + domain.shutdown().await; + result? +} + +#[tokio::test] +async fn pending_cli_does_not_hide_a_route_refresh_failure() -> Result<()> { + let domain = ExecutionDomain::new(tokio::runtime::Handle::current()); + let result = domain + .run(async { + let error = cli_with_refresh(std::future::pending::<()>(), async { + anyhow::bail!("route probe refused") + }) + .await + .unwrap_err(); + ensure!(error.to_string() == "route probe refused"); + Ok::<_, anyhow::Error>(()) + }) + .await; + domain.shutdown().await; + result? +} + async fn view(server: &Server, operation: &str) -> Result { let args = vec![ "fleet".into(), @@ -231,7 +275,9 @@ async fn owned() -> Result<()> { .await?; let (node, mut command, request) = prepare(&f, &server).await?; command.extend(["--wait".into(), "--timeout-seconds".into(), "5".into()]); - let (exit, value) = routed(&f, server.cli(&command, "maintainer")).await??; + let (exit, value) = routed(&f, server.cli(&command, "maintainer")) + .await + .with_context(|| format!("owned prepare wait; service {:?}", service.status()))??; ensure!(exit == 0, "maintenance prepare wait {exit}: {value}"); let ready: RaftFleetMaintenanceStatus = serde_json::from_value(value)?; ready.validate()?; @@ -254,7 +300,9 @@ async fn owned() -> Result<()> { "--timeout-seconds".into(), "5".into(), ]); - let (exit, value) = routed(&f, server.cli(&command, "canceller")).await??; + let (exit, value) = routed(&f, server.cli(&command, "canceller")) + .await + .with_context(|| format!("owned cancellation wait; service {:?}", service.status()))??; ensure!(exit == 0, "maintenance cancellation wait {exit}: {value}"); let status: RaftFleetMaintenanceStatus = serde_json::from_value(value)?; status.validate()?; diff --git a/crates/cellar/tests/support/raft_operator_native.rs b/crates/cellar/tests/support/raft_operator_native.rs index 47f1051ca..ffd2707be 100644 --- a/crates/cellar/tests/support/raft_operator_native.rs +++ b/crates/cellar/tests/support/raft_operator_native.rs @@ -507,7 +507,10 @@ async fn cancel() -> Result<()> { let mut waiting = op_args("wait", &begin.operation_id); waiting.extend(["--timeout-seconds".into(), "1".into()]); let (exit, timed_out) = server.cli(&waiting, "reader").await?; - ensure!(exit == 6 && timed_out["error"]["code"] == "wait_timeout"); + ensure!( + exit == 6 && timed_out["error"]["code"] == "wait_timeout", + "CLI wait result {exit}: {timed_out}" + ); ensure!(f.s.f.catalog.read(TIMEOUT).await?.revision() == before.revision()); let current = f.s.f diff --git a/crates/cellar/tests/support/raft_operator_restart_native.rs b/crates/cellar/tests/support/raft_operator_restart_native.rs index 02f8ce078..9bdfe0e4f 100644 --- a/crates/cellar/tests/support/raft_operator_restart_native.rs +++ b/crates/cellar/tests/support/raft_operator_restart_native.rs @@ -279,11 +279,24 @@ async fn resume_primary( service: FleetRestartService, host: RaftCells, ) -> Result<()> { - let target_id = host.observe_node(TIMEOUT).await?.node.identity.node; + let target_id = host + .observe_node(TIMEOUT) + .await + .context("observe returned node identity")? + .node + .identity + .node; let node = hex::encode(target_id.bytes()); let reader = Fixture::open_catalog(&f.s.f.domain, f.s.f.store.clone())?; - let mut attestation = attestation(&f.s.f, &f.spare, &host).await?; - let mut profile = host.observe_node(TIMEOUT).await?.node.profile; + let mut attestation = attestation(&f.s.f, &f.spare, &host) + .await + .context("read returned node attestation")?; + let mut profile = host + .observe_node(TIMEOUT) + .await + .context("observe returned node profile")? + .node + .profile; profile.draining = false; attestation.profile_digest = fleet_capacity_profile_digest(&profile)?; // A new intended profile uses a new capacity revision. Reusing the old @@ -327,7 +340,10 @@ async fn resume_primary( f.spare.clone(), None, )?; - controller.reconcile(TIMEOUT).await?; + controller + .reconcile(TIMEOUT) + .await + .context("initialize manual resume controller")?; let routes = Fixture::open_catalog(&f.s.f.domain, f.s.f.store.clone())?; f.s.f .domain @@ -335,7 +351,9 @@ async fn resume_primary( TIMEOUT, refresh_routes_once(f, &routes), )) - .await???; + .await? + .context("initial resume route refresh deadline")? + .context("initial resume route refresh")?; let before = reader.read(TIMEOUT).await?.revision(); let rejected = controller.reconcile(TIMEOUT).await; let error = rejected @@ -363,13 +381,22 @@ async fn resume_primary( .run(cellar::asyncrt::timeout(TIMEOUT, async { let mut refreshed = None; loop { - let revision = routes.read(TIMEOUT).await?.revision(); + let revision = routes + .read(TIMEOUT) + .await + .context("read resume proof Root")? + .revision(); if refreshed != Some(revision) { - refreshed = Some(refresh_routes_once(f, &routes).await?); + refreshed = Some( + refresh_routes_once(f, &routes) + .await + .context("refresh resume proof routes")?, + ); } let progress = controller .reconcile(TIMEOUT) - .await? + .await + .context("advance manual resume controller")? .context("owned resume operation")?; { let mut seen = seen.lock().map_err(|_| anyhow::anyhow!("progress mutex"))?; diff --git a/crates/cellar/tests/support/raft_outbox_native.rs b/crates/cellar/tests/support/raft_outbox_native.rs index 165e5c5b6..4f7459680 100644 --- a/crates/cellar/tests/support/raft_outbox_native.rs +++ b/crates/cellar/tests/support/raft_outbox_native.rs @@ -260,18 +260,46 @@ impl Cluster { } async fn read(&self, index: usize, minimum: Option) -> Result { - Ok(self - .cell(index)? - .query( - self.definitions[index].get.clone(), - vec![], - QueryOptions { - consistency: minimum.map_or(QueryConsistency::Stale, QueryConsistency::AtLeast), - timeout: Duration::from_millis(1000), - ..Default::default() - }, - ) - .await?) + let deadline = tokio::time::Instant::now() + Duration::from_millis(1000); + tokio::time::timeout_at(deadline, async { + loop { + let result = self + .cell(index)? + .query( + self.definitions[index].get.clone(), + vec![], + QueryOptions { + consistency: minimum + .clone() + .map_or(QueryConsistency::Stale, QueryConsistency::AtLeast), + timeout: deadline + .saturating_duration_since(tokio::time::Instant::now()), + ..Default::default() + }, + ) + .await; + if !matches!(result, Err(CommandFailure::Overloaded)) { + return Ok::<_, anyhow::Error>(result?); + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await? + } + + async fn status(&self, index: usize) -> Result { + // Status shares admission with the scheduler's own observation. Keep + // its original SDK bound while waiting for a fresh successful sample. + tokio::time::timeout(Duration::from_secs(5), async { + loop { + let result = self.cell(index)?.status().await; + if !matches!(result, Err(CommandFailure::Overloaded)) { + return Ok::<_, anyhow::Error>(result?); + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await? } async fn settled(&self, sequence: u64) -> Result { @@ -314,7 +342,7 @@ impl Cluster { let leader = self.leader().await?; tokio::time::timeout(Duration::from_secs(10), async { loop { - let status = self.cell(leader)?.status().await?; + let status = self.status(leader).await?; if status.command_ready && status.archived >= status.applied.index { return Ok::<_, anyhow::Error>(()); } @@ -626,7 +654,7 @@ pub async fn leadership_race() -> Result<()> { loop { let read = f.destination.read(target, None).await?; if i64::from_be_bytes(read.result.as_slice().try_into()?) == 9 { - let status = f.destination.cell(target)?.status().await?; + let status = f.destination.status(target).await?; if status.archived < read.version.position.index { return Ok::<_, anyhow::Error>(()); } @@ -681,8 +709,13 @@ pub async fn replicated_progress() -> Result<()> { ensure!(f.source.store.archive_unavailable.load(Ordering::SeqCst)); f.source .read(leader, Some(settled.version_token.clone())) - .await?; - let status = f.source.cell(leader)?.status().await?; + .await + .context("source settled minimum read")?; + let status = f + .source + .status(leader) + .await + .context("source terminal progress status")?; ensure!( status.applied.index >= settled.version.position.index && status.archived < settled.version.position.index, @@ -693,7 +726,11 @@ pub async fn replicated_progress() -> Result<()> { ); // Source cleanup committed while its promised S3 completion response is // unavailable. Stop the producer and recover progress on another voter. - let report = f.source.stop(leader).await?; + let report = f + .source + .stop(leader) + .await + .context("stop producer after settled progress")?; ensure!( !report.unfinished.is_empty(), "missing ambiguous S3 completion in close report" diff --git a/crates/cellar/tests/support/raft_repair_native.rs b/crates/cellar/tests/support/raft_repair_native.rs index 507e6203a..a673b3f79 100644 --- a/crates/cellar/tests/support/raft_repair_native.rs +++ b/crates/cellar/tests/support/raft_repair_native.rs @@ -453,7 +453,11 @@ impl Fixture { } pub(super) async fn learner_before_promotion() -> Result<()> { - let f = Fixture::new(false).await.context("start learner fixture")?; + let mut f = Fixture::new(false).await.context("start learner fixture")?; + // The advertised peer must own a real protocol listener even while its + // learner remains unactivated. Otherwise unrelated sends wait for transport + // timeouts while the test is trying to observe a committed abort. + let learner = standby::registered(&mut f).await?; f.fail_obsolete().await.context("stop obsolete voter")?; let started = f.begin().await?; let current = f.handles[f.primary].status().await?; @@ -484,6 +488,11 @@ pub(super) async fn learner_before_promotion() -> Result<()> { matches!(result, Err(CommandFailure::Unavailable)), "empty learner promoted: {result:?}" ); + ensure!(matches!( + learner.status().await, + Err(CommandFailure::UnsupportedCapability) + )); + assert_eq!(f.calls[3].load(Ordering::SeqCst), 0); assert_eq!( f.handles[f.primary].repair_status().await?.unwrap().phase, RepairPhase::Seeding @@ -511,7 +520,8 @@ pub(super) async fn learner_before_promotion() -> Result<()> { } pub(super) async fn stale_plan_idempotence() -> Result<()> { - let f = Fixture::new(false).await?; + let mut f = Fixture::new(false).await?; + let _learner = standby::registered(&mut f).await?; f.fail_obsolete().await?; // Admission starts synchronously even while its returned future is unpolled. let lost = f.handles[f.primary].begin_repair(f.plan.clone(), Duration::from_secs(5)); diff --git a/crates/cellar/tests/support/raft_restart_operator_cancel_native.rs b/crates/cellar/tests/support/raft_restart_operator_cancel_native.rs index ed149555d..b0c97ab0e 100644 --- a/crates/cellar/tests/support/raft_restart_operator_cancel_native.rs +++ b/crates/cellar/tests/support/raft_restart_operator_cancel_native.rs @@ -34,14 +34,21 @@ pub(super) async fn ready( s.f.domain .run(cellar::asyncrt::timeout(TIMEOUT, async { loop { - routes(s).await?; + routes(s).await.context("operator readiness discovery")?; match controller.reconcile(TIMEOUT).await { Ok(Some(progress)) => { if let Some(ready) = progress.ready { return Ok::<_, anyhow::Error>(ready); } } - Err(FleetCatalogError::Unconfirmed | FleetCatalogError::Overloaded) + // The driver owns durable child identities across turns. + // Incomplete physical registration is not readiness; keep + // reconciling that same operation within this call's bound. + Err( + FleetCatalogError::Unconfirmed + | FleetCatalogError::Overloaded + | FleetCatalogError::InventoryIncomplete, + ) | Err(FleetCatalogError::Observation( InspectionError::Unavailable | InspectionError::Overloaded, )) => (), diff --git a/crates/cellar/tests/support/raft_restart_operator_native.rs b/crates/cellar/tests/support/raft_restart_operator_native.rs index 117dcc049..681bc1aa4 100644 --- a/crates/cellar/tests/support/raft_restart_operator_native.rs +++ b/crates/cellar/tests/support/raft_restart_operator_native.rs @@ -327,7 +327,14 @@ async fn preparation(recreate: bool, concurrent: bool, churn: bool) -> Result<() } Err(FleetCatalogError::Conflict | FleetCatalogError::PayloadConflict) if concurrent => {} - Err(FleetCatalogError::Unconfirmed | FleetCatalogError::Overloaded) + // Recreated controllers must obtain a complete live + // inventory before returning the same durable intent's + // Ready proof. Refusal is not preparation progress. + Err( + FleetCatalogError::Unconfirmed + | FleetCatalogError::Overloaded + | FleetCatalogError::InventoryIncomplete, + ) | Err(FleetCatalogError::Observation( InspectionError::Unavailable | InspectionError::Overloaded, )) => {} diff --git a/crates/cellar/tests/support/raft_restart_resume_native.rs b/crates/cellar/tests/support/raft_restart_resume_native.rs index 92a1fa514..32ac4a7f1 100644 --- a/crates/cellar/tests/support/raft_restart_resume_native.rs +++ b/crates/cellar/tests/support/raft_restart_resume_native.rs @@ -58,18 +58,62 @@ async fn prepare_resume( s: &Setup, request: &FleetRestartResumeRequest, ) -> Result { - trust(s).await?; - let root = s.f.catalog.read(TIMEOUT).await?; - Ok(s.f - .catalog - .prepare_restart_resume( - &root, - request, - std::slice::from_ref(&s.namespace), - &s.spare, - TIMEOUT, - ) - .await?) + trust(s).await.context("restart resume peer discovery")?; + // Returning engines register asynchronously. Construct the verification + // from a fresh, complete inventory within the existing call budget. + s.f.domain + .run(cellar::asyncrt::timeout(TIMEOUT, async { + loop { + let observed = async { + let root = s.f.catalog.read(TIMEOUT).await?; + s.f.catalog + .prepare_restart_resume( + &root, + request, + std::slice::from_ref(&s.namespace), + &s.spare, + TIMEOUT, + ) + .await + } + .await; + match observed { + Ok(walk) => return Ok(walk), + Err(FleetCatalogError::InventoryIncomplete | FleetCatalogError::Overloaded) => { + cellar::asyncrt::sleep(Duration::from_millis(10)).await; + } + Err(error) => { + return Err( + anyhow::Error::from(error).context("restart resume proof construction") + ); + } + } + } + })) + .await? + .context("restart resume inventory preparation deadline")? +} + +async fn coherent_inspection( + s: &Setup, + cell: &RaftCellHandle, +) -> Result { + // inspect() itself has a five-second budget. A mixed runtime/image sample + // is refused with Overloaded; keep that same bound across fresh samples. + s.f.domain + .run(cellar::asyncrt::timeout(TIMEOUT, async { + loop { + match cell.inspect().await { + Ok(view) => return Ok(view), + Err(CommandFailure::Overloaded) => { + cellar::asyncrt::sleep(Duration::from_millis(10)).await; + } + Err(error) => return Err(error.into()), + } + } + })) + .await? + .context("coherent restart inspection deadline")? } async fn consume_resume(s: &Setup, walk: &mut FleetRestartResumeVerification) -> Result { @@ -386,14 +430,19 @@ async fn fleet_restart_resume_root_drift_requires_fresh_same_scope_consent() -> #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn fleet_restart_resume_rechecks_and_hands_off_regained_leadership() -> Result<()> { - let (s, p, request) = Box::pin(setup(false)).await?; - let mut warm = prepare_resume(&s, &request).await?; - consume_resume(&s, &mut warm).await?; + let (s, p, request) = Box::pin(setup(false)) + .await + .context("regained leader setup")?; + let mut warm = prepare_resume(&s, &request) + .await + .context("initial resume proof")?; + consume_resume(&s, &mut warm) + .await + .context("initial resume histories")?; drop(warm); let source = primary(&s.f, &s.cells).await?; ensure!(source != p); - let state = s.cells[source] - .inspect() + let state = coherent_inspection(&s, &s.cells[source]) .await? .member .context("surviving leader")?; @@ -409,8 +458,7 @@ async fn fleet_restart_resume_rechecks_and_hands_off_regained_leadership() -> Re .await?; s.f.domain .run(cellar::asyncrt::timeout(TIMEOUT, async { - while !s.cells[p] - .inspect() + while !coherent_inspection(&s, &s.cells[p]) .await? .member .is_some_and(|m| m.is_leader) @@ -420,27 +468,40 @@ async fn fleet_restart_resume_rechecks_and_hands_off_regained_leadership() -> Re Ok::<_, anyhow::Error>(()) })) .await???; - pending(&s, p, true).await?; - let mut walk = prepare_resume(&s, &request).await?; - ensure!(consume_resume(&s, &mut walk).await? > 0); + pending(&s, p, true) + .await + .context("regained leader paused state")?; + let mut walk = prepare_resume(&s, &request) + .await + .context("regained leader fresh proof")?; + ensure!( + consume_resume(&s, &mut walk) + .await + .context("regained leader handoff proof")? + > 0 + ); let admission = walk .admit_resume(&request, TIMEOUT) .await .context("publish refreshed resume consent")?; - trust(&s).await?; + trust(&s) + .await + .context("regained leader resume discovery")?; s.f.catalog .resume_restart_maintenance(&admission, &s.spare, TIMEOUT) - .await?; - pending(&s, p, false).await?; + .await + .context("guarded regained leader resume")?; + pending(&s, p, false) + .await + .context("regained leader resumed state")?; ensure!( - !s.cells[p] - .inspect() + !coherent_inspection(&s, &s.cells[p]) .await? .member .context("returned member")? .is_leader ); - s.close().await + s.close().await.context("regained leader joined cleanup") } #[tokio::test(flavor = "multi_thread", worker_threads = 4)] diff --git a/crates/cellar/tests/support/raft_restart_retained_runtime.rs b/crates/cellar/tests/support/raft_restart_retained_runtime.rs index fdf5597eb..18ccfc011 100644 --- a/crates/cellar/tests/support/raft_restart_retained_runtime.rs +++ b/crates/cellar/tests/support/raft_restart_retained_runtime.rs @@ -171,7 +171,11 @@ async fn verified(dynamic: bool, owned: bool) -> Result<()> { Ok(Some(cellar::application::consensus::verification::FleetRestartRuntimeProgress::HandoffRequested { from, to, .. })) => { ensure!(dynamic && from == r.request.plan.replacement.member && from != to); } - Err(FleetCatalogError::Unconfirmed | FleetCatalogError::Overloaded) + // Handoff can overlap a fresh physical inventory cut. Keep + // the pending history until an actual Verified turn; an + // incomplete cut is neither progress nor a matched vote. + Err(FleetCatalogError::Unconfirmed | FleetCatalogError::Overloaded + | FleetCatalogError::InventoryIncomplete) | Err(FleetCatalogError::Observation(cellar::application::consensus::InspectionError::Unavailable | cellar::application::consensus::InspectionError::Overloaded)) => { cellar::asyncrt::sleep(Duration::from_millis(10)).await; diff --git a/crates/cellar/tests/support/raft_restart_return_native.rs b/crates/cellar/tests/support/raft_restart_return_native.rs index 27f647ba0..59e5fd45e 100644 --- a/crates/cellar/tests/support/raft_restart_return_native.rs +++ b/crates/cellar/tests/support/raft_restart_return_native.rs @@ -7,16 +7,13 @@ mod resume; use cellar::application::consensus::verification::FleetRestartReturnRequest; async fn offline(s: &mut Setup, p: usize) -> Result<(Running, FleetRestartReturnRequest)> { - let (gate, request, root) = consent(s, p).await?; + let (gate, request, root) = consent(s, p).await.context("restart return stop consent")?; s.f.catalog .admit_restart_stop(&root, &request, TIMEOUT) .await?; trust(s).await?; let cut = s.f.catalog.read(TIMEOUT).await?; - let result = - s.f.catalog - .stop_restart_maintenance(&cut, &request, &s.spare, TIMEOUT) - .await; + let result = submit_stop(s, &s.f.catalog, &cut, &request).await?; ensure!( result.is_ok() || matches!( @@ -27,7 +24,9 @@ async fn offline(s: &mut Setup, p: usize) -> Result<(Running, FleetRestartReturn result.as_ref().err() ); let mut target = s.nodes.remove(p); - joined(&mut target.node, &s.f).await?; + joined(&mut target.node, &s.f) + .await + .context("restart return old process join")?; let root = s.f.catalog.read(TIMEOUT).await?; let mut identity = request_identity(180, root.revision(), gate.identity.operation); let offline = FleetRestartOfflineRequest { @@ -82,14 +81,20 @@ async fn configuration( } async fn returned(primary_target: bool, lose_confirmations: bool) -> Result<()> { - let mut s = Box::pin(Setup::new()).await?; - let leader = primary(&s.f, &s.cells).await?; + let mut s = Box::pin(Setup::new()) + .await + .context("restart return setup")?; + let leader = primary(&s.f, &s.cells) + .await + .context("restart return initial primary")?; let p = if primary_target { leader } else { (leader + 1) % 3 }; - let (mut target, request) = offline(&mut s, p).await?; + let (mut target, request) = offline(&mut s, p) + .await + .context("restart return offline predecessor")?; let old = target.host.observe_node(TIMEOUT).await; ensure!(old.is_err()); let root = s.f.catalog.read(TIMEOUT).await?; @@ -124,7 +129,9 @@ async fn returned(primary_target: bool, lose_confirmations: bool) -> Result<()> .filter(|(i, _)| *i != p) .map(|(_, c)| c.clone()) .collect::>(); - write_read(&s.f, &survivors, 2, 5, 12).await?; + write_read(&s.f, &survivors, 2, 5, 12) + .await + .context("restart return survivor write before startup")?; let node = public_node::node_named( target.root.path(), s.f.definition.application.clone(), @@ -210,16 +217,24 @@ async fn returned(primary_target: bool, lose_confirmations: bool) -> Result<()> ); let revision = root.revision(); for survivor in &s.nodes { - s.f.remote(&survivor.host, request.node, revision).await?; - s.f.remote(&host, survivor.spec.node, revision).await?; + s.f.remote(&survivor.host, request.node, revision) + .await + .context("restart return install returned peer")?; + s.f.remote(&host, survivor.spec.node, revision) + .await + .context("restart return install survivor peer")?; } - write_read(&s.f, &survivors, 3, 2, 14).await?; + write_read(&s.f, &survivors, 3, 2, 14) + .await + .context("restart return survivor write after startup")?; let wanted = survivors[0] .inspect() - .await? + .await + .context("restart return survivor progress")? .member .context("survivor progress")? .applied; + let seen = std::sync::Mutex::new((0u64, None)); s.f.domain .run(cellar::asyncrt::timeout(TIMEOUT, async { loop { @@ -231,6 +246,25 @@ async fn returned(primary_target: bool, lose_confirmations: bool) -> Result<()> Err(CommandFailure::Overloaded) => None, Err(error) => return Err(error.into()), }; + { + let mut seen = seen + .lock() + .map_err(|_| anyhow::anyhow!("restart return observation mutex"))?; + if let Some(view) = observed.as_ref() { + seen.1 = Some(view.member.as_ref().map(|m| { + ( + m.applied, + m.committed, + m.membership_revision, + m.term, + m.leader, + m.is_leader, + ) + })); + } else { + seen.0 += 1; + } + } if observed .and_then(|view| view.member) .is_some_and(|m| m.applied.index > wanted.index || m.applied == wanted) @@ -240,7 +274,14 @@ async fn returned(primary_target: bool, lose_confirmations: bool) -> Result<()> cellar::asyncrt::sleep(Duration::from_millis(10)).await; } })) - .await???; + .await? + .with_context(|| { + format!( + "restart return catch-up: wanted={wanted:?}, seen={:?}", + seen.lock() + ) + })? + .context("restart return inspect sample")?; ensure!(host.observe_node(TIMEOUT).await?.node.profile.draining); ensure!(reopened .begin_evacuation( @@ -255,7 +296,7 @@ async fn returned(primary_target: bool, lose_confirmations: bool) -> Result<()> target.node = node; target.host = host; s.nodes.push(target); - s.close().await + s.close().await.context("restart return joined cleanup") } #[tokio::test(flavor = "multi_thread", worker_threads = 4)] diff --git a/crates/cellar/tests/support/raft_restart_stop_native.rs b/crates/cellar/tests/support/raft_restart_stop_native.rs index 61c134a69..a881dd4d3 100644 --- a/crates/cellar/tests/support/raft_restart_stop_native.rs +++ b/crates/cellar/tests/support/raft_restart_stop_native.rs @@ -6,7 +6,7 @@ use super::*; mod rejoin; use cellar::application::consensus::verification::{ FleetChange, FleetMutation, FleetRestartOfflineRequest, FleetRestartReadyRequest, - FleetRestartStopRequest, FleetSnapshot, FleetTable, + FleetRestartStopAdmission, FleetRestartStopRequest, FleetSnapshot, FleetTable, }; async fn consent( @@ -58,10 +58,17 @@ async fn trust(s: &Setup) -> Result<()> { } async fn joined(node: &mut cellar::sdk::Node, f: &Fixture) -> Result<()> { - let outcome = f + let completed = f .domain .run(cellar::asyncrt::timeout(TIMEOUT, node.finished())) - .await???; + .await?; + let outcome = completed.with_context(|| { + format!( + "restart stopped process join: node={}, closed={}", + node.info().node, + node.is_closed() + ) + })??; let NodeOutcome::Stopped(report) = outcome else { anyhow::bail!("maintenance did not autonomously stop"); }; @@ -70,6 +77,33 @@ async fn joined(node: &mut cellar::sdk::Node, f: &Fixture) -> Result<()> { Ok(()) } +// Overload is a retryable observation, not proof of stop or closure. Reuse the +// exact admitted request within the original stop-call budget; callers still +// validate the final verdict and join the actual process below. +async fn submit_stop( + s: &Setup, + catalog: &FleetCatalog, + snapshot: &FleetSnapshot, + request: &FleetRestartStopRequest, +) -> Result> { + s.f.domain + .run(cellar::asyncrt::timeout(TIMEOUT, async { + loop { + match catalog + .stop_restart_maintenance(snapshot, request, &s.spare, TIMEOUT) + .await + { + Err(FleetCatalogError::Overloaded) => { + cellar::asyncrt::sleep(Duration::from_millis(10)).await; + } + result => return result, + } + } + })) + .await? + .context("maintenance stop submission deadline") +} + async fn stop_and_confirm(primary_target: bool, lose_replies: bool) -> Result<()> { let mut s = Box::pin(Setup::new()).await?; let leader = primary(&s.f, &s.cells).await?; @@ -116,9 +150,7 @@ async fn stop_and_confirm(primary_target: bool, lose_replies: bool) -> Result<() )); ensure!(catalog.read(TIMEOUT).await?.revision() == cut.revision()); trust(&s).await?; - let outcome = catalog - .stop_restart_maintenance(&cut, &request, &s.spare, TIMEOUT) - .await; + let outcome = submit_stop(&s, &catalog, &cut, &request).await?; ensure!( outcome.is_ok() || matches!( @@ -250,16 +282,15 @@ async fn fleet_restart_stop_renews_expired_readiness_against_current_sql() -> Re s.f.domain .run(async { cellar::asyncrt::sleep(Duration::from_millis(5001)).await }) .await?; - let result = - s.f.catalog - .stop_restart_maintenance(&root, &request, &s.spare, TIMEOUT) - .await; + let result = submit_stop(&s, &s.f.catalog, &root, &request).await?; ensure!( result.is_ok() || matches!( result, Err(FleetCatalogError::Unconfirmed | FleetCatalogError::Closed) - ) + ), + "expired-readiness stop refusal: {:?}", + result.as_ref().err().map(ToString::to_string) ); let mut target = s.nodes.remove(p); joined(&mut target.node, &s.f).await?; diff --git a/crates/cellar/tests/support/raft_standby_native.rs b/crates/cellar/tests/support/raft_standby_native.rs index af7085c2e..9ecbc1e78 100644 --- a/crates/cellar/tests/support/raft_standby_native.rs +++ b/crates/cellar/tests/support/raft_standby_native.rs @@ -58,7 +58,7 @@ pub(in super::super) async fn registered_standby_refuses_quorumless_s3_activatio f.cluster.close().await } -async fn registered(f: &mut Fixture) -> Result { +pub(super) async fn registered(f: &mut Fixture) -> Result { let service = f .cluster .join_standby( diff --git a/docs/19-testing-qualification.md b/docs/19-testing-qualification.md index 6cca3ea6c..1e674a55c 100644 --- a/docs/19-testing-qualification.md +++ b/docs/19-testing-qualification.md @@ -32,6 +32,32 @@ Run independent test cases serially, as CI does. Node fixtures own several concurrent runtimes and short leases; unrelated fixtures competing for the runner can expire those leases. Concurrency inside each fixture stays enabled. +SDK CI runs the serial native campaign independently from Python ABI/package +qualification in both Python threading modes. Each campaign retains the original +commands, assertions and job ceiling; individual request and test deadlines stay +unchanged. Native CI sets `CARGO_PROFILE_TEST_OPT_LEVEL=1`, +`CARGO_PROFILE_TEST_DEBUG_ASSERTIONS=true` and +`CARGO_PROFILE_TEST_OVERFLOW_CHECKS=true`. Use these same settings when +reproducing its bounded native campaign locally. + +The packaged Node delivery restart campaign uses the same test-profile settings +for its repeated cold-start fixture. Its 250 ms handler budget and all 100 fresh +store/process cases remain unchanged. CI builds the test artifact before the +campaign and flushes completed build output to keep compiler writeback out of +the fixture's disk queue. Every fixture still performs its normal FULL syncs. +The live RPC campaigns use the same build/writeback separation without changing +their one-second node leases, per-request deadlines or runtime cleanup checks. +Their disposable x64 Linux runners also bound ongoing dirty writeback to 4 MiB +in the background and 16 MiB overall, with one-second expiry and 500 ms writeback +intervals. This limits interference between buffered writes and FULL-sync +commits; it does not disable syncs or filesystem barriers. An always-run cleanup +step restores the runner's original writeback policy. + +The [shared durable fixture bucket](../tools/qualification/durable_bucket.rs) +keeps node-record CAS independent of application-data writers using FULL-sync +SQLite stores. Ordinary reads, CAS, listings and recovery use one coherent +namespace, including the dense filesystem cold-recovery fixture. + The allocator startup test uses a fresh process with allocator environment overrides removed. Linux must start background purging; macOS must leave that unsupported build default disabled. Forcing `background_thread:true` on macOS diff --git a/docs/design/multi-raft/snapshot-audit.md b/docs/design/multi-raft/snapshot-audit.md index 5f8b14c23..8a887fa6d 100644 --- a/docs/design/multi-raft/snapshot-audit.md +++ b/docs/design/multi-raft/snapshot-audit.md @@ -121,6 +121,8 @@ The [cancellation record](evidence/MR-10-abort-progress.json), [peer-admission r - The existing runtime persists the snapshot, applied marker, configuration and required HardState before emitting `SnapshotInstalled` or releasing dependent messages. - Physical completion also supplies an opaque verified reference bound to the exact ticket/digest/image; mechanical fixtures explicitly use no such capability. - `PreparedSnapshot` checks the durable engine snapshot and exact applied image before publication; sealing bytes or merely submitting a completion does not make state queryable. +- After `SnapshotInstalled`, the runtime can accept another snapshot while the SQL owner is still publishing the preceding image. The host retains at most one next snapshot event, including its original runtime admission, until that publication completes. It starts no second download or installation before releasing the preceding publication owner. +- An overlap before durable installation, a changed replica identity or a second deferred event still closes admission. Reads remain blocked by the incoming snapshot, and every installation and publication retains its exact ticket and image checks. - `ImageApplier` separately retains the durable snapshot generation and the latest applied generation. - Recovery validates/adopts both before retiring other inventory. diff --git a/tools/rpc/host/peer_faults/storage.rs b/tools/qualification/durable_bucket.rs similarity index 99% rename from tools/rpc/host/peer_faults/storage.rs rename to tools/qualification/durable_bucket.rs index 17e9b7830..13d769c09 100644 --- a/tools/rpc/host/peer_faults/storage.rs +++ b/tools/qualification/durable_bucket.rs @@ -13,7 +13,7 @@ use std::{ sync::Arc, }; -pub(super) fn open(root: &std::path::Path, names: &[&str]) -> anyhow::Result { +pub(crate) fn open(root: &std::path::Path, names: &[&str]) -> anyhow::Result { let bucket = Bucket::open_dev(&root.join("fleet.sqlite"))?; let mut nodes = BTreeMap::new(); for name in names { diff --git a/tools/rpc/README.md b/tools/rpc/README.md index bc34135a1..ebeae2f30 100644 --- a/tools/rpc/README.md +++ b/tools/rpc/README.md @@ -1666,6 +1666,11 @@ The surviving caller runs in a separate process with a distinct node directory and the same development bucket. Peer calls use the production signed transport with fleet durability enabled. +Both processes open the same `sqlite-full-per-node-key` namespace described +above. The owner's and successor's authority records each use their canonical +FULL-synchronous file, including after the owner is killed. Cell state, receipt +recovery and the one-second lease retain their normal durability requirements. + ```text before_handler admit RPC -> hold before business method -> SIGKILL owner @@ -1801,6 +1806,9 @@ The suite runs all 16 caller/provider pairs at four boundaries across three authenticated nodes, reusing the native SDK's real log-transport and object-store fault gates. Distinct acknowledged warmup commands establish both replica members before any fault is armed. Lease renewal and reads remain independent of data proof. +Proof and proof-wait fixtures use the shared canonical FULL-synchronous bucket: +each node key has its own SQLite writer, while application data shares the +ordinary durable data store. Lease CAS and ordinary reads see the same records. | Point | Original handler at the barrier | Coordinator action | | --- | --- | --- | diff --git a/tools/rpc/host/crash/mod.rs b/tools/rpc/host/crash/mod.rs index 8bbd675d0..cfbd70ab4 100644 --- a/tools/rpc/host/crash/mod.rs +++ b/tools/rpc/host/crash/mod.rs @@ -12,7 +12,12 @@ pub use gate::{Counter, Gate, Point}; use anyhow::{ensure, Context as _, Result}; use cellar::{bucket::Bucket, sdk::ShutdownMode}; use serde_json::{json, Value}; -use std::{collections::BTreeSet, path::PathBuf, sync::Arc, time::Duration}; +use std::{ + collections::BTreeSet, + path::{Path, PathBuf}, + sync::Arc, + time::Duration, +}; use super::{application::Composite, fleet}; @@ -23,13 +28,31 @@ pub struct Server { pub ready: PathBuf, } +fn durable_bucket(database: &Path) -> Result { + // The coordinator and the killed owner must reopen the same canonical + // namespace. Independent authority writes retain FULL sync without waiting + // behind the other node's application-data transaction. + ensure!( + database + .file_name() + .is_some_and(|name| name == "fleet.sqlite"), + "crash fixture requires its canonical fleet database" + ); + crate::peer_faults::durable_fixture_bucket( + database + .parent() + .context("crash fixture bucket directory")?, + &["crash-owner", "crash-next"], + ) +} + pub async fn serve(app: Arc, server: Server) -> Result { // Only the disposable crash-owner process installs this observer. Native // applications and the qualification coordinator keep their SQLite hooks. sqlite::install()?; app.crash .evidence_file(server.ready.with_file_name("barrier.json")); - let bucket = Bucket::open_dev(&server.bucket)?; + let bucket = durable_bucket(&server.bucket)?; let controls = app.durability.clone(); let (mut node, url) = fleet::start_controlled(&server.data, app, &server.name, bucket, Some(controls)).await?; @@ -98,7 +121,7 @@ async fn case( class: &'static str, ) -> Result { let directory = tempfile::tempdir()?; - let bucket = Bucket::open_dev(&directory.path().join("fleet.sqlite"))?; + let bucket = durable_bucket(&directory.path().join("fleet.sqlite"))?; let (mut owner, owner_url) = process::Owner::start(directory.path(), "crash-owner").await?; let worker = get(client, &owner_url, "/node/process").await?; owner.worker( diff --git a/tools/rpc/host/peer_faults/mod.rs b/tools/rpc/host/peer_faults/mod.rs index f433c2617..e2526bd7c 100644 --- a/tools/rpc/host/peer_faults/mod.rs +++ b/tools/rpc/host/peer_faults/mod.rs @@ -6,6 +6,9 @@ mod framing; mod matrix; mod receiver; mod relay; +#[path = "../../../qualification/durable_bucket.rs"] mod storage; pub use matrix::run; + +pub(crate) use storage::open as durable_fixture_bucket; diff --git a/tools/rpc/host/profile/matrix.rs b/tools/rpc/host/profile/matrix.rs index b211fb654..b044ec946 100644 --- a/tools/rpc/host/profile/matrix.rs +++ b/tools/rpc/host/profile/matrix.rs @@ -2,7 +2,6 @@ //! Every vector traverses real generated clients, native cells and peer routing. use super::*; -use cellar::bucket::Bucket; use serde_json::{json, Value}; async fn matrix( @@ -85,7 +84,12 @@ pub(super) async fn run( vectors: Arc>, ) -> Result { let directory = tempfile::tempdir()?; - let bucket = Bucket::open_dev(&directory.path().join("fleet.sqlite"))?; + // Node-key CAS stays durable and visible through the shared namespace, but + // cannot queue behind application-data writers and consume the lease TTL. + let bucket = crate::peer_faults::durable_fixture_bucket( + directory.path(), + &["profile-owner", "profile-source"], + )?; let client = reqwest::Client::builder() .no_proxy() .timeout(Duration::from_secs(20)) diff --git a/tools/rpc/host/proof/mod.rs b/tools/rpc/host/proof/mod.rs index 925ac75d1..2396b4355 100644 --- a/tools/rpc/host/proof/mod.rs +++ b/tools/rpc/host/proof/mod.rs @@ -41,7 +41,7 @@ mod faults; mod fault_tests; use anyhow::{ensure, Context as _, Result}; -use cellar::{bucket::Bucket, sdk::*}; +use cellar::sdk::*; use faults::{Gate, Peer, Release, Store}; use futures_util::FutureExt as _; use serde_json::{json, Value}; @@ -230,7 +230,10 @@ async fn case( // captured evidence no longer needs live cursors into this bounded log. app.audit.reset_after_node_drain(); let directory = tempfile::tempdir()?; - let bucket = Bucket::open_dev(&directory.path().join("fleet.sqlite"))?; + let bucket = crate::peer_faults::durable_fixture_bucket( + directory.path(), + &["proof-owner", "proof-ingress", "proof-replica"], + )?; let controls = Controls::default(); let (mut owner, owner_url) = fleet::start_controlled( &directory.path().join("owner"), diff --git a/tools/rpc/host/proof/recovery.rs b/tools/rpc/host/proof/recovery.rs index 9746a252d..5afb7327e 100644 --- a/tools/rpc/host/proof/recovery.rs +++ b/tools/rpc/host/proof/recovery.rs @@ -20,40 +20,48 @@ fn transient_transport(error: &reqwest::Error) -> bool { } pub(crate) async fn read(client: &reqwest::Client, url: &str, target: &str) -> Result { - tokio::time::timeout(Duration::from_secs(20), async { - loop { - // This is a new read observation, never a command retry. A fenced - // route can close transport before its typed refusal arrives. - // Keep builder, decoding and arbitrary application errors fatal. - let response = async { - let response = client - .get(format!("{url}/rust/read?target={target}")) - .send() - .await?; - let status = response.status(); - Ok::<_, reqwest::Error>((status, response.text().await?)) - } - .await; - let (status, text) = match response { - Ok(observation) => observation, - Err(error) if transient_transport(&error) => { - tokio::time::sleep(Duration::from_millis(20)).await; - continue; - } - Err(error) => return Err(error.into()), - }; - if status.is_success() { - return Ok::<_, anyhow::Error>(serde_json::from_str(&text)?); - } - ensure!( + // Recovery must not borrow the fenced owner's execution domain. + let domain = cellar::sdk::ExecutionDomain::new(tokio::runtime::Handle::current()); + let result = domain + .run(async { + cellar::asyncrt::timeout(Duration::from_secs(20), async { + loop { + // This is a new read observation, never a command retry. A fenced + // route can close transport before its typed refusal arrives. + // Keep builder, decoding and arbitrary application errors fatal. + let response = async { + let response = client + .get(format!("{url}/rust/read?target={target}")) + .send() + .await?; + let status = response.status(); + Ok::<_, reqwest::Error>((status, response.text().await?)) + } + .await; + let (status, text) = match response { + Ok(observation) => observation, + Err(error) if transient_transport(&error) => { + cellar::asyncrt::sleep(Duration::from_millis(20)).await; + continue; + } + Err(error) => return Err(error.into()), + }; + if status.is_success() { + return Ok::<_, anyhow::Error>(serde_json::from_str(&text)?); + } + ensure!( transient(status, &text), "recovery read failed outside a transient RPC admission refusal: {status}: {text}" ); - tokio::time::sleep(Duration::from_millis(20)).await; - } - }) - .await - .context("proof recovery did not reopen read admission")? + cellar::asyncrt::sleep(Duration::from_millis(20)).await; + } + }) + .await + .context("proof recovery did not reopen read admission")? + }) + .await; + domain.shutdown().await; + result? } #[cfg(test)] diff --git a/tools/rpc/host/proof/wait.rs b/tools/rpc/host/proof/wait.rs index dc6a42f29..82eb15c59 100644 --- a/tools/rpc/host/proof/wait.rs +++ b/tools/rpc/host/proof/wait.rs @@ -91,7 +91,10 @@ async fn case_inner(app: Arc, client: &reqwest::Client, case: Case) - lease_trace.mark("proof-wait case start"); app.audit.reset_after_node_drain(); let directory = tempfile::tempdir()?; - let bucket = Bucket::open_dev(&directory.path().join("fleet.sqlite"))?; + let bucket = crate::peer_faults::durable_fixture_bucket( + directory.path(), + &["proof-owner", "proof-ingress", "proof-replica"], + )?; let controls = Controls::default(); let (owner, owner_url) = fleet::start_controlled( &directory.path().join("owner"), diff --git a/tools/rpc/host/saturation/mod.rs b/tools/rpc/host/saturation/mod.rs index 766ddc2fa..3e8f65bb6 100644 --- a/tools/rpc/host/saturation/mod.rs +++ b/tools/rpc/host/saturation/mod.rs @@ -5,7 +5,7 @@ mod gate; mod matrix; use anyhow::{ensure, Context as _, Result}; -use cellar::{bucket::Bucket, sdk::*}; +use cellar::sdk::*; use serde_json::{json, Value}; use std::{sync::Arc, time::Duration}; @@ -118,7 +118,10 @@ async fn request(client: &reqwest::Client, base: &str, path: &str) -> Result, remote: bool, limit: Limit, side: Side) -> Result { app.audit.reset_after_node_drain(); let directory = tempfile::tempdir()?; - let bucket = Bucket::open_dev(&directory.path().join("fleet.sqlite"))?; + // Saturate RPC resources without serializing independent node authority + // behind application-object writes. Every logical client still observes + // the same canonical FULL-synchronous records. + let bucket = crate::peer_faults::durable_fixture_bucket(directory.path(), &[SOURCE, OWNER])?; let client = reqwest::Client::builder() .timeout(Duration::from_secs(20)) .build()?; diff --git a/tools/rpc/host/tests/peer_storage.rs b/tools/rpc/host/tests/peer_storage.rs index 271a5c3b2..c721d50d4 100644 --- a/tools/rpc/host/tests/peer_storage.rs +++ b/tools/rpc/host/tests/peer_storage.rs @@ -2,7 +2,7 @@ //! Node authority remains durable and visible through every logical client. #![allow(clippy::disallowed_methods)] -#[path = "../peer_faults/storage.rs"] +#[path = "../../../qualification/durable_bucket.rs"] mod storage; use anyhow::{ensure, Result}; diff --git a/tools/rpc/host/tests/proof_recovery.rs b/tools/rpc/host/tests/proof_recovery.rs index 54a9c980b..a118b36c4 100644 --- a/tools/rpc/host/tests/proof_recovery.rs +++ b/tools/rpc/host/tests/proof_recovery.rs @@ -10,9 +10,11 @@ use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; #[tokio::test] async fn recovery_reads_retry_disconnect_and_typed_refusal_inside_one_deadline() -> Result<()> { + let domain = cellar::sdk::ExecutionDomain::new(tokio::runtime::Handle::current()); + let result = domain.run(async { let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; let url = format!("http://{}", listener.local_addr()?); - let server = tokio::spawn(async move { + let server = cellar::asyncrt::spawn(async move { for index in 0..3 { let (mut socket, _) = listener.accept().await?; let mut request = Vec::new(); @@ -43,7 +45,7 @@ async fn recovery_reads_retry_disconnect_and_typed_refusal_inside_one_deadline() Ok::<_, anyhow::Error>(()) }); let client = reqwest::Client::new(); - let observed = tokio::time::timeout( + let observed = cellar::asyncrt::timeout( Duration::from_secs(1), recovery::read(&client, &url, "python"), ) @@ -55,5 +57,8 @@ async fn recovery_reads_retry_disconnect_and_typed_refusal_inside_one_deadline() let observed = observed??; ensure!(observed == serde_json::json!({"value":"7"})); served??; - Ok(()) + Ok::<_, anyhow::Error>(()) + }).await; + domain.shutdown().await; + result? } diff --git a/tools/rpc/host/worker/matrix.rs b/tools/rpc/host/worker/matrix.rs index db1abff79..cb2ce06a5 100644 --- a/tools/rpc/host/worker/matrix.rs +++ b/tools/rpc/host/worker/matrix.rs @@ -3,10 +3,7 @@ use super::{application::Fixture, backend::Backend, trace::Trace}; use crate::{application::Composite, crash::process::wait_groups_gone, fleet}; use anyhow::{ensure, Context as _, Result}; -use cellar::{ - bucket::Bucket, - sdk::{NodeOutcome, ShutdownMode}, -}; +use cellar::sdk::{NodeOutcome, ShutdownMode}; use cellar_javascript::{BridgeUsage, JavaScriptApplication, Options}; use serde_json::{json, Value}; use std::{ @@ -214,7 +211,7 @@ async fn live_case( trace: trace.clone(), provider_class: backend.class(), }); - let bucket = Bucket::open_dev(&path.join("fleet.sqlite"))?; + let bucket = crate::peer_faults::durable_fixture_bucket(path, &[OWNER, SOURCE])?; let client = reqwest::Client::builder() .timeout(Duration::from_secs(20)) .build()?; @@ -456,7 +453,7 @@ async fn recovered( trace, provider_class: backend.class(), }); - let bucket = Bucket::open_dev(&path.join("fleet.sqlite"))?; + let bucket = crate::peer_faults::durable_fixture_bucket(path, &[OWNER, SOURCE])?; let (node, url) = fleet::start(&path.join("owner"), fixture, OWNER, bucket).await?; let client = reqwest::Client::builder() .timeout(Duration::from_secs(20)) diff --git a/tools/rpc/host/worker/python/case.rs b/tools/rpc/host/worker/python/case.rs index f30a1d2da..be0f12095 100644 --- a/tools/rpc/host/worker/python/case.rs +++ b/tools/rpc/host/worker/python/case.rs @@ -8,10 +8,7 @@ use super::super::{ use super::{probes, recovery, Case, Trigger, DELTA, OWNER, SOURCE}; use crate::{application::Composite, fleet}; use anyhow::{ensure, Context as _, Result}; -use cellar::{ - bucket::Bucket, - sdk::{NodeOutcome, ShutdownMode}, -}; +use cellar::sdk::{NodeOutcome, ShutdownMode}; use cellar_python::PythonApplication; use serde_json::{json, Value}; use std::{ @@ -34,7 +31,7 @@ pub(super) async fn run( trace: trace.clone(), provider_class: "PythonCounter", }); - let bucket = Bucket::open_dev(&root.join("fleet.sqlite"))?; + let bucket = crate::peer_faults::durable_fixture_bucket(root, &[OWNER, SOURCE])?; let client = reqwest::Client::builder() .timeout(Duration::from_secs(20)) .build()?; @@ -111,7 +108,8 @@ pub(super) async fn run( } else { let stopping = owner.take().context("Python owner already stopped")?.shutdown(ShutdownMode::Preserve); let (stopped, returned) = tokio::join!(stopping, tokio::time::timeout(Duration::from_secs(12), &mut call)); - let NodeOutcome::Stopped(proof) = stopped? else { anyhow::bail!("Python owner failed during shutdown"); }; + let outcome = stopped?; + let NodeOutcome::Stopped(proof) = outcome else { anyhow::bail!("Python owner failed during shutdown: {outcome:?}; case={spec:?}"); }; ensure!(proof.drain_timed_out && !proof.application_work_drained && !proof.cells_drained && !proof.clean_reload_prepared && proof.pending_application_events > 0 && proof.pending_dispatches > 0 && proof.log_quiesced && proof.local_durability_closed, "Python forced shutdown proof differs: {proof:?}"); diff --git a/tools/rpc/host/worker/python/recovery.rs b/tools/rpc/host/worker/python/recovery.rs index 1ef967e0a..150270d5f 100644 --- a/tools/rpc/host/worker/python/recovery.rs +++ b/tools/rpc/host/worker/python/recovery.rs @@ -1,10 +1,10 @@ // Copyright 2026 Cellar Project Authors. Licensed under Apache-2.0. use super::super::{application::Fixture, matrix::request, trace::Trace}; -use super::{DELTA, OWNER}; +use super::{DELTA, OWNER, SOURCE}; use crate::{application::Composite, fleet}; use anyhow::{ensure, Context as _, Result}; -use cellar::{bucket::Bucket, sdk::ShutdownMode}; +use cellar::sdk::ShutdownMode; use cellar_python::PythonApplication; use serde_json::{json, Value}; use std::{path::Path, sync::Arc}; @@ -24,7 +24,7 @@ pub(super) async fn run( trace: trace.clone(), provider_class: "PythonCounter", }); - let bucket = Bucket::open_dev(&root.join("fleet.sqlite"))?; + let bucket = crate::peer_faults::durable_fixture_bucket(root, &[OWNER, SOURCE])?; let (node, url) = fleet::start(&root.join("owner"), app, OWNER, bucket).await?; let result = async { let before = before.to_string();