Fix Raft membership fence and CI qualification failures

.github/workflows/javascript-sdk.yml

@@ -17,6 +17,7 @@ on:
       - 'tools/javascript/**'
       - 'tools/sandbox/**'
       - 'tools/rpc/**'
+      - 'tools/qualification/**'
       - 'tools/cellar-rpc-codegen/**'
       - 'proto/**'
       - 'docs/design/named-typed-rpc/*.proto'
@@ -242,6 +243,11 @@ jobs:
         if: matrix.profile == 'packages'
         env:
           RUST_LOG: timing=debug,cellar::runtime=debug,store=debug
+          # Keep the existing handler deadlines and all debug/overflow checks.
+          # Optimize this repeated cold-restart artifact, as in native SDK CI.
+          CARGO_PROFILE_TEST_OPT_LEVEL: '1'
+          CARGO_PROFILE_TEST_DEBUG_ASSERTIONS: 'true'
+          CARGO_PROFILE_TEST_OVERFLOW_CHECKS: 'true'
         run: |
           cargo build --locked -p cellar-javascript --no-default-features --features node --bin cellar-node-example
           mkdir -p .cellar
@@ -254,6 +260,10 @@ jobs:
           .cellar/node-delivery/launch.sh --help
           export CELLAR_TEST_NODE_ENTRY="$PWD/.cellar/node-delivery/app/dist/app.js"
           export CELLAR_TEST_NODE_EXECUTABLE="$PWD/.cellar/node-delivery/bin/node"
+          cargo test --locked -p cellar-javascript --no-default-features --features node --test node_runtime --no-run
+          # Finish build writeback before measuring fresh FULL-sync fixtures.
+          # Keep their real fsyncs and the existing handler deadline unchanged.
+          sync -f "${CARGO_TARGET_DIR:-$PWD/target}"
           # Fresh stores and Node processes in every case. Stop at the first
           # failure; these repetitions never retry a failed request or test.
           for iteration in {1..100}; do

.github/workflows/native-sdk.yml

@@ -104,6 +104,7 @@ jobs:
           git rev-parse HEAD > "$RUNNER_TEMP/filesystem-release/source.txt"
       - name: Build the release CLI and filesystem application
         run: |
+          set -o pipefail
           cargo build --release --locked -p cellar --bin cellar --bin cellar-native-examples \
             2>&1 | tee "$RUNNER_TEMP/filesystem-release/build.log"
           python3 - <<'PY'
@@ -123,6 +124,7 @@ jobs:
           PY
       - name: Exercise the release filesystem application and restart
         run: |
+          set -o pipefail
           python3 examples/focused_smoke.py --cellar target/release/cellar \
             --example filesystem --profile release \
             2>&1 | tee "$RUNNER_TEMP/filesystem-release/smoke.log"
@@ -150,10 +152,10 @@ jobs:
       measure: ${{ inputs.fleet_measure }}
   check:
     if: github.event_name != 'workflow_dispatch' || inputs.mode == 'qualification'
-    name: Rust framework and SDK (${{ matrix.python }})
+    name: Python ABI and SDK (${{ matrix.python }})
     runs-on: ubuntu-22.04
-    # ABI/package qualification plus serial fleet tests and restart examples
-    # exceed 90 minutes on shared runners; individual test bounds remain intact.
+    # ABI/package qualification and examples retain their existing ceiling.
+    # Native campaigns run independently with all per-test bounds unchanged.
     timeout-minutes: 120
     strategy:
       fail-fast: false
@@ -331,34 +333,24 @@ jobs:
           cargo run -p cellar --example python-package-qualification --locked -- \
             "$RUNNER_TEMP/python-package" "$RUNNER_TEMP" "$RUNNER_TEMP/python-package-v2" \
             --supervisor "$PWD/target/debug/cellar"
-      - name: Test and lint
-        env:
-          # Failed native fixtures retain synthetic recovery observations here.
-          TMPDIR: ${{ runner.temp }}/native-failure-fixtures
-          CELLAR_FS_FAULT_REPORT: ${{ runner.temp }}/filesystem-proof-fault.json
+      - name: Doctests and Clippy
         run: |
-          mkdir -p "$TMPDIR"
-          # Fleet fixtures own concurrent runtimes and short authority leases.
-          # Run independent cases serially to keep their resource envelope stable.
-          # The opt-in RPC qualifier owns every guest runtime, including V8.
-          # Keep this native/Python job independent of JavaScript engines.
-          cargo test --workspace --exclude cellar-python --exclude cellar-javascript --exclude cellar-rpc-qualification --all-features --lib --tests --locked -- --test-threads=1
-          cargo test -p cellar --bin cellar-perf-cells --bin cellar-perf-load --locked -- --test-threads=1
           cargo test --workspace --exclude cellar-javascript --exclude cellar-rpc-qualification --all-features --doc --locked
           cargo clippy --workspace --exclude cellar-javascript --exclude cellar-rpc-qualification --all-targets --all-features --locked -- -D warnings
-      - name: Preserve filesystem proof and recovery ledger
-        uses: actions/upload-artifact@v6
-        if: always()
-        with:
-          name: filesystem-proof-fault-${{ matrix.python }}-${{ github.run_id }}-${{ github.run_attempt }}
-          path: ${{ runner.temp }}/filesystem-proof-fault.json
-          if-no-files-found: ignore
       - name: Qualify dense filesystem writes and cold recovery
         env:
+          # Match the native campaign while retaining its correctness checks.
+          CARGO_PROFILE_TEST_OPT_LEVEL: "1"
+          CARGO_PROFILE_TEST_DEBUG_ASSERTIONS: "true"
+          CARGO_PROFILE_TEST_OVERFLOW_CHECKS: "true"
           CELLAR_FS_CAPACITY_MIB: "10"
           CELLAR_FS_CAPACITY_REPORT: ${{ runner.temp }}/filesystem-dense-capacity.json
           RUST_LOG: 'warn,cellar::actor::effects=info,store[node_lease_store]=debug'
         run: |
+          set -o pipefail
+          cargo test --locked -p cellar --all-features --test native_node --no-run
+          # Complete build writeback before the live durability qualification.
+          sync -f "${CARGO_TARGET_DIR:-$PWD/target}"
           cargo test --locked -p cellar --all-features --test native_node \
             filesystem_dense_capacity_cold_recovery -- --ignored --nocapture \
             2>&1 | tee "$RUNNER_TEMP/filesystem-dense-capacity.log"
@@ -373,6 +365,9 @@ jobs:
           if-no-files-found: ignore
       - name: Test Axum feature modes and generated contract
         run: |
+          cargo test -p cellar-axum --no-default-features --locked --no-run
+          # Keep compiler writeback outside the live one-second lease checks.
+          sync -f "${CARGO_TARGET_DIR:-$PWD/target}"
           cargo test -p cellar-axum --no-default-features --locked
           cargo check -p cellar-axum-counter --no-default-features --bins --locked
           cargo run -p cellar-axum-counter --bin export-openapi --locked -- "$RUNNER_TEMP/counter-openapi.json"
@@ -396,6 +391,65 @@ jobs:
           python3 examples/platform_smoke.py --cellar target/debug/cellar \
             --embedded target/debug/cellar-example-embedded --with-containers
 
+      - uses: actions/upload-artifact@v6
+        if: always()
+        with:
+          name: python-churn-${{ matrix.python }}-${{ github.run_id }}-${{ github.run_attempt }}
+          path: |
+            ${{ runner.temp }}/python-churn/**/*.json
+            ${{ runner.temp }}/python-churn/**/*.log
+            ${{ runner.temp }}/python-churn/report.md
+          if-no-files-found: ignore
+          retention-days: 7
+
+  native_tests:
+    if: github.event_name != 'workflow_dispatch' || inputs.mode == 'qualification'
+    name: Native framework tests (${{ matrix.python }})
+    runs-on: ubuntu-22.04
+    env:
+      # Exercise native protocol deadlines with optimized code, retaining all
+      # debug assertions and integer overflow checks in the test artifact.
+      CARGO_PROFILE_TEST_OPT_LEVEL: "1"
+      CARGO_PROFILE_TEST_DEBUG_ASSERTIONS: "true"
+      CARGO_PROFILE_TEST_OVERFLOW_CHECKS: "true"
+    # Separate the serial native campaign from ABI/package qualification;
+    # preserve the original job ceiling and every existing test command.
+    timeout-minutes: 120
+    strategy:
+      fail-fast: false
+      matrix:
+        python: ['3.14', '3.14t']
+    steps:
+      - uses: actions/checkout@v6
+      - name: Install Rust
+        run: |
+          rustup toolchain install "$RUST_VERSION" --profile minimal --component rustfmt
+          rustup default "$RUST_VERSION"
+      - uses: actions/setup-python@v6
+        with:
+          python-version: ${{ matrix.python }}
+      - name: Select embedded Python
+        run: echo "PYO3_PYTHON=$(command -v python)" >> "$GITHUB_ENV"
+      - name: Native framework tests
+        env:
+          # Failed native fixtures retain synthetic recovery observations here.
+          TMPDIR: ${{ runner.temp }}/native-failure-fixtures
+          CELLAR_FS_FAULT_REPORT: ${{ runner.temp }}/filesystem-proof-fault.json
+        run: |
+          mkdir -p "$TMPDIR"
+          # Fleet fixtures own concurrent runtimes and short authority leases.
+          # Run independent cases serially to keep their resource envelope stable.
+          # The opt-in RPC qualifier owns every guest runtime, including V8.
+          # Keep this native/Python job independent of JavaScript engines.
+          cargo test --workspace --exclude cellar-python --exclude cellar-javascript --exclude cellar-rpc-qualification --all-features --lib --tests --locked -- --test-threads=1
+          cargo test -p cellar --bin cellar-perf-cells --bin cellar-perf-load --locked -- --test-threads=1
+      - name: Preserve filesystem proof and recovery ledger
+        uses: actions/upload-artifact@v6
+        if: always()
+        with:
+          name: filesystem-proof-fault-${{ matrix.python }}-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/filesystem-proof-fault.json
+          if-no-files-found: ignore
       - name: Preserve native failure observations
         uses: actions/upload-artifact@v6
         if: always()
@@ -417,16 +471,27 @@ jobs:
           if-no-files-found: ignore
           retention-days: 7
 
-      - uses: actions/upload-artifact@v6
-        if: always()
-        with:
-          name: python-churn-${{ matrix.python }}-${{ github.run_id }}-${{ github.run_attempt }}
-          path: |
-            ${{ runner.temp }}/python-churn/**/*.json
-            ${{ runner.temp }}/python-churn/**/*.log
-            ${{ runner.temp }}/python-churn/report.md
-          if-no-files-found: ignore
-          retention-days: 7
+
+  qualification:
+    if: always() && (github.event_name != 'workflow_dispatch' || inputs.mode == 'qualification')
+    needs: [check, native_tests]
+    # Preserve the existing required contexts across the campaign split. A
+    # failed, cancelled or skipped campaign must fail the original gate.
+    name: Rust framework and SDK (${{ matrix.python }})
+    runs-on: ubuntu-22.04
+    timeout-minutes: 5
+    strategy:
+      fail-fast: false
+      matrix:
+        python: ['3.14', '3.14t']
+    steps:
+      - name: Require both qualification campaigns
+        env:
+          SDK_RESULT: ${{ needs.check.result }}
+          NATIVE_RESULT: ${{ needs.native_tests.result }}
+        run: |
+          test "$SDK_RESULT" = success
+          test "$NATIVE_RESULT" = success
 
   python_load:
     if: github.event_name == 'workflow_dispatch' && inputs.mode == 'python-load'

.github/workflows/typed-rpc.yml

@@ -7,6 +7,7 @@ on:
       - 'crates/**'
       - 'packages/cellar-sdk/**'
       - 'tools/rpc/**'
+      - 'tools/qualification/**'
       - 'tools/javascript/**'
       - 'tools/cellar-rpc-codegen/**'
       - 'tools/python/**'
@@ -88,6 +89,27 @@ jobs:
           python3 -m unittest discover -s tools/perf -p test_python_fleet.py
           cargo clippy --locked -p cellar-rpc-qualification --all-targets --features full -- -D warnings
           cargo test --locked -p cellar-rpc-qualification --features full -- --test-threads=1
+      - name: Finish build writeback before live qualification
+        run: |
+          cargo build --locked -p cellar-rpc-qualification --features full
+          # Bound ongoing dirty writeback on this disposable x64 runner. A
+          # one-time flush alone still allowed >1s commits in the full matrix.
+          # FULL sync, barriers and the one-second authority lease stay enabled.
+          snapshot="$RUNNER_TEMP/cellar-rpc-writeback.conf"
+          : > "$snapshot"
+          for setting in dirty_background dirty; do
+            bytes=$(sysctl -n "vm.${setting}_bytes")
+            if [ "$bytes" -eq 0 ]; then
+              key="vm.${setting}_ratio"
+            else
+              key="vm.${setting}_bytes"
+            fi
+            printf '%s=%s\n' "$key" "$(sysctl -n "$key")" >> "$snapshot"
+          done
+          sysctl vm.dirty_expire_centisecs vm.dirty_writeback_centisecs >> "$snapshot"
+          sudo sysctl -w vm.dirty_background_bytes=4194304 vm.dirty_bytes=16777216 \
+            vm.dirty_expire_centisecs=100 vm.dirty_writeback_centisecs=50
+          sync -f "$CARGO_TARGET_DIR"
       - name: Live portable Protobuf profile
         if: matrix.campaign == 'focused'
         run: |
@@ -135,6 +157,13 @@ jobs:
           python3 tools/rpc/check_python_mode.py --expected "$CELLAR_RPC_PYTHON_MODE" .cellar/rpc/all.json
           python3 tools/rpc/check_provenance.py .cellar/rpc/all.json
           python3 tools/rpc/check_provider_limits.py .cellar/rpc/all.json
+      - name: Restore runner writeback policy
+        if: always()
+        run: |
+          snapshot="$RUNNER_TEMP/cellar-rpc-writeback.conf"
+          if [ -f "$snapshot" ]; then
+            sudo sysctl -p "$snapshot"
+          fi
       - name: Retain lifecycle evidence
         if: always()
         uses: actions/upload-artifact@v6

crates/cellar-python/tests/bridge.py

@@ -319,7 +319,16 @@ async def wrong_context():
     if request.path == "/library":
         # A real native extension (_hashlib) plus pure Python library imports.
         return Response.json(hashlib.sha256(b"cellar").hexdigest())
-    return await ctx.namespace("COUNTERS").get_by_name("one").fetch(request)
+    cell = ctx.namespace("COUNTERS").get_by_name("one")
+    if request.path == "/proof/state":
+        try:
+            return await cell.fetch(request)
+        except Ambiguous as error:
+            # Only this fixed, read-only recovery route can be reconciled. A
+            # failed/reset callee never establishes a mutation's rollback.
+            assert error.code == "Ambiguous" and error.outcome == "unknown"
+            return Response.json({"code": error.code, "outcome": error.outcome}, status=503)
+    return await cell.fetch(request)
 
 
 class Counter:

crates/cellar-python/tests/support/durability.rs

@@ -16,12 +16,23 @@ async fn restored_state(client: &reqwest::Client, url: &str) -> Result<Value> {
             if status.is_success() {
                 return Ok::<_, anyhow::Error>(serde_json::from_str(&body)?);
             }
-            ensure!(
-                status.is_server_error()
-                    && body.contains("cellar output-gate durability proof failed for ")
-                    && body.contains("ltx durability timed out for "),
-                "restored proof read failed outside the pending proof reset: {status}: {body}"
-            );
+            if status == reqwest::StatusCode::SERVICE_UNAVAILABLE {
+                // Python preserves the native refusal on this read-only route.
+                // It is an unknown outcome, never rollback or an acknowledged
+                // mutation. Reject unrelated errors instead of retrying a 500.
+                ensure!(
+                    serde_json::from_str::<Value>(&body)?
+                        == json!({"code":"Ambiguous","outcome":"unknown"}),
+                    "recovery read lost its native refusal: {body}"
+                );
+            } else {
+                ensure!(
+                    status.is_server_error()
+                        && body.contains("cellar output-gate durability proof failed for ")
+                        && body.contains("ltx durability timed out for "),
+                    "restored proof read failed outside the pending proof reset: {status}: {body}"
+                );
+            }
             tokio::time::sleep(Duration::from_millis(20)).await;
         }
     })
@@ -198,6 +209,17 @@ pub(super) fn child() -> Result<()> {
                     "an unproven mutation handler was blindly replayed: {attempts}"
                 );
                 let state = restored_state(&client, &url).await?;
+                let restored_attempts: Value = client
+                    .get(format!("{url}/proof/attempts"))
+                    .send()
+                    .await?
+                    .error_for_status()?
+                    .json()
+                    .await?;
+                ensure!(
+                    restored_attempts == attempts,
+                    "recovery read replayed a mutation: {restored_attempts}"
+                );
                 ensure!(
                     state["sync"] == 1 && state["output"] == 1,
                     "restored proof access lost previously acknowledged state: {state}"

crates/cellar/application/consensus/resource_metrics.rs

@@ -55,7 +55,7 @@ fn budget(text: &mut Text, prefix: &str, labels: &str, usage: RaftBudgetUsage) {
 }
 
 impl RaftResourceUsage {
-    /// Bounded label vocabularies, with 111 samples when SQL, image, group and
+    /// Bounded label vocabularies, with 113 samples when SQL, image, group and
     /// transport sources are live, plus 13 for a configured controller.
     /// A live extent inventory adds 13 samples.
     /// Unavailable sources expose only their availability gauge,