From 76aae7a3a2f1ac84b4b3b20fa137166a5c1d2abb Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Sat, 7 Nov 2026 22:10:57 -0500 Subject: [PATCH 01/66] Fix quorum and request ordering; establish deterministic correctness baseline (#1) --- .github/workflows/correctness.yml | 34 +++++ .gitmodules | 2 +- CMakeLists.txt | 15 ++- README.md | 6 + docs/correctness-baseline.md | 80 ++++++++++++ miniraft/raft.cpp | 12 +- test/CMakeLists.txt | 3 + test/test_correctness.cpp | 203 ++++++++++++++++++++++++++++++ 8 files changed, 344 insertions(+), 11 deletions(-) create mode 100644 .github/workflows/correctness.yml create mode 100644 docs/correctness-baseline.md create mode 100644 test/test_correctness.cpp diff --git a/.github/workflows/correctness.yml b/.github/workflows/correctness.yml new file mode 100644 index 0000000..f78c895 --- /dev/null +++ b/.github/workflows/correctness.yml @@ -0,0 +1,34 @@ +name: Consensus correctness +on: [push, pull_request] +permissions: + contents: read +jobs: + core: + runs-on: ubuntu-latest + strategy: + matrix: + build_type: [Debug, Release] + steps: + - uses: actions/checkout@v4 + - run: sudo apt-get update && sudo apt-get install -y libcmocka-dev + - name: Configure core without networking dependencies + run: >- + cmake -S . -B build/core + -DMINIRAFT_BUILD_NETWORK=OFF + -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} + - run: cmake --build build/core --parallel 2 + - run: ctest --test-dir build/core --output-on-failure + sanitizers: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - run: sudo apt-get update && sudo apt-get install -y libcmocka-dev liburing-dev libssl-dev libsqlite3-dev + - name: Configure full build with ASan and UBSan + run: >- + cmake -S . -B build/full -DCMAKE_BUILD_TYPE=Debug + -DCMAKE_CXX_FLAGS="-fsanitize=address,undefined -fno-sanitize-recover=all" + -DCMAKE_EXE_LINKER_FLAGS="-fsanitize=address,undefined" + - run: cmake --build build/full --parallel 2 + - run: ctest --test-dir build/full --output-on-failure diff --git a/.gitmodules b/.gitmodules index bc42759..0e292cc 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "coroio"] path = coroio - url = ../coroio.git + url = https://github.com/resetius/coroio.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 3e108ad..a421d79 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,10 +5,15 @@ if (EXISTS ${CMAKE_BINARY_DIR}/local.cmake) include(${CMAKE_BINARY_DIR}/local.cmake) endif () -enable_testing() +include(CTest) +option(MINIRAFT_BUILD_NETWORK "Build coroio networking and examples" ON) -add_subdirectory(coroio/coroio) add_subdirectory(miniraft) -add_subdirectory(miniraft/net) -add_subdirectory(examples) -add_subdirectory(test) \ No newline at end of file +if(MINIRAFT_BUILD_NETWORK) + add_subdirectory(coroio/coroio) + add_subdirectory(miniraft/net) + add_subdirectory(examples) +endif() +if(BUILD_TESTING) + add_subdirectory(test) +endif() diff --git a/README.md b/README.md index 8daf485..f70f0ee 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,11 @@ # MiniRaft-CPP +This fork's first milestone fixes strict-majority commit selection and FIFO +request completion, and adds deterministic consensus failure tests. +See the [correctness baseline and review](docs/correctness-baseline.md) for +validation, build instructions and remaining production release gates. +The milestone is experimental; durable crash recovery is not established. + [![Number of GitHub stars](https://img.shields.io/github/stars/resetius/miniraft-cpp)](https://github.com/resetius/miniraft-cpp/stargazers) ![GitHub commit activity](https://img.shields.io/github/commit-activity/m/resetius/miniraft-cpp) [![GitHub license which is BSD-2-Clause license](https://img.shields.io/github/license/resetius/miniraft-cpp)](https://github.com/resetius/miniraft-cpp) diff --git a/docs/correctness-baseline.md b/docs/correctness-baseline.md new file mode 100644 index 0000000..2fbae1b --- /dev/null +++ b/docs/correctness-baseline.md @@ -0,0 +1,80 @@ +# Correctness baseline: v0.1.0-alpha.1 + +This fork starts from upstream a2848b843b3910664216a00605a406b5516e0f8c. +This is an experimental correctness milestone, not a production-readiness claim. + +## Fixes + +* Select the strict-majority replication index for both odd and even cluster + sizes. For four nodes, two replicas must never commit an entry. +* Process request queues from the front, matching std::queue::pop(). This + preserves FIFO replies for reads, strong reads and writes, and allows an + earlier committed write to complete while a later write remains pending. +* Resolve the pinned coroio submodule against its upstream URL so forks clone + successfully. The pinned dependency commit is unchanged. +* Support MINIRAFT_BUILD_NETWORK=OFF and BUILD_TESTING=OFF independently. + +## Deterministic failure harness + +test/test_correctness.cpp uses virtual time and an explicit message queue. +It uses no sockets, wall clock, sleeps or nondeterministic scheduling. +The eight tests cover: + +1. Every replica count for cluster sizes one through eight. +2. FIFO normal-read replies. +3. FIFO strong-read replies. +4. FIFO write replies. +5. Partial write completion without blocking earlier requests. +6. Dropped replication traffic followed by retry, reversal and duplication. +7. Leader isolation, majority election and commit, delayed old-term messages, + conflicting minority-log replacement and partition healing. +8. A four-node partition where two replicas cannot commit, followed by healing. + +Cluster scenarios check commit progress and equality of committed log prefixes. +These are selected deterministic schedules, not exhaustive state-space exploration. +Restart and storage crash injection are not covered by the simulator yet. + +## Reproduce + +Install a C++20 compiler, CMake, pkg-config and cmocka development files. +For the full build, initialize submodules and install coroio dependencies +(on Ubuntu: liburing-dev, libssl-dev and libsqlite3-dev). + +```sh +cmake -S . -B build/core -DMINIRAFT_BUILD_NETWORK=OFF -DCMAKE_BUILD_TYPE=Debug +cmake --build build/core --parallel 4 +ctest --test-dir build/core --output-on-failure + +git submodule update --init --recursive +cmake -S . -B build/full -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_CXX_FLAGS="-fsanitize=address,undefined -fno-sanitize-recover=all" \ + -DCMAKE_EXE_LINKER_FLAGS="-fsanitize=address,undefined" +cmake --build build/full --parallel 4 +ctest --test-dir build/full --output-on-failure +``` + +## Review and validation + +The implementing agent performed a source self-review of the final diff, +majority arithmetic, queue ownership and completion ordering, build options, +dependency pin and deterministic test schedules. No independent human or +second-agent review has been obtained. The release is explicitly an alpha. + +Local validation: the 27 upstream core tests, eight new correctness tests and +three upstream transport tests pass. The full suite also passes with ASan and +UBSan. The new tests fail against the upstream implementation: majority and +read-order assertions fail, and queued write handling aborts. Local validation +is separate from hosted GitHub Actions results. + +## Remaining release gates before production + +* Replace stream-only persistence with a defined durable write protocol, ordered + synchronization, recovery validation and crash-injection testing. Current + TDiskState is not validated for power-loss durability. +* Audit strong-read quorum freshness, pending operations across leadership + changes, client retry/deduplication semantics and malformed message handling. + FIFO tests do not establish linearizability. +* Add deterministic node restart and broader generated fault schedules, checking + election safety and committed application histories. +* Implement and validate snapshots, compaction and safe membership changes. +* Establish real-network load/failure testing, backpressure and operational APIs. diff --git a/miniraft/raft.cpp b/miniraft/raft.cpp index cb72b2e..a39b24e 100644 --- a/miniraft/raft.cpp +++ b/miniraft/raft.cpp @@ -80,7 +80,9 @@ TVolatileState& TVolatileState::CommitAdvance(int nservers, const IState& state, Indices.push_back(0); } std::sort(Indices.begin(), Indices.end()); - auto commitIndex = std::max(CommitIndex, Indices[nservers / 2]); + // Ascending order: a strict majority must have replicated this index. + // For even cluster sizes, the upper median would accept only half. + auto commitIndex = std::max(CommitIndex, Indices[(nservers - 1) / 2]); if (state.LogTerm(commitIndex) == state.CurrentTerm) { CommitSeqno = std::max(CommitSeqno, seqno); CommitIndex = commitIndex; @@ -647,8 +649,8 @@ void TRequestProcessor::ProcessCommitted() { void TRequestProcessor::ProcessWaiting() { auto lastApplied = Rsm->LastAppliedIndex; - while (!Waiting.empty() && Waiting.back().Index <= lastApplied) { - auto w = Waiting.back(); Waiting.pop(); + while (!Waiting.empty() && Waiting.front().Index <= lastApplied) { + auto w = Waiting.front(); Waiting.pop(); TMessageHolder reply; auto cookie = w.Command->Cookie;; if (w.Command->Flags & TCommandRequest::EWrite) { @@ -667,8 +669,8 @@ void TRequestProcessor::ProcessWaiting() { } auto seqno = Raft->CommitSeqno(); - while (!StrongWaiting.empty() && StrongWaiting.back().Index <= lastApplied && StrongWaiting.back().Seqno <= seqno) { - auto w = StrongWaiting.back(); StrongWaiting.pop(); + while (!StrongWaiting.empty() && StrongWaiting.front().Index <= lastApplied && StrongWaiting.front().Seqno <= seqno) { + auto w = StrongWaiting.front(); StrongWaiting.pop(); TMessageHolder reply; assert (!(w.Command->Flags & TCommandRequest::EWrite)); auto cookie = w.Command->Cookie; diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index b681e0c..ce2fc09 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -13,5 +13,8 @@ endmacro() if (CMOCKA_FOUND) ut(test_raft test_raft.cpp miniraft) +ut(test_correctness test_correctness.cpp miniraft) +if(MINIRAFT_BUILD_NETWORK) ut(test_read_write test_read_write.cpp miniraft.net) +endif() endif () diff --git a/test/test_correctness.cpp b/test/test_correctness.cpp new file mode 100644 index 0000000..92c1789 --- /dev/null +++ b/test/test_correctness.cpp @@ -0,0 +1,203 @@ +#include +#include +#include +#include +#include +#include +#include +extern "C" { +#include +} + +namespace { +struct TEndpoint : INode { + std::function)> Deliver; + explicit TEndpoint(std::function)> f) : Deliver(std::move(f)) {} + void Send(TMessageHolder m) override { Deliver(std::move(m)); } + void Drain() override {} +}; + +// Virtual time and a manually scheduled network: no sockets, sleeps or wall clock. +struct TCluster { + ITimeSource::Time Now{}; + std::vector> Nodes; + std::deque> Packets; + std::set Isolated; + explicit TCluster(uint32_t count) { + for (uint32_t id = 1; id <= count; ++id) { + TNodeDict peers; + for (uint32_t dst = 1; dst <= count; ++dst) { + if (dst != id) peers[dst] = std::make_shared([this](auto m) { + Packets.push_back(std::move(m)); + }); + } + Nodes.push_back(std::make_shared(std::make_shared(), id, peers)); + } + } + void Drain(bool reverse = false, bool duplicate = false) { + size_t budget = 10000; + while (!Packets.empty()) { + assert_true(budget-- > 0); + auto m = reverse ? Packets.back() : Packets.front(); + if (reverse) Packets.pop_back(); else Packets.pop_front(); + auto ex = m.Cast(); + if (Isolated.count(ex->Src) || Isolated.count(ex->Dst)) continue; + Nodes.at(ex->Dst - 1)->Process(Now, m); + if (duplicate) Nodes.at(ex->Dst - 1)->Process(Now, m); + } + } + void Elect(uint32_t id) { + Now += std::chrono::seconds(10); + auto& node = Nodes.at(id - 1); + node->ProcessTimeout(Now); + Drain(); + node->ProcessTimeout(Now); + Drain(); + assert_int_equal(node->CurrentStateName(), EState::LEADER); + } + void Replicate(uint32_t id, bool reverse = false, bool duplicate = false) { + for (int round = 0; round < 12; ++round) { + Now += std::chrono::milliseconds(500); + Nodes.at(id - 1)->ProcessTimeout(Now); + Drain(reverse, duplicate); + } + } + void AssertCommittedPrefix() { + for (auto& a : Nodes) for (auto& b : Nodes) { + auto end = std::min(a->GetVolatileState()->CommitIndex, b->GetVolatileState()->CommitIndex); + for (uint64_t i = 1; i <= end; ++i) { + auto x = a->GetState()->Get(i), y = b->GetState()->Get(i); + assert_int_equal(x->Len, y->Len); + assert_memory_equal(x.Mes, y.Mes, x->Len); + } + } + } +}; + +void test_majority_all_cluster_sizes(void**) { + for (int count = 1; count <= 8; ++count) { + TState state; + state.Append(NewHoldedMessage()); + for (int copies = 1; copies <= count; ++copies) { + TVolatileState v; + for (int peer = 1; peer < copies; ++peer) v.MatchIndex[peer + 1] = 1; + v.CommitAdvance(count, state); + assert_int_equal(v.CommitIndex, copies >= count / 2 + 1 ? 1 : 0); + } + } +} + +void CheckRequests(uint32_t flags, bool partial) { + auto state = std::make_shared(); + auto raft = std::make_shared(state, 1, TNodeDict{}); + raft->Become(EState::LEADER); + auto rsm = std::make_shared(); + TRequestProcessor processor(raft, rsm, {}); + std::vector replies; + auto client = std::make_shared([&](auto m) { + replies.push_back(m.template Cast()->Cookie); + }); + for (uint32_t cookie : {11, 22}) { + auto command = NewHoldedMessage(sizeof(TCommandRequest) + sizeof(int64_t)); + int64_t readIndex = 0; + memcpy(command->Data, &readIndex, sizeof(readIndex)); + command->Flags = flags; + command->Cookie = cookie; + processor.OnCommandRequest(command, client); + } + if (partial) { + // Only the first write is committed; a later request must not block it. + const_cast(raft->GetVolatileState())->CommitIndex = 1; + processor.ProcessCommitted(); + processor.ProcessWaiting(); + assert_int_equal(replies.size(), 1); + assert_int_equal(replies[0], 11); + } + raft->ProcessTimeout(ITimeSource::Time{}); + processor.ProcessCommitted(); + processor.ProcessWaiting(); + assert_int_equal(replies.size(), 2); + assert_int_equal(replies[0], 11); + assert_int_equal(replies[1], 22); + processor.ProcessWaiting(); + assert_int_equal(replies.size(), 2); +} +void test_read_fifo(void**) { CheckRequests(0, false); } +void test_strong_read_fifo(void**) { CheckRequests(TCommandRequest::EConsistent, false); } +void test_write_fifo(void**) { CheckRequests(TCommandRequest::EWrite, false); } +void test_partial_write_completion(void**) { CheckRequests(TCommandRequest::EWrite, true); } + +void test_loss_reorder_duplicate_and_recovery(void**) { + TCluster cluster(3); + cluster.Elect(1); + auto index = cluster.Nodes[0]->Append(NewHoldedMessage()); + cluster.Nodes[0]->ProcessTimeout(cluster.Now += std::chrono::seconds(1)); + cluster.Packets.clear(); // Drop a complete replication round. + assert_true(cluster.Nodes[0]->GetVolatileState()->CommitIndex < index); + cluster.Replicate(1, true, true); + for (auto& node : cluster.Nodes) assert_int_equal(node->GetVolatileState()->CommitIndex, index); + cluster.AssertCommittedPrefix(); +} + +void test_partition_leader_replacement_and_heal(void**) { + TCluster cluster(3); + cluster.Elect(1); + auto committed = cluster.Nodes[0]->Append(NewHoldedMessage()); + cluster.Replicate(1); + cluster.Nodes[0]->ProcessTimeout(cluster.Now += std::chrono::seconds(1)); + auto delayed = std::move(cluster.Packets); + cluster.Packets.clear(); + cluster.Isolated.insert(1); + auto lost = NewHoldedMessage(); + lost->Flags = 8; // Distinguish the minority's uncommitted entry. + auto minority = cluster.Nodes[0]->Append(lost); + cluster.Replicate(1); + assert_int_equal(cluster.Nodes[0]->GetVolatileState()->CommitIndex, committed); + cluster.Elect(2); + auto replacement = cluster.Nodes[1]->Append(NewHoldedMessage()); + cluster.Replicate(2); + assert_true(cluster.Nodes[1]->GetVolatileState()->CommitIndex >= replacement); + cluster.AssertCommittedPrefix(); + cluster.Isolated.clear(); + // Old-term traffic arrives after the new leader has committed its entry. + cluster.Packets = std::move(delayed); + cluster.Drain(true, true); + cluster.Replicate(2, true, true); + assert_int_equal(cluster.Nodes[0]->CurrentStateName(), EState::FOLLOWER); + for (auto& node : cluster.Nodes) { + assert_int_equal(node->GetVolatileState()->CommitIndex, replacement); + assert_int_equal(node->GetState()->Get(minority)->Flags, TLogEntry::EStub); + } + cluster.AssertCommittedPrefix(); +} + +void test_even_cluster_partition_requires_majority(void**) { + TCluster cluster(4); + cluster.Elect(1); + cluster.Replicate(1); + auto previous = cluster.Nodes[0]->GetVolatileState()->CommitIndex; + cluster.Isolated = {3, 4}; + auto index = cluster.Nodes[0]->Append(NewHoldedMessage()); + cluster.Replicate(1); + assert_int_equal(cluster.Nodes[0]->GetVolatileState()->CommitIndex, previous); + assert_int_equal(cluster.Nodes[1]->GetVolatileState()->CommitIndex, previous); + cluster.Isolated.clear(); + cluster.Replicate(1, true, true); + for (auto& node : cluster.Nodes) assert_int_equal(node->GetVolatileState()->CommitIndex, index); + cluster.AssertCommittedPrefix(); +} +} + +int main() { + const CMUnitTest tests[] = { + cmocka_unit_test(test_majority_all_cluster_sizes), + cmocka_unit_test(test_read_fifo), + cmocka_unit_test(test_strong_read_fifo), + cmocka_unit_test(test_write_fifo), + cmocka_unit_test(test_partial_write_completion), + cmocka_unit_test(test_loss_reorder_duplicate_and_recovery), + cmocka_unit_test(test_partition_leader_replacement_and_heal), + cmocka_unit_test(test_even_cluster_partition_requires_majority), + }; + return cmocka_run_group_tests(tests, nullptr, nullptr); +} From 781ce32fe375b719464c5ac2214a43a60d50574e Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Sat, 7 Nov 2026 22:24:37 -0500 Subject: [PATCH 02/66] Reject corrupt storage on recovery and fix truncation state (#2) * Reject corrupt persistence on recovery and fix log truncation state * Fail-stop on live storage errors instead of escaping into network handlers * Cover replacement after live truncation without reopening storage --- README.md | 2 + docs/persistence-recovery.md | 64 ++++++++++ miniraft/persist.cpp | 182 ++++++++++++++++++--------- miniraft/persist.h | 4 +- test/CMakeLists.txt | 1 + test/test_persist_recovery.cpp | 221 +++++++++++++++++++++++++++++++++ 6 files changed, 413 insertions(+), 61 deletions(-) create mode 100644 docs/persistence-recovery.md create mode 100644 test/test_persist_recovery.cpp diff --git a/README.md b/README.md index f70f0ee..4adef06 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,8 @@ request completion, and adds deterministic consensus failure tests. See the [correctness baseline and review](docs/correctness-baseline.md) for validation, build instructions and remaining production release gates. The milestone is experimental; durable crash recovery is not established. +The [persistence recovery follow-up](docs/persistence-recovery.md) rejects +incomplete or structurally corrupt storage and fixes truncation/restart behavior. [![Number of GitHub stars](https://img.shields.io/github/stars/resetius/miniraft-cpp)](https://github.com/resetius/miniraft-cpp/stargazers) ![GitHub commit activity](https://img.shields.io/github/commit-activity/m/resetius/miniraft-cpp) diff --git a/docs/persistence-recovery.md b/docs/persistence-recovery.md new file mode 100644 index 0000000..6c27856 --- /dev/null +++ b/docs/persistence-recovery.md @@ -0,0 +1,64 @@ +# Persistence recovery validation: v0.1.0-alpha.2 + +This follow-up closes unsafe recovery behavior in the original three-file +TDiskState format. It is still experimental storage, not power-loss durability. + +## Fixed behavior + +Previously an incorrectly sized state file silently initialized fresh metadata, +potentially forgetting the node's current term and vote. Missing files were +recreated, short indexes could crash startup, and entry lengths were trusted +before allocation. Recovery now rejects these conditions with std::runtime_error +without changing or recreating the stored files. + +A new store is initialized only when all three paths are absent. Existing stores +must contain all three files and an exactly sized metadata record with a +nonzero current term. Recovery validates the complete selected log prefix: +index extent, contiguous offsets, entry types, minimum lengths and file bounds. +Unreferenced trailing bytes remain allowed because log truncation is logical. +The file format and public storage interface are unchanged. + +RemoveLast now updates LastLogTerm, including when the log becomes empty. +Append explicitly seeks the write position, so appending after truncation to +zero overwrites the beginning rather than writing into an obsolete suffix. +File-open failures no longer fall back to truncating an existing file. Stream +write/flush errors are detected, and buffered entries/index are flushed before metadata. + +## Validation and review + +Ten new persistence tests cover truncated/oversized metadata, each missing +storage file, short indexes/entries, invalid entry headers and offsets, zero +terms, excessive log counts and truncation/restart/replacement. Rejected recovery +is checked against a byte-for-byte snapshot of every file to verify no mutation. +An additional POSIX subprocess test verifies that a live storage read failure +terminates the node rather than escaping as an ordinary network exception. +The original implementation failed eight of the initial nine tests, including +a caught segmentation fault on a short index and stale LastLogTerm after removal. + +Local Debug/Release core tests and the full ASan/UBSan suite pass: 27 existing +core tests, eight consensus correctness tests, eleven recovery tests and three +transport tests (49 total on POSIX; the subprocess test is excluded on Windows). CI runs the same targets on pushes and PRs. + +Self-review checked recovery's non-mutating rejection paths, length arithmetic +before allocation, signed file positions, offset continuity, existing-format +compatibility, output positions after truncation and fail-stop behavior in the +presence of the network layer's generic exception handlers. No independent reviewer +approval is claimed. + +## Operational contract and remaining gaps + +Use one writer per storage set. Startup validation throws std::runtime_error. +Detected live storage read/write/flush errors call std::terminate: the network +layer catches ordinary exceptions, so throwing alone would allow a node with +unpersisted state to continue processing. +Recovery rejection requires restoring verified state or deliberately rebuilding +the node through a safe cluster procedure; never delete metadata alone to force +startup. Previously created empty or partial file sets are deliberately rejected. + +Validation detects structural damage, not all corruption: there are no checksums +or record generations, so a bit flip that leaves valid fields can go undetected. +The three-file format still lacks fsync, atomic metadata publication, directory +synchronization and a crash-safe replacement protocol. In-place updates can +destroy an older valid version on power failure. This change rejects detectable +damage rather than attempting speculative repair. Crash injection, snapshots, +membership changes and strong-read linearizability remain separate work. diff --git a/miniraft/persist.cpp b/miniraft/persist.cpp index 7ad69a8..a02f68b 100644 --- a/miniraft/persist.cpp +++ b/miniraft/persist.cpp @@ -1,70 +1,123 @@ #include "persist.h" +#include +#include +#include #include TDiskState::TDiskState(const std::string& name, uint32_t id) - : Entries(Open(name + ".entries." + std::to_string(id))) - , Index(Open(name + ".index." + std::to_string(id))) - , State(Open(name + ".state." + std::to_string(id))) { - // TODO: read all an validate - State.seekg(0, std::ios_base::end); - auto size = State.tellg(); - if (size == sizeof(LastLogIndex) + sizeof(CurrentTerm) + sizeof(VotedFor)) { - State.seekg(0); - State.read((char*)&LastLogIndex, sizeof(LastLogIndex)); - State.read((char*)&CurrentTerm, sizeof(CurrentTerm)); - State.read((char*)&VotedFor, sizeof(VotedFor)); - } else { + const auto suffix = "." + std::to_string(id); + const auto entriesPath = name + ".entries" + suffix; + const auto indexPath = name + ".index" + suffix; + const auto statePath = name + ".state" + suffix; + const int existing = std::filesystem::exists(entriesPath) + + std::filesystem::exists(indexPath) + std::filesystem::exists(statePath); + // Never turn a partially missing store into a fresh Raft voter. + if (existing != 0 && existing != 3) { + throw std::runtime_error("Incomplete Raft storage: " + name); + } + const bool fresh = existing == 0; + Entries = Open(entriesPath, fresh); + Index = Open(indexPath, fresh); + State = Open(statePath, fresh); + if (fresh) { Commit(); + Recovering = false; + return; } - if (LastLogIndex > 0) { - LastLogTerm = Get(LastLogIndex)->Term; + + State.seekg(0, std::ios::end); + const auto size = State.tellg(); + if (size != std::streamoff(sizeof(LastLogIndex) + sizeof(CurrentTerm) + sizeof(VotedFor))) { + throw std::runtime_error("Invalid Raft metadata size: " + statePath); + } + State.seekg(0); + if (!State.read(reinterpret_cast(&LastLogIndex), sizeof(LastLogIndex)) || + !State.read(reinterpret_cast(&CurrentTerm), sizeof(CurrentTerm)) || + !State.read(reinterpret_cast(&VotedFor), sizeof(VotedFor)) || CurrentTerm == 0) { + throw std::runtime_error("Invalid Raft metadata: " + statePath); } -} -std::fstream TDiskState::Open(const std::string& name) -{ - auto flags = std::ios::in | std::ios::out | std::ios::binary; - std::fstream f(name, flags); - if (!f.is_open()) { - std::ofstream tmp(name); - tmp.close(); - f.open(name, flags); + Index.seekg(0, std::ios::end); + const auto indexSize = Index.tellg(); + if (indexSize < 0 || LastLogIndex > uint64_t(indexSize) / sizeof(uint64_t)) { + throw std::runtime_error("Truncated Raft index: " + indexPath); } + // Validate the entire selected prefix, not only its last entry. Trailing + // bytes can be left by logical truncation and are deliberately ignored. + uint64_t expectedOffset = 0; + for (uint64_t i = 1; i <= LastLogIndex; ++i) { + uint64_t offset; + Index.seekg((i - 1) * sizeof(offset)); + if (!Index.read(reinterpret_cast(&offset), sizeof(offset)) || offset != expectedOffset) { + throw std::runtime_error("Invalid Raft log offset: " + indexPath); + } + auto entry = Get(i); + expectedOffset += entry->Len; + LastLogTerm = entry->Term; + } + Recovering = false; +} - if (!f.is_open()) { - throw std::runtime_error("Cannot open file: " + name); +std::fstream TDiskState::Open(const std::string& name, bool create) +{ + if (create) { + std::ofstream file(name, std::ios::binary); + if (!file) throw std::runtime_error("Cannot create file: " + name); } + std::fstream file(name, std::ios::in | std::ios::out | std::ios::binary); + // An open failure must never fall back to truncating an existing file. + if (!file) throw std::runtime_error("Cannot open file: " + name); + return file; +} - return f; +[[noreturn]] void TDiskState::Fail(const char* message) const +{ + if (Recovering) throw std::runtime_error(message); + // The network layer catches ordinary exceptions and keeps the node alive. + // Continuing after a live storage failure could acknowledge unpersisted + // votes or logs. Fail-stop instead; startup validation remains catchable. + std::cerr << "Fatal Raft storage error: " << message << '\n'; + std::terminate(); } TMessageHolder TDiskState::Read() const { - decltype(TMessage::Type) type; - decltype(TMessage::Len) len; - if (!Entries.read((char*)&type, sizeof(type))) { - throw std::runtime_error("Error on read 1"); + const auto start = Entries.tellg(); + Entries.seekg(0, std::ios::end); + const auto end = Entries.tellg(); + if (start < 0 || end < start || end - start < std::streamoff(sizeof(TLogEntry))) { + Fail("Truncated Raft log entry"); } - if (!Entries.read((char*)&len, sizeof(len))) { - throw std::runtime_error("Error on read 2"); + Entries.seekg(start); + uint32_t type, len; + if (!Entries.read(reinterpret_cast(&type), sizeof(type)) || + !Entries.read(reinterpret_cast(&len), sizeof(len)) || + type != uint32_t(EMessageType::LOG_ENTRY) || len < sizeof(TLogEntry) || + uint64_t(len) > uint64_t(end - start)) { + Fail("Invalid Raft log entry header"); } - auto mes = NewHoldedMessage(type, len); - if (!Entries.read((char*)mes->Value, len - sizeof(TMessage))) { - throw std::runtime_error("Error on read 3"); + // Bound the allocation and subtraction by the validated file extent. + auto message = NewHoldedMessage(type, len); + if (!Entries.read(reinterpret_cast(message->Value), len - sizeof(TMessage))) { + Fail("Cannot read Raft log entry"); } - return mes; + return message; } void TDiskState::Write(TMessageHolder message) { - Entries.write((char*)message.Mes, message->Len); + if (!Entries.write(reinterpret_cast(message.Mes), message->Len)) { + Fail("Cannot write Raft log entry"); + } } void TDiskState::RemoveLast() { if (LastLogIndex > 0) { - LastLogIndex--; + const auto previousTerm = LastLogIndex == 1 ? 0 : Get(LastLogIndex - 1)->Term; + --LastLogIndex; + LastLogTerm = previousTerm; Commit(); } } @@ -72,43 +125,52 @@ void TDiskState::RemoveLast() void TDiskState::Append(TMessageHolder entry) { uint64_t offset = 0; - if (Get(LastLogIndex)) { // TODO: optimize + if (Get(LastLogIndex)) { offset = Entries.tellg(); } - + // In particular, after truncation to zero the stream may still point into + // the old suffix. Always position the output explicitly. + Entries.seekp(offset); Write(entry); - Index.seekg(LastLogIndex * sizeof(offset)); - Index.write((char*)&offset, sizeof(offset)); + Index.seekp(LastLogIndex * sizeof(offset)); + if (!Index.write(reinterpret_cast(&offset), sizeof(offset))) { + Fail("Cannot write Raft log index"); + } LastLogTerm = entry->Term; - LastLogIndex ++; + ++LastLogIndex; Commit(); } TMessageHolder TDiskState::Get(int64_t index) const { - if (index > LastLogIndex || index < 1) { - return {}; + if (index > LastLogIndex || index < 1) return {}; + uint64_t offset; + Index.seekg((index - 1) * sizeof(offset)); + if (!Index.read(reinterpret_cast(&offset), sizeof(offset))) { + Fail("Cannot read Raft log index"); } - - uint64_t offset = 0; - Index.seekg((index-1) * sizeof(offset)); - if (!Index.read((char*)&offset, sizeof(offset))) { - return {}; + Entries.seekg(0, std::ios::end); + const auto end = Entries.tellg(); + if (end < 0 || offset > uint64_t(end)) { + Fail("Raft log offset outside file"); } - Entries.seekg(offset); - auto entry = Read().Cast(); - return entry; + return Read().Cast(); } void TDiskState::Commit() { - State.seekg(0); - if (!State.write((char*)&LastLogIndex, sizeof(LastLogIndex))) { abort(); } - if (!State.write((char*)&CurrentTerm, sizeof(CurrentTerm))) { abort(); } - if (!State.write((char*)&VotedFor, sizeof(VotedFor))) { abort(); } - State.flush(); - Index.flush(); + // Publish buffered log data before metadata. Stream flushing alone is not + // a power-loss durability protocol; fsync/atomic publication remain TODO. Entries.flush(); + Index.flush(); + if (!Entries || !Index) Fail("Cannot flush Raft log"); + State.seekp(0); + if (!State.write(reinterpret_cast(&LastLogIndex), sizeof(LastLogIndex)) || + !State.write(reinterpret_cast(&CurrentTerm), sizeof(CurrentTerm)) || + !State.write(reinterpret_cast(&VotedFor), sizeof(VotedFor))) { + Fail("Cannot write Raft metadata"); + } + State.flush(); + if (!State) Fail("Cannot flush Raft metadata"); } - diff --git a/miniraft/persist.h b/miniraft/persist.h index 71c92b1..202242c 100644 --- a/miniraft/persist.h +++ b/miniraft/persist.h @@ -14,7 +14,9 @@ struct TDiskState : IState { void Commit() override; private: - std::fstream Open(const std::string& name); + bool Recovering = true; + [[noreturn]] void Fail(const char* message) const; + std::fstream Open(const std::string& name, bool create); TMessageHolder Read() const; void Write(TMessageHolder); diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index ce2fc09..944ddb6 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -14,6 +14,7 @@ endmacro() if (CMOCKA_FOUND) ut(test_raft test_raft.cpp miniraft) ut(test_correctness test_correctness.cpp miniraft) +ut(test_persist_recovery test_persist_recovery.cpp miniraft) if(MINIRAFT_BUILD_NETWORK) ut(test_read_write test_read_write.cpp miniraft.net) endif() diff --git a/test/test_persist_recovery.cpp b/test/test_persist_recovery.cpp new file mode 100644 index 0000000..38b8b4b --- /dev/null +++ b/test/test_persist_recovery.cpp @@ -0,0 +1,221 @@ +#include +#include +#include +#include +#include +#include +#ifndef _WIN32 +#include +#include +#endif +#include +#include +#include +extern "C" { +#include +} + +namespace { +namespace fs = std::filesystem; +struct TFiles { + fs::path Directory; + std::string Name; + TFiles() { + static unsigned counter = 0; + Directory = fs::temp_directory_path() / + ("miniraft-recovery-" + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count()) + + "-" + std::to_string(counter++)); + assert_true(fs::create_directory(Directory)); + Name = (Directory / "raft").string(); + } + ~TFiles() { std::error_code ignored; fs::remove_all(Directory, ignored); } + fs::path Path(const std::string& kind) const { return Name + "." + kind + ".1"; } + void Seed() { + TDiskState state(Name, 1); + state.CurrentTerm = 7; + state.VotedFor = 2; + for (uint64_t term : {2, 5}) { + auto entry = NewHoldedMessage(); + entry->Term = term; + state.Append(entry); + } + } + template void Overwrite(const std::string& kind, uint64_t offset, T value) { + std::fstream file(Path(kind), std::ios::in | std::ios::out | std::ios::binary); + file.seekp(offset); + file.write(reinterpret_cast(&value), sizeof(value)); + file.flush(); + assert_true(bool(file)); + } + void RejectRecovery() { + auto contents = [&] { + std::map result; + for (auto& file : fs::directory_iterator(Directory)) { + std::ifstream input(file.path(), std::ios::binary); + result[file.path().filename().string()] = + std::string(std::istreambuf_iterator(input), {}); + } + return result; + }; + const auto before = contents(); + bool rejected = false; + try { TDiskState state(Name, 1); } + catch (const std::runtime_error&) { rejected = true; } + assert_true(rejected); + assert_true(before == contents()); + } +}; + +void test_truncated_metadata_is_not_reset(void**) { + for (uint64_t size : {0, 1, 8, 16, 19}) { + TFiles files; + files.Seed(); + fs::resize_file(files.Path("state"), size); + files.RejectRecovery(); + assert_int_equal(fs::file_size(files.Path("state")), size); + } +} +void test_oversized_metadata_is_rejected(void**) { + TFiles files; + files.Seed(); + fs::resize_file(files.Path("state"), 21); + files.RejectRecovery(); + assert_int_equal(fs::file_size(files.Path("state")), 21); +} +void test_missing_storage_is_not_recreated(void**) { + for (const std::string kind : {"state", "index", "entries"}) { + TFiles files; + files.Seed(); + fs::remove(files.Path(kind)); + files.RejectRecovery(); + assert_false(fs::exists(files.Path(kind))); + } +} +void test_short_index_is_rejected(void**) { + TFiles files; + files.Seed(); + fs::resize_file(files.Path("index"), sizeof(uint64_t)); + files.RejectRecovery(); +} +void test_short_entry_is_rejected(void**) { + TFiles files; + files.Seed(); + fs::resize_file(files.Path("entries"), sizeof(TLogEntry) + 4); + files.RejectRecovery(); +} +void test_invalid_entry_header_is_rejected(void**) { + for (uint32_t len : {0u, 7u, 8u, uint32_t(sizeof(TLogEntry) - 1), UINT32_MAX}) { + TFiles files; + files.Seed(); + files.Overwrite("entries", sizeof(uint32_t), len); + files.RejectRecovery(); + } + TFiles files; + files.Seed(); + files.Overwrite("entries", 0, uint32_t(EMessageType::COMMAND_REQUEST)); + files.RejectRecovery(); +} +void test_invalid_offsets_are_rejected(void**) { + for (uint64_t offset : {uint64_t(0), uint64_t(1), UINT64_MAX}) { + TFiles files; + files.Seed(); + files.Overwrite("index", sizeof(uint64_t), offset); + files.RejectRecovery(); + } +} +void test_invalid_term_is_rejected(void**) { + TFiles files; + files.Seed(); + files.Overwrite("state", sizeof(uint64_t), uint64_t(0)); + files.RejectRecovery(); +} +void test_invalid_log_count_is_rejected(void**) { + TFiles files; + files.Seed(); + files.Overwrite("state", 0, UINT64_MAX); + files.RejectRecovery(); +} +void test_truncate_updates_last_term_and_restarts(void**) { + TFiles files; + files.Seed(); + { + TDiskState state(files.Name, 1); + state.RemoveLast(); + assert_int_equal(state.LastLogIndex, 1); + assert_int_equal(state.LastLogTerm, 2); + } + { + TDiskState state(files.Name, 1); + assert_int_equal(state.CurrentTerm, 7); + assert_int_equal(state.VotedFor, 2); + assert_int_equal(state.LastLogTerm, 2); + state.RemoveLast(); + assert_int_equal(state.LastLogIndex, 0); + assert_int_equal(state.LastLogTerm, 0); + } + { + // Logical truncation leaves a suffix on disk. Reopen it as an empty + // log, then ensure the replacement really starts at byte zero. + TDiskState state(files.Name, 1); + assert_int_equal(state.LastLogIndex, 0); + assert_int_equal(state.LastLogTerm, 0); + auto entry = NewHoldedMessage(); + entry->Term = 7; + state.Append(entry); + assert_int_equal(state.Get(1)->Term, 7); + } + TDiskState recovered(files.Name, 1); + assert_int_equal(recovered.LastLogIndex, 1); + assert_int_equal(recovered.LastLogTerm, 7); + assert_int_equal(recovered.Get(1)->Term, 7); + assert_int_equal(recovered.VotedFor, 2); + // Also replace an empty log without reopening: the shared stream position + // is still at the end of the removed entry, rather than at byte zero. + recovered.RemoveLast(); + auto replacement = NewHoldedMessage(); + replacement->Term = 6; + recovered.Append(replacement); + assert_int_equal(recovered.Get(1)->Term, 6); + assert_int_equal(recovered.LastLogTerm, 6); +} +#ifndef _WIN32 +void test_live_storage_failure_is_fatal(void**) { + TFiles files; + files.Seed(); + const auto child = fork(); + assert_true(child >= 0); + if (child == 0) { + std::set_terminate([] { _exit(73); }); + try { + TDiskState state(files.Name, 1); + fs::resize_file(files.Path("entries"), 0); + state.Get(1); + } catch (...) { _exit(74); } + _exit(75); + } + int status = 0; + assert_int_equal(waitpid(child, &status, 0), child); + assert_true(WIFEXITED(status)); + assert_int_equal(WEXITSTATUS(status), 73); +} +#endif +} + +int main() { + const CMUnitTest tests[] = { + cmocka_unit_test(test_truncated_metadata_is_not_reset), + cmocka_unit_test(test_oversized_metadata_is_rejected), + cmocka_unit_test(test_missing_storage_is_not_recreated), + cmocka_unit_test(test_short_index_is_rejected), + cmocka_unit_test(test_short_entry_is_rejected), + cmocka_unit_test(test_invalid_entry_header_is_rejected), + cmocka_unit_test(test_invalid_offsets_are_rejected), + cmocka_unit_test(test_invalid_term_is_rejected), + cmocka_unit_test(test_invalid_log_count_is_rejected), + cmocka_unit_test(test_truncate_updates_last_term_and_restarts), +#ifndef _WIN32 + cmocka_unit_test(test_live_storage_failure_is_fatal), +#endif + }; + return cmocka_run_group_tests(tests, nullptr, nullptr); +} From 4e706a4c83abf70d213d6194c1a6dc8670f925aa Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Sat, 7 Nov 2026 22:52:29 -0500 Subject: [PATCH 03/66] Add opt-in durable Linux storage with atomic image publication (#3) * Add Linux durable state with atomic publication and crash-boundary tests * Verify Raft election and application replay across durable-state restart --- README.md | 3 + docs/durable-state.md | 101 +++++++++++++ examples/kv.cpp | 10 +- miniraft/CMakeLists.txt | 3 +- miniraft/durable.cpp | 246 ++++++++++++++++++++++++++++++ miniraft/durable.h | 27 ++++ test/CMakeLists.txt | 3 + test/test_durable.cpp | 321 ++++++++++++++++++++++++++++++++++++++++ 8 files changed, 711 insertions(+), 3 deletions(-) create mode 100644 docs/durable-state.md create mode 100644 miniraft/durable.cpp create mode 100644 miniraft/durable.h create mode 100644 test/test_durable.cpp diff --git a/README.md b/README.md index 4adef06..fba176a 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,9 @@ validation, build instructions and remaining production release gates. The milestone is experimental; durable crash recovery is not established. The [persistence recovery follow-up](docs/persistence-recovery.md) rejects incomplete or structurally corrupt storage and fixes truncation/restart behavior. +For new Linux stores, the opt-in [durable state backend](docs/durable-state.md) +adds checksummed atomic publication with file and directory synchronization. +The KV example selects it with `--durable`; legacy `--persist` is unchanged. [![Number of GitHub stars](https://img.shields.io/github/stars/resetius/miniraft-cpp)](https://github.com/resetius/miniraft-cpp/stargazers) ![GitHub commit activity](https://img.shields.io/github/commit-activity/m/resetius/miniraft-cpp) diff --git a/docs/durable-state.md b/docs/durable-state.md new file mode 100644 index 0000000..73d270a --- /dev/null +++ b/docs/durable-state.md @@ -0,0 +1,101 @@ +# Linux durable state: v0.1.0-alpha.3 + +TDurableState is an opt-in storage backend with synchronous durable publication. +It closes the missing file-sync and atomic-publication gap for new stores, under +the filesystem assumptions below. The legacy TDiskState backend is unchanged +and still does not provide power-loss durability. + +## Publication protocol + +Each Append, RemoveLast and Commit publishes one complete image containing the +current term, vote and selected log prefix: + +1. Serialize the complete state into a temporary file in the same directory. +2. fsync the temporary file and check the result. +3. Rename the temporary file over the published image atomically. +4. fsync the containing directory and check the result. +5. Return to the caller, allowing Raft to acknowledge the update. + +An older image is never overwritten in place. A crash before rename leaves the +previous published image; a process crash after rename exposes the complete new +image. An unacknowledged update may therefore appear after recovery, which is +allowed. No acknowledged operation returns before file and directory sync finish. +Startup syncs the selected image and directory again before serving requests. +If a crash/filesystem error leaves the published name missing, startup rejects +the store rather than inventing a new term or vote. + +Live publication errors terminate the process, since the networking layer catches +ordinary exceptions and would otherwise keep an unsafe node running. An error +after rename has an uncertain outcome; clients must not assume it rolled back. +Startup errors are exceptions and do not fall back to older/temp images. + +## Format and ownership + +The image has version magic, payload length and CRC32 followed by term, vote, +entry count and length-prefixed log entries. Recovery checks the checksum before +decoding entries and validates bounds and term ordering. CRC32 detects accidental +corruption but does not protect against malicious modification or all collisions. + +The files are NAME.durable.ID, NAME.durable.ID.tmp and NAME.durable.ID.lock. +A nonblocking advisory flock holds the lock for the object's lifetime. Use one +object/thread to mutate a store, and do not delete/replace its lock while running. +Returned/appended entries are copied so external mutation cannot change the +selected state behind the publisher's back. Temporary files are never recovery +candidates; the next publication replaces a stale temp file while holding the lock. + +An existing lock with no published image is rejected, including an interrupted +first initialization. This conservative choice can require operator recovery +even if the node had not acknowledged any updates. Do not delete an existing +store's files to force startup. + +Legacy entries/index/state files with the same base name are rejected. There is +no automatic migration or downgrade. Use this backend for fresh clusters until +a separately validated offline migration tool exists; do not replace an existing +node's state with an empty store under the same identity. A corrupt main image is never replaced by a +seemingly valid temp image. + +## Use + +```cpp +#include +auto state = std::make_shared("state", nodeId); +``` + +On Linux, the KV example accepts --durable in place of --persist. Run it in a +dedicated directory without legacy state files. --persist and the SQL example +continue to use the legacy backend. Other platforms build the library but +TDurableState construction throws an explicit unsupported-platform exception. + +## Validation and self-review + +Twelve new tests cover publication ordering, 20 process-crash boundaries across +append/truncation/vote/replacement operations, four immediate exits after returned +updates, five interrupted initializations, a live injected failure, an actual +write error using a child-only file-size limit, corrupted/truncated images, +single-writer exclusion, missing images, legacy-store refusal, entry ownership and Raft election/application replay across +a storage restart. +There are 31 child-process scenarios; none relies on sleeps or random scheduling. + +All 61 tests pass locally under ASan/UBSan, and the core suite passes in Debug +and Release builds. These process-crash tests do not simulate physical power +loss or prove a storage controller honors flush commands. Hosted CI executes the +same targets; its results are separate from local validation. + +Self-review checked sync-before-rename-before-ack ordering, suffix replacement, +directory durability, lock lifetime, failure propagation, format parsing and +recovery's refusal to adopt temporary images. No independent approval is claimed. + +## Limits and assumptions + +This first implementation rewrites the entire log on every update, uses memory +proportional to the log, and blocks the calling thread during sync. It establishes +a conservative protocol baseline, not NuRaft-level throughput. An efficient WAL, +group commit, snapshots and compaction remain necessary for substantial workloads. + +The durability contract assumes Linux, a pre-existing durable parent directory, +and a local filesystem/storage stack honoring fsync and same-directory atomic +rename, with exclusive application ownership of the files. Network filesystems, +arbitrary external file deletion, broken hardware/flush semantics and real power +cut testing are outside this milestone. CRC32 is not a substitute for backups. +Storage durability also does not establish end-to-end linearizability or exactly +once application behavior; those consensus/client audits remain open. diff --git a/examples/kv.cpp b/examples/kv.cpp index 990d72f..cce07e3 100644 --- a/examples/kv.cpp +++ b/examples/kv.cpp @@ -2,6 +2,7 @@ #include #include +#include #include "kv.h" @@ -152,7 +153,7 @@ NNet::TFuture Client(TPoller& poller, TSocket socket, uint32_t flags) { } void usage(const char* prog) { - std::cerr << prog << "--client|--server --id myid --node ip:port:id [--node ip:port:id ...] [--persist] [--stale] [--consistent]" << "\n"; + std::cerr << prog << "--client|--server --id myid --node ip:port:id [--node ip:port:id ...] [--persist|--durable] [--stale] [--consistent]" << "\n"; exit(0); } @@ -164,6 +165,7 @@ int main(int argc, char** argv) { uint32_t id = 0; bool server = false; bool persist = false; + bool durable = false; uint32_t flags = 0; for (int i = 1; i < argc; i++) { if (!strcmp(argv[i], "--server")) { @@ -175,6 +177,9 @@ int main(int argc, char** argv) { id = atoi(argv[++i]); } else if (!strcmp(argv[i], "--persist")) { persist = true; + } else if (!strcmp(argv[i], "--durable")) { + persist = true; + durable = true; } else if (!strcmp(argv[i], "--stale")) { flags |= TCommandRequest::EStale; } else if (!strcmp(argv[i], "--consistent")) { @@ -211,7 +216,8 @@ int main(int argc, char** argv) { std::shared_ptr rsm = std::make_shared(); std::shared_ptr state = std::make_shared(); if (persist) { - state = std::make_shared("state", myHost.Id); + if (durable) state = std::make_shared("state", myHost.Id); + else state = std::make_shared("state", myHost.Id); } auto raft = std::make_shared(state, myHost.Id, nodes); TPoller::TSocket socket(NNet::TAddress{myHost.Address, myHost.Port}, loop.Poller()); diff --git a/miniraft/CMakeLists.txt b/miniraft/CMakeLists.txt index a26c8c4..6345728 100644 --- a/miniraft/CMakeLists.txt +++ b/miniraft/CMakeLists.txt @@ -2,7 +2,8 @@ add_library(miniraft messages.cpp raft.cpp persist.cpp + durable.cpp ) target_include_directories(miniraft PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/..) -target_compile_features(miniraft PUBLIC cxx_std_20) \ No newline at end of file +target_compile_features(miniraft PUBLIC cxx_std_20) diff --git a/miniraft/durable.cpp b/miniraft/durable.cpp new file mode 100644 index 0000000..f9196e2 --- /dev/null +++ b/miniraft/durable.cpp @@ -0,0 +1,246 @@ +#include "durable.h" +#include +#include +#include +#include +#include +#include +#ifdef __linux__ +#include +#include +#include +#include +#include +#include +#endif + +namespace { +using TBytes = std::vector; +constexpr std::array Magic{'M','R','F','T','D','0','0','1'}; +void Put(TBytes& bytes, uint64_t value, unsigned width) { + for (unsigned i = 0; i < width; ++i) bytes.push_back(uint8_t(value >> (8 * i))); +} +uint64_t Take(const TBytes& bytes, size_t& position, unsigned width) { + if (position > bytes.size() || width > bytes.size() - position) + throw std::runtime_error("Truncated durable state"); + uint64_t value = 0; + for (unsigned i = 0; i < width; ++i) value |= uint64_t(bytes[position++]) << (8 * i); + return value; +} +uint32_t Checksum(const TBytes& bytes, size_t start) { + uint32_t crc = UINT32_MAX; + for (size_t i = start; i < bytes.size(); ++i) { + crc ^= bytes[i]; + for (int bit = 0; bit < 8; ++bit) crc = (crc >> 1) ^ (0xedb88320u & (0u - (crc & 1u))); + } + return ~crc; +} +TMessageHolder Clone(TMessageHolder entry) { + if (!entry.Mes || entry->Len < sizeof(TLogEntry) || entry->Type != uint32_t(EMessageType::LOG_ENTRY)) + throw std::runtime_error("Invalid durable log entry"); + auto copy = NewHoldedMessage(entry->Len); + memcpy(copy.Mes, entry.Mes, entry->Len); + return copy; +} +#ifdef __linux__ +struct TFd { + int Value = -1; + TFd() = default; + explicit TFd(int fd) : Value(fd) {} + ~TFd() { if (Value >= 0) close(Value); } + TFd(const TFd&) = delete; + TFd& operator=(const TFd&) = delete; +}; +void Require(bool ok, const char* operation) { + if (!ok) throw std::system_error(errno, std::generic_category(), operation); +} +void Sync(int fd) { + int result; + do { result = fsync(fd); } while (result < 0 && errno == EINTR); + Require(result == 0, "fsync durable state"); +} +void WriteAll(int fd, const TBytes& bytes) { + size_t done = 0; + while (done < bytes.size()) { + auto n = write(fd, bytes.data() + done, bytes.size() - done); + if (n < 0 && errno == EINTR) continue; + Require(n > 0, "write durable state"); + done += size_t(n); + } +} +TBytes ReadAll(int fd) { + struct stat info; + Require(fstat(fd, &info) == 0, "stat durable state"); + if (!S_ISREG(info.st_mode) || info.st_size < 20 || uint64_t(info.st_size) > SIZE_MAX) + throw std::runtime_error("Invalid durable state file size/type"); + TBytes bytes(size_t(info.st_size)); + size_t done = 0; + while (done < bytes.size()) { + auto n = read(fd, bytes.data() + done, bytes.size() - done); + if (n < 0 && errno == EINTR) continue; + if (n == 0) throw std::runtime_error("Truncated durable state file"); + Require(n > 0, "read durable state"); + done += size_t(n); + } + return bytes; +} +#endif +} + +struct TDurableState::TImpl { + std::vector> Log; + THook Hook; + std::string File, Temp; +#ifdef __linux__ + TFd Directory, Lock; +#endif + void Stage(EStage stage) { if (Hook) Hook(stage); } +}; + +TDurableState::TDurableState(const std::string& name, uint32_t id, THook hook) + : Impl(std::make_unique()) +{ +#ifndef __linux__ + throw std::runtime_error("TDurableState currently requires Linux"); +#else + const auto base = std::filesystem::path(name); + auto directory = base.parent_path(); + if (directory.empty()) directory = "."; + Impl->File = base.filename().string() + ".durable." + std::to_string(id); + Impl->Temp = Impl->File + ".tmp"; + Impl->Hook = std::move(hook); + Impl->Directory.Value = open(directory.c_str(), O_RDONLY | O_DIRECTORY | O_CLOEXEC); + Require(Impl->Directory.Value >= 0, "open durable directory"); + // Never interpret a legacy store as a fresh node. Migration must be explicit. + for (const auto* kind : {"entries", "index", "state"}) { + if (std::filesystem::exists(name + "." + kind + "." + std::to_string(id))) + throw std::runtime_error("Legacy storage exists; durable migration is not automatic"); + } + const auto lockName = Impl->File + ".lock"; + Impl->Lock.Value = openat(Impl->Directory.Value, lockName.c_str(), + O_RDWR | O_CREAT | O_EXCL | O_CLOEXEC | O_NOFOLLOW, 0600); + const bool newLock = Impl->Lock.Value >= 0; + if (!newLock && errno == EEXIST) + Impl->Lock.Value = openat(Impl->Directory.Value, lockName.c_str(), O_RDWR | O_CLOEXEC | O_NOFOLLOW); + Require(Impl->Lock.Value >= 0, "open durable lock"); + Require(flock(Impl->Lock.Value, LOCK_EX | LOCK_NB) == 0, "lock durable state"); + TFd file(openat(Impl->Directory.Value, Impl->File.c_str(), O_RDONLY | O_CLOEXEC | O_NOFOLLOW)); + if (file.Value < 0) { + Require(errno == ENOENT, "open durable state"); + if (!newLock) throw std::runtime_error("Published durable state is missing"); + Publish(); + return; + } + const auto bytes = ReadAll(file.Value); + if (!std::equal(Magic.begin(), Magic.end(), bytes.begin())) + throw std::runtime_error("Unknown durable state format"); + size_t position = Magic.size(); + const auto length = Take(bytes, position, 8); + const auto crc = Take(bytes, position, 4); + if (length != bytes.size() - position || crc != Checksum(bytes, position)) + throw std::runtime_error("Durable state length/checksum mismatch"); + CurrentTerm = Take(bytes, position, 8); + VotedFor = Take(bytes, position, 4); + const auto count = Take(bytes, position, 8); + if (!CurrentTerm || count > (bytes.size() - position) / 20) + throw std::runtime_error("Invalid durable state metadata"); + for (uint64_t i = 0; i < count; ++i) { + const auto term = Take(bytes, position, 8); + const auto flags = Take(bytes, position, 8); + const auto length = Take(bytes, position, 4); + if (length > bytes.size() - position || length > UINT32_MAX - sizeof(TLogEntry) || + !term || term < LastLogTerm || term > CurrentTerm) + throw std::runtime_error("Invalid durable log entry"); + auto entry = NewHoldedMessage(sizeof(TLogEntry) + length); + entry->Term = term; + entry->Flags = flags; + memcpy(entry->Data, bytes.data() + position, length); + position += length; + Impl->Log.push_back(entry); + LastLogTerm = term; + } + if (position != bytes.size()) throw std::runtime_error("Trailing durable state data"); + LastLogIndex = count; + // A crash after rename but before directory fsync may leave a visible new + // image. Stabilize the selected image and its name before serving requests. + Sync(file.Value); + Sync(Impl->Directory.Value); + // A leftover temp is never a recovery candidate, even if its checksum is valid. +#endif +} + +TDurableState::~TDurableState() = default; + +void TDurableState::Publish() +{ +#ifndef __linux__ + throw std::runtime_error("TDurableState currently requires Linux"); +#else + if (!CurrentTerm || LastLogIndex != Impl->Log.size() || + LastLogTerm != (Impl->Log.empty() ? 0 : Impl->Log.back()->Term)) + throw std::runtime_error("Invalid durable state fields"); + TBytes payload; + Put(payload, CurrentTerm, 8); + Put(payload, VotedFor, 4); + Put(payload, LastLogIndex, 8); + uint64_t previousTerm = 0; + for (auto entry : Impl->Log) { + if (!entry->Term || entry->Term < previousTerm || entry->Term > CurrentTerm) + throw std::runtime_error("Invalid durable log term"); + previousTerm = entry->Term; + Put(payload, entry->Term, 8); + Put(payload, entry->Flags, 8); + Put(payload, entry->Len - sizeof(TLogEntry), 4); + payload.insert(payload.end(), entry->Data, entry->Data + entry->Len - sizeof(TLogEntry)); + } + TBytes bytes(Magic.begin(), Magic.end()); + Put(bytes, payload.size(), 8); + Put(bytes, Checksum(payload, 0), 4); + bytes.insert(bytes.end(), payload.begin(), payload.end()); + TFd temp(openat(Impl->Directory.Value, Impl->Temp.c_str(), + O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC | O_NOFOLLOW, 0600)); + Require(temp.Value >= 0, "create durable temporary image"); + Impl->Stage(EStage::TempCreated); + WriteAll(temp.Value, bytes); + Impl->Stage(EStage::Written); + Sync(temp.Value); + Impl->Stage(EStage::FileSynced); + Require(renameat(Impl->Directory.Value, Impl->Temp.c_str(), + Impl->Directory.Value, Impl->File.c_str()) == 0, "publish durable image"); + Impl->Stage(EStage::Renamed); + Sync(Impl->Directory.Value); + Impl->Stage(EStage::DirectorySynced); +#endif +} + +void TDurableState::Commit() +{ + try { Publish(); } + catch (const std::exception& error) { + std::cerr << "Fatal durable storage error: " << error.what() << '\n'; + std::terminate(); + } catch (...) { std::terminate(); } +} + +void TDurableState::Append(TMessageHolder entry) +{ + Impl->Log.push_back(Clone(entry)); + LastLogIndex = Impl->Log.size(); + LastLogTerm = Impl->Log.back()->Term; + Commit(); +} + +void TDurableState::RemoveLast() +{ + if (Impl->Log.empty()) return; + Impl->Log.pop_back(); + LastLogIndex = Impl->Log.size(); + LastLogTerm = Impl->Log.empty() ? 0 : Impl->Log.back()->Term; + Commit(); +} + +TMessageHolder TDurableState::Get(int64_t index) const +{ + if (index < 1 || uint64_t(index) > Impl->Log.size()) return {}; + return Clone(Impl->Log[index - 1]); +} diff --git a/miniraft/durable.h b/miniraft/durable.h new file mode 100644 index 0000000..4c5cad7 --- /dev/null +++ b/miniraft/durable.h @@ -0,0 +1,27 @@ +#pragma once +#include "state.h" +#include +#include +#include + +// Linux: atomic full-image replacement, with file and directory fsync. +// O(log size) per update; intended as a conservative durability baseline. +class TDurableState : public IState { +public: + enum class EStage { TempCreated, Written, FileSynced, Renamed, DirectorySynced }; + // Diagnostics/fault injection. Throwing from this hook during a live + // operation is fatal, just like an I/O failure. + using THook = std::function; + TDurableState(const std::string& name, uint32_t id, THook hook = {}); + ~TDurableState() override; + TDurableState(const TDurableState&) = delete; + TDurableState& operator=(const TDurableState&) = delete; + void RemoveLast() override; + void Append(TMessageHolder entry) override; + TMessageHolder Get(int64_t index) const override; + void Commit() override; +private: + struct TImpl; + std::unique_ptr Impl; + void Publish(); +}; diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 944ddb6..578174a 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -15,6 +15,9 @@ if (CMOCKA_FOUND) ut(test_raft test_raft.cpp miniraft) ut(test_correctness test_correctness.cpp miniraft) ut(test_persist_recovery test_persist_recovery.cpp miniraft) +if(CMAKE_SYSTEM_NAME STREQUAL "Linux") +ut(test_durable test_durable.cpp miniraft) +endif() if(MINIRAFT_BUILD_NETWORK) ut(test_read_write test_read_write.cpp miniraft.net) endif() diff --git a/test/test_durable.cpp b/test/test_durable.cpp new file mode 100644 index 0000000..1be7c11 --- /dev/null +++ b/test/test_durable.cpp @@ -0,0 +1,321 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +extern "C" { +#include +} + +namespace { +namespace fs = std::filesystem; +using TStage = TDurableState::EStage; +struct TFiles { + fs::path Directory; + std::string Name; + TFiles() { + static unsigned next = 0; + Directory = fs::temp_directory_path() / ("miniraft-durable-" + + std::to_string(getpid()) + "-" + std::to_string(next++)); + assert_true(fs::create_directory(Directory)); + Name = (Directory / "raft").string(); + } + ~TFiles() { std::error_code ignored; fs::remove_all(Directory, ignored); } + fs::path Image() const { return Name + ".durable.1"; } + void Seed(); +}; +TMessageHolder Entry(uint64_t term, const std::string& data) { + auto entry = NewHoldedMessage(sizeof(TLogEntry) + data.size()); + entry->Term = term; + memcpy(entry->Data, data.data(), data.size()); + return entry; +} +void TFiles::Seed() { + TDurableState state(Name, 1); + state.CurrentTerm = 3; + state.VotedFor = 2; + state.Append(Entry(1, "first")); + state.Append(Entry(2, "second")); +} +std::string Data(TDurableState& state, int64_t index) { + auto entry = state.Get(index); + return std::string(entry->Data, entry->Len - sizeof(TLogEntry)); +} +enum class EOperation { Append, Truncate, Vote, Replace }; +void Operate(TDurableState& state, EOperation operation) { + if (operation == EOperation::Truncate) state.RemoveLast(); + else if (operation == EOperation::Vote) { + state.CurrentTerm = 4; + state.VotedFor = 3; + state.Commit(); + } else state.Append(Entry(3, "replacement")); +} +void Check(TDurableState& state, EOperation operation, bool updated) { + assert_int_equal(state.CurrentTerm, updated && operation == EOperation::Vote ? 4 : 3); + assert_int_equal(state.VotedFor, updated && operation == EOperation::Vote ? 3 : 2); + const auto oldCount = operation == EOperation::Replace ? 1 : 2; + const auto count = !updated ? oldCount : operation == EOperation::Truncate ? 1 : + operation == EOperation::Vote ? 2 : oldCount + 1; + assert_int_equal(state.LastLogIndex, count); + assert_string_equal(Data(state, 1).c_str(), "first"); + if (count >= 2) { + const bool replaced = updated && operation == EOperation::Replace; + assert_string_equal(Data(state, 2).c_str(), replaced ? "replacement" : "second"); + assert_int_equal(state.Get(2)->Term, replaced ? 3 : 2); + } + if (count == 3) assert_string_equal(Data(state, 3).c_str(), "replacement"); + assert_int_equal(state.LastLogTerm, state.Get(count)->Term); +} +void Wait(pid_t child, int expected) { + int status; + assert_int_equal(waitpid(child, &status, 0), child); + assert_true(WIFEXITED(status)); + assert_int_equal(WEXITSTATUS(status), expected); +} + +void test_crash_at_every_publication_boundary(void**) { + for (auto operation : {EOperation::Append, EOperation::Truncate, EOperation::Vote, EOperation::Replace}) { + for (auto stage : {TStage::TempCreated, TStage::Written, TStage::FileSynced, + TStage::Renamed, TStage::DirectorySynced}) { + TFiles files; + files.Seed(); + const auto child = fork(); + assert_true(child >= 0); + if (child == 0) { + try { + bool armed = false; + TDurableState state(files.Name, 1, [&](auto point) { + if (armed && point == stage) _exit(90); + }); + if (operation == EOperation::Replace) state.RemoveLast(); + armed = true; + Operate(state, operation); + } catch (...) { _exit(2); } + _exit(3); + } + Wait(child, 90); + TDurableState recovered(files.Name, 1); + Check(recovered, operation, stage >= TStage::Renamed); + } + } +} + +void test_acknowledged_updates_survive_immediate_exit(void**) { + for (auto operation : {EOperation::Append, EOperation::Truncate, EOperation::Vote, EOperation::Replace}) { + TFiles files; + files.Seed(); + const auto child = fork(); + assert_true(child >= 0); + if (child == 0) { + try { + TDurableState state(files.Name, 1); + if (operation == EOperation::Replace) state.RemoveLast(); + Operate(state, operation); + _exit(0); // No destructors or stream flushing after acknowledgement. + } catch (...) { _exit(2); } + } + Wait(child, 0); + TDurableState recovered(files.Name, 1); + Check(recovered, operation, true); + } +} + +void test_publication_order(void**) { + TFiles files; + files.Seed(); + std::vector stages; + TDurableState state(files.Name, 1, [&](auto stage) { stages.push_back(stage); }); + state.CurrentTerm = 4; + state.Commit(); + const std::vector expected{TStage::TempCreated, TStage::Written, TStage::FileSynced, + TStage::Renamed, TStage::DirectorySynced}; + assert_true(stages == expected); +} + +void test_live_publication_failure_is_fatal(void**) { + TFiles files; + files.Seed(); + const auto child = fork(); + assert_true(child >= 0); + if (child == 0) { + std::set_terminate([] { _exit(73); }); + try { + TDurableState state(files.Name, 1, [](auto stage) { + if (stage == TStage::FileSynced) throw std::runtime_error("injected failure"); + }); + state.CurrentTerm = 4; + state.Commit(); + } catch (...) { _exit(74); } + _exit(75); + } + Wait(child, 73); + TDurableState recovered(files.Name, 1); + assert_int_equal(recovered.CurrentTerm, 3); + assert_int_equal(recovered.VotedFor, 2); +} + +void test_write_error_is_fatal_and_preserves_image(void**) { + TFiles files; + files.Seed(); + const auto child = fork(); + assert_true(child >= 0); + if (child == 0) { + std::set_terminate([] { _exit(73); }); + try { + TDurableState state(files.Name, 1); + struct rlimit limit; + if (getrlimit(RLIMIT_FSIZE, &limit) != 0) _exit(76); + limit.rlim_cur = 0; + if (setrlimit(RLIMIT_FSIZE, &limit) != 0) _exit(76); + std::signal(SIGXFSZ, SIG_IGN); + state.Append(Entry(3, "must not be acknowledged")); + } catch (...) { _exit(74); } + _exit(75); + } + Wait(child, 73); + TDurableState recovered(files.Name, 1); + Check(recovered, EOperation::Append, false); +} + +void test_interrupted_initialization_is_not_reset(void**) { + for (auto stage : {TStage::TempCreated, TStage::Written, TStage::FileSynced, + TStage::Renamed, TStage::DirectorySynced}) { + TFiles files; + const auto child = fork(); + assert_true(child >= 0); + if (child == 0) { + try { + TDurableState state(files.Name, 1, [&](auto point) { + if (point == stage) _exit(90); + }); + } catch (...) { _exit(2); } + _exit(3); + } + Wait(child, 90); + bool rejected = false; + try { + TDurableState recovered(files.Name, 1); + assert_int_equal(recovered.CurrentTerm, 1); + assert_int_equal(recovered.VotedFor, 0); + assert_int_equal(recovered.LastLogIndex, 0); + } catch (const std::runtime_error&) { rejected = true; } + assert_int_equal(rejected, stage < TStage::Renamed); + } +} + +void Reject(const TFiles& files) { + bool rejected = false; + try { TDurableState state(files.Name, 1); } + catch (const std::runtime_error&) { rejected = true; } + assert_true(rejected); +} +void test_corrupt_images_are_rejected(void**) { + for (const auto offset : {0, 8, 16, 20, 30, 45}) { + TFiles files; + files.Seed(); + std::fstream file(files.Image(), std::ios::in | std::ios::out | std::ios::binary); + file.seekg(offset); + char byte; + file.read(&byte, 1); + byte ^= 1; + file.seekp(offset); + file.write(&byte, 1); + file.close(); + Reject(files); + } + for (const auto size : {0, 7, 19, 21, 40}) { + TFiles files; + files.Seed(); + fs::resize_file(files.Image(), size); + Reject(files); + } +} +void test_single_writer_lock(void**) { + TFiles files; + files.Seed(); + { + TDurableState writer(files.Name, 1); + Reject(files); + } + TDurableState afterClose(files.Name, 1); + assert_int_equal(afterClose.CurrentTerm, 3); +} +void test_missing_image_is_not_recreated(void**) { + TFiles files; + files.Seed(); + fs::remove(files.Image()); + Reject(files); + assert_false(fs::exists(files.Image())); +} +void test_legacy_store_is_not_silently_migrated(void**) { + TFiles files; + { TDiskState legacy(files.Name, 1); } + Reject(files); + assert_false(fs::exists(files.Image())); +} +void test_get_and_append_do_not_share_mutable_entries(void**) { + TFiles files; + { + TDurableState state(files.Name, 1); + auto entry = Entry(1, "data"); + state.Append(entry); + entry->Term = 100; + auto result = state.Get(1); + assert_int_equal(result->Term, 1); + result->Term = 99; + assert_int_equal(state.Get(1)->Term, 1); + state.Commit(); + } + TDurableState state(files.Name, 1); + assert_int_equal(state.Get(1)->Term, 1); +} +void test_raft_election_and_application_replay_after_restart(void**) { + TFiles files; + uint64_t term = 0, index = 0; + for (int restart = 0; restart < 2; ++restart) { + auto state = std::make_shared(files.Name, 1); + auto raft = std::make_shared(state, 1, TNodeDict{}); + raft->ProcessTimeout(ITimeSource::Time{}); + assert_int_equal(raft->CurrentStateName(), EState::LEADER); + assert_true(state->CurrentTerm > term); + term = state->CurrentTerm; + if (restart == 0) { + index = raft->Append(Entry(term, "application-command")); + raft->ProcessTimeout(ITimeSource::Time{}); + } + assert_true(raft->GetVolatileState()->CommitIndex >= index); + assert_string_equal(Data(*state, index).c_str(), "application-command"); + auto rsm = std::make_shared(); + TRequestProcessor processor(raft, rsm, {}); + processor.ProcessCommitted(); + assert_int_equal(rsm->LastAppliedIndex, state->LastLogIndex); + } +} +} + +int main() { + const CMUnitTest tests[] = { + cmocka_unit_test(test_crash_at_every_publication_boundary), + cmocka_unit_test(test_acknowledged_updates_survive_immediate_exit), + cmocka_unit_test(test_publication_order), + cmocka_unit_test(test_live_publication_failure_is_fatal), + cmocka_unit_test(test_write_error_is_fatal_and_preserves_image), + cmocka_unit_test(test_interrupted_initialization_is_not_reset), + cmocka_unit_test(test_corrupt_images_are_rejected), + cmocka_unit_test(test_single_writer_lock), + cmocka_unit_test(test_missing_image_is_not_recreated), + cmocka_unit_test(test_legacy_store_is_not_silently_migrated), + cmocka_unit_test(test_get_and_append_do_not_share_mutable_entries), + cmocka_unit_test(test_raft_election_and_application_replay_after_restart), + }; + return cmocka_run_group_tests(tests, nullptr, nullptr); +} From d114d5d95c758d741a4e8624151fe23227d54418 Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Sun, 8 Nov 2026 08:32:34 -0500 Subject: [PATCH 04/66] Require fresh majority confirmation for strong reads and bind them to leadership term (#4) --- README.md | 2 + docs/strong-reads.md | 88 ++++++++++++++++ miniraft/raft.cpp | 61 +++++++++-- miniraft/raft.h | 13 ++- test/test_correctness.cpp | 209 +++++++++++++++++++++++++++++++++++++- 5 files changed, 358 insertions(+), 15 deletions(-) create mode 100644 docs/strong-reads.md diff --git a/README.md b/README.md index fba176a..395bd8d 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,8 @@ incomplete or structurally corrupt storage and fixes truncation/restart behavior For new Linux stores, the opt-in [durable state backend](docs/durable-state.md) adds checksummed atomic publication with file and directory synchronization. The KV example selects it with `--durable`; legacy `--persist` is unchanged. +The [strong-read follow-up](docs/strong-reads.md) requires a fresh majority for +`--consistent` reads and rejects pending reads when their leadership term ends. [![Number of GitHub stars](https://img.shields.io/github/stars/resetius/miniraft-cpp)](https://github.com/resetius/miniraft-cpp/stargazers) ![GitHub commit activity](https://img.shields.io/github/commit-activity/m/resetius/miniraft-cpp) diff --git a/docs/strong-reads.md b/docs/strong-reads.md new file mode 100644 index 0000000..0b9cfcf --- /dev/null +++ b/docs/strong-reads.md @@ -0,0 +1,88 @@ +# Fresh-quorum strong reads: v0.1.0-alpha.4 + +This milestone fixes a reproduced stale-leader read bug in EConsistent and the +KV example's --consistent mode. Default reads and explicit stale reads keep +their previous weaker semantics. + +## Reproduced failure + +In a five-node cluster, node 1 had previously replicated its log to all peers. +A partition left it connected only to node 2. Nodes 3, 4 and 5 elected a new +leader and committed another entry. A strong read arriving at node 1 then +received a successful stale reply after one fresh response from node 2. + +The implementation combined historical MatchIndex values with the sequence +number of a single fresh response. Replication quorum was incorrectly treated +as proof that a fresh quorum had confirmed this read's leadership. + +## Read protocol + +Read confirmation now has per-peer acknowledgement watermarks separate from +log replication indices. A strong read reserves a unique nonzero uint64_t +barrier, then sends AppendEntries RPCs whose sequences are newer than it. +Successful responses from configured peers advance their individual watermarks; +duplicate replies cannot add voters. Pre-read replies cannot reach the new +barrier. Sequence allocation checks exhaustion rather than wrapping to zero. + +The confirmed watermark is the strict-majority order statistic of the leader's +own sequence and each peer's acknowledged sequence, including zero for peers +that have not replied. This works for both odd and even cluster sizes. A +five-node read therefore needs the leader plus two distinct fresh peers. + +Confirmation is usable only while the node is leader and an entry from its +current term is committed. The local state machine must also have applied the +read's target log prefix before executing the read. The target remains the +last local log index at read admission, a conservative choice that can wait +behind an uncommitted tail rather than optimizing read latency. + +Each pending strong read records its leadership term. Losing leadership or +changing term rejects it with ErrorCode=1 and its original cookie. This is a +conservative cancellation policy; operations are not automatically replayed. +Term checking also works if the application does not poll between stepping +down and being elected again. A later read cannot inherit an earlier read's +quorum. If EConsistent and EStale are both set, EConsistent takes precedence. + +The argument assumes a fixed configured voter set, correct non-Byzantine peers, +and serialized access to Raft/application state. It uses RPC ordering rather +than clock leases. It does not establish complete end-to-end linearizability +for the library or application. + +## Validation and self-review + +Eight new deterministic tests cover: + +* Fresh-majority thresholds for cluster sizes one through eight, with duplicate + responses (including single-node progress). +* The five-node stale-leader partition and a new majority's committed write. +* A second read requiring fresh confirmation after a previous read completes. +* Leadership loss after confirmation but before application service. +* Conflicting consistent/stale flags. +* Delayed pre-read replies, followed by partial and complete fresh quorums. +* A new leader waiting for its current-term commit and local application. +* An old-term read rejected after stepping down and being re-elected, before + accepting a new read. + +The first five tests all fail against alpha.3. The existing single-node FIFO +test now elects a real leader and commits its initial no-op, satisfying the +strong-read protocol's preconditions instead of merely forcing the role flag. +All 69 tests pass locally under ASan/UBSan, with core Debug/Release verification. +Hosted CI runs the same targets; its results are separate from local checks. + +Self-review checked distinct-voter counting, strict-majority arithmetic, +monotonic nonzero barriers, term isolation, duplicate/delayed replies, current +term commitment and application progress. No independent approval is claimed. + +## Compatibility and remaining work + +The wire layout is unchanged. TRaft::ApproveRead and TRaft::CommitSeqno remain; +CommitSeqno now reports a fresh-majority read watermark rather than the largest +sequence attached to a log acknowledgement. Internal TVolatileState removes +CommitSeqno, adds acknowledgement maps, and CommitAdvance no longer accepts a +sequence argument. This alpha does not promise internal-structure or mixed +version cluster compatibility. + +Pending write/default-read handling across leadership changes, disconnect +cleanup, retry/deduplication, admission limits, transport validation and broader +generated history/linearizability testing remain separate gaps. Quorum loss can +leave a strong read waiting indefinitely; deadlines and cancellation APIs are +not introduced here. diff --git a/miniraft/raft.cpp b/miniraft/raft.cpp index a39b24e..0111cf8 100644 --- a/miniraft/raft.cpp +++ b/miniraft/raft.cpp @@ -68,7 +68,7 @@ TVolatileState& TVolatileState::Vote(uint32_t nodeId) return *this; } -TVolatileState& TVolatileState::CommitAdvance(int nservers, const IState& state, uint64_t seqno) +TVolatileState& TVolatileState::CommitAdvance(int nservers, const IState& state) { auto lastIndex = state.LastLogIndex; Indices.clear(); Indices.reserve(nservers); @@ -84,7 +84,6 @@ TVolatileState& TVolatileState::CommitAdvance(int nservers, const IState& state, // For even cluster sizes, the upper median would accept only half. auto commitIndex = std::max(CommitIndex, Indices[(nservers - 1) / 2]); if (state.LogTerm(commitIndex) == state.CurrentTerm) { - CommitSeqno = std::max(CommitSeqno, seqno); CommitIndex = commitIndex; } return *this; @@ -253,7 +252,12 @@ void TRaft::OnAppendEntries(TMessageHolder message) { } auto nodeId = message->Src; + if (!Nodes.contains(nodeId)) return; if (message->Success) { + if (message->Seqno > 0 && message->Seqno <= VolatileState->LastSentSeqno[nodeId]) { + auto& acknowledged = VolatileState->ReadAckSeqno[nodeId]; + acknowledged = std::max(acknowledged, message->Seqno); + } auto matchIndex = std::max(VolatileState->MatchIndex[nodeId], message->MatchIndex); (*VolatileState) .SetMatchIndex(nodeId, matchIndex) @@ -261,7 +265,7 @@ void TRaft::OnAppendEntries(TMessageHolder message) { .SetRpcDue(nodeId, ITimeSource::Time{}) .SetBatchSize(nodeId, 1024) .SetBackOff(nodeId, 1) - .CommitAdvance(Nservers, *State, message->Seqno); + .CommitAdvance(Nservers, *State); } else { auto backOff = std::max(VolatileState->BackOff[nodeId], 1); auto nextIndex = VolatileState->NextIndex[nodeId] > backOff @@ -295,7 +299,7 @@ TMessageHolder TRaft::CreateAppendEntries(uint32_t nodeId } auto mes = NewHoldedMessage( - TMessageEx {.Src = Id, .Dst = nodeId, .Term = State->CurrentTerm, .Seqno = Seqno++}, + TMessageEx {.Src = Id, .Dst = nodeId, .Term = State->CurrentTerm, .Seqno = NextSeqno()}, TAppendEntriesRequest { .PrevLogIndex = prevIndex, .PrevLogTerm = State->LogTerm(prevIndex), @@ -311,6 +315,7 @@ TMessageHolder TRaft::CreateAppendEntries(uint32_t nodeId mes.Payload[j++] = State->Get(i); } } + VolatileState->LastSentSeqno[nodeId] = mes->Seqno; return mes; } @@ -410,7 +415,10 @@ void TRaft::LeaderTimeout(ITimeSource::Time now) { } uint64_t TRaft::ApproveRead() { - int seqno = Seqno; + if (StateName != EState::LEADER) throw std::logic_error("Only a leader can approve a read"); + // Reserve a unique barrier even for a single-node cluster. Every request + // emitted below is newer than this barrier; old replies cannot satisfy it. + const uint64_t seqno = NextSeqno(); for (auto& [id, node] : Nodes) { node->Send(CreateAppendEntries(id)); } @@ -418,7 +426,21 @@ uint64_t TRaft::ApproveRead() { } uint64_t TRaft::CommitSeqno() const { - return VolatileState->CommitSeqno; + if (StateName != EState::LEADER || VolatileState->CommitIndex == 0 || + State->LogTerm(VolatileState->CommitIndex) != State->CurrentTerm) return 0; + std::vector acknowledgements{Seqno - 1}; // leader's own confirmation + acknowledgements.reserve(Nservers); + for (const auto& [id, _] : Nodes) { + auto it = VolatileState->ReadAckSeqno.find(id); + acknowledgements.push_back(it == VolatileState->ReadAckSeqno.end() ? 0 : it->second); + } + std::sort(acknowledgements.begin(), acknowledgements.end()); + return acknowledgements[(Nservers - 1) / 2]; +} + +uint64_t TRaft::NextSeqno() { + if (Seqno == UINT64_MAX) throw std::overflow_error("Raft RPC sequence exhausted"); + return Seqno++; } void TRaft::ProcessTimeout(ITimeSource::Time now) { @@ -561,7 +583,8 @@ void TRequestProcessor::OnReadRequest(TMessageHolder command, c assert(!(flags & TCommandRequest::EWrite)); // stale read, default read (from leader) - if ((flags & TCommandRequest::EStale) || (!(flags & TCommandRequest::EConsistent) && stateName == EState::LEADER)) { + if (!(flags & TCommandRequest::EConsistent) && + ((flags & TCommandRequest::EStale) || stateName == EState::LEADER)) { assert(Waiting.empty() || Waiting.back().Index <= Raft->GetLastIndex()); Waiting.emplace(TWaiting{Raft->GetLastIndex(), 0, std::move(command), replyTo}); return; @@ -572,9 +595,23 @@ void TRequestProcessor::OnReadRequest(TMessageHolder command, c } // Consistent read + RejectObsoleteStrongReads(); + const auto term = Raft->GetState()->CurrentTerm; + const auto index = Raft->GetLastIndex(); auto seqno = Raft->ApproveRead(); - assert(StrongWaiting.empty() || (StrongWaiting.back().Index <= Raft->GetLastIndex() && StrongWaiting.back().Seqno <= seqno)); - StrongWaiting.emplace(TWaiting{Raft->GetLastIndex(), seqno, std::move(command), replyTo}); + assert(StrongWaiting.empty() || (StrongWaiting.back().Index <= index && StrongWaiting.back().Seqno <= seqno)); + StrongWaiting.emplace(TWaiting{index, seqno, std::move(command), replyTo, term}); +} + +void TRequestProcessor::RejectObsoleteStrongReads() { + while (!StrongWaiting.empty() && + (Raft->CurrentStateName() != EState::LEADER || + StrongWaiting.front().Term != Raft->GetState()->CurrentTerm)) { + auto waiting = std::move(StrongWaiting.front()); + StrongWaiting.pop(); + if (waiting.ReplyTo) waiting.ReplyTo->Send(NewHoldedMessage( + TCommandResponse{.Cookie = waiting.Command->Cookie, .ErrorCode = 1})); + } } void TRequestProcessor::OnWriteRequest(TMessageHolder command, const std::shared_ptr& replyTo) { @@ -668,8 +705,10 @@ void TRequestProcessor::ProcessWaiting() { w.ReplyTo->Send(std::move(reply)); } - auto seqno = Raft->CommitSeqno(); - while (!StrongWaiting.empty() && StrongWaiting.front().Index <= lastApplied && StrongWaiting.front().Seqno <= seqno) { + while (!StrongWaiting.empty()) { + RejectObsoleteStrongReads(); + if (StrongWaiting.empty() || StrongWaiting.front().Index > lastApplied || + StrongWaiting.front().Seqno > Raft->CommitSeqno()) break; auto w = StrongWaiting.front(); StrongWaiting.pop(); TMessageHolder reply; assert (!(w.Command->Flags & TCommandRequest::EWrite)); diff --git a/miniraft/raft.h b/miniraft/raft.h index c6bb923..249731c 100644 --- a/miniraft/raft.h +++ b/miniraft/raft.h @@ -41,7 +41,9 @@ using TNodeDict = std::unordered_map>; struct TVolatileState { uint64_t CommitIndex = 0; - uint64_t CommitSeqno = 0; + // Successful current-term RPC acknowledgements, separately from log indices. + std::unordered_map ReadAckSeqno; + std::unordered_map LastSentSeqno; uint32_t LeaderId = 0; std::unordered_map NextIndex; std::unordered_map MatchIndex; @@ -55,7 +57,7 @@ struct TVolatileState { std::vector Indices; TVolatileState& Vote(uint32_t id); - TVolatileState& CommitAdvance(int nservers, const IState& state, uint64_t seqno = 0); + TVolatileState& CommitAdvance(int nservers, const IState& state); TVolatileState& SetCommitIndex(int index); TVolatileState& SetElectionDue(ITimeSource::Time); TVolatileState& SetNextIndex(uint32_t id, uint64_t nextIndex); @@ -91,6 +93,8 @@ class TRaft { uint32_t GetLeaderId() const; uint64_t GetLastIndex() const; uint64_t ApproveRead(); + // Highest barrier confirmed by a fresh majority in this leadership term. + // Zero means no confirmation (including before a current-term commit). uint64_t CommitSeqno() const; // ut @@ -141,6 +145,7 @@ class TRaft { TMessageHolder CreateVote(uint32_t nodeId); TMessageHolder CreateAppendEntries(uint32_t nodeId); ITimeSource::Time MakeElection(ITimeSource::Time now); + uint64_t NextSeqno(); uint32_t Id; TNodeDict Nodes; @@ -149,7 +154,7 @@ class TRaft { int Nservers; std::shared_ptr State; std::unique_ptr VolatileState; - uint64_t Seqno = 0; // for matching responses + uint64_t Seqno = 1; // zero is reserved for an unconfirmed read EState StateName; uint32_t Seed = 31337; @@ -174,6 +179,7 @@ class TRequestProcessor { void Forward(TMessageHolder message, const std::shared_ptr& replyTo); void OnReadRequest(TMessageHolder message, const std::shared_ptr& replyTo); void OnWriteRequest(TMessageHolder message, const std::shared_ptr& replyTo); + void RejectObsoleteStrongReads(); std::shared_ptr Raft; std::shared_ptr Rsm; @@ -184,6 +190,7 @@ class TRequestProcessor { uint64_t Seqno = 0; TMessageHolder Command; std::shared_ptr ReplyTo; + uint64_t Term = 0; }; std::queue Waiting; std::queue StrongWaiting; diff --git a/test/test_correctness.cpp b/test/test_correctness.cpp index 92c1789..0aa3ff6 100644 --- a/test/test_correctness.cpp +++ b/test/test_correctness.cpp @@ -72,8 +72,206 @@ struct TCluster { } } } + TMessageHolder DeliverOne(uint32_t src, uint32_t dst) { + for (auto it = Packets.begin(); it != Packets.end(); ++it) { + auto ex = it->Cast(); + if (ex->Src == src && ex->Dst == dst) { + auto message = *it; + Packets.erase(it); + Nodes.at(dst - 1)->Process(Now, message); + return message; + } + } + fail_msg("Expected a queued packet from %u to %u", src, dst); + return {}; + } + void RoundTrip(uint32_t leader, uint32_t peer, bool duplicate = false) { + DeliverOne(leader, peer); + auto response = DeliverOne(peer, leader); + if (duplicate) Nodes.at(leader - 1)->Process(Now, response); + } }; +struct TReadClient { + struct TReply { uint32_t Cookie; uint32_t Error; }; + std::vector Replies; + std::shared_ptr Rsm = std::make_shared(); + TRequestProcessor Processor; + std::shared_ptr Endpoint; + explicit TReadClient(std::shared_ptr raft) + : Processor(raft, Rsm, {}) + , Endpoint(std::make_shared([this](auto message) { + auto response = message.template Cast(); + Replies.push_back({response->Cookie, response->ErrorCode}); + })) {} + void Read(uint32_t cookie, uint32_t flags = TCommandRequest::EConsistent) { + auto command = NewHoldedMessage(sizeof(TCommandRequest) + sizeof(int64_t)); + int64_t index = 0; + memcpy(command->Data, &index, sizeof(index)); + command->Cookie = cookie; + command->Flags = flags; + Processor.OnCommandRequest(command, Endpoint); + } + void Poll() { Processor.ProcessCommitted(); Processor.ProcessWaiting(); } +}; + +void test_strong_reads_require_fresh_majority_in_all_cluster_sizes(void**) { + for (uint32_t count = 1; count <= 8; ++count) { + TCluster cluster(count); + cluster.Elect(1); + cluster.Replicate(1); + TReadClient client(cluster.Nodes[0]); + client.Read(11); + client.Poll(); + assert_int_equal(client.Replies.size(), count == 1 ? 1 : 0); + for (uint32_t peer = 2; peer <= count; ++peer) { + cluster.RoundTrip(1, peer, true); + client.Poll(); + assert_int_equal(client.Replies.size(), peer >= count / 2 + 1 ? 1 : 0); + } + assert_int_equal(client.Replies[0].Cookie, 11); + assert_int_equal(client.Replies[0].Error, 0); + } +} + +void test_stale_leader_with_one_follower_cannot_serve_strong_read(void**) { + TCluster cluster(5); + cluster.Elect(1); + cluster.Replicate(1); + // The majority elects a new leader and commits a later entry while the + // old leader remains connected to one old-term follower. + cluster.Isolated = {1, 2}; + cluster.Elect(3); + auto latest = cluster.Nodes[2]->Append(NewHoldedMessage()); + cluster.Replicate(3); + assert_true(cluster.Nodes[2]->GetVolatileState()->CommitIndex >= latest); + cluster.Isolated.clear(); + TReadClient client(cluster.Nodes[0]); + client.Read(11); + cluster.RoundTrip(1, 2, true); + client.Poll(); + assert_true(client.Replies.empty()); + // A current-term leader's follower reveals the higher term. Pending reads + // must fail, rather than serving the old leader's state. + cluster.RoundTrip(1, 3); + client.Poll(); + assert_int_equal(client.Replies.size(), 1); + assert_int_equal(client.Replies[0].Error, 1); +} + +void test_new_read_cannot_reuse_previous_read_quorum(void**) { + TCluster cluster(5); + cluster.Elect(1); + cluster.Replicate(1); + TReadClient client(cluster.Nodes[0]); + client.Read(11); + cluster.RoundTrip(1, 2); + cluster.RoundTrip(1, 3); + client.Poll(); + assert_int_equal(client.Replies.size(), 1); + client.Read(22); + cluster.RoundTrip(1, 2, true); + client.Poll(); + assert_int_equal(client.Replies.size(), 1); + cluster.RoundTrip(1, 3); + client.Poll(); + assert_int_equal(client.Replies.size(), 2); + assert_int_equal(client.Replies[1].Cookie, 22); +} + +void test_ready_read_is_cancelled_on_leadership_change(void**) { + TCluster cluster(3); + cluster.Elect(1); + cluster.Replicate(1); + TReadClient client(cluster.Nodes[0]); + client.Read(11); + cluster.RoundTrip(1, 2); + cluster.Elect(2); // Lose leadership before the application processes reads. + client.Poll(); + assert_int_equal(client.Replies.size(), 1); + assert_int_equal(client.Replies[0].Error, 1); +} + +void test_consistent_flag_takes_precedence_over_stale(void**) { + TCluster cluster(3); + cluster.Elect(1); + cluster.Replicate(1); + TReadClient client(cluster.Nodes[0]); + client.Read(11, TCommandRequest::EConsistent | TCommandRequest::EStale); + client.Poll(); + assert_true(client.Replies.empty()); + cluster.RoundTrip(1, 2); + client.Poll(); + assert_int_equal(client.Replies.size(), 1); + assert_int_equal(client.Replies[0].Error, 0); +} + +void test_delayed_pre_read_replies_cannot_confirm_new_read(void**) { + TCluster cluster(5); + cluster.Elect(1); + cluster.Replicate(1); + cluster.Nodes[0]->ProcessTimeout(cluster.Now += TTimeout::Election); + for (uint32_t peer = 2; peer <= 5; ++peer) cluster.DeliverOne(1, peer); + TReadClient client(cluster.Nodes[0]); + client.Read(11); + for (uint32_t peer = 2; peer <= 5; ++peer) cluster.DeliverOne(peer, 1); + client.Poll(); + assert_true(client.Replies.empty()); + cluster.RoundTrip(1, 2, true); + client.Poll(); + assert_true(client.Replies.empty()); + cluster.RoundTrip(1, 3); + client.Poll(); + assert_int_equal(client.Replies.size(), 1); +} + +void test_new_leader_waits_for_current_term_commit_and_application(void**) { + TCluster cluster(3); + for (auto& node : cluster.Nodes) { + node->GetState()->Append(NewHoldedMessage()); + const_cast(node->GetVolatileState())->CommitIndex = 1; + } + cluster.Elect(1); + // First RPCs probe the old prefix; the new-term no-op is not committed yet. + assert_int_equal(cluster.Nodes[0]->GetVolatileState()->CommitIndex, 1); + assert_int_equal(cluster.Nodes[0]->CommitSeqno(), 0); + TReadClient client(cluster.Nodes[0]); + client.Read(11); + client.Poll(); + assert_true(client.Replies.empty()); + cluster.RoundTrip(1, 2); + assert_int_equal(cluster.Nodes[0]->GetVolatileState()->CommitIndex, 2); + // Fresh confirmation alone is insufficient until the local state machine + // has applied the read's target prefix. + client.Processor.ProcessWaiting(); + assert_true(client.Replies.empty()); + client.Poll(); + assert_int_equal(client.Replies.size(), 1); + assert_int_equal(client.Replies[0].Error, 0); +} + +void test_old_term_read_is_rejected_after_reelection(void**) { + TCluster cluster(3); + cluster.Elect(1); + cluster.Replicate(1); + TReadClient client(cluster.Nodes[0]); + client.Read(11); + cluster.Elect(2); + cluster.Elect(1); + cluster.Replicate(1); + // No application poll observed the intermediate follower state. The term + // tag must still reject the old operation before accepting another read. + client.Read(22); + assert_int_equal(client.Replies.size(), 1); + assert_int_equal(client.Replies[0].Cookie, 11); + assert_int_equal(client.Replies[0].Error, 1); + cluster.RoundTrip(1, 2); + client.Poll(); + assert_int_equal(client.Replies.size(), 2); + assert_int_equal(client.Replies[1].Cookie, 22); + assert_int_equal(client.Replies[1].Error, 0); +} + void test_majority_all_cluster_sizes(void**) { for (int count = 1; count <= 8; ++count) { TState state; @@ -90,7 +288,8 @@ void test_majority_all_cluster_sizes(void**) { void CheckRequests(uint32_t flags, bool partial) { auto state = std::make_shared(); auto raft = std::make_shared(state, 1, TNodeDict{}); - raft->Become(EState::LEADER); + if (flags & TCommandRequest::EConsistent) raft->ProcessTimeout(ITimeSource::Time{}); + else raft->Become(EState::LEADER); auto rsm = std::make_shared(); TRequestProcessor processor(raft, rsm, {}); std::vector replies; @@ -198,6 +397,14 @@ int main() { cmocka_unit_test(test_loss_reorder_duplicate_and_recovery), cmocka_unit_test(test_partition_leader_replacement_and_heal), cmocka_unit_test(test_even_cluster_partition_requires_majority), + cmocka_unit_test(test_strong_reads_require_fresh_majority_in_all_cluster_sizes), + cmocka_unit_test(test_stale_leader_with_one_follower_cannot_serve_strong_read), + cmocka_unit_test(test_new_read_cannot_reuse_previous_read_quorum), + cmocka_unit_test(test_ready_read_is_cancelled_on_leadership_change), + cmocka_unit_test(test_consistent_flag_takes_precedence_over_stale), + cmocka_unit_test(test_delayed_pre_read_replies_cannot_confirm_new_read), + cmocka_unit_test(test_new_leader_waits_for_current_term_commit_and_application), + cmocka_unit_test(test_old_term_read_is_rejected_after_reelection), }; return cmocka_run_group_tests(tests, nullptr, nullptr); } From e117c18c6c94c11dc186ba4573642589cc876d17 Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Sun, 8 Nov 2026 08:42:20 -0500 Subject: [PATCH 05/66] Bind pending writes to leadership terms and clean up request ownership (#5) --- README.md | 2 + docs/pending-requests.md | 87 ++++++++++++ examples/client.cpp | 9 +- examples/kv.cpp | 9 +- examples/sql.cpp | 9 +- miniraft/net/server.cpp | 3 + miniraft/raft.cpp | 105 ++++++++++----- miniraft/raft.h | 12 +- test/test_correctness.cpp | 277 ++++++++++++++++++++++++++++++++++++++ 9 files changed, 471 insertions(+), 42 deletions(-) create mode 100644 docs/pending-requests.md diff --git a/README.md b/README.md index 395bd8d..380a20e 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,8 @@ adds checksummed atomic publication with file and directory synchronization. The KV example selects it with `--durable`; legacy `--persist` is unchanged. The [strong-read follow-up](docs/strong-reads.md) requires a fresh majority for `--consistent` reads and rejects pending reads when their leadership term ends. +The [pending request follow-up](docs/pending-requests.md) prevents replacement +write acknowledgements and cleans up obsolete or disconnected client requests. [![Number of GitHub stars](https://img.shields.io/github/stars/resetius/miniraft-cpp)](https://github.com/resetius/miniraft-cpp/stargazers) ![GitHub commit activity](https://img.shields.io/github/commit-activity/m/resetius/miniraft-cpp) diff --git a/docs/pending-requests.md b/docs/pending-requests.md new file mode 100644 index 0000000..dd77914 --- /dev/null +++ b/docs/pending-requests.md @@ -0,0 +1,87 @@ +# Pending request lifecycle: v0.1.0-alpha.5 + +This milestone prevents pending writes from receiving a replacement command's +reply after leadership changes. It also releases disconnected clients and +unused application replies, and makes pre-election requests progress when the +local node becomes leader. + +## Reproduced failure + +A leader accepted an uncommitted command at index 3. After a partition, another +leader replaced that index with a different command and committed it. When the +old leader rejoined, its request processor returned the replacement command's +successful reply to the original client. Requests and application replies were +matched only by index, with no leadership term or lifecycle validation. A +replacement no-op could instead leave no application reply and hit an assertion. + +## Completion and cancellation contract + +Local writes, default leader reads and strong reads now belong to the leadership +term in which they were admitted. Losing leadership or changing term rejects +pending operations with ErrorCode=1 and the original cookie. Re-election cannot +redeem an earlier term's request. Pruning runs before admission, application, +completion and leader-wait queue processing; unchanged terms/roles avoid rescans. + +Write application replies are retained only while a matching local request is +pending. Both index and term must match before a success is returned. Replies +for follower/background application are released immediately. Completion, +leadership cancellation and disconnect cleanup remove cached replies and their +pending-write records. The retained reply cache is bounded by pending writes, +although admission limits for the request queues themselves remain open. + +Cancellation concerns the response, not the replicated command. Committed +commands still apply even if their client disconnected or their leader changed. +An error can therefore mean UNKNOWN OUTCOME: the write may already have +committed. It is not a rollback indication and must not trigger a blind retry of +a non-idempotent operation. Request IDs and replicated deduplication are not +implemented. The KV, SQL and debug clients now display errors rather than +printing a successful commit for nonzero ErrorCode responses. + +CleanUp removes a disconnected endpoint from ordinary, strong-read and +leader-wait queues, plus forwarded-request routing. Disconnect cleanup sends no +reply and does not retract entries already appended to the log. + +Explicit stale reads are independent of leadership terms and target the known +committed prefix, rather than an uncommitted tail. Default reads remain weaker +than strong reads but are cancelled if their admitting leadership ends. FIFO +queueing can still delay a stale read behind an earlier pending request. + +## Leader discovery and timer progress + +A newly elected leader records its own ID. Higher-term stepdown clears stale +leader routing until AppendEntries identifies the new leader. Requests waiting +for leader discovery can execute locally after election, including a single-node +election. Invalid or self-directed follower routes remain queued rather than +asserting or dereferencing a missing peer. + +The server's idle timer now runs request-state checks, committed application and +completion processing after Raft timeouts, using the same ordering as inbound +events. Progress therefore does not require an unrelated incoming message. + +## Validation and self-review + +Twelve new deterministic tests cover replacement by another command or no-op, +stepdown before application, already-committed but unanswered writes, re-election +without intermediate application polls, default reads of lost tails, requests +queued before single/three-node elections, disconnects from all local queues, +cached reply release, unused follower replies and stale reads of a committed +prefix. Weak references verify ownership is actually released. The initial ten +tests all fail against alpha.4. + +All 81 tests pass locally under ASan/UBSan, with core Debug/Release verification. +Hosted CI executes the same targets; its results are separate from local checks. +The wire format and public request-processor methods are unchanged. + +Self-review checked index-and-term matching, reply-cache bounds, cancellation +before log application, disconnect ownership, leader routing and timer/inbound +ordering. Access must remain serialized and callbacks must not re-enter the +processor. No independent reviewer approval is claimed. + +## Remaining gaps + +Exactly-once retries, durable request IDs, forwarded-request lifecycle across +leader failures, queue admission limits, deadlines, transport validation and +broader generated history/linearizability testing remain separate work. This +change does not promise successful completion during quorum loss or establish +library-wide linearizability. Unanswered writes after process crashes also have +unknown outcomes. diff --git a/examples/client.cpp b/examples/client.cpp index b666086..9620c54 100644 --- a/examples/client.cpp +++ b/examples/client.cpp @@ -24,8 +24,13 @@ TVoidSuspendedTask ClientReader(TPoller& poller, TSocket& socket) { auto commandResponse = response.template Cast(); auto len = commandResponse->Len - sizeof(TCommandResponse); std::string_view data(commandResponse->Data, len); - std::cout << "Ok, commitIndex: " << commandResponse->Index << " " - << data << " " << dt.count() << "\n"; + if (commandResponse->ErrorCode) { + std::cout << "Request failed (error " << commandResponse->ErrorCode + << "). A write may still have committed.\n"; + } else { + std::cout << "Ok, commitIndex: " << commandResponse->Index << " " + << data << " " << dt.count() << "\n"; + } inflight --; } } catch (const std::exception& ex) { diff --git a/examples/kv.cpp b/examples/kv.cpp index cce07e3..3ecabc5 100644 --- a/examples/kv.cpp +++ b/examples/kv.cpp @@ -140,9 +140,12 @@ NNet::TFuture Client(TPoller& poller, TSocket socket, uint32_t flags) { auto res = reply.template Cast(); auto len = res->Len - sizeof(TCommandResponse); std::string_view data(res->Data, len); - std::cout << "Ok, commitIndex: " << res->Index << "\n"; - if (!data.empty()) { - std::cout << data << "\n"; + if (res->ErrorCode) { + std::cout << "Request failed (error " << res->ErrorCode + << "). A write may still have committed.\n"; + } else { + std::cout << "Ok, commitIndex: " << res->Index << "\n"; + if (!data.empty()) std::cout << data << "\n"; } std::cout << "> " << std::flush; } diff --git a/examples/sql.cpp b/examples/sql.cpp index 356089f..8354293 100644 --- a/examples/sql.cpp +++ b/examples/sql.cpp @@ -236,9 +236,12 @@ NNet::TFuture Client(TPoller& poller, TSocket socket) { auto res = reply.template Cast(); auto len = res->Len - sizeof(TCommandResponse); std::string_view data(res->Data, len); - std::cout << "commitIndex: " << res->Index << "\n"; - if (!data.empty()) { - std::cout << data << "\n"; + if (res->ErrorCode) { + std::cout << "Request failed (error " << res->ErrorCode + << "). A write may still have committed.\n"; + } else { + std::cout << "commitIndex: " << res->Index << "\n"; + if (!data.empty()) std::cout << data << "\n"; } } } catch (const std::exception& ex) { diff --git a/miniraft/net/server.cpp b/miniraft/net/server.cpp index 6064e8b..99d17fc 100644 --- a/miniraft/net/server.cpp +++ b/miniraft/net/server.cpp @@ -252,6 +252,9 @@ NNet::TVoidTask TRaftServer::Idle() { auto sleep = std::chrono::milliseconds(100); while (true) { Raft->ProcessTimeout(TimeSource->Now()); + RequestProcessor->CheckStateChange(); + RequestProcessor->ProcessCommitted(); + RequestProcessor->ProcessWaiting(); DrainNodes(); auto t1 = TimeSource->Now(); if (t1 > t0 + dt) { diff --git a/miniraft/raft.cpp b/miniraft/raft.cpp index 0111cf8..b139bee 100644 --- a/miniraft/raft.cpp +++ b/miniraft/raft.cpp @@ -361,6 +361,7 @@ void TRaft::Process(ITimeSource::Time now, TMessageHolder message, con State->VotedFor = 0; State->Commit(); StateName = EState::FOLLOWER; + VolatileState->LeaderId = 0; if (VolatileState->ElectionDue <= now || VolatileState->ElectionDue == ITimeSource::Max) { VolatileState->ElectionDue = MakeElection(now); } @@ -473,6 +474,7 @@ void TRaft::ProcessTimeout(ITimeSource::Time now) { auto nextVolatileState = std::make_unique(TVolatileState { .CommitIndex = VolatileState->CommitIndex, + .LeaderId = Id, .NextIndex = nextIndex, .RpcDue = rpcDue, .ElectionDue = ITimeSource::Max, @@ -524,6 +526,7 @@ uint64_t TRaft::GetLastIndex() const { } void TRequestProcessor::CheckStateChange() { + PruneRequests(); if (WaitingStateChange.empty()) { return; } @@ -531,7 +534,8 @@ void TRequestProcessor::CheckStateChange() { auto stateName = Raft->CurrentStateName(); auto leaderId = Raft->GetLeaderId(); - if (stateName == EState::CANDIDATE || leaderId == 0) { + if (stateName == EState::CANDIDATE || (stateName != EState::LEADER && + (leaderId == 0 || leaderId == Raft->GetId() || !Nodes.contains(leaderId)))) { return; } @@ -557,13 +561,13 @@ void TRequestProcessor::Forward(TMessageHolder command, const s auto stateName = Raft->CurrentStateName(); auto leaderId = Raft->GetLeaderId(); - if (stateName == EState::CANDIDATE || leaderId == 0) { + if (stateName == EState::CANDIDATE || leaderId == 0 || + leaderId == Raft->GetId() || !Nodes.contains(leaderId)) { WaitingStateChange.emplace(TWaiting{0, 0, std::move(command), replyTo}); return; } if (stateName == EState::FOLLOWER) { - assert(leaderId != Raft->GetId()); // Forward command->Cookie = std::max(1, ForwardCookie); Nodes[leaderId]->Send(std::move(command)); @@ -585,8 +589,10 @@ void TRequestProcessor::OnReadRequest(TMessageHolder command, c // stale read, default read (from leader) if (!(flags & TCommandRequest::EConsistent) && ((flags & TCommandRequest::EStale) || stateName == EState::LEADER)) { - assert(Waiting.empty() || Waiting.back().Index <= Raft->GetLastIndex()); - Waiting.emplace(TWaiting{Raft->GetLastIndex(), 0, std::move(command), replyTo}); + const bool stale = flags & TCommandRequest::EStale; + const auto index = stale ? Raft->GetVolatileState()->CommitIndex : Raft->GetLastIndex(); + const auto term = stale ? 0 : Raft->GetState()->CurrentTerm; + Waiting.emplace(TWaiting{index, 0, std::move(command), replyTo, term}); return; } @@ -595,7 +601,6 @@ void TRequestProcessor::OnReadRequest(TMessageHolder command, c } // Consistent read - RejectObsoleteStrongReads(); const auto term = Raft->GetState()->CurrentTerm; const auto index = Raft->GetLastIndex(); auto seqno = Raft->ApproveRead(); @@ -603,14 +608,33 @@ void TRequestProcessor::OnReadRequest(TMessageHolder command, c StrongWaiting.emplace(TWaiting{index, seqno, std::move(command), replyTo, term}); } -void TRequestProcessor::RejectObsoleteStrongReads() { - while (!StrongWaiting.empty() && - (Raft->CurrentStateName() != EState::LEADER || - StrongWaiting.front().Term != Raft->GetState()->CurrentTerm)) { - auto waiting = std::move(StrongWaiting.front()); - StrongWaiting.pop(); - if (waiting.ReplyTo) waiting.ReplyTo->Send(NewHoldedMessage( - TCommandResponse{.Cookie = waiting.Command->Cookie, .ErrorCode = 1})); +void TRequestProcessor::PruneRequests(const std::shared_ptr& disconnected) { + const auto term = Raft->GetState()->CurrentTerm; + const bool leader = Raft->CurrentStateName() == EState::LEADER; + if (!disconnected) { + if (RequestsTerm == term && RequestsLeader == leader) return; + RequestsTerm = term; + RequestsLeader = leader; + } + for (auto* queue : {&Waiting, &StrongWaiting, &WaitingStateChange}) { + std::queue retained; + while (!queue->empty()) { + auto waiting = std::move(queue->front()); + queue->pop(); + const bool remove = disconnected ? waiting.ReplyTo == disconnected : + waiting.Term != 0 && (!leader || waiting.Term != term); + if (!remove) { + retained.push(std::move(waiting)); + continue; + } + if (waiting.Command->Flags & TCommandRequest::EWrite) { + PendingWriteTerms.erase(waiting.Index); + WriteAnswers.erase(waiting.Index); + } + if (!disconnected && waiting.ReplyTo) waiting.ReplyTo->Send(NewHoldedMessage( + TCommandResponse{.Cookie = waiting.Command->Cookie, .ErrorCode = 1})); + } + queue->swap(retained); } } @@ -620,11 +644,11 @@ void TRequestProcessor::OnWriteRequest(TMessageHolder command, assert((flags & TCommandRequest::EWrite)); if (stateName == EState::LEADER) { + const auto term = Raft->GetState()->CurrentTerm; uint64_t index = Raft->Append(std::move(Rsm->Prepare(command))); if (replyTo) { - assert(Waiting.empty() || Waiting.back().Index <= index); - // TODO: cleanup these queues on state change - Waiting.emplace(TWaiting{index, 0, std::move(command), replyTo}); + PendingWriteTerms[index] = term; + Waiting.emplace(TWaiting{index, 0, std::move(command), replyTo, term}); } } else { Forward(std::move(command), replyTo); @@ -632,6 +656,7 @@ void TRequestProcessor::OnWriteRequest(TMessageHolder command, } void TRequestProcessor::OnCommandRequest(TMessageHolder command, const std::shared_ptr& replyTo) { + PruneRequests(); auto flags = command->Flags; if (!(flags & TCommandRequest::EWrite)) { @@ -657,6 +682,8 @@ void TRequestProcessor::OnCommandResponse(TMessageHolder comma } void TRequestProcessor::CleanUp(const std::shared_ptr& replyTo) { + if (!replyTo) return; + PruneRequests(replyTo); auto jt = Client2Cookie.find(replyTo); if (jt == Client2Cookie.end()) { return; @@ -668,6 +695,7 @@ void TRequestProcessor::CleanUp(const std::shared_ptr& replyTo) { } void TRequestProcessor::ProcessCommitted() { + PruneRequests(); auto commitIndex = Raft->GetVolatileState()->CommitIndex; auto& state = Raft->GetState(); for (auto i = Rsm->LastAppliedIndex+1; i <= commitIndex; i++) { @@ -676,28 +704,39 @@ void TRequestProcessor::ProcessCommitted() { continue; } auto reply = Rsm->Write(entry, i); - WriteAnswers.emplace(TAnswer { - .Index = i, - .Reply = reply ? reply : NewHoldedMessage(TCommandResponse {.Index = i}) - }); + auto pending = PendingWriteTerms.find(i); + if (pending != PendingWriteTerms.end() && pending->second == entry->Term) { + WriteAnswers.emplace(i, TAnswer { + .Term = entry->Term, + .Reply = reply ? reply : NewHoldedMessage(TCommandResponse {.Index = i}) + }); + } } Rsm->LastAppliedIndex = commitIndex; } void TRequestProcessor::ProcessWaiting() { + PruneRequests(); auto lastApplied = Rsm->LastAppliedIndex; - while (!Waiting.empty() && Waiting.front().Index <= lastApplied) { + while (!Waiting.empty()) { + if (Waiting.front().Term != 0 && (Raft->CurrentStateName() != EState::LEADER || + Waiting.front().Term != Raft->GetState()->CurrentTerm)) { + PruneRequests(); + continue; + } + if (Waiting.front().Index > lastApplied) break; auto w = Waiting.front(); Waiting.pop(); TMessageHolder reply; - auto cookie = w.Command->Cookie;; + auto cookie = w.Command->Cookie; if (w.Command->Flags & TCommandRequest::EWrite) { - while (!WriteAnswers.empty() && WriteAnswers.front().Index < w.Index) { - WriteAnswers.pop(); + auto answer = WriteAnswers.find(w.Index); + if (answer != WriteAnswers.end() && answer->second.Term == w.Term) { + reply = answer->second.Reply.Cast(); + } else { + reply = NewHoldedMessage(TCommandResponse{.ErrorCode = 1}); } - assert(!WriteAnswers.empty()); - auto answer = std::move(WriteAnswers.front()); WriteAnswers.pop(); - assert(answer.Index == w.Index); - reply = std::move(answer.Reply.Cast()); + PendingWriteTerms.erase(w.Index); + WriteAnswers.erase(w.Index); } else { reply = Rsm->Read(std::move(w.Command), w.Index).Cast(); } @@ -706,8 +745,12 @@ void TRequestProcessor::ProcessWaiting() { } while (!StrongWaiting.empty()) { - RejectObsoleteStrongReads(); - if (StrongWaiting.empty() || StrongWaiting.front().Index > lastApplied || + if (Raft->CurrentStateName() != EState::LEADER || + StrongWaiting.front().Term != Raft->GetState()->CurrentTerm) { + PruneRequests(); + continue; + } + if (StrongWaiting.front().Index > lastApplied || StrongWaiting.front().Seqno > Raft->CommitSeqno()) break; auto w = StrongWaiting.front(); StrongWaiting.pop(); TMessageHolder reply; diff --git a/miniraft/raft.h b/miniraft/raft.h index 249731c..9b90845 100644 --- a/miniraft/raft.h +++ b/miniraft/raft.h @@ -179,7 +179,7 @@ class TRequestProcessor { void Forward(TMessageHolder message, const std::shared_ptr& replyTo); void OnReadRequest(TMessageHolder message, const std::shared_ptr& replyTo); void OnWriteRequest(TMessageHolder message, const std::shared_ptr& replyTo); - void RejectObsoleteStrongReads(); + void PruneRequests(const std::shared_ptr& disconnected = {}); std::shared_ptr Raft; std::shared_ptr Rsm; @@ -190,6 +190,7 @@ class TRequestProcessor { uint64_t Seqno = 0; TMessageHolder Command; std::shared_ptr ReplyTo; + // Zero is reserved for requests not owned by a leadership term. uint64_t Term = 0; }; std::queue Waiting; @@ -197,10 +198,15 @@ class TRequestProcessor { std::queue WaitingStateChange; struct TAnswer { - uint64_t Index; + uint64_t Term; TMessageHolder Reply; }; - std::queue WriteAnswers; + // Retain application replies only for live local writes, matched by index + // and term. An overwritten index must never redeem an older request. + std::unordered_map PendingWriteTerms; + std::unordered_map WriteAnswers; + uint64_t RequestsTerm = 0; + bool RequestsLeader = false; uint32_t ForwardCookie = 1; std::unordered_map> Cookie2Client; std::unordered_map, std::unordered_set> Client2Cookie; diff --git a/test/test_correctness.cpp b/test/test_correctness.cpp index 0aa3ff6..18b769d 100644 --- a/test/test_correctness.cpp +++ b/test/test_correctness.cpp @@ -115,6 +115,271 @@ struct TReadClient { void Poll() { Processor.ProcessCommitted(); Processor.ProcessWaiting(); } }; +struct TEchoRsm : IRsm { + std::vector Applied; + std::weak_ptr LastReply; + TMessageHolder Read(TMessageHolder, uint64_t index) override { + return NewHoldedMessage(TCommandResponse{.Index = index}); + } + TMessageHolder Prepare(TMessageHolder command) override { + const auto size = command->Len - sizeof(TCommandRequest); + auto entry = NewHoldedMessage(sizeof(TLogEntry) + size); + memcpy(entry->Data, command->Data, size); + return entry; + } + TMessageHolder Write(TMessageHolder entry, uint64_t index) override { + const auto size = entry->Len - sizeof(TLogEntry); + Applied.emplace_back(entry->Data, size); + LastAppliedIndex = index; + auto reply = NewHoldedMessage(sizeof(TCommandResponse) + size); + reply->Index = index; + memcpy(reply->Data, entry->Data, size); + LastReply = reply.RawData; + return reply; + } +}; +struct TWriteClient { + struct TReply { uint32_t Cookie, Error; std::string Data; }; + std::vector Replies; + std::shared_ptr Rsm = std::make_shared(); + TRequestProcessor Processor; + std::shared_ptr Endpoint; + explicit TWriteClient(std::shared_ptr raft) + : Processor(raft, Rsm, {}) + , Endpoint(std::make_shared([this](auto message) { + auto response = message.template Cast(); + Replies.push_back({response->Cookie, response->ErrorCode, + std::string(response->Data, response->Len - sizeof(TCommandResponse))}); + })) {} + void Write(uint32_t cookie, const std::string& value) { + auto command = NewHoldedMessage(sizeof(TCommandRequest) + value.size()); + command->Cookie = cookie; + command->Flags = TCommandRequest::EWrite; + memcpy(command->Data, value.data(), value.size()); + Processor.OnCommandRequest(command, Endpoint); + } + void Read(uint32_t cookie, uint32_t flags = 0) { + auto command = NewHoldedMessage(); + command->Cookie = cookie; + command->Flags = flags; + Processor.OnCommandRequest(command, Endpoint); + } + void Poll() { Processor.ProcessCommitted(); Processor.ProcessWaiting(); } +}; +TMessageHolder LogValue(const std::string& value) { + auto entry = NewHoldedMessage(sizeof(TLogEntry) + value.size()); + memcpy(entry->Data, value.data(), value.size()); + return entry; +} + +void test_replaced_write_never_receives_another_commands_reply(void**) { + TCluster cluster(3); + cluster.Elect(1); + cluster.Replicate(1); + cluster.Nodes[0]->Append(LogValue("uncommitted-padding")); + TWriteClient client(cluster.Nodes[0]); + client.Write(11, "lost-command"); // index 3, old term + cluster.Isolated = {1}; + cluster.Elect(2); // new-term no-op at index 2 + cluster.Nodes[1]->Append(LogValue("replacement-command")); // index 3, new term + cluster.Replicate(2); + cluster.Isolated.clear(); + cluster.Replicate(2); + client.Poll(); + assert_int_equal(client.Replies.size(), 1); + assert_int_equal(client.Replies[0].Cookie, 11); + assert_int_equal(client.Replies[0].Error, 1); + assert_true(client.Replies[0].Data.empty()); + assert_int_equal(client.Rsm->Applied.size(), 1); + assert_string_equal(client.Rsm->Applied[0].c_str(), "replacement-command"); + client.Poll(); + assert_int_equal(client.Replies.size(), 1); +} + +void test_stepdown_rejects_write_without_waiting_for_application(void**) { + TCluster cluster(3); + cluster.Elect(1); + cluster.Replicate(1); + TWriteClient client(cluster.Nodes[0]); + client.Write(11, "pending"); + cluster.Elect(2); + client.Processor.CheckStateChange(); + assert_int_equal(client.Replies.size(), 1); + assert_int_equal(client.Replies[0].Error, 1); +} + +void test_committed_but_unanswered_write_has_unknown_outcome_on_stepdown(void**) { + TCluster cluster(3); + cluster.Elect(1); + cluster.Replicate(1); + TWriteClient client(cluster.Nodes[0]); + client.Write(11, "committed"); + cluster.Replicate(1); + client.Processor.ProcessCommitted(); // application reply cached, not sent + assert_false(client.Rsm->LastReply.expired()); + cluster.Elect(2); + client.Poll(); + assert_int_equal(client.Replies.size(), 1); + assert_int_equal(client.Replies[0].Error, 1); + assert_int_equal(client.Rsm->Applied.size(), 1); + assert_string_equal(client.Rsm->Applied[0].c_str(), "committed"); + assert_true(client.Rsm->LastReply.expired()); + client.Poll(); + assert_int_equal(client.Rsm->Applied.size(), 1); + assert_int_equal(client.Replies.size(), 1); +} + +void test_old_write_is_rejected_after_unobserved_reelection(void**) { + TCluster cluster(3); + cluster.Elect(1); + cluster.Replicate(1); + TWriteClient client(cluster.Nodes[0]); + client.Write(11, "lost"); + cluster.Elect(2); + cluster.Elect(1); + cluster.Replicate(1); + client.Write(22, "new"); + assert_int_equal(client.Replies.size(), 1); + assert_int_equal(client.Replies[0].Cookie, 11); + assert_int_equal(client.Replies[0].Error, 1); + cluster.Replicate(1); + client.Poll(); + assert_int_equal(client.Replies.size(), 2); + assert_int_equal(client.Replies[1].Cookie, 22); + assert_int_equal(client.Replies[1].Error, 0); + assert_string_equal(client.Replies[1].Data.c_str(), "new"); +} + +void test_default_read_of_lost_tail_is_rejected_on_stepdown(void**) { + TCluster cluster(3); + cluster.Elect(1); + cluster.Replicate(1); + cluster.Nodes[0]->Append(LogValue("lost-1")); + cluster.Nodes[0]->Append(LogValue("lost-2")); + TWriteClient client(cluster.Nodes[0]); + client.Read(11); + cluster.Elect(2); + client.Processor.CheckStateChange(); + assert_int_equal(client.Replies.size(), 1); + assert_int_equal(client.Replies[0].Error, 1); +} + +void test_request_waiting_for_leader_runs_when_local_node_is_elected(void**) { + for (uint32_t count : {1, 3}) { + TCluster cluster(count); + TWriteClient client(cluster.Nodes[0]); + client.Write(0, "waiting-for-election"); + cluster.Elect(1); + client.Processor.CheckStateChange(); + assert_int_equal(cluster.Nodes[0]->GetLastIndex(), 2); + cluster.Replicate(1); + client.Poll(); + assert_int_equal(client.Replies.size(), 1); + assert_int_equal(client.Replies[0].Error, 0); + assert_string_equal(client.Replies[0].Data.c_str(), "waiting-for-election"); + } +} + +void test_write_replaced_by_noop_is_rejected_without_application_reply(void**) { + TCluster cluster(3); + cluster.Elect(1); + cluster.Replicate(1); + TWriteClient client(cluster.Nodes[0]); + client.Write(11, "lost"); + cluster.Elect(2); + cluster.Replicate(2); + client.Poll(); + assert_int_equal(client.Replies.size(), 1); + assert_int_equal(client.Replies[0].Cookie, 11); + assert_int_equal(client.Replies[0].Error, 1); + assert_true(client.Rsm->Applied.empty()); +} + +void test_stale_read_does_not_wait_for_uncommitted_tail(void**) { + TCluster cluster(3); + cluster.Elect(1); + cluster.Replicate(1); + cluster.Nodes[0]->Append(LogValue("uncommitted")); + TWriteClient client(cluster.Nodes[0]); + client.Read(11, TCommandRequest::EStale); + client.Poll(); + assert_int_equal(client.Replies.size(), 1); + assert_int_equal(client.Replies[0].Error, 0); + assert_true(client.Rsm->Applied.empty()); +} + +void test_disconnect_removes_local_write_and_read_waiters(void**) { + TCluster cluster(3); + cluster.Elect(1); + cluster.Replicate(1); + TWriteClient client(cluster.Nodes[0]); + client.Write(11, "still-may-commit"); + client.Read(22); + client.Read(33, TCommandRequest::EConsistent); + std::vector otherReplies; + auto other = std::make_shared([&](auto message) { + auto reply = message.template Cast(); + assert_int_equal(reply->ErrorCode, 0); + otherReplies.push_back(reply->Cookie); + }); + auto command = NewHoldedMessage(sizeof(TCommandRequest) + 5); + command->Cookie = 44; + command->Flags = TCommandRequest::EWrite; + memcpy(command->Data, "other", 5); + client.Processor.OnCommandRequest(command, other); + std::weak_ptr endpoint = client.Endpoint; + client.Processor.CleanUp(client.Endpoint); + client.Endpoint.reset(); + assert_true(endpoint.expired()); + cluster.Replicate(1); + client.Poll(); + assert_true(client.Replies.empty()); + assert_int_equal(client.Rsm->Applied.size(), 2); + assert_int_equal(otherReplies.size(), 1); + assert_int_equal(otherReplies[0], 44); + assert_true(client.Rsm->LastReply.expired()); +} + +void test_disconnect_removes_requests_waiting_for_leader(void**) { + TCluster cluster(3); + TWriteClient client(cluster.Nodes[0]); + client.Write(0, "disconnected-before-election"); + std::weak_ptr endpoint = client.Endpoint; + client.Processor.CleanUp(client.Endpoint); + client.Endpoint.reset(); + assert_true(endpoint.expired()); + cluster.Elect(1); + client.Processor.CheckStateChange(); + assert_int_equal(cluster.Nodes[0]->GetLastIndex(), 1); +} + +void test_disconnect_releases_cached_write_reply(void**) { + TCluster cluster(3); + cluster.Elect(1); + cluster.Replicate(1); + TWriteClient client(cluster.Nodes[0]); + client.Write(11, "committed"); + cluster.Replicate(1); + client.Processor.ProcessCommitted(); + assert_false(client.Rsm->LastReply.expired()); + client.Processor.CleanUp(client.Endpoint); + assert_true(client.Rsm->LastReply.expired()); + client.Poll(); + assert_true(client.Replies.empty()); +} + +void test_follower_application_does_not_retain_unused_replies(void**) { + TCluster cluster(3); + cluster.Elect(1); + cluster.Replicate(1); + TWriteClient follower(cluster.Nodes[1]); + cluster.Nodes[0]->Append(LogValue("background-command")); + cluster.Replicate(1); + follower.Processor.ProcessCommitted(); + assert_int_equal(follower.Rsm->Applied.size(), 1); + assert_true(follower.Rsm->LastReply.expired()); +} + void test_strong_reads_require_fresh_majority_in_all_cluster_sizes(void**) { for (uint32_t count = 1; count <= 8; ++count) { TCluster cluster(count); @@ -405,6 +670,18 @@ int main() { cmocka_unit_test(test_delayed_pre_read_replies_cannot_confirm_new_read), cmocka_unit_test(test_new_leader_waits_for_current_term_commit_and_application), cmocka_unit_test(test_old_term_read_is_rejected_after_reelection), + cmocka_unit_test(test_replaced_write_never_receives_another_commands_reply), + cmocka_unit_test(test_stepdown_rejects_write_without_waiting_for_application), + cmocka_unit_test(test_committed_but_unanswered_write_has_unknown_outcome_on_stepdown), + cmocka_unit_test(test_old_write_is_rejected_after_unobserved_reelection), + cmocka_unit_test(test_default_read_of_lost_tail_is_rejected_on_stepdown), + cmocka_unit_test(test_request_waiting_for_leader_runs_when_local_node_is_elected), + cmocka_unit_test(test_disconnect_removes_local_write_and_read_waiters), + cmocka_unit_test(test_disconnect_removes_requests_waiting_for_leader), + cmocka_unit_test(test_disconnect_releases_cached_write_reply), + cmocka_unit_test(test_follower_application_does_not_retain_unused_replies), + cmocka_unit_test(test_write_replaced_by_noop_is_rejected_without_application_reply), + cmocka_unit_test(test_stale_read_does_not_wait_for_uncommitted_tail), }; return cmocka_run_group_tests(tests, nullptr, nullptr); } From eee8ddc03e99ca5782815a17233341fb1d867881 Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Sun, 8 Nov 2026 08:54:38 -0500 Subject: [PATCH 06/66] Preserve forwarded client cookies and validate reply routing context (#6) * Preserve forwarded client cookies and bind routes to leader and peer lifecycle * Clarify forwarding marker value and peer lifecycle terminology --- README.md | 2 + docs/forwarding.md | 89 ++++++++++++++ miniraft/messages.h | 1 + miniraft/net/server.cpp | 6 +- miniraft/raft.cpp | 82 ++++++++----- miniraft/raft.h | 19 ++- test/CMakeLists.txt | 1 + test/test_correctness.cpp | 47 ++++++++ test/test_forwarding.cpp | 242 ++++++++++++++++++++++++++++++++++++++ 9 files changed, 451 insertions(+), 38 deletions(-) create mode 100644 docs/forwarding.md create mode 100644 test/test_forwarding.cpp diff --git a/README.md b/README.md index 380a20e..d65dc86 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,8 @@ The [strong-read follow-up](docs/strong-reads.md) requires a fresh majority for `--consistent` reads and rejects pending reads when their leadership term ends. The [pending request follow-up](docs/pending-requests.md) prevents replacement write acknowledgements and cleans up obsolete or disconnected client requests. +The [forwarding follow-up](docs/forwarding.md) preserves client cookies and binds +forwarded replies to their original leader, term and outbound peer. [![Number of GitHub stars](https://img.shields.io/github/stars/resetius/miniraft-cpp)](https://github.com/resetius/miniraft-cpp/stargazers) ![GitHub commit activity](https://img.shields.io/github/commit-activity/m/resetius/miniraft-cpp) diff --git a/docs/forwarding.md b/docs/forwarding.md new file mode 100644 index 0000000..fea4ab3 --- /dev/null +++ b/docs/forwarding.md @@ -0,0 +1,89 @@ +# Forwarded request lifecycle: v0.1.0-alpha.6 + +This milestone separates client cookies from forwarding correlation and gives +forwarded attempts an explicit leader/term/peer lifecycle. + +## Reproduced failures + +Followers previously interpreted every nonzero client cookie as an already +forwarded request and rejected it. For zero-cookie requests they replaced the +cookie with an internal routing token, then returned that token to the client +instead of restoring zero. They also mutated the caller's request buffer and +retained forwarding routes when terms/roles changed, allowing late replies to +complete obsolete attempts. + +## Routing contract + +EForwarded is a dedicated internal request flag. Clients may use any uint32_t +cookie, including zero and UINT32_MAX. A follower copies a request, sets this +one-hop flag and assigns a distinct nonzero routing cookie. A forwarded request +that reaches another follower fails instead of being relayed again. + +The routing record holds the original cookie, client endpoint, target peer, +leader ID and origin term. Responses are accepted only from that configured +outbound peer. Before delivery, the route is removed and the response is copied +with the original cookie restored. Caller-owned request/response buffers are +not changed. Duplicate, unknown, wrong-peer and unattributed responses cannot +complete another attempt. + +Routes are rejected with ErrorCode=1 and their original cookie when the origin +changes term, leaves follower state, observes a different leader, or receives a +peer-unavailable notification. Client disconnect removes only that client's +routes silently. Late responses for removed routes are ignored. Routing cookies +are never reused within a processor, even after success, cancellation or peer +reconnection. Exhausting the 32-bit wire space rejects further forwarding rather +than wrapping; a larger session/correlation protocol remains future work. + +Cancellation is local bookkeeping. A write already sent or buffered by the +transport may still execute. No automatic retry, rollback or exactly-once +execution guarantee is introduced. ErrorCode=1 can indicate an unknown write +outcome, so non-idempotent writes must not be blindly retried. + +## Transport integration and compatibility + +Custom adapters must call OnCommandResponse(response, peer), passing the same +configured INode object used for forwarding, and call OnPeerDisconnected(peer) +when the outbound connection is reported unavailable. The old one-argument +response call still compiles but intentionally ignores unattributed responses. +The included server supplies peer context and reports outbound read/unavailable +errors. Repeated failure notifications are idempotent. + +Configured-peer context is a routing check, not cryptographic authentication or +Byzantine fault tolerance. Untrusted inbound client sockets are not accepted as +the outbound peer that owns a response. Write-side failure notification timing +still depends on the transport detecting/reporting connection loss; blackholed +connections without such notification have no deadline here. + +Construct a new processor only with fresh transport sessions: the cookie counter +is per processor, and this milestone does not provide a durable cross-processor +session generation. Message struct sizes are unchanged, but EForwarded reserves +flag value 0x08. Upgrade custom adapters and all cluster nodes together; mixed-version +operation is not validated. + +## Validation and self-review + +Twelve forwarding tests cover full-range client cookies, zero-cookie restoration, +request-copy/one-hop semantics, out-of-order and duplicate replies, term and +candidate transitions, stale replies arriving before an application poll, +selective client disconnect, leader discovery, wrong/missing peer context, +error propagation and peer failure/reconnection with late replies. Nine of the +initial ten tests fail against alpha.5; the selective-disconnect baseline passes. + +An additional three-node deterministic test forwards a write and strong read +through actual Raft replication/application, including reversed and duplicate +consensus traffic. Together with existing tests, all 94 pass locally under +ASan/UBSan, with core Debug/Release verification. Hosted CI executes the same +targets; its results are separate from local checks. There are no sleeps or +wall-clock dependencies in the new tests. + +Self-review checked original-cookie restoration, copy ownership, distinct token +allocation, route invalidation before response delivery, source-peer matching, +disconnect selectivity and adapter call sites. No independent approval is +claimed. Access remains serialized; callbacks must not re-enter the processor. + +## Remaining gaps + +Durable request IDs/deduplication, deadlines, admission limits, a larger +session/correlation namespace, broader transport validation and generated +history/linearizability testing remain open. This change does not prove +library-wide linearizability or guarantee completion during quorum loss. diff --git a/miniraft/messages.h b/miniraft/messages.h index 830030b..88b239a 100644 --- a/miniraft/messages.h +++ b/miniraft/messages.h @@ -97,6 +97,7 @@ struct TCommandRequest: public TMessage { // read semantics, default: read from leader w/o ping check, possible stale reads if there are 2 leaders EStale = 2, // stale read, can read from follower EConsistent = 4, // strong consistent read (wait for pings, low latency, no stale read) + EForwarded = 8, // internal one-hop marker, independent of client cookies }; uint32_t Flags = ENone; uint32_t Cookie = 0; diff --git a/miniraft/net/server.cpp b/miniraft/net/server.cpp index 99d17fc..5eac8b8 100644 --- a/miniraft/net/server.cpp +++ b/miniraft/net/server.cpp @@ -127,7 +127,7 @@ NNet::TVoidTask TRaftServer::InboundConnection(TSocket socket) { if (auto maybeCommandRequest = mes.template Maybe()) { RequestProcessor->OnCommandRequest(std::move(maybeCommandRequest.Cast()), client); } else if (auto maybeCommandResponse = mes.template Maybe()) { - RequestProcessor->OnCommandResponse(std::move(maybeCommandResponse.Cast())); + RequestProcessor->OnCommandResponse(std::move(maybeCommandResponse.Cast()), client); } else { Raft->Process(TimeSource->Now(), std::move(mes), client); } @@ -176,7 +176,7 @@ NNet::TVoidTask TRaftServer::OutboundServe(std::shared_ptrSock()).Read(); if (auto maybeCommandResponse = mes.template Maybe()) { - RequestProcessor->OnCommandResponse(std::move(maybeCommandResponse.Cast())); + RequestProcessor->OnCommandResponse(std::move(maybeCommandResponse.Cast()), node); DrainNodes(); } else { std::cerr << "Wrong message type: " << mes->Type << std::endl; @@ -187,6 +187,8 @@ NNet::TVoidTask TRaftServer::OutboundServe(std::shared_ptrOnPeerDisconnected(node); + DrainNodes(); co_await Poller.Sleep(std::chrono::milliseconds(1000)); } } diff --git a/miniraft/raft.cpp b/miniraft/raft.cpp index b139bee..eb3f21f 100644 --- a/miniraft/raft.cpp +++ b/miniraft/raft.cpp @@ -553,7 +553,7 @@ void TRequestProcessor::Forward(TMessageHolder command, const s return; } - if (command->Cookie) { + if (command->Flags & TCommandRequest::EForwarded) { // already forwarded replyTo->Send(NewHoldedMessage(TCommandResponse{.Cookie = command->Cookie, .ErrorCode = 1})); return; @@ -568,12 +568,19 @@ void TRequestProcessor::Forward(TMessageHolder command, const s } if (stateName == EState::FOLLOWER) { - // Forward - command->Cookie = std::max(1, ForwardCookie); - Nodes[leaderId]->Send(std::move(command)); - Cookie2Client[ForwardCookie] = replyTo; - Client2Cookie[replyTo].emplace(ForwardCookie); - ForwardCookie++; + if (NextForwardCookie > UINT32_MAX) { + replyTo->Send(NewHoldedMessage(TCommandResponse{.Cookie = command->Cookie, .ErrorCode = 1})); + return; + } + const auto cookie = uint32_t(NextForwardCookie++); + auto forwarded = NewHoldedMessage(command->Len); + memcpy(forwarded.Mes, command.Mes, command->Len); + forwarded->Cookie = cookie; + forwarded->Flags |= TCommandRequest::EForwarded; + auto peer = Nodes.at(leaderId); + Forwarded.emplace(cookie, TForwarded{replyTo, peer, command->Cookie, + leaderId, Raft->GetState()->CurrentTerm}); + peer->Send(std::move(forwarded)); return; } @@ -610,11 +617,14 @@ void TRequestProcessor::OnReadRequest(TMessageHolder command, c void TRequestProcessor::PruneRequests(const std::shared_ptr& disconnected) { const auto term = Raft->GetState()->CurrentTerm; - const bool leader = Raft->CurrentStateName() == EState::LEADER; + const auto state = Raft->CurrentStateName(); + const bool leader = state == EState::LEADER; + const auto leaderId = Raft->GetLeaderId(); if (!disconnected) { - if (RequestsTerm == term && RequestsLeader == leader) return; + if (RequestsTerm == term && RequestsState == state && RequestsLeaderId == leaderId) return; RequestsTerm = term; - RequestsLeader = leader; + RequestsState = state; + RequestsLeaderId = leaderId; } for (auto* queue : {&Waiting, &StrongWaiting, &WaitingStateChange}) { std::queue retained; @@ -636,6 +646,22 @@ void TRequestProcessor::PruneRequests(const std::shared_ptr& disconnected } queue->swap(retained); } + CancelForwarded(disconnected); +} + +void TRequestProcessor::CancelForwarded(const std::shared_ptr& disconnected, bool peerFailure) { + for (auto it = Forwarded.begin(); it != Forwarded.end();) { + const auto& route = it->second; + const bool remove = disconnected ? + (peerFailure ? route.Peer == disconnected : route.Client == disconnected) : + Raft->CurrentStateName() != EState::FOLLOWER || + route.Term != Raft->GetState()->CurrentTerm || route.LeaderId != Raft->GetLeaderId(); + if (!remove) { ++it; continue; } + auto pending = std::move(it->second); + it = Forwarded.erase(it); + if (!disconnected || peerFailure) pending.Client->Send(NewHoldedMessage( + TCommandResponse{.Cookie = pending.OriginalCookie, .ErrorCode = 1})); + } } void TRequestProcessor::OnWriteRequest(TMessageHolder command, const std::shared_ptr& replyTo) { @@ -666,32 +692,26 @@ void TRequestProcessor::OnCommandRequest(TMessageHolder command } } -void TRequestProcessor::OnCommandResponse(TMessageHolder command) { - // forwarded - auto it = Cookie2Client.find(command->Cookie); - if (it == Cookie2Client.end()) { - return; - } - it->second->Send(std::move(command)); - auto jt = Client2Cookie.find(it->second); - jt->second.erase(command->Cookie); - if (jt->second.empty()) { - Client2Cookie.erase(jt); - } - Cookie2Client.erase(it); +void TRequestProcessor::OnCommandResponse(TMessageHolder command, + const std::shared_ptr& peer) { + PruneRequests(); + auto it = Forwarded.find(command->Cookie); + if (it == Forwarded.end() || !peer || it->second.Peer != peer) return; + auto pending = std::move(it->second); + Forwarded.erase(it); + auto reply = NewHoldedMessage(command->Len); + memcpy(reply.Mes, command.Mes, command->Len); + reply->Cookie = pending.OriginalCookie; + pending.Client->Send(std::move(reply)); +} + +void TRequestProcessor::OnPeerDisconnected(const std::shared_ptr& peer) { + if (peer) CancelForwarded(peer, true); } void TRequestProcessor::CleanUp(const std::shared_ptr& replyTo) { if (!replyTo) return; PruneRequests(replyTo); - auto jt = Client2Cookie.find(replyTo); - if (jt == Client2Cookie.end()) { - return; - } - for (auto cookie : jt->second) { - Cookie2Client.erase(cookie); - } - Client2Cookie.erase(jt); } void TRequestProcessor::ProcessCommitted() { diff --git a/miniraft/raft.h b/miniraft/raft.h index 9b90845..9865b10 100644 --- a/miniraft/raft.h +++ b/miniraft/raft.h @@ -170,7 +170,9 @@ class TRequestProcessor { void CheckStateChange(); void OnCommandRequest(TMessageHolder message, const std::shared_ptr& replyTo); - void OnCommandResponse(TMessageHolder message); + // Unattributed responses are ignored; pass the configured outbound peer. + void OnCommandResponse(TMessageHolder message, const std::shared_ptr& peer = {}); + void OnPeerDisconnected(const std::shared_ptr& peer); void ProcessCommitted(); void ProcessWaiting(); void CleanUp(const std::shared_ptr& replyTo); @@ -180,6 +182,7 @@ class TRequestProcessor { void OnReadRequest(TMessageHolder message, const std::shared_ptr& replyTo); void OnWriteRequest(TMessageHolder message, const std::shared_ptr& replyTo); void PruneRequests(const std::shared_ptr& disconnected = {}); + void CancelForwarded(const std::shared_ptr& disconnected = {}, bool peerFailure = false); std::shared_ptr Raft; std::shared_ptr Rsm; @@ -206,9 +209,15 @@ class TRequestProcessor { std::unordered_map PendingWriteTerms; std::unordered_map WriteAnswers; uint64_t RequestsTerm = 0; - bool RequestsLeader = false; - uint32_t ForwardCookie = 1; - std::unordered_map> Cookie2Client; - std::unordered_map, std::unordered_set> Client2Cookie; + EState RequestsState = EState::NONE; + uint32_t RequestsLeaderId = 0; + struct TForwarded { + std::shared_ptr Client, Peer; + uint32_t OriginalCookie, LeaderId; + uint64_t Term; + }; + // Never reuse a wire cookie within this processor, including after failure. + uint64_t NextForwardCookie = 1; + std::unordered_map Forwarded; }; diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 578174a..998def8 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -14,6 +14,7 @@ endmacro() if (CMOCKA_FOUND) ut(test_raft test_raft.cpp miniraft) ut(test_correctness test_correctness.cpp miniraft) +ut(test_forwarding test_forwarding.cpp miniraft) ut(test_persist_recovery test_persist_recovery.cpp miniraft) if(CMAKE_SYSTEM_NAME STREQUAL "Linux") ut(test_durable test_durable.cpp miniraft) diff --git a/test/test_correctness.cpp b/test/test_correctness.cpp index 18b769d..8a3dfce 100644 --- a/test/test_correctness.cpp +++ b/test/test_correctness.cpp @@ -172,6 +172,52 @@ TMessageHolder LogValue(const std::string& value) { return entry; } +void test_forwarded_write_and_strong_read_complete_through_real_raft(void**) { + TCluster cluster(3); + cluster.Elect(1); + cluster.Replicate(1); + std::deque> requests; + std::deque> responses; + auto peer = std::make_shared([&](auto message) { + requests.push_back(message.template Cast()); + }); + auto returnPath = std::make_shared([&](auto message) { + responses.push_back(message.template Cast()); + }); + auto leaderRsm = std::make_shared(); + auto originRsm = std::make_shared(); + TRequestProcessor leader(cluster.Nodes[0], leaderRsm, {}); + TRequestProcessor origin(cluster.Nodes[1], originRsm, TNodeDict{{1, peer}}); + std::vector cookies; + auto client = std::make_shared([&](auto message) { + auto response = message.template Cast(); + assert_int_equal(response->ErrorCode, 0); + cookies.push_back(response->Cookie); + }); + for (uint32_t cookie : {42, 77}) { + auto command = NewHoldedMessage(sizeof(TCommandRequest) + 5); + command->Cookie = cookie; + command->Flags = cookie == 42 ? TCommandRequest::EWrite : TCommandRequest::EConsistent; + memcpy(command->Data, "value", 5); + origin.OnCommandRequest(command, client); + assert_int_equal(requests.size(), 1); + leader.OnCommandRequest(requests.front(), returnPath); + requests.pop_front(); + cluster.Replicate(1, true, true); + leader.ProcessCommitted(); + leader.ProcessWaiting(); + assert_int_equal(responses.size(), 1); + origin.OnCommandResponse(responses.front(), peer); + origin.OnCommandResponse(responses.front(), peer); + responses.pop_front(); + assert_int_equal(cookies.back(), cookie); + } + assert_int_equal(cookies.size(), 2); + assert_int_equal(leaderRsm->Applied.size(), 1); + assert_string_equal(leaderRsm->Applied[0].c_str(), "value"); + assert_true(originRsm->Applied.empty()); +} + void test_replaced_write_never_receives_another_commands_reply(void**) { TCluster cluster(3); cluster.Elect(1); @@ -682,6 +728,7 @@ int main() { cmocka_unit_test(test_follower_application_does_not_retain_unused_replies), cmocka_unit_test(test_write_replaced_by_noop_is_rejected_without_application_reply), cmocka_unit_test(test_stale_read_does_not_wait_for_uncommitted_tail), + cmocka_unit_test(test_forwarded_write_and_strong_read_complete_through_real_raft), }; return cmocka_run_group_tests(tests, nullptr, nullptr); } diff --git a/test/test_forwarding.cpp b/test/test_forwarding.cpp new file mode 100644 index 0000000..4e8ab9b --- /dev/null +++ b/test/test_forwarding.cpp @@ -0,0 +1,242 @@ +#include +#include +#include +#include +#include +#include +extern "C" { +#include +} + +namespace { +struct TEndpoint : INode { + std::function)> Callback; + explicit TEndpoint(std::function)> cb) : Callback(std::move(cb)) {} + void Send(TMessageHolder message) override { Callback(std::move(message)); } + void Drain() override {} +}; +struct TClient { + struct TReply { uint32_t Cookie, Error; std::string Data; }; + std::vector Replies; + std::shared_ptr Endpoint = std::make_shared([this](auto message) { + auto response = message.template Cast(); + Replies.push_back({response->Cookie, response->ErrorCode, + std::string(response->Data, response->Len - sizeof(TCommandResponse))}); + }); +}; +struct TFollower { + std::vector> Sent; + std::shared_ptr Leader = std::make_shared([this](auto message) { + if (message->Type == uint32_t(EMessageType::COMMAND_REQUEST)) + Sent.push_back(message.template Cast()); + }); + std::shared_ptr Other = std::make_shared([this](auto message) { + if (message->Type == uint32_t(EMessageType::COMMAND_REQUEST)) + Sent.push_back(message.template Cast()); + }); + TNodeDict Peers{{1, Leader}, {3, Other}}; + std::shared_ptr Raft = std::make_shared(std::make_shared(), 2, Peers); + std::shared_ptr Rsm = std::make_shared(); + TRequestProcessor Processor{Raft, Rsm, Peers}; + explicit TFollower(bool known = true) { if (known) LearnLeader(1, 3); } + void LearnLeader(uint32_t id, uint64_t term) { + auto message = NewHoldedMessage(TMessageEx{.Src = id, .Dst = 2, .Term = term}, + TAppendEntriesRequest{.LeaderId = id}); + Raft->Process(ITimeSource::Time{}, message); + } + TMessageHolder Request(TClient& client, uint32_t cookie, uint32_t extra = 0) { + auto request = NewHoldedMessage(sizeof(TCommandRequest) + 5); + request->Cookie = cookie; + request->Flags = TCommandRequest::EWrite | extra; + memcpy(request->Data, "value", 5); + Processor.OnCommandRequest(request, client.Endpoint); + return request; + } + TMessageHolder Response(size_t sent, uint32_t error = 0) { + auto response = NewHoldedMessage(sizeof(TCommandResponse) + 6); + response->Cookie = Sent.at(sent)->Cookie; + response->ErrorCode = error; + memcpy(response->Data, "answer", 6); + return response; + } + void Reply(size_t sent, uint32_t error = 0) { + Processor.OnCommandResponse(Response(sent, error), Leader); + } +}; + +void test_nonzero_client_cookies_are_forwarded(void**) { + for (uint32_t cookie : {42u, UINT32_MAX}) { + TFollower follower; + TClient client; + follower.Request(client, cookie); + assert_int_equal(follower.Sent.size(), 1); + follower.Reply(0); + assert_int_equal(client.Replies.size(), 1); + assert_int_equal(client.Replies[0].Cookie, cookie); + assert_int_equal(client.Replies[0].Error, 0); + } +} +void test_zero_client_cookie_is_restored(void**) { + TFollower follower; + TClient client; + follower.Request(client, 0); + assert_int_equal(follower.Sent.size(), 1); + follower.Reply(0); + assert_int_equal(client.Replies.size(), 1); + assert_int_equal(client.Replies[0].Cookie, 0); + assert_string_equal(client.Replies[0].Data.c_str(), "answer"); +} +void test_forwarding_copies_request_and_marks_one_hop(void**) { + TFollower follower; + TClient client; + auto original = follower.Request(client, 0); + assert_int_equal(original->Cookie, 0); + assert_int_equal(original->Flags, TCommandRequest::EWrite); + assert_true(follower.Sent[0]->Flags & TCommandRequest::EForwarded); + assert_true(follower.Sent[0].RawData != original.RawData); +} +void test_out_of_order_and_duplicate_replies_preserve_client_routes(void**) { + TFollower follower; + TClient first, second; + follower.Request(first, 0); + follower.Request(second, 0); + assert_int_not_equal(follower.Sent[0]->Cookie, follower.Sent[1]->Cookie); + follower.Reply(1); + follower.Reply(1); + assert_true(first.Replies.empty()); + assert_int_equal(second.Replies.size(), 1); + assert_int_equal(second.Replies[0].Cookie, 0); + follower.Reply(0); + assert_int_equal(first.Replies.size(), 1); + assert_int_equal(first.Replies[0].Cookie, 0); +} +void test_forwarded_request_is_rejected_on_term_change(void**) { + TFollower follower; + TClient client; + follower.Request(client, 0); + follower.LearnLeader(3, 4); + follower.Processor.CheckStateChange(); + assert_int_equal(client.Replies.size(), 1); + assert_int_equal(client.Replies[0].Cookie, 0); + assert_int_equal(client.Replies[0].Error, 1); + follower.Reply(0); + assert_int_equal(client.Replies.size(), 1); +} +void test_candidate_transition_rejects_forwarded_requests(void**) { + TFollower follower; + TClient client; + follower.Request(client, 0); + follower.Raft->ProcessTimeout(ITimeSource::Time{} + std::chrono::seconds(20)); + assert_int_equal(follower.Raft->CurrentStateName(), EState::CANDIDATE); + follower.Processor.CheckStateChange(); + assert_int_equal(client.Replies.size(), 1); + assert_int_equal(client.Replies[0].Error, 1); +} +void test_late_reply_prunes_stale_route_before_application_poll(void**) { + TFollower follower; + TClient client; + follower.Request(client, 0); + follower.LearnLeader(3, 4); + follower.Reply(0); // No CheckStateChange/ProcessWaiting in between. + assert_int_equal(client.Replies.size(), 1); + assert_int_equal(client.Replies[0].Error, 1); + assert_int_equal(client.Replies[0].Cookie, 0); +} +void test_client_disconnect_releases_only_its_forwarded_route(void**) { + TFollower follower; + TClient first, second; + follower.Request(first, 0); + follower.Request(second, 0); + std::weak_ptr endpoint = first.Endpoint; + follower.Processor.CleanUp(first.Endpoint); + first.Endpoint.reset(); + assert_true(endpoint.expired()); + follower.Reply(0); + assert_true(first.Replies.empty()); + follower.Reply(1); + assert_int_equal(second.Replies.size(), 1); + assert_int_equal(second.Replies[0].Error, 0); +} +void test_already_forwarded_request_is_not_relayed_again(void**) { + TFollower follower; + TClient client; + follower.Request(client, 0, TCommandRequest::EForwarded); + assert_true(follower.Sent.empty()); + assert_int_equal(client.Replies.size(), 1); + assert_int_equal(client.Replies[0].Error, 1); +} +void test_nonzero_cookie_waits_for_leader_discovery(void**) { + TFollower follower(false); + TClient client; + follower.Request(client, 77); + assert_true(client.Replies.empty()); + assert_true(follower.Sent.empty()); + follower.LearnLeader(1, 3); + follower.Processor.CheckStateChange(); + assert_int_equal(follower.Sent.size(), 1); + follower.Reply(0); + assert_int_equal(client.Replies[0].Cookie, 77); +} +void test_response_requires_original_peer_and_preserves_wire_message(void**) { + TFollower follower; + TClient client; + follower.Request(client, 0); + auto response = follower.Response(0, 77); + const auto cookie = response->Cookie; + follower.Processor.OnCommandResponse(response); + follower.Processor.OnCommandResponse(response, follower.Other); + assert_true(client.Replies.empty()); + follower.Processor.OnCommandResponse(response, follower.Leader); + assert_int_equal(client.Replies.size(), 1); + assert_int_equal(client.Replies[0].Cookie, 0); + assert_int_equal(client.Replies[0].Error, 77); + assert_int_equal(response->Cookie, cookie); +} +void test_peer_disconnect_cancels_once_and_does_not_reuse_cookies(void**) { + TFollower follower; + TClient first, second; + follower.Request(first, 42); + follower.Request(second, 43); + follower.Processor.OnPeerDisconnected(follower.Other); + assert_true(first.Replies.empty()); + assert_true(second.Replies.empty()); + follower.Processor.OnPeerDisconnected(follower.Leader); + follower.Processor.OnPeerDisconnected(follower.Leader); + assert_int_equal(first.Replies.size(), 1); + assert_int_equal(second.Replies.size(), 1); + assert_int_equal(first.Replies[0].Cookie, 42); + assert_int_equal(second.Replies[0].Cookie, 43); + assert_int_equal(first.Replies[0].Error, 1); + assert_int_equal(second.Replies[0].Error, 1); + std::weak_ptr endpoint = first.Endpoint; + first.Endpoint.reset(); + assert_true(endpoint.expired()); + TClient afterReconnect; + follower.Request(afterReconnect, 44); + assert_true(follower.Sent[2]->Cookie > follower.Sent[1]->Cookie); + follower.Reply(0); // Late old-connection reply cannot redeem the new route. + assert_true(afterReconnect.Replies.empty()); + follower.Reply(2); + assert_int_equal(afterReconnect.Replies.size(), 1); + assert_int_equal(afterReconnect.Replies[0].Cookie, 44); + assert_int_equal(afterReconnect.Replies[0].Error, 0); +} +} + +int main() { + const CMUnitTest tests[] = { + cmocka_unit_test(test_nonzero_client_cookies_are_forwarded), + cmocka_unit_test(test_zero_client_cookie_is_restored), + cmocka_unit_test(test_forwarding_copies_request_and_marks_one_hop), + cmocka_unit_test(test_out_of_order_and_duplicate_replies_preserve_client_routes), + cmocka_unit_test(test_forwarded_request_is_rejected_on_term_change), + cmocka_unit_test(test_candidate_transition_rejects_forwarded_requests), + cmocka_unit_test(test_late_reply_prunes_stale_route_before_application_poll), + cmocka_unit_test(test_client_disconnect_releases_only_its_forwarded_route), + cmocka_unit_test(test_already_forwarded_request_is_not_relayed_again), + cmocka_unit_test(test_nonzero_cookie_waits_for_leader_discovery), + cmocka_unit_test(test_response_requires_original_peer_and_preserves_wire_message), + cmocka_unit_test(test_peer_disconnect_cancels_once_and_does_not_reuse_cookies), + }; + return cmocka_run_group_tests(tests, nullptr, nullptr); +} From b36e5b195ade3b47cb908b41b9a67ba77cfbb923 Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Sun, 8 Nov 2026 09:11:40 -0500 Subject: [PATCH 07/66] Bound network frames and replication batches; validate message allocations and casts (#7) --- README.md | 2 + docs/network-frames.md | 99 ++++++++++++++ miniraft/messages.cpp | 44 +++++++ miniraft/messages.h | 27 +++- miniraft/net/server.cpp | 49 ++----- miniraft/net/server.h | 58 +++++++-- miniraft/raft.cpp | 24 +++- test/CMakeLists.txt | 2 + test/test_message_safety.cpp | 95 ++++++++++++++ test/test_network_frames.cpp | 241 +++++++++++++++++++++++++++++++++++ test/test_raft.cpp | 4 +- 11 files changed, 583 insertions(+), 62 deletions(-) create mode 100644 docs/network-frames.md create mode 100644 test/test_message_safety.cpp create mode 100644 test/test_network_frames.cpp diff --git a/README.md b/README.md index d65dc86..95115b6 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,8 @@ The [pending request follow-up](docs/pending-requests.md) prevents replacement write acknowledgements and cleans up obsolete or disconnected client requests. The [forwarding follow-up](docs/forwarding.md) preserves client cookies and binds forwarded replies to their original leader, term and outbound peer. +The [network framing follow-up](docs/network-frames.md) handles fragmented input +and enforces type, frame, batch and AppendEntries payload bounds. [![Number of GitHub stars](https://img.shields.io/github/stars/resetius/miniraft-cpp)](https://github.com/resetius/miniraft-cpp/stargazers) ![GitHub commit activity](https://img.shields.io/github/commit-activity/m/resetius/miniraft-cpp) diff --git a/docs/network-frames.md b/docs/network-frames.md new file mode 100644 index 0000000..0879509 --- /dev/null +++ b/docs/network-frames.md @@ -0,0 +1,99 @@ +# Bounded network framing: v0.1.0-alpha.7 + +This milestone fixes fragmented TCP header handling and validates structural +frame bounds before allocation, typed casts or delivery to Raft. + +## Reproduced failures + +The reader assumed each four-byte header field arrived in one ReadSome call, +rejecting valid byte-fragmented TCP input. It trusted the declared length and +AppendEntries count before allocation, accepted unknown/unimplemented types and +short fixed-size messages, and recursively accepted arbitrary child messages. +All six initial deterministic regressions fail against alpha.6. + +The message factory also only asserted an eight-byte allocation minimum even +when placement-constructing a larger derived type. Undersized typed allocations +could overwrite their buffers, particularly when assertions were disabled. + +## Decoder and encoder contract + +Headers and bodies now use exact-read loops. Before allocating, the decoder +checks the type, its minimum/fixed size, the frame cap and remaining batch +budget. Vote/AppendEntries control frames have exact sizes. Log entries and +command request/response frames have typed minimum sizes and variable data. +Unknown types and unimplemented snapshot types are rejected. + +AppendEntries children must be log entries with no nested payloads. Entry count +and minimum required bytes are checked before allocating the holder array; every +child consumes the same aggregate byte budget. Nesting depth is bounded to the +root and one log-entry level. EOF/truncation is an error; a failed reader cannot +resume at a later frame. Caller adapters must discard the failed stream. + +The writer validates the entire outgoing tree before its first byte, including +Nentries/PayloadSize agreement, required payload storage, child types and byte +budgets. Valid consecutive frames retain their boundaries under partial I/O. +The bundled server closes rejected inbound streams and resets failed outbound +connections after the reader unwinds, rather than parsing more bytes from the +damaged stream. Associated forwarded attempts receive the existing failure +notification; write outcomes may remain unknown. + +## Default limits and replication progress + +* Maximum frame size: 16 MiB, including its header. +* Maximum complete AppendEntries batch: 64 MiB, including root and child headers. +* Maximum AppendEntries child count: 4096. + +Raft log proposals are checked against the frame cap before mutating the log or +the proposed entry's term. Batch construction now observes the same aggregate +cap and continues with the remaining entries after acknowledgement. This avoids +building a valid-looking replication batch that every peer would reject. + +Application Prepare implementations can expand commands. If their prepared log +entry exceeds the cap, the proposal is rejected before append; an accepted +command-frame length alone does not guarantee its prepared entry fits. + +TMessageLimits can be supplied to standalone reader/writer adapters. The bundled +Raft sender/admission path uses shared defaults; coordinated deployment-level +limit configuration is not introduced here. Stricter custom adapter limits can +reject the sender's batches. Existing stores with oversized entries are not +supported by the new default replication cap and require review before upgrade. +The wire layout and native-endian representation are otherwise unchanged. + +## Typed message safety + +Factories reject allocations smaller than sizeof(T), including in Release builds. +Cast checks null, declared minimum size and the concrete protocol type; Maybe +and IsEx no longer expose short or empty messages as larger structures. A generic +TMessage factory can still hold opaque bytes for validation. Local callers must +provide genuine buffers and keep frames immutable during I/O: mutable length +fields are not an allocation-capacity capability, and forged local holders are +outside this contract. + +## Validation and self-review + +Fourteen deterministic stream tests exercise one-byte fragmentation, all header +truncation offsets, fixed-frame body truncation offsets, invalid/oversized lengths, +unknown types, wrong/nested children, entry count and aggregate bounds, exact +budget boundaries, reader poisoning, pre-write validation and adjacent frames. +They use immediate fake I/O, with no sockets, sleeps or wall-clock dependencies. + +Four core tests cover typed allocation bounds, checked casts, bounded multi-batch +replication progress and rejection before persistent-state mutation. An older +election test fixture was corrected to store vote requests as requests instead of +casting them to vote responses; it still checks the same expected wire bytes. + +All 112 tests pass locally under ASan/UBSan, with core Debug/Release verification. +Hosted CI runs the same targets; its results are separate from local checks. +Self-review checked subtraction/aggregation bounds, non-recursive child shape, +pre-allocation validation, pre-write validation, batch continuation and failed +connection reset ordering. No independent reviewer approval is claimed. + +## Remaining gaps + +This is structural framing validation, not authentication or complete validation +of consensus fields. Sender/destination membership, index/term semantics and +application payload lengths (KV/SQL/custom commands) still need their own checks. +Per-connection caps do not bound aggregate memory across unlimited connections; +admission limits, deadlines and broader fuzzing remain open. Native wire encoding, +TLS lifecycle stress, retry deduplication and generated history checking also +remain separate work. No library-wide security or linearizability claim is made. diff --git a/miniraft/messages.cpp b/miniraft/messages.cpp index f6152ec..c8751f1 100644 --- a/miniraft/messages.cpp +++ b/miniraft/messages.cpp @@ -1 +1,45 @@ #include "messages.h" + +void ValidateMessageLimits(const TMessageLimits& limits) { + if (limits.MaxFrameBytes < sizeof(TMessage) || limits.MaxBatchBytes < sizeof(TMessage)) + throw std::invalid_argument("Message byte limits cannot hold a header"); +} + +void ValidateMessageHeader(uint32_t type, uint32_t length, const TMessageLimits& limits) { + uint32_t minimum; + bool fixed = true; + switch (EMessageType(type)) { + case EMessageType::LOG_ENTRY: minimum = sizeof(TLogEntry); fixed = false; break; + case EMessageType::REQUEST_VOTE_REQUEST: minimum = sizeof(TRequestVoteRequest); break; + case EMessageType::REQUEST_VOTE_RESPONSE: minimum = sizeof(TRequestVoteResponse); break; + case EMessageType::APPEND_ENTRIES_REQUEST: minimum = sizeof(TAppendEntriesRequest); break; + case EMessageType::APPEND_ENTRIES_RESPONSE: minimum = sizeof(TAppendEntriesResponse); break; + case EMessageType::COMMAND_REQUEST: minimum = sizeof(TCommandRequest); fixed = false; break; + case EMessageType::COMMAND_RESPONSE: minimum = sizeof(TCommandResponse); fixed = false; break; + default: throw TMessageError("Unknown or unsupported message type"); + } + if (length < minimum || (fixed && length != minimum) || length > limits.MaxFrameBytes) + throw TMessageError("Invalid message length"); +} + +void ValidateMessageTree(const TMessageHolder& message, const TMessageLimits& limits) { + ValidateMessageLimits(limits); + uint64_t remaining = limits.MaxBatchBytes; + auto validate = [&](const auto& frame, bool child) { + if (!frame) throw TMessageError("Missing message"); + ValidateMessageHeader(frame->Type, frame->Len, limits); + if (frame->Len > remaining) throw TMessageError("Message batch exceeds byte limit"); + remaining -= frame->Len; + if (child && frame->Type != uint32_t(EMessageType::LOG_ENTRY)) + throw TMessageError("AppendEntries children must be log entries"); + if (frame->Type != uint32_t(EMessageType::APPEND_ENTRIES_REQUEST) && frame.PayloadSize) + throw TMessageError("Unexpected message payload"); + }; + validate(message, false); + if (message->Type != uint32_t(EMessageType::APPEND_ENTRIES_REQUEST)) return; + auto copy = message; + const auto count = copy.Cast()->Nentries; + if (count > limits.MaxEntries || count != message.PayloadSize || (count && !message.Payload)) + throw TMessageError("Invalid AppendEntries entry count"); + for (uint32_t i = 0; i < count; ++i) validate(message.Payload[i], true); +} diff --git a/miniraft/messages.h b/miniraft/messages.h index 88b239a..b8ee088 100644 --- a/miniraft/messages.h +++ b/miniraft/messages.h @@ -1,5 +1,6 @@ #pragma once #include +#include #include #include @@ -7,6 +8,17 @@ #include #include #include +#include + +struct TMessageLimits { + uint32_t MaxFrameBytes = 16 * 1024 * 1024; + uint64_t MaxBatchBytes = 64 * 1024 * 1024; + uint32_t MaxEntries = 4096; +}; + +struct TMessageError : std::runtime_error { using std::runtime_error::runtime_error; }; +void ValidateMessageLimits(const TMessageLimits& limits); +void ValidateMessageHeader(uint32_t type, uint32_t length, const TMessageLimits& limits = {}); enum class EMessageType : uint32_t { NONE = 0, @@ -128,7 +140,7 @@ struct TMessageHolder { T* Mes; std::shared_ptr RawData; - uint32_t PayloadSize; + uint32_t PayloadSize = 0; std::shared_ptr[]> Payload; TMessageHolder() @@ -174,12 +186,15 @@ struct TMessageHolder { } bool IsEx() const { - return (2 <= Mes->Type && Mes->Type <= 5); + return Mes && Mes->Len >= sizeof(TMessageEx) && (2 <= Mes->Type && Mes->Type <= 5); } template requires std::derived_from TMessageHolder Cast() { + if (!Mes || Mes->Len < sizeof(U) || + (U::MessageType != EMessageType::NONE && Mes->Type != uint32_t(U::MessageType))) + throw std::bad_cast(); return TMessageHolder(static_cast(Mes), RawData, PayloadSize, Payload); } @@ -201,16 +216,19 @@ struct TMessageHolder { } }; - return Mes->Type == static_cast(U::MessageType) + return Mes && Mes->Len >= sizeof(U) && Mes->Type == static_cast(U::MessageType) ? Maybe { *this } : Maybe { }; } }; +// Validate the complete frame/batch before emitting any bytes. +void ValidateMessageTree(const TMessageHolder& message, const TMessageLimits& limits = {}); + template requires std::derived_from T* NewMessage(uint32_t type, uint32_t len) { - assert(len >= sizeof(TMessage)); + if (len < sizeof(T)) throw std::invalid_argument("Message allocation is smaller than its type"); char* data = new char[len]; T* mes = new (data) T; mes->Type = type; @@ -233,7 +251,6 @@ TMessageHolder NewHoldedMessage() { template TMessageHolder NewHoldedMessage(uint32_t size) { - assert(size >= sizeof(T)); return NewHoldedMessage(static_cast(T::MessageType), size); } diff --git a/miniraft/net/server.cpp b/miniraft/net/server.cpp index 5eac8b8..8e76913 100644 --- a/miniraft/net/server.cpp +++ b/miniraft/net/server.cpp @@ -9,44 +9,6 @@ #include "server.h" -template -NNet::TValueTask TMessageWriter::Write(TMessageHolder message) { - co_await NNet::TByteWriter(Socket).Write(message.Mes, message->Len); - - auto payload = std::move(message.Payload); - for (uint32_t i = 0; i < message.PayloadSize; ++i) { - co_await Write(std::move(payload[i])); - } - - co_return; -} - -template -NNet::TValueTask> TMessageReader::Read() { - decltype(TMessage::Type) type; - decltype(TMessage::Len) len; - auto s = co_await Socket.ReadSome(&type, sizeof(type)); - if (s != sizeof(type)) { - throw std::runtime_error("Connection closed"); - } - s = co_await Socket.ReadSome(&len, sizeof(len)); - if (s != sizeof(len)) { - throw std::runtime_error("Connection closed"); - } - auto mes = NewHoldedMessage(type, len); - co_await NNet::TByteReader(Socket).Read(mes->Value, len - sizeof(TMessage)); - auto maybeAppendEntries = mes.Maybe(); - if (maybeAppendEntries) { - auto appendEntries = maybeAppendEntries.Cast(); - auto nentries = appendEntries->Nentries; - mes.InitPayload(nentries); - for (uint32_t i = 0; i < nentries; i++) { - mes.Payload[i] = co_await Read(); - } - } - co_return mes; -} - template void TNode::Send(TMessageHolder message) { Messages.emplace_back(std::move(message)); @@ -82,6 +44,15 @@ NNet::TVoidSuspendedTask TNode::DoDrain() { co_return; } +template +void TNode::Disconnect() { + Connected = false; + if (Drainer) { Drainer.destroy(); Drainer = {}; } + if (Connector) { Connector.destroy(); Connector = {}; } + std::destroy_at(&Socket); + std::construct_at(&Socket); +} + template void TNode::Connect() { if (Address && (!Connector || Connector.done())) { @@ -170,6 +141,7 @@ NNet::TVoidTask TRaftServer::OutboundServe(std::shared_ptrIsConnected(); try { if (!node->IsConnected()) { throw std::runtime_error("Not connected"); @@ -188,6 +160,7 @@ NNet::TVoidTask TRaftServer::OutboundServe(std::shared_ptrOnPeerDisconnected(node); + if (wasConnected) node->Disconnect(); DrainNodes(); co_await Poller.Sleep(std::chrono::milliseconds(1000)); } diff --git a/miniraft/net/server.h b/miniraft/net/server.h index 298d6ca..4f02dda 100644 --- a/miniraft/net/server.h +++ b/miniraft/net/server.h @@ -16,27 +16,61 @@ template class TMessageReader { public: - TMessageReader(TSocket& socket) - : Socket(socket) - { } - - NNet::TValueTask> Read(); + TMessageReader(TSocket& socket, TMessageLimits limits = {}) + : Socket(socket), Limits(limits) { ValidateMessageLimits(Limits); } + + NNet::TValueTask> Read() { + if (Failed) throw TMessageError("Cannot reuse a failed frame reader"); + try { + uint64_t remaining = Limits.MaxBatchBytes; + co_return co_await ReadFrame(false, remaining); + } catch (...) { + Failed = true; + throw; + } + } private: + NNet::TValueTask> ReadFrame(bool logOnly, uint64_t& remaining) { + TMessage header; + co_await NNet::TByteReader(Socket).Read(&header, sizeof(header)); + ValidateMessageHeader(header.Type, header.Len, Limits); + if (logOnly && header.Type != uint32_t(EMessageType::LOG_ENTRY)) + throw TMessageError("AppendEntries children must be log entries"); + if (header.Len > remaining) throw TMessageError("Message batch exceeds byte limit"); + remaining -= header.Len; + auto message = NewHoldedMessage(header.Type, header.Len); + co_await NNet::TByteReader(Socket).Read(message->Value, header.Len - sizeof(TMessage)); + if (header.Type == uint32_t(EMessageType::APPEND_ENTRIES_REQUEST)) { + const auto count = message.template Cast()->Nentries; + if (count > Limits.MaxEntries || uint64_t(count) * sizeof(TLogEntry) > remaining) + throw TMessageError("AppendEntries count exceeds limits"); + message.InitPayload(count); + for (uint32_t i = 0; i < count; ++i) + message.Payload[i] = co_await ReadFrame(true, remaining); + } + co_return message; + } TSocket& Socket; + TMessageLimits Limits; + bool Failed = false; }; template class TMessageWriter { public: - TMessageWriter(TSocket& socket) - : Socket(socket) - { } - - NNet::TValueTask Write(TMessageHolder message); - + TMessageWriter(TSocket& socket, TMessageLimits limits = {}) + : Socket(socket), Limits(limits) { ValidateMessageLimits(Limits); } + + NNet::TValueTask Write(TMessageHolder message) { + ValidateMessageTree(message, Limits); + co_await NNet::TByteWriter(Socket).Write(message.Mes, message->Len); + for (uint32_t i = 0; i < message.PayloadSize; ++i) + co_await Write(message.Payload[i]); + } private: TSocket& Socket; + TMessageLimits Limits; }; struct THost { @@ -87,6 +121,8 @@ class TNode: public INode { void Send(TMessageHolder message) override; void Drain() override; + // Called after a failed outbound read, once its reader has unwound. + void Disconnect(); bool IsConnected() { return Connected; } TSocket& Sock() { return Socket; diff --git a/miniraft/raft.cpp b/miniraft/raft.cpp index eb3f21f..81b586f 100644 --- a/miniraft/raft.cpp +++ b/miniraft/raft.cpp @@ -298,6 +298,18 @@ TMessageHolder TRaft::CreateAppendEntries(uint32_t nodeId lastIndex = prevIndex; } + const TMessageLimits limits; + uint64_t bytes = sizeof(TAppendEntriesRequest); + std::vector> entries; + for (auto i = prevIndex + 1; i <= lastIndex && entries.size() < limits.MaxEntries; ++i) { + auto entry = State->Get(i); + ValidateMessageHeader(entry->Type, entry->Len, limits); + if (entry->Len > limits.MaxBatchBytes - bytes) break; + bytes += entry->Len; + entries.push_back(std::move(entry)); + } + lastIndex = prevIndex + entries.size(); + auto mes = NewHoldedMessage( TMessageEx {.Src = Id, .Dst = nodeId, .Term = State->CurrentTerm, .Seqno = NextSeqno()}, TAppendEntriesRequest { @@ -308,12 +320,9 @@ TMessageHolder TRaft::CreateAppendEntries(uint32_t nodeId .Nentries = static_cast(lastIndex - prevIndex), }); - if (lastIndex - prevIndex > 0) { - mes.InitPayload(lastIndex - prevIndex); - uint32_t j = 0; - for (auto i = prevIndex+1; i <= lastIndex; i++) { - mes.Payload[j++] = State->Get(i); - } + if (!entries.empty()) { + mes.InitPayload(entries.size()); + for (uint32_t i = 0; i < entries.size(); ++i) mes.Payload[i] = entries[i]; } VolatileState->LastSentSeqno[nodeId] = mes->Seqno; return mes; @@ -512,6 +521,9 @@ ITimeSource::Time TRaft::MakeElection(ITimeSource::Time now) { } uint64_t TRaft::Append(TMessageHolder entry) { + if (!entry || entry->Type != uint32_t(EMessageType::LOG_ENTRY)) + throw TMessageError("Invalid proposed log entry"); + ValidateMessageHeader(entry->Type, entry->Len); entry->Term = State->CurrentTerm; State->Append(std::move(entry)); return State->LastLogIndex; diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 998def8..86b796e 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -15,11 +15,13 @@ if (CMOCKA_FOUND) ut(test_raft test_raft.cpp miniraft) ut(test_correctness test_correctness.cpp miniraft) ut(test_forwarding test_forwarding.cpp miniraft) +ut(test_message_safety test_message_safety.cpp miniraft) ut(test_persist_recovery test_persist_recovery.cpp miniraft) if(CMAKE_SYSTEM_NAME STREQUAL "Linux") ut(test_durable test_durable.cpp miniraft) endif() if(MINIRAFT_BUILD_NETWORK) ut(test_read_write test_read_write.cpp miniraft.net) +ut(test_network_frames test_network_frames.cpp miniraft.net) endif() endif () diff --git a/test/test_message_safety.cpp b/test/test_message_safety.cpp new file mode 100644 index 0000000..fd58c1b --- /dev/null +++ b/test/test_message_safety.cpp @@ -0,0 +1,95 @@ +#include +#include +#include +#include +#include +extern "C" { +#include +} + +namespace { +void test_typed_allocations_reject_undersized_storage(void**) { + for (uint32_t size : {0u, 7u, 8u, uint32_t(sizeof(TRequestVoteRequest) - 1)}) { + for (bool explicitType : {false, true}) { + bool rejected = false; + try { + if (explicitType) NewHoldedMessage(uint32_t(EMessageType::REQUEST_VOTE_REQUEST), size); + else NewHoldedMessage(size); + } catch (const std::invalid_argument&) { rejected = true; } + assert_true(rejected); + } + } +} +void test_short_wrong_type_and_empty_casts_are_rejected(void**) { + auto shortFrame = NewHoldedMessage(uint32_t(EMessageType::REQUEST_VOTE_REQUEST), sizeof(TMessage)); + auto wrongType = NewHoldedMessage(uint32_t(EMessageType::LOG_ENTRY), sizeof(TRequestVoteRequest)); + for (auto message : {shortFrame, wrongType, TMessageHolder{}}) { + assert_false(message.IsEx()); + assert_false(message.Maybe()); + bool rejected = false; + try { message.Cast(); } catch (const std::bad_cast&) { rejected = true; } + assert_true(rejected); + } +} +struct TEndpoint : INode { + std::vector> Sent; + void Send(TMessageHolder message) override { + if (message->Type == uint32_t(EMessageType::APPEND_ENTRIES_REQUEST)) + Sent.push_back(message.Cast()); + } + void Drain() override {} +}; +void test_raft_batches_respect_budget_and_continue_replication(void**) { + auto state = std::make_shared(); + state->CurrentTerm = 2; + auto entry = NewHoldedMessage(1024 * 1024); + entry->Term = 2; + for (int i = 0; i < 100; ++i) state->Append(entry); + auto peer = std::make_shared(); + TRaft raft(state, 1, {{2, peer}}); + raft.Become(EState::LEADER); + auto* progress = const_cast(raft.GetVolatileState()); + progress->BatchSize[2] = 1024; + raft.ProcessTimeout(ITimeSource::Time{}); + assert_int_equal(peer->Sent.size(), 1); + auto first = peer->Sent[0]; + ValidateMessageTree(first); + assert_true(first->Nentries > 0); + assert_true(first->Nentries < 100); + assert_int_equal(first->Nentries, first.PayloadSize); + auto reply = NewHoldedMessage(TMessageEx{.Src = 2, .Dst = 1, .Term = 2, .Seqno = first->Seqno}, + TAppendEntriesResponse{.MatchIndex = first->Nentries, .Success = 1}); + raft.Process(ITimeSource::Time{}, reply); + raft.ProcessTimeout(ITimeSource::Time{}); + assert_int_equal(peer->Sent.size(), 2); + auto second = peer->Sent[1]; + ValidateMessageTree(second); + assert_int_equal(second->PrevLogIndex, first->Nentries); + assert_int_equal(first->Nentries + second->Nentries, 100); + assert_true(second->Nentries > 0); +} +void test_invalid_proposal_does_not_mutate_persistent_state(void**) { + auto state = std::make_shared(); + TRaft raft(state, 1, {}); + raft.Become(EState::LEADER); + for (uint32_t length : {0u, uint32_t(sizeof(TLogEntry) - 1), TMessageLimits{}.MaxFrameBytes + 1}) { + auto entry = NewHoldedMessage(); + entry->Len = length; + entry->Term = 99; + bool rejected = false; + try { raft.Append(entry); } catch (const TMessageError&) { rejected = true; } + assert_true(rejected); + assert_int_equal(state->LastLogIndex, 0); + assert_int_equal(entry->Term, 99); + } +} +} +int main() { + const CMUnitTest tests[] = { + cmocka_unit_test(test_typed_allocations_reject_undersized_storage), + cmocka_unit_test(test_short_wrong_type_and_empty_casts_are_rejected), + cmocka_unit_test(test_raft_batches_respect_budget_and_continue_replication), + cmocka_unit_test(test_invalid_proposal_does_not_mutate_persistent_state), + }; + return cmocka_run_group_tests(tests, nullptr, nullptr); +} diff --git a/test/test_network_frames.cpp b/test/test_network_frames.cpp new file mode 100644 index 0000000..40b43a7 --- /dev/null +++ b/test/test_network_frames.cpp @@ -0,0 +1,241 @@ +#include +#include +#include +#include +#include +extern "C" { +#include +} + +namespace { +// Immediate deterministic I/O: no sockets, sleeps, pollers or wall clock. +struct TStream { + std::vector Bytes; + size_t Position = 0; + size_t Chunk = SIZE_MAX; + size_t MaxRequested = 0; + unsigned Reads = 0; + std::vector Output; + NNet::TValueTask ReadSome(void* out, size_t length) { + ++Reads; + MaxRequested = std::max(MaxRequested, length); + const auto count = std::min({length, Chunk, Bytes.size() - Position}); + if (count) memcpy(out, Bytes.data() + Position, count); + Position += count; + co_return count; + } + NNet::TValueTask WriteSome(const void* data, size_t length) { + const auto count = std::min(length, Chunk); + auto bytes = static_cast(data); + Output.insert(Output.end(), bytes, bytes + count); + co_return count; + } + void Add(uint32_t type, uint32_t length) { + const char* a = reinterpret_cast(&type); + const char* b = reinterpret_cast(&length); + Bytes.insert(Bytes.end(), a, a + sizeof(type)); + Bytes.insert(Bytes.end(), b, b + sizeof(length)); + } + template void Add(TMessageHolder message) { + auto data = reinterpret_cast(message.Mes); + Bytes.insert(Bytes.end(), data, data + message->Len); + } +}; +TMessageHolder Read(TStream& stream) { + auto task = TMessageReader(stream).Read(); + assert_true(task.done()); + return task.await_resume(); +} +bool Rejected(TStream& stream) { + try { Read(stream); } catch (const std::runtime_error&) { return true; } + return false; +} + +void test_fragmented_header_and_body_are_read_exactly(void**) { + TStream stream; + stream.Chunk = 1; + auto original = NewHoldedMessage(TRequestVoteRequest{.LastLogIndex = 7, .LastLogTerm = 2, .CandidateId = 1}); + stream.Add(original); + bool complete = false; + try { + auto received = Read(stream).Cast(); + assert_int_equal(received->LastLogIndex, 7); + assert_int_equal(received->LastLogTerm, 2); + assert_int_equal(stream.Position, original->Len); + complete = true; + } catch (const std::runtime_error&) {} + assert_true(complete); +} +void test_short_fixed_frame_is_rejected(void**) { + TStream stream; + stream.Add(uint32_t(EMessageType::REQUEST_VOTE_REQUEST), sizeof(TMessage)); + assert_true(Rejected(stream)); +} +void test_unknown_and_unimplemented_types_are_rejected(void**) { + for (auto type : {0u, 6u, 7u, 10u, UINT32_MAX}) { + TStream stream; + stream.Add(type, sizeof(TMessage)); + assert_true(Rejected(stream)); + } +} +void test_append_entries_rejects_non_log_child(void**) { + TStream stream; + stream.Add(NewHoldedMessage(TAppendEntriesRequest{.Nentries = 1})); + stream.Add(NewHoldedMessage()); + assert_true(Rejected(stream)); +} +void test_append_entries_rejects_nested_append(void**) { + TStream stream; + stream.Add(NewHoldedMessage(TAppendEntriesRequest{.Nentries = 1})); + stream.Add(NewHoldedMessage()); + assert_true(Rejected(stream)); +} +void test_oversized_frame_is_rejected_before_body_read(void**) { + TStream stream; + stream.Add(uint32_t(EMessageType::COMMAND_REQUEST), 16 * 1024 * 1024 + 1); + assert_true(Rejected(stream)); + assert_true(stream.MaxRequested <= sizeof(TMessage)); +} +void test_all_truncated_headers_are_rejected(void**) { + for (size_t size = 0; size < sizeof(TMessage); ++size) { + TStream stream; + stream.Add(uint32_t(EMessageType::COMMAND_REQUEST), sizeof(TCommandRequest)); + stream.Bytes.resize(size); + stream.Chunk = 1; + assert_true(Rejected(stream)); + } +} +void test_invalid_lengths_are_rejected_before_body_read(void**) { + for (auto length : {0u, 1u, 7u, 8u, 15u, UINT32_MAX}) { + TStream stream; + stream.Add(uint32_t(EMessageType::COMMAND_REQUEST), length); + assert_true(Rejected(stream)); + assert_int_equal(stream.Reads, 1); + } + TStream stream; + stream.Add(uint32_t(EMessageType::REQUEST_VOTE_REQUEST), sizeof(TRequestVoteRequest) + 1); + assert_true(Rejected(stream)); + assert_int_equal(stream.Reads, 1); +} +void test_all_truncated_bodies_are_rejected(void**) { + auto message = NewHoldedMessage(); + for (size_t size = sizeof(TMessage); size < message->Len; ++size) { + TStream stream; + stream.Add(message); + stream.Bytes.resize(size); + stream.Chunk = 3; + assert_true(Rejected(stream)); + } +} +void test_count_and_minimum_batch_budget_are_checked_before_children(void**) { + for (auto count : {4097u, UINT32_MAX}) { + TStream stream; + stream.Add(NewHoldedMessage(TAppendEntriesRequest{.Nentries = count})); + assert_true(Rejected(stream)); + assert_int_equal(stream.Reads, 2); + } + TStream stream; + stream.Add(NewHoldedMessage(TAppendEntriesRequest{.Nentries = 1})); + TMessageLimits limits; + limits.MaxBatchBytes = sizeof(TAppendEntriesRequest) + sizeof(TLogEntry) - 1; + auto task = TMessageReader(stream, limits).Read(); + bool rejected = false; + try { task.await_resume(); } catch (const TMessageError&) { rejected = true; } + assert_true(rejected); + assert_int_equal(stream.Reads, 2); +} +void test_aggregate_budget_is_shared_between_entries(void**) { + for (bool enough : {false, true}) { + TStream stream; + stream.Add(NewHoldedMessage(TAppendEntriesRequest{.Nentries = 2})); + stream.Add(NewHoldedMessage(sizeof(TLogEntry) + 1)); + stream.Add(NewHoldedMessage(sizeof(TLogEntry) + 1)); + TMessageLimits limits; + limits.MaxBatchBytes = stream.Bytes.size() - (enough ? 0 : 1); + auto task = TMessageReader(stream, limits).Read(); + bool rejected = false; + try { + auto message = task.await_resume(); + assert_int_equal(message.PayloadSize, 2); + } catch (const TMessageError&) { rejected = true; } + assert_int_equal(rejected, !enough); + if (!enough) assert_int_equal(stream.Position, + sizeof(TAppendEntriesRequest) + sizeof(TLogEntry) + 1 + sizeof(TMessage)); + } +} +void test_failed_reader_cannot_resume_at_another_frame(void**) { + TStream stream; + stream.Add(10, sizeof(TMessage)); + stream.Add(NewHoldedMessage()); + TMessageReader reader(stream); + for (int attempt = 0; attempt < 2; ++attempt) { + auto task = reader.Read(); + bool rejected = false; + try { task.await_resume(); } catch (const TMessageError&) { rejected = true; } + assert_true(rejected); + assert_int_equal(stream.Position, sizeof(TMessage)); + } +} +void test_writer_validates_entire_batch_before_first_byte(void**) { + for (int invalid = 0; invalid < 7; ++invalid) { + auto root = NewHoldedMessage(TAppendEntriesRequest{.Nentries = 1}); + root.InitPayload(1); + root.Payload[0] = NewHoldedMessage(); + TMessageLimits limits; + if (invalid == 0) root->Nentries = 2; + if (invalid == 1) root.Payload[0] = NewHoldedMessage(); + if (invalid == 2) root.Payload[0] = NewHoldedMessage(); + if (invalid == 3) limits.MaxBatchBytes = sizeof(TAppendEntriesRequest) + sizeof(TLogEntry) - 1; + if (invalid == 4) root.Payload.reset(); + if (invalid == 5) root.Payload[0] = {}; + if (invalid == 6) root.Payload[0].InitPayload(1); + TStream stream; + auto task = TMessageWriter(stream, limits).Write(root); + bool rejected = false; + try { task.await_resume(); } catch (const TMessageError&) { rejected = true; } + assert_true(rejected); + assert_true(stream.Output.empty()); + } +} +void test_fragmented_roundtrip_keeps_consecutive_frame_boundaries(void**) { + auto root = NewHoldedMessage(TAppendEntriesRequest{.Nentries = 2}); + root.InitPayload(2); + root.Payload[0] = NewHoldedMessage(); + root.Payload[1] = NewHoldedMessage(sizeof(TLogEntry) + 3); + memcpy(root.Payload[1].Cast()->Data, "abc", 3); + TStream stream; + stream.Chunk = 1; + auto batch = TMessageWriter(stream).Write(root); + batch.await_resume(); + auto command = TMessageWriter(stream).Write(NewHoldedMessage()); + command.await_resume(); + stream.Bytes = stream.Output; + auto received = Read(stream); + assert_int_equal(received.PayloadSize, 2); + auto entry = received.Payload[1].Cast(); + assert_memory_equal(entry->Data, "abc", 3); + auto next = Read(stream).Cast(); + assert_int_equal(next->Len, sizeof(TCommandRequest)); + assert_int_equal(stream.Position, stream.Bytes.size()); +} +} + +int main() { + const CMUnitTest tests[] = { + cmocka_unit_test(test_fragmented_header_and_body_are_read_exactly), + cmocka_unit_test(test_short_fixed_frame_is_rejected), + cmocka_unit_test(test_unknown_and_unimplemented_types_are_rejected), + cmocka_unit_test(test_append_entries_rejects_non_log_child), + cmocka_unit_test(test_append_entries_rejects_nested_append), + cmocka_unit_test(test_oversized_frame_is_rejected_before_body_read), + cmocka_unit_test(test_all_truncated_headers_are_rejected), + cmocka_unit_test(test_invalid_lengths_are_rejected_before_body_read), + cmocka_unit_test(test_all_truncated_bodies_are_rejected), + cmocka_unit_test(test_count_and_minimum_batch_budget_are_checked_before_children), + cmocka_unit_test(test_aggregate_budget_is_shared_between_entries), + cmocka_unit_test(test_failed_reader_cannot_resume_at_another_frame), + cmocka_unit_test(test_writer_validates_entire_batch_before_first_byte), + cmocka_unit_test(test_fragmented_roundtrip_keeps_consecutive_frame_boundaries), + }; + return cmocka_run_group_tests(tests, nullptr, nullptr); +} diff --git a/test/test_raft.cpp b/test/test_raft.cpp index eb66e0b..2b219fd 100644 --- a/test/test_raft.cpp +++ b/test/test_raft.cpp @@ -425,9 +425,9 @@ void test_follower_append_entries_empty_to_empty_log(void**) { } void test_candidate_initiate_election(void**) { - std::vector> messages; + std::vector> messages; auto onSend = [&](auto message) { - messages.emplace_back(message.template Cast()); + messages.emplace_back(message.template Cast()); }; auto ts = std::make_shared(); auto raft = MakeRaft(onSend, 3); From cbbbfa1d6125b9084f0ab124d61690e588179e16 Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Sun, 8 Nov 2026 13:49:32 -0500 Subject: [PATCH 08/66] Validate consensus RPCs and correlate bounded replication acknowledgements (#8) --- README.md | 2 + docs/consensus-validation.md | 41 +++++ miniraft/raft.cpp | 71 ++++++++- miniraft/raft.h | 7 +- test/CMakeLists.txt | 1 + test/test_raft.cpp | 20 +-- test/test_rpc_validation.cpp | 283 +++++++++++++++++++++++++++++++++++ 7 files changed, 409 insertions(+), 16 deletions(-) create mode 100644 docs/consensus-validation.md create mode 100644 test/test_rpc_validation.cpp diff --git a/README.md b/README.md index 95115b6..a0283a7 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,8 @@ The [forwarding follow-up](docs/forwarding.md) preserves client cookies and bind forwarded replies to their original leader, term and outbound peer. The [network framing follow-up](docs/network-frames.md) handles fragmented input and enforces type, frame, batch and AppendEntries payload bounds. +The [consensus validation follow-up](docs/consensus-validation.md) checks RPC +identities and indices, protects committed prefixes and correlates acknowledgements. [![Number of GitHub stars](https://img.shields.io/github/stars/resetius/miniraft-cpp)](https://github.com/resetius/miniraft-cpp/stargazers) ![GitHub commit activity](https://img.shields.io/github/commit-activity/m/resetius/miniraft-cpp) diff --git a/docs/consensus-validation.md b/docs/consensus-validation.md new file mode 100644 index 0000000..7a77536 --- /dev/null +++ b/docs/consensus-validation.md @@ -0,0 +1,41 @@ +# Consensus RPC validation: v0.1.0-alpha.8 + +Validate complete RPCs before changing terms, votes or logs. Unknown senders, +wrong destinations, inconsistent leader/candidate IDs, invalid booleans, +impossible log metadata and malformed payloads are ignored. IDs must be positive; +peer configuration cannot include self, zero or null endpoints. + +Log indices are limited to INT64_MAX to match IState's signed indexing API. +AppendEntries checks index addition and nonzero, nondecreasing entry terms no +greater than the RPC term. The whole batch is checked before mutation, including +conflicts with committed entries. Follower commit advancement is clamped to the +prefix matched by that RPC, while retaining already-established commit progress. +SetCommitIndex preserves uint64_t width. Election/log-index exhaustion raises an +overflow error before wrapping or changing the vote/log. + +Each peer retains at most 256 outstanding AppendEntries sequence/end-index +records. Success must identify a retained RPC and exactly its sent end index. +Processing a response discards that record and older records, preventing duplicate +or reordered failures from repeating backoff. NextIndex never falls below known +MatchIndex+1. Evicted replies are ignored; newer retained replies/retries can +still make progress. Very delayed peers under sustained request load can lose +old correlation records; admission control remains future work. + +Eighteen deterministic tests cover identity, index/term bounds, false/unsent +acknowledgements, reordering, duplicates, follower commit bounds, committed-prefix +protection, batch atomic rejection, configuration, bounded response history and +counter exhaustion. All initial fourteen fail against alpha.7. Existing fixtures +with log terms greater than their RPC term, or mismatched candidate/sender IDs, +were corrected while preserving their intended reconciliation/election cases. + +All 130 tests pass locally under ASan/UBSan, with core Debug/Release verification. +Hosted CI runs the same targets. Self-review checked validation ordering, bounded +correlation ownership, prefix safety, full-width arithmetic and progress after +rejection/eviction. No independent approval is claimed. + +These are consistency checks, not peer authentication or Byzantine tolerance: +an untrusted connection can still claim a configured sender ID. Authentication, +application payload validation, deadlines, admission limits, retry deduplication +and broader generated histories remain open. Custom adapters must preserve RPC +sequence numbers and destination IDs. The wire format is unchanged; stricter +validation intentionally rejects formerly tolerated invalid messages. diff --git a/miniraft/raft.cpp b/miniraft/raft.cpp index 81b586f..36add82 100644 --- a/miniraft/raft.cpp +++ b/miniraft/raft.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include @@ -129,7 +130,7 @@ TVolatileState& TVolatileState::SetLeaderId(uint32_t id) { return *this; } -TVolatileState& TVolatileState::SetCommitIndex(int index) +TVolatileState& TVolatileState::SetCommitIndex(uint64_t index) { CommitIndex = index; return *this; @@ -145,7 +146,10 @@ TRaft::TRaft(std::shared_ptr state, int node, const TNodeDict& nodes) , VolatileState(std::make_unique()) , StateName(EState::FOLLOWER) { - for (auto [id, _] : Nodes) { + if (node <= 0 || !State) throw std::invalid_argument("Raft requires a positive ID and state"); + for (const auto& [id, peer] : Nodes) { + if (id == 0 || id == Id || !peer) + throw std::invalid_argument("Invalid Raft peer configuration"); VolatileState->NextIndex[id] = 1; } } @@ -230,7 +234,7 @@ void TRaft::OnAppendEntries(ITimeSource::Time now, TMessageHolderLeaderCommit); + commitIndex = std::max(commitIndex, std::min(message->LeaderCommit, matchIndex)); } auto reply = NewHoldedMessage( @@ -253,6 +257,17 @@ void TRaft::OnAppendEntries(TMessageHolder message) { auto nodeId = message->Src; if (!Nodes.contains(nodeId)) return; + auto pending = VolatileState->PendingAppendRpcs.find(nodeId); + if (pending == VolatileState->PendingAppendRpcs.end()) return; + auto& requests = pending->second; + auto request = std::find_if(requests.begin(), requests.end(), [&](const auto& rpc) { + return rpc.Seqno == message->Seqno; + }); + if (request == requests.end() || + (message->Success && message->MatchIndex != request->LastIndex)) return; + // A newer response supersedes older RPCs. Reordered/duplicate failures + // must not undo progress or repeatedly increase backoff. + requests.erase(requests.begin(), std::next(request)); if (message->Success) { if (message->Seqno > 0 && message->Seqno <= VolatileState->LastSentSeqno[nodeId]) { auto& acknowledged = VolatileState->ReadAckSeqno[nodeId]; @@ -261,7 +276,7 @@ void TRaft::OnAppendEntries(TMessageHolder message) { auto matchIndex = std::max(VolatileState->MatchIndex[nodeId], message->MatchIndex); (*VolatileState) .SetMatchIndex(nodeId, matchIndex) - .SetNextIndex(nodeId, message->MatchIndex+1) + .SetNextIndex(nodeId, matchIndex+1) .SetRpcDue(nodeId, ITimeSource::Time{}) .SetBatchSize(nodeId, 1024) .SetBackOff(nodeId, 1) @@ -272,7 +287,7 @@ void TRaft::OnAppendEntries(TMessageHolder message) { ? VolatileState->NextIndex[nodeId] - backOff : 0; (*VolatileState) - .SetNextIndex(nodeId, std::max((uint64_t)1, nextIndex)) + .SetNextIndex(nodeId, std::max(VolatileState->MatchIndex[nodeId] + 1, nextIndex)) .SetRpcDue(nodeId, ITimeSource::Time{}) .SetBatchSize(nodeId, 1) .SetBackOff(nodeId, std::min(32768, backOff << 1)); @@ -325,6 +340,9 @@ TMessageHolder TRaft::CreateAppendEntries(uint32_t nodeId for (uint32_t i = 0; i < entries.size(); ++i) mes.Payload[i] = entries[i]; } VolatileState->LastSentSeqno[nodeId] = mes->Seqno; + auto& pending = VolatileState->PendingAppendRpcs[nodeId]; + pending.push_back({mes->Seqno, lastIndex}); + if (pending.size() > TVolatileState::MaxPendingAppendRpcs) pending.pop_front(); return mes; } @@ -363,6 +381,7 @@ void TRaft::Become(EState newStateName) { } void TRaft::Process(ITimeSource::Time now, TMessageHolder message, const std::shared_ptr& replyTo) { + if (!ValidateRpc(message)) return; if (message.IsEx()) { auto messageEx = message.Cast(); if (messageEx->Term > State->CurrentTerm) { @@ -392,6 +411,46 @@ void TRaft::Process(ITimeSource::Time now, TMessageHolder message, con } } +bool TRaft::ValidateRpc(TMessageHolder message) const { + if (!message || !message.IsEx()) return false; + try { ValidateMessageTree(message); } catch (const TMessageError&) { return false; } + auto header = message.Cast(); + if (header->Dst != Id || !Nodes.contains(header->Src)) return false; + constexpr uint64_t maxIndex = INT64_MAX; + switch (EMessageType(message->Type)) { + case EMessageType::REQUEST_VOTE_REQUEST: { + auto vote = message.Cast(); + return vote->CandidateId == header->Src && vote->LastLogIndex <= maxIndex && + vote->LastLogTerm <= header->Term && + ((vote->LastLogIndex == 0) == (vote->LastLogTerm == 0)); + } + case EMessageType::REQUEST_VOTE_RESPONSE: + return message.Cast()->VoteGranted <= 1; + case EMessageType::APPEND_ENTRIES_RESPONSE: { + auto reply = message.Cast(); + return reply->Success <= 1 && reply->MatchIndex <= maxIndex && + (reply->Success ? reply->MatchIndex <= State->LastLogIndex : reply->MatchIndex == 0); + } + case EMessageType::APPEND_ENTRIES_REQUEST: { + auto append = message.Cast(); + if (append->LeaderId != header->Src || append->PrevLogIndex > maxIndex || + append->LeaderCommit > maxIndex || append->Nentries > maxIndex - append->PrevLogIndex || + append->PrevLogTerm > header->Term || + ((append->PrevLogIndex == 0) != (append->PrevLogTerm == 0))) return false; + auto previousTerm = append->PrevLogTerm; + for (uint32_t i = 0; i < message.PayloadSize; ++i) { + auto entry = message.Payload[i].Cast(); + if (!entry->Term || entry->Term < previousTerm || entry->Term > header->Term) return false; + previousTerm = entry->Term; + const auto index = append->PrevLogIndex + i + 1; + if (index <= VolatileState->CommitIndex && State->LogTerm(index) != entry->Term) return false; + } + return true; + } + default: return false; + } +} + void TRaft::FollowerTimeout(ITimeSource::Time now) { if (VolatileState->ElectionDue <= now) { Become(EState::CANDIDATE); @@ -456,6 +515,7 @@ uint64_t TRaft::NextSeqno() { void TRaft::ProcessTimeout(ITimeSource::Time now) { if (StateName == EState::CANDIDATE || StateName == EState::FOLLOWER) { if (VolatileState->ElectionDue <= now) { + if (State->CurrentTerm == UINT64_MAX) throw std::overflow_error("Raft term exhausted"); auto nextVolatileState = std::make_unique(); for (auto [id, _] : Nodes) { nextVolatileState->NextIndex[id] = 1; @@ -521,6 +581,7 @@ ITimeSource::Time TRaft::MakeElection(ITimeSource::Time now) { } uint64_t TRaft::Append(TMessageHolder entry) { + if (State->LastLogIndex >= uint64_t(INT64_MAX)) throw std::overflow_error("Raft log index exhausted"); if (!entry || entry->Type != uint32_t(EMessageType::LOG_ENTRY)) throw TMessageError("Invalid proposed log entry"); ValidateMessageHeader(entry->Type, entry->Len); diff --git a/miniraft/raft.h b/miniraft/raft.h index 9865b10..cb70861 100644 --- a/miniraft/raft.h +++ b/miniraft/raft.h @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -40,10 +41,13 @@ struct TDummyRsm: public IRsm { using TNodeDict = std::unordered_map>; struct TVolatileState { + struct TAppendRpc { uint64_t Seqno, LastIndex; }; + static constexpr size_t MaxPendingAppendRpcs = 256; uint64_t CommitIndex = 0; // Successful current-term RPC acknowledgements, separately from log indices. std::unordered_map ReadAckSeqno; std::unordered_map LastSentSeqno; + std::unordered_map> PendingAppendRpcs; uint32_t LeaderId = 0; std::unordered_map NextIndex; std::unordered_map MatchIndex; @@ -58,7 +62,7 @@ struct TVolatileState { TVolatileState& Vote(uint32_t id); TVolatileState& CommitAdvance(int nservers, const IState& state); - TVolatileState& SetCommitIndex(int index); + TVolatileState& SetCommitIndex(uint64_t index); TVolatileState& SetElectionDue(ITimeSource::Time); TVolatileState& SetNextIndex(uint32_t id, uint64_t nextIndex); TVolatileState& SetMatchIndex(uint32_t id, uint64_t matchIndex); @@ -146,6 +150,7 @@ class TRaft { TMessageHolder CreateAppendEntries(uint32_t nodeId); ITimeSource::Time MakeElection(ITimeSource::Time now); uint64_t NextSeqno(); + bool ValidateRpc(TMessageHolder message) const; uint32_t Id; TNodeDict Nodes; diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 86b796e..48c87fe 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -16,6 +16,7 @@ ut(test_raft test_raft.cpp miniraft) ut(test_correctness test_correctness.cpp miniraft) ut(test_forwarding test_forwarding.cpp miniraft) ut(test_message_safety test_message_safety.cpp miniraft) +ut(test_rpc_validation test_rpc_validation.cpp miniraft) ut(test_persist_recovery test_persist_recovery.cpp miniraft) if(CMAKE_SYSTEM_NAME STREQUAL "Linux") ut(test_durable test_durable.cpp miniraft) diff --git a/test/test_raft.cpp b/test/test_raft.cpp index 2b219fd..8808490 100644 --- a/test/test_raft.cpp +++ b/test/test_raft.cpp @@ -272,14 +272,14 @@ void test_follower_append_entries_7a(void**) { }; auto ts = std::make_shared(); auto raft = MakeRaft(onSend, 3, TState{ - /*.CurrentTerm =*/ 1, + /*.CurrentTerm =*/ 6, /*.VotedFor =*/ 2, /*.Log =*/ MakeLog({1,1,1,4,4,5,5,6,6}) }); auto mes = NewHoldedMessage(TMessageEx { .Src = 2, .Dst = 1, - .Term = 1, + .Term = 6, }, TAppendEntriesRequest { .PrevLogIndex = 9, .PrevLogTerm = 6, @@ -305,14 +305,14 @@ void test_follower_append_entries_7b(void**) { }; auto ts = std::make_shared(); auto raft = MakeRaft(onSend, 3, TState{ - /*.CurrentTerm =*/ 1, + /*.CurrentTerm =*/ 6, /*.VotedFor =*/ 2, /*.Log =*/ MakeLog({1,1,1,4}) }); auto mes = NewHoldedMessage(TMessageEx { .Src = 2, .Dst = 1, - .Term = 1, + .Term = 6, }, TAppendEntriesRequest { .PrevLogIndex = 4, .PrevLogTerm = 4, @@ -338,14 +338,14 @@ void test_follower_append_entries_7c(void**) { }; auto ts = std::make_shared(); auto raft = MakeRaft(onSend, 3, TState{ - /*.CurrentTerm =*/ 1, + /*.CurrentTerm =*/ 6, /*.VotedFor =*/ 2, /*.Log =*/ MakeLog({1,1,1,4,4,5,5,6,6,6,6}) }); auto mes = NewHoldedMessage(TMessageEx { .Src = 2, .Dst = 1, - .Term = 1, + .Term = 6, }, TAppendEntriesRequest { .PrevLogIndex = 9, .PrevLogTerm = 6, @@ -371,7 +371,7 @@ void test_follower_append_entries_7f(void**) { }; auto ts = std::make_shared(); auto raft = MakeRaft(onSend, 3, TState{ - /*.CurrentTerm =*/ 1, + /*.CurrentTerm =*/ 3, /*.VotedFor =*/ 2, /*.Log =*/ MakeLog({1,1,1,2,2,2,3,3,3,3,3}) }); @@ -465,8 +465,8 @@ void test_candidate_vote_request_small_term(void**) { .Dst = 1, .Term = 0, }, TRequestVoteRequest { - .LastLogIndex = 1, - .LastLogTerm = 1, + .LastLogIndex = 0, + .LastLogTerm = 0, .CandidateId = 2, }); raft->Process(ts->Now(), std::move(req)); @@ -557,7 +557,7 @@ void test_candidate_vote_after_start(void**) { // request with higher term => follower req = NewHoldedMessage(TMessageEx { - .Src = 2, + .Src = 3, .Dst = 1, .Term = 3, }, TRequestVoteRequest { diff --git a/test/test_rpc_validation.cpp b/test/test_rpc_validation.cpp new file mode 100644 index 0000000..8163d51 --- /dev/null +++ b/test/test_rpc_validation.cpp @@ -0,0 +1,283 @@ +#include +#include +#include +#include +#include +extern "C" { +#include +} + +namespace { +struct TEndpoint : INode { + std::vector> Sent; + void Send(TMessageHolder message) override { Sent.push_back(message); } + void Drain() override {} +}; +struct TFixture { + std::shared_ptr State = std::make_shared(); + std::shared_ptr Peer = std::make_shared(); + std::shared_ptr Other = std::make_shared(); + TRaft Raft{State, 1, {{2, Peer}, {3, Other}}}; + ITimeSource::Time Now{}; + TFixture() { State->CurrentTerm = 3; State->VotedFor = 1; } + void Append(uint64_t term) { + auto entry = NewHoldedMessage(); + entry->Term = term; + State->Append(entry); + } + TMessageHolder Send() { + Raft.Become(EState::LEADER); + Raft.ProcessTimeout(Now); + return Peer->Sent.back().Cast(); + } + TMessageHolder Ack(uint64_t sequence, uint64_t index, bool success = true) { + return NewHoldedMessage(TMessageEx{.Src = 2, .Dst = 1, .Term = 3, .Seqno = sequence}, + TAppendEntriesResponse{.MatchIndex = index, .Success = uint32_t(success)}); + } + TMessageHolder Request(uint64_t prev, uint64_t prevTerm, + uint64_t commit, std::initializer_list terms = {}) { + auto request = NewHoldedMessage(TMessageEx{.Src = 2, .Dst = 1, .Term = 4}, + TAppendEntriesRequest{.PrevLogIndex = prev, .PrevLogTerm = prevTerm, + .LeaderCommit = commit, .LeaderId = 2, .Nentries = uint32_t(terms.size())}); + request.InitPayload(terms.size()); + uint32_t i = 0; + for (auto term : terms) { + auto entry = NewHoldedMessage(); + entry->Term = term; + request.Payload[i++] = entry; + } + return request; + } +}; +void test_unknown_sender_cannot_change_term(void**) { + TFixture fixture; + auto response = NewHoldedMessage(TMessageEx{.Src = 99, .Dst = 1, .Term = 100}, + TRequestVoteResponse{.VoteGranted = 1}); + fixture.Raft.Process(fixture.Now, response); + assert_int_equal(fixture.State->CurrentTerm, 3); + assert_int_equal(fixture.State->VotedFor, 1); +} +void test_wrong_destination_cannot_change_term(void**) { + TFixture fixture; + auto response = NewHoldedMessage(TMessageEx{.Src = 2, .Dst = 3, .Term = 100}, + TRequestVoteResponse{.VoteGranted = 1}); + fixture.Raft.Process(fixture.Now, response); + assert_int_equal(fixture.State->CurrentTerm, 3); +} +void test_candidate_identity_must_match_sender(void**) { + TFixture fixture; + auto request = NewHoldedMessage(TMessageEx{.Src = 2, .Dst = 1, .Term = 4}, + TRequestVoteRequest{.LastLogIndex = 0, .LastLogTerm = 0, .CandidateId = 3}); + fixture.Raft.Process(fixture.Now, request); + assert_int_equal(fixture.State->CurrentTerm, 3); + assert_int_equal(fixture.State->VotedFor, 1); + assert_true(fixture.Peer->Sent.empty()); +} +void test_out_of_range_ack_cannot_wrap_next_index(void**) { + TFixture fixture; + fixture.Append(3); + auto request = fixture.Send(); + fixture.Raft.Process(fixture.Now, fixture.Ack(request->Seqno, UINT64_MAX)); + assert_int_equal(fixture.Raft.GetVolatileState()->NextIndex.at(2), 1); + assert_int_equal(fixture.Raft.GetVolatileState()->CommitIndex, 0); + fixture.Raft.Process(fixture.Now, fixture.Ack(request->Seqno, 1)); + assert_int_equal(fixture.Raft.GetVolatileState()->NextIndex.at(2), 2); +} +void test_unknown_sequence_cannot_commit(void**) { + TFixture fixture; + fixture.Append(3); + auto request = fixture.Send(); + fixture.Raft.Process(fixture.Now, fixture.Ack(request->Seqno + 100, 1)); + assert_int_equal(fixture.Raft.GetVolatileState()->CommitIndex, 0); + fixture.Raft.Process(fixture.Now, fixture.Ack(request->Seqno, 1)); + assert_int_equal(fixture.Raft.GetVolatileState()->CommitIndex, 1); +} +void test_ack_cannot_confirm_an_entry_not_in_its_rpc(void**) { + TFixture fixture; + fixture.Append(1); + fixture.Append(3); + auto request = fixture.Send(); + assert_int_equal(request->Nentries, 1); + fixture.Raft.Process(fixture.Now, fixture.Ack(request->Seqno, 2)); + assert_int_equal(fixture.Raft.GetVolatileState()->CommitIndex, 0); + assert_int_equal(fixture.Raft.GetVolatileState()->NextIndex.at(2), 1); +} +void test_old_failure_cannot_regress_new_success(void**) { + TFixture fixture; + fixture.Append(3); + auto first = fixture.Send(); + fixture.Now += TTimeout::Rpc + std::chrono::milliseconds(1); + auto second = fixture.Send(); + fixture.Raft.Process(fixture.Now, fixture.Ack(second->Seqno, 1)); + fixture.Raft.Process(fixture.Now, fixture.Ack(first->Seqno, 0, false)); + assert_int_equal(fixture.Raft.GetVolatileState()->NextIndex.at(2), 2); + assert_int_equal(fixture.Raft.GetVolatileState()->BatchSize.at(2), 1024); +} +void test_duplicate_failure_does_not_repeat_backoff(void**) { + TFixture fixture; + fixture.Append(3); + auto request = fixture.Send(); + auto response = fixture.Ack(request->Seqno, 0, false); + fixture.Raft.Process(fixture.Now, response); + auto backoff = fixture.Raft.GetVolatileState()->BackOff.at(2); + fixture.Raft.Process(fixture.Now, response); + assert_int_equal(fixture.Raft.GetVolatileState()->BackOff.at(2), backoff); +} +void test_follower_commits_only_prefix_matched_by_rpc(void**) { + TFixture fixture; + fixture.Append(1); + fixture.Append(2); + fixture.Append(2); + auto request = fixture.Request(1, 1, 9); + fixture.Raft.Process(fixture.Now, request); + assert_int_equal(fixture.Raft.GetVolatileState()->CommitIndex, 1); + assert_int_equal(fixture.State->LastLogIndex, 3); +} +void test_commit_index_preserves_full_width(void**) { + TVolatileState state; + const uint64_t index = (uint64_t(1) << 32) + 123; + state.SetCommitIndex(index); + assert_int_equal(state.CommitIndex, index); +} +void test_invalid_batch_terms_do_not_partially_mutate_log(void**) { + TFixture fixture; + fixture.Append(1); + auto request = fixture.Request(1, 1, 0, {3, 2}); + fixture.Raft.Process(fixture.Now, request); + assert_int_equal(fixture.State->CurrentTerm, 3); + assert_int_equal(fixture.State->LastLogIndex, 1); +} +void test_committed_prefix_cannot_be_replaced(void**) { + TFixture fixture; + fixture.Append(1); + fixture.Append(2); + const_cast(fixture.Raft.GetVolatileState())->CommitIndex = 2; + fixture.Raft.Process(fixture.Now, fixture.Request(1, 1, 2, {3})); + assert_int_equal(fixture.State->LogTerm(2), 2); + assert_int_equal(fixture.State->CurrentTerm, 3); + assert_int_equal(fixture.Raft.GetVolatileState()->CommitIndex, 2); +} +void test_impossible_vote_log_metadata_is_ignored(void**) { + for (int invalid = 0; invalid < 3; ++invalid) { + TFixture fixture; + auto request = NewHoldedMessage(TMessageEx{.Src = 2, .Dst = 1, .Term = 4}, + TRequestVoteRequest{.LastLogIndex = 1, .LastLogTerm = 3, .CandidateId = 2}); + if (invalid == 0) request->LastLogIndex = UINT64_MAX; + if (invalid == 1) request->LastLogTerm = 5; + if (invalid == 2) request->LastLogIndex = 0; + fixture.Raft.Process(fixture.Now, request); + assert_int_equal(fixture.State->CurrentTerm, 3); + assert_int_equal(fixture.State->VotedFor, 1); + } +} +void test_invalid_peer_configuration_is_rejected(void**) { + auto state = std::make_shared(); + auto peer = std::make_shared(); + for (int invalid = 0; invalid < 4; ++invalid) { + bool rejected = false; + try { + if (invalid == 0) { TRaft raft(state, 0, {}); } + if (invalid == 1) { TRaft raft(state, 1, {{0, peer}}); } + if (invalid == 2) { TRaft raft(state, 1, {{1, peer}}); } + if (invalid == 3) { TRaft raft(state, 1, {{2, {}}}); } + } catch (const std::invalid_argument&) { rejected = true; } + assert_true(rejected); + } +} +void test_invalid_requests_and_boolean_fields_are_ignored(void**) { + for (int invalid = 0; invalid < 8; ++invalid) { + TFixture fixture; + fixture.Append(1); + auto request = fixture.Request(1, 1, 0, {3}); + if (invalid == 0) request->Src = 99; + if (invalid == 1) request->LeaderId = 3; + if (invalid == 2) request->PrevLogIndex = UINT64_MAX; + if (invalid == 3) request->LeaderCommit = UINT64_MAX; + if (invalid == 4) request.Payload[0].Cast()->Term = 0; + if (invalid == 5) request.Payload[0].Cast()->Term = 5; + if (invalid == 6) request->Nentries = 2; + if (invalid == 7) request->PrevLogIndex = uint64_t(INT64_MAX); + fixture.Raft.Process(fixture.Now, request); + assert_int_equal(fixture.State->CurrentTerm, 3); + assert_int_equal(fixture.State->LastLogIndex, 1); + assert_true(fixture.Peer->Sent.empty()); + } + TFixture fixture; + auto vote = NewHoldedMessage(TMessageEx{.Src = 2, .Dst = 1, .Term = 100}, + TRequestVoteResponse{.VoteGranted = 2}); + fixture.Raft.Process(fixture.Now, vote); + assert_int_equal(fixture.State->CurrentTerm, 3); + auto append = fixture.Ack(1, 0); + append->Term = 100; + append->Success = 2; + fixture.Raft.Process(fixture.Now, append); + assert_int_equal(fixture.State->CurrentTerm, 3); +} +void test_old_evicted_response_is_ignored_and_latest_progresses(void**) { + TFixture fixture; + fixture.Append(3); + auto first = fixture.Send(); + for (int i = 0; i < 300; ++i) { + fixture.Now += TTimeout::Rpc + std::chrono::milliseconds(1); + fixture.Send(); + } + auto latest = fixture.Peer->Sent.back().Cast(); + fixture.Raft.Process(fixture.Now, fixture.Ack(first->Seqno, 1)); + assert_int_equal(fixture.Raft.GetVolatileState()->CommitIndex, 0); + fixture.Raft.Process(fixture.Now, fixture.Ack(latest->Seqno, 1)); + assert_int_equal(fixture.Raft.GetVolatileState()->CommitIndex, 1); +} +void test_term_exhaustion_does_not_wrap_or_change_vote(void**) { + TFixture fixture; + fixture.State->CurrentTerm = UINT64_MAX; + bool rejected = false; + try { fixture.Raft.ProcessTimeout(fixture.Now); } + catch (const std::overflow_error&) { rejected = true; } + assert_true(rejected); + assert_int_equal(fixture.State->CurrentTerm, UINT64_MAX); + assert_int_equal(fixture.State->VotedFor, 1); + assert_int_equal(fixture.Raft.CurrentStateName(), EState::FOLLOWER); +} +void test_lower_inflight_probe_cannot_regress_matched_progress(void**) { + TFixture fixture; + fixture.Append(3); + fixture.Append(3); + auto* progress = const_cast(fixture.Raft.GetVolatileState()); + progress->NextIndex[2] = 3; // newly elected leader initially probes its tail + auto first = fixture.Send(); + fixture.Now += TTimeout::Rpc + std::chrono::milliseconds(1); + auto higher = fixture.Send(); + fixture.Raft.Process(fixture.Now, fixture.Ack(first->Seqno, 0, false)); + fixture.Now += TTimeout::Rpc + std::chrono::milliseconds(1); + auto lower = fixture.Send(); + assert_int_equal(higher->PrevLogIndex, 2); + assert_int_equal(lower->PrevLogIndex, 1); + fixture.Raft.Process(fixture.Now, fixture.Ack(higher->Seqno, 2)); + fixture.Raft.Process(fixture.Now, fixture.Ack(lower->Seqno, 1)); + assert_int_equal(fixture.Raft.GetVolatileState()->MatchIndex.at(2), 2); + assert_int_equal(fixture.Raft.GetVolatileState()->NextIndex.at(2), 3); +} +} +int main() { + const CMUnitTest tests[] = { + cmocka_unit_test(test_unknown_sender_cannot_change_term), + cmocka_unit_test(test_wrong_destination_cannot_change_term), + cmocka_unit_test(test_candidate_identity_must_match_sender), + cmocka_unit_test(test_out_of_range_ack_cannot_wrap_next_index), + cmocka_unit_test(test_unknown_sequence_cannot_commit), + cmocka_unit_test(test_ack_cannot_confirm_an_entry_not_in_its_rpc), + cmocka_unit_test(test_old_failure_cannot_regress_new_success), + cmocka_unit_test(test_duplicate_failure_does_not_repeat_backoff), + cmocka_unit_test(test_follower_commits_only_prefix_matched_by_rpc), + cmocka_unit_test(test_commit_index_preserves_full_width), + cmocka_unit_test(test_invalid_batch_terms_do_not_partially_mutate_log), + cmocka_unit_test(test_committed_prefix_cannot_be_replaced), + cmocka_unit_test(test_impossible_vote_log_metadata_is_ignored), + cmocka_unit_test(test_invalid_peer_configuration_is_rejected), + cmocka_unit_test(test_invalid_requests_and_boolean_fields_are_ignored), + cmocka_unit_test(test_old_evicted_response_is_ignored_and_latest_progresses), + cmocka_unit_test(test_term_exhaustion_does_not_wrap_or_change_vote), + cmocka_unit_test(test_lower_inflight_probe_cannot_regress_matched_progress), + }; + return cmocka_run_group_tests(tests, nullptr, nullptr); +} From ca86106b60096797459cf904b263196a7eab5e78 Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Sun, 8 Nov 2026 13:58:12 -0500 Subject: [PATCH 09/66] Validate application payloads before admission and bound RSM data access (#9) --- .github/workflows/correctness.yml | 2 +- README.md | 2 + docs/application-payloads.md | 49 ++++++++ examples/CMakeLists.txt | 4 +- examples/kv.cpp | 73 +----------- examples/kv.h | 1 + examples/kv_messages.h | 22 ++++ examples/kv_rsm.cpp | 71 ++++++++++++ examples/payload_validation.h | 11 ++ examples/sql.cpp | 187 +----------------------------- examples/sql_rsm.cpp | 144 +++++++++++++++++++++++ examples/sql_rsm.h | 63 ++++++++++ miniraft/messages.h | 8 +- miniraft/raft.cpp | 26 ++++- miniraft/raft.h | 3 + test/CMakeLists.txt | 6 + test/test_app_payloads.cpp | 178 ++++++++++++++++++++++++++++ test/test_sql_payloads.cpp | 73 ++++++++++++ 18 files changed, 661 insertions(+), 262 deletions(-) create mode 100644 docs/application-payloads.md create mode 100644 examples/kv_messages.h create mode 100644 examples/kv_rsm.cpp create mode 100644 examples/payload_validation.h create mode 100644 examples/sql_rsm.cpp create mode 100644 examples/sql_rsm.h create mode 100644 test/test_app_payloads.cpp create mode 100644 test/test_sql_payloads.cpp diff --git a/.github/workflows/correctness.yml b/.github/workflows/correctness.yml index f78c895..a9f2c6c 100644 --- a/.github/workflows/correctness.yml +++ b/.github/workflows/correctness.yml @@ -10,7 +10,7 @@ jobs: build_type: [Debug, Release] steps: - uses: actions/checkout@v4 - - run: sudo apt-get update && sudo apt-get install -y libcmocka-dev + - run: sudo apt-get update && sudo apt-get install -y libcmocka-dev libsqlite3-dev - name: Configure core without networking dependencies run: >- cmake -S . -B build/core diff --git a/README.md b/README.md index a0283a7..77c65ca 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,8 @@ The [network framing follow-up](docs/network-frames.md) handles fragmented input and enforces type, frame, batch and AppendEntries payload bounds. The [consensus validation follow-up](docs/consensus-validation.md) checks RPC identities and indices, protects committed prefixes and correlates acknowledgements. +The [application payload follow-up](docs/application-payloads.md) validates KV, +SQL and dummy-RSM data before access and provides pre-admission validation hooks. [![Number of GitHub stars](https://img.shields.io/github/stars/resetius/miniraft-cpp)](https://github.com/resetius/miniraft-cpp/stargazers) ![GitHub commit activity](https://img.shields.io/github/commit-activity/m/resetius/miniraft-cpp) diff --git a/docs/application-payloads.md b/docs/application-payloads.md new file mode 100644 index 0000000..08743ff --- /dev/null +++ b/docs/application-payloads.md @@ -0,0 +1,49 @@ +# Application payload validation: v0.1.0-alpha.9 + +KV key/value lengths and SQL query lengths are checked against the actual +remaining bytes before preparing, reading or applying a command. Checks use the +payload's offset rather than sizeof(derived structure), preserving legacy native +layout/tail padding. Invalid direct application writes do not advance application +progress or mutate KV state. + +IRsm gains a default no-op const Validate hook. TRequestProcessor calls it before +queueing, preparing or forwarding requests. KV, SQL and the dummy RSM implement +their shape checks. Custom applications must supply their own validator: the +default preserves source compatibility but cannot infer custom payload formats. +Admission validation exceptions propagate to the inbound handler, closing that +client connection before the invalid operation is admitted. + +Read-time standard exceptions are returned as ErrorCode=2 with the original +cookie, allowing other requests and timer processing to continue. Exceptions +while applying a committed entry terminate the process: continuing after a +partial or invalid committed application is unsafe. This is an intentional +behavioral change for embedders whose Write callback throws. + +The dummy RSM requires a complete read index and copies only the log entry's +data into its reply, fixing a header-size error that overread by 16 bytes. +Message factories take size_t allocation sizes and reject values above UINT32_MAX +before narrowing/allocation. KV list and SQL reply construction enforce the +existing frame cap. Blank/malformed KV CLI commands no longer dereference missing +tokens or attempt to send an empty request. + +KV/SQL state machines are split from their CLI drivers so the real application +methods are directly testable without networking. SQL tests use SQLite's genuine +:memory: database; file-backed naming remains unchanged. SQLite development files +are installed in the core CI job. SQL tests are optional when SQLite is absent. + +Fifteen new tests cover malformed lengths, valid KV/SQL round trips, rejection +before admission/forwarding, no mutation/progress on rejection, oversized factory +arguments, dummy read bounds, recoverable read failures and fatal committed +application failures. All 145 tests pass locally on Linux with SQLite under +ASan/UBSan; core Release checks pass. Hosted CI executes the same targets. +The fatal-application subprocess test is excluded on Windows. + +Self-review checked validation before data access and admission, widening before +length addition, native-layout offsets, response allocation bounds, CLI tokens +and exception routing. No independent reviewer approval is claimed. + +This is payload memory-safety validation, not SQL authorization or complete state +machine correctness. SQL read-only enforcement, nondeterministic SQL, transaction +and metadata semantics, result-set memory quotas, authentication, deadlines, +admission limits and retry deduplication remain open. Callback access remains +serialized and non-reentrant. Write outcomes after interruption can be unknown. diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 6c4cc94..9fcecb3 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -14,7 +14,9 @@ target(debug_client client.cpp) target(debug_server server.cpp) target(debug_percentiles percentiles2.cpp) target(kv kv.cpp) +target_sources(kv PRIVATE kv_rsm.cpp) if (SQLite3_FOUND) target(sql sql.cpp) -endif() \ No newline at end of file +target_sources(sql PRIVATE sql_rsm.cpp) +endif() diff --git a/examples/kv.cpp b/examples/kv.cpp index 3ecabc5..227a7d4 100644 --- a/examples/kv.cpp +++ b/examples/kv.cpp @@ -6,73 +6,7 @@ #include "kv.h" -struct TKvEntry { - uint16_t KeySize = 0; - uint16_t ValSize = 0; - char Data[0]; -}; - -struct TKvLogEntry: public TLogEntry, public TKvEntry -{ -}; - -struct TWriteKv: public TCommandRequest, public TKvEntry -{ -}; - -struct TReadKv: public TCommandRequest { - uint16_t KeySize = 0; - char Data[0]; -}; - -TMessageHolder TKv::Read(TMessageHolder message, uint64_t index) { - auto readKv = message.Cast(); - if (readKv->KeySize == 0) { - std::string data; - for (auto& [k, v] : H) { - data += k + " = " + v + "\n"; - } - auto res = NewHoldedMessage(sizeof(TCommandResponse) + data.size()); - memcpy(res->Data, data.data(), data.size()); - res->Index = index; - return res; - } else { - std::string_view k(readKv->Data, readKv->KeySize); - auto it = H.find(std::string(k)); - if (it == H.end()) { - auto res = NewHoldedMessage(sizeof(TCommandResponse)); - res->Index = index; - return res; - } else { - auto res = NewHoldedMessage(sizeof(TCommandResponse)+it->second.size()); - res->Index = index; - memcpy(res->Data, it->second.data(), it->second.size()); - return res; - } - } -} - -TMessageHolder TKv::Write(TMessageHolder message, uint64_t index) { - if (LastAppliedIndex < index) { - auto entry = message.Cast(); - std::string_view k(entry->TKvEntry::Data, entry->KeySize); - if (entry->ValSize) { - std::string_view v(entry->TKvEntry::Data + entry->KeySize, entry->ValSize); - H[std::string(k)] = std::string(v); - } else { - H.erase(std::string(k)); - } - LastAppliedIndex = index; - } - return {}; -} - -TMessageHolder TKv::Prepare(TMessageHolder command) { - auto dataSize = command->Len - sizeof(TCommandRequest); - auto entry = NewHoldedMessage(sizeof(TLogEntry)+dataSize); - memcpy(entry->Data, command->Data, dataSize); - return entry; -} +#include "kv_messages.h" template NNet::TFuture Client(TPoller& poller, TSocket socket, uint32_t flags) { @@ -96,11 +30,13 @@ NNet::TFuture Client(TPoller& poller, TSocket socket, uint32_t flags) { strLine += std::string_view(line.Part1.data(), line.Part1.size()); strLine += std::string_view(line.Part2.data(), line.Part2.size()); auto prefix = strtok((char*)strLine.data(), sep); + if (!prefix) { std::cout << "> " << std::flush; continue; } TMessageHolder req; if (!strcmp(prefix, "set")) { auto key = strtok(nullptr, sep); auto val = strtok(nullptr, sep); + if (!key || !val) { std::cout << "Expected set \n"; continue; } auto keySize = strlen(key); auto valSize = strlen(val); auto mes = NewHoldedMessage(sizeof(TWriteKv) + keySize + valSize); @@ -112,6 +48,7 @@ NNet::TFuture Client(TPoller& poller, TSocket socket, uint32_t flags) { req = mes; } else if (!strcmp(prefix, "get")) { auto key = strtok(nullptr, sep); + if (!key) { std::cout << "Expected a key\n"; continue; } auto size = strlen(key); auto mes = NewHoldedMessage(sizeof(TReadKv) + size); mes->Flags = flags; @@ -124,6 +61,7 @@ NNet::TFuture Client(TPoller& poller, TSocket socket, uint32_t flags) { req = mes; } else if (!strcmp(prefix, "del")) { auto key = strtok(nullptr, sep); + if (!key) { std::cout << "Expected a key\n"; continue; } auto size = strlen(key); auto mes = NewHoldedMessage(sizeof(TWriteKv) + size); mes->Flags = TCommandRequest::EWrite; @@ -133,6 +71,7 @@ NNet::TFuture Client(TPoller& poller, TSocket socket, uint32_t flags) { req = mes; } else { std::cout << "Cannot parse command: " << strLine << "\n"; + continue; } co_await TMessageWriter(socket).Write(std::move(req)); diff --git a/examples/kv.h b/examples/kv.h index c0e337b..c584ee4 100644 --- a/examples/kv.h +++ b/examples/kv.h @@ -7,6 +7,7 @@ class TKv: public IRsm { public: + void Validate(TMessageHolder message) const override; TMessageHolder Read(TMessageHolder message, uint64_t index) override; TMessageHolder Write(TMessageHolder message, uint64_t index) override; TMessageHolder Prepare(TMessageHolder message) override; diff --git a/examples/kv_messages.h b/examples/kv_messages.h new file mode 100644 index 0000000..27b048d --- /dev/null +++ b/examples/kv_messages.h @@ -0,0 +1,22 @@ +#pragma once +#include + +struct TKvEntry { + uint16_t KeySize = 0; + uint16_t ValSize = 0; + char Data[0]; +}; + +struct TKvLogEntry: public TLogEntry, public TKvEntry +{ +}; + +struct TWriteKv: public TCommandRequest, public TKvEntry +{ +}; + +struct TReadKv: public TCommandRequest { + uint16_t KeySize = 0; + char Data[0]; +}; + diff --git a/examples/kv_rsm.cpp b/examples/kv_rsm.cpp new file mode 100644 index 0000000..061232f --- /dev/null +++ b/examples/kv_rsm.cpp @@ -0,0 +1,71 @@ +#include "kv.h" +#include "kv_messages.h" +#include "payload_validation.h" +#include + +void TKv::Validate(TMessageHolder message) const { + if (!message || message->Len < sizeof(TCommandRequest)) throw TMessageError("Short KV command"); + if (message->Flags & TCommandRequest::EWrite) { + auto write = message.Cast(); + ValidatePayloadRange(message, write->TKvEntry::Data, uint64_t(write->KeySize) + write->ValSize); + } else { + auto read = message.Cast(); + ValidatePayloadRange(message, read->Data, read->KeySize); + } +} + +TMessageHolder TKv::Read(TMessageHolder message, uint64_t index) { + auto readKv = message.Cast(); + ValidatePayloadRange(message, readKv->Data, readKv->KeySize); + if (readKv->KeySize == 0) { + std::string data; + for (auto& [k, v] : H) { + if (k.size() + v.size() + 4 > TMessageLimits{}.MaxFrameBytes - sizeof(TCommandResponse) - data.size()) + throw TMessageError("KV list response exceeds frame limit"); + data += k + " = " + v + "\n"; + } + auto res = NewHoldedMessage(sizeof(TCommandResponse) + data.size()); + memcpy(res->Data, data.data(), data.size()); + res->Index = index; + return res; + } else { + std::string_view k(readKv->Data, readKv->KeySize); + auto it = H.find(std::string(k)); + if (it == H.end()) { + auto res = NewHoldedMessage(sizeof(TCommandResponse)); + res->Index = index; + return res; + } else { + auto res = NewHoldedMessage(sizeof(TCommandResponse)+it->second.size()); + res->Index = index; + memcpy(res->Data, it->second.data(), it->second.size()); + return res; + } + } +} + +TMessageHolder TKv::Write(TMessageHolder message, uint64_t index) { + if (LastAppliedIndex < index) { + auto entry = message.Cast(); + ValidatePayloadRange(message, entry->TKvEntry::Data, uint64_t(entry->KeySize) + entry->ValSize); + std::string_view k(entry->TKvEntry::Data, entry->KeySize); + if (entry->ValSize) { + std::string_view v(entry->TKvEntry::Data + entry->KeySize, entry->ValSize); + H[std::string(k)] = std::string(v); + } else { + H.erase(std::string(k)); + } + LastAppliedIndex = index; + } + return {}; +} + +TMessageHolder TKv::Prepare(TMessageHolder command) { + auto write = command.Cast(); + ValidatePayloadRange(command, write->TKvEntry::Data, uint64_t(write->KeySize) + write->ValSize); + auto dataSize = command->Len - sizeof(TCommandRequest); + auto entry = NewHoldedMessage(sizeof(TLogEntry)+dataSize); + memcpy(entry->Data, command->Data, dataSize); + return entry; +} + diff --git a/examples/payload_validation.h b/examples/payload_validation.h new file mode 100644 index 0000000..349716e --- /dev/null +++ b/examples/payload_validation.h @@ -0,0 +1,11 @@ +#pragma once +#include + +// Derived message structs may contain tail padding. Bound from the actual +// payload offset, preserving the existing native-layout wire representation. +template +void ValidatePayloadRange(const TMessageHolder& message, const char* data, uint64_t size) { + const auto offset = size_t(data - reinterpret_cast(message.Mes)); + if (offset > message->Len || size > message->Len - offset) + throw TMessageError("Application payload length exceeds message"); +} diff --git a/examples/sql.cpp b/examples/sql.cpp index 8354293..930d9d9 100644 --- a/examples/sql.cpp +++ b/examples/sql.cpp @@ -4,192 +4,9 @@ #include #include #include +#include "payload_validation.h" -struct TSqlEntry { - uint32_t QuerySize = 0; - char Query[0]; -}; - -struct TSqlLogEntry: public TLogEntry, public TSqlEntry -{ -}; - -struct TWriteSql: public TCommandRequest, public TSqlEntry -{ -}; - -struct TReadSql: public TCommandRequest, public TSqlEntry -{ -}; - -struct TRow { - std::vector> Values; -}; - -struct TResult { - std::vector Cols; - std::vector Rows; - - void Clear() { - Cols.clear(); - Rows.clear(); - } - - bool Empty() { - return Cols.empty(); - } -}; - -class TSql: public IRsm { -public: - TSql(const std::string& path, int serverId); - ~TSql(); - - // select - TMessageHolder Read(TMessageHolder message, uint64_t index) override; - // insert, update, create - TMessageHolder Write(TMessageHolder message, uint64_t index) override; - // convert request to log message - TMessageHolder Prepare(TMessageHolder message) override; - -private: - bool Execute(const std::string& q); - static int Process(void* self, int ncols, char** values, char** cols); - TMessageHolder Reply(const std::string& ans, uint64_t index); - - TResult Result; - std::string LastError; - sqlite3* Db = nullptr; -}; - -TSql::TSql(const std::string& path, int serverId) -{ - std::string dbPath = path + "." + std::to_string(serverId); - if (sqlite3_open(dbPath.c_str(), &Db) != SQLITE_OK) { - std::cerr << "Cannot open db: `" << dbPath << "', " - << "error: " << sqlite3_errmsg(Db) - << std::endl; - throw std::runtime_error("Cannot open db"); - } - Execute(R"__(CREATE TABLE IF NOT EXISTS raft_metadata_ (key TEXT PRIMARY KEY, value TEXT))__"); - Execute(R"__(SELECT value FROM raft_metadata_ WHERE key = 'LastAppliedIndex')__"); - if (!Result.Empty()) { - LastAppliedIndex = std::stoi(*Result.Rows[0].Values[0]); - } - std::cerr << "LastAppliedIndex: " << LastAppliedIndex << std::endl; -} - -TSql::~TSql() -{ - if (Db) { - if (sqlite3_close(Db) != SQLITE_OK) { - std::cerr << "Failed to close db, error:" << sqlite3_errmsg(Db) << std::endl; - } - } -} - -int TSql::Process(void* self, int ncols, char** values, char** cols) { - TSql* this_ = (TSql*)self; - if (this_->Result.Cols.empty()) { - for (int i = 0; i < ncols; i++) { - this_->Result.Cols.emplace_back(cols[i]); - } - } - TRow row; - for (int i = 0; i < ncols; i++) { - if (values[i]) { - row.Values.emplace_back(values[i]); - } else { - row.Values.emplace_back(std::nullopt); - } - } - this_->Result.Rows.emplace_back(std::move(row)); - return 0; -} - -bool TSql::Execute(const std::string& q) { - char* err = nullptr; - std::cerr << "Execute: " << q << std::endl; - Result.Clear(); - LastError.clear(); - if (sqlite3_exec(Db, q.c_str(), Process, this, &err) != SQLITE_OK) { - std::cerr << "Cannot apply changes, error: " << err << std::endl; - LastError = err; - sqlite3_free(err); - return false; - } - std::cerr << "OK" << std::endl; - return true; -} - -TMessageHolder TSql::Read(TMessageHolder message, uint64_t index) { - auto readSql = message.Cast(); - if (!Execute(std::string(readSql->Query, readSql->QuerySize))) { - return Reply(LastError, index); - } else { - std::string text; - for (int i = 0; i < Result.Cols.size(); i++) { - text += Result.Cols[i]; - if (i != Result.Cols.size() - 1) { - text += ","; - } - } - text += "\n"; - for (int j = 0; j < Result.Rows.size(); j++) { - for (int i = 0; i < Result.Cols.size(); i++) { - text += Result.Rows[j].Values[i] ? *Result.Rows[j].Values[i] : "null"; - if (i != Result.Cols.size() - 1) { - text += ","; - } - } - text += "\n"; - } - return Reply(text, index); - } -} - -TMessageHolder TSql::Write(TMessageHolder message, uint64_t index) { - // TODO: index + 1 == LastAppliedIndex - - std::string err; - if (LastAppliedIndex < index) { - auto entry = message.Cast(); - std::cerr << "Execute write of size: " << entry->QuerySize << std::endl; - std::string updateLastApplied; - updateLastApplied += "INSERT INTO raft_metadata_ (key, value) VALUES ('LastAppliedIndex','" + std::to_string(index) + "')\n"; - updateLastApplied += "ON CONFLICT(key) DO UPDATE SET value = '" + std::to_string(index) + "';\n"; - std::string q = "BEGIN TRANSACTION;\n"; - q += std::string(entry->Query, entry->QuerySize); - if (q.back() != ';') { - q += ";\n"; - } - q += updateLastApplied; - q += "COMMIT;"; - if (Execute(q)) { - LastAppliedIndex = index; - } else { - err = LastError; - Execute("ROLLBACK;"); - Execute(updateLastApplied); // need to update LastAppliedIndex in order not to execute failed query aqain - } - } - return Reply(err, index); -} - -TMessageHolder TSql::Reply(const std::string& ans, uint64_t index) -{ - auto res = NewHoldedMessage(sizeof(TCommandResponse)+ans.size()); - res->Index = index; - memcpy(res->Data, ans.data(), ans.size()); - return res; -} - -TMessageHolder TSql::Prepare(TMessageHolder command) { - auto dataSize = command->Len - sizeof(TCommandRequest); - auto entry = NewHoldedMessage(sizeof(TLogEntry)+dataSize); - memcpy(entry->Data, command->Data, dataSize); - return entry; -} +#include "sql_rsm.h" template NNet::TFuture Client(TPoller& poller, TSocket socket) { diff --git a/examples/sql_rsm.cpp b/examples/sql_rsm.cpp new file mode 100644 index 0000000..1ac73a4 --- /dev/null +++ b/examples/sql_rsm.cpp @@ -0,0 +1,144 @@ +#include "sql_rsm.h" +#include "payload_validation.h" +#include + +void TSql::Validate(TMessageHolder message) const { + auto query = message.Cast(); + ValidatePayloadRange(message, query->Query, query->QuerySize); +} + +TSql::TSql(const std::string& path, int serverId) +{ + std::string dbPath = path == ":memory:" ? path : path + "." + std::to_string(serverId); + if (sqlite3_open(dbPath.c_str(), &Db) != SQLITE_OK) { + std::cerr << "Cannot open db: `" << dbPath << "', " + << "error: " << sqlite3_errmsg(Db) + << std::endl; + throw std::runtime_error("Cannot open db"); + } + Execute(R"__(CREATE TABLE IF NOT EXISTS raft_metadata_ (key TEXT PRIMARY KEY, value TEXT))__"); + Execute(R"__(SELECT value FROM raft_metadata_ WHERE key = 'LastAppliedIndex')__"); + if (!Result.Empty()) { + LastAppliedIndex = std::stoi(*Result.Rows[0].Values[0]); + } + std::cerr << "LastAppliedIndex: " << LastAppliedIndex << std::endl; +} + +TSql::~TSql() +{ + if (Db) { + if (sqlite3_close(Db) != SQLITE_OK) { + std::cerr << "Failed to close db, error:" << sqlite3_errmsg(Db) << std::endl; + } + } +} + +int TSql::Process(void* self, int ncols, char** values, char** cols) { + TSql* this_ = (TSql*)self; + if (this_->Result.Cols.empty()) { + for (int i = 0; i < ncols; i++) { + this_->Result.Cols.emplace_back(cols[i]); + } + } + TRow row; + for (int i = 0; i < ncols; i++) { + if (values[i]) { + row.Values.emplace_back(values[i]); + } else { + row.Values.emplace_back(std::nullopt); + } + } + this_->Result.Rows.emplace_back(std::move(row)); + return 0; +} + +bool TSql::Execute(const std::string& q) { + char* err = nullptr; + std::cerr << "Execute: " << q << std::endl; + Result.Clear(); + LastError.clear(); + if (sqlite3_exec(Db, q.c_str(), Process, this, &err) != SQLITE_OK) { + std::cerr << "Cannot apply changes, error: " << err << std::endl; + LastError = err; + sqlite3_free(err); + return false; + } + std::cerr << "OK" << std::endl; + return true; +} + +TMessageHolder TSql::Read(TMessageHolder message, uint64_t index) { + auto readSql = message.Cast(); + ValidatePayloadRange(message, readSql->Query, readSql->QuerySize); + if (!Execute(std::string(readSql->Query, readSql->QuerySize))) { + return Reply(LastError, index); + } else { + std::string text; + for (int i = 0; i < Result.Cols.size(); i++) { + text += Result.Cols[i]; + if (i != Result.Cols.size() - 1) { + text += ","; + } + } + text += "\n"; + for (int j = 0; j < Result.Rows.size(); j++) { + for (int i = 0; i < Result.Cols.size(); i++) { + text += Result.Rows[j].Values[i] ? *Result.Rows[j].Values[i] : "null"; + if (i != Result.Cols.size() - 1) { + text += ","; + } + } + text += "\n"; + } + return Reply(text, index); + } +} + +TMessageHolder TSql::Write(TMessageHolder message, uint64_t index) { + // TODO: index + 1 == LastAppliedIndex + + std::string err; + if (LastAppliedIndex < index) { + auto entry = message.Cast(); + ValidatePayloadRange(message, entry->Query, entry->QuerySize); + std::cerr << "Execute write of size: " << entry->QuerySize << std::endl; + std::string updateLastApplied; + updateLastApplied += "INSERT INTO raft_metadata_ (key, value) VALUES ('LastAppliedIndex','" + std::to_string(index) + "')\n"; + updateLastApplied += "ON CONFLICT(key) DO UPDATE SET value = '" + std::to_string(index) + "';\n"; + std::string q = "BEGIN TRANSACTION;\n"; + q += std::string(entry->Query, entry->QuerySize); + if (q.back() != ';') { + q += ";\n"; + } + q += updateLastApplied; + q += "COMMIT;"; + if (Execute(q)) { + LastAppliedIndex = index; + } else { + err = LastError; + Execute("ROLLBACK;"); + Execute(updateLastApplied); // need to update LastAppliedIndex in order not to execute failed query aqain + } + } + return Reply(err, index); +} + +TMessageHolder TSql::Reply(const std::string& ans, uint64_t index) +{ + if (ans.size() > TMessageLimits{}.MaxFrameBytes - sizeof(TCommandResponse)) + throw TMessageError("SQL response exceeds frame limit"); + auto res = NewHoldedMessage(sizeof(TCommandResponse)+ans.size()); + res->Index = index; + memcpy(res->Data, ans.data(), ans.size()); + return res; +} + +TMessageHolder TSql::Prepare(TMessageHolder command) { + auto write = command.Cast(); + ValidatePayloadRange(command, write->Query, write->QuerySize); + auto dataSize = command->Len - sizeof(TCommandRequest); + auto entry = NewHoldedMessage(sizeof(TLogEntry)+dataSize); + memcpy(entry->Data, command->Data, dataSize); + return entry; +} + diff --git a/examples/sql_rsm.h b/examples/sql_rsm.h new file mode 100644 index 0000000..ded9e92 --- /dev/null +++ b/examples/sql_rsm.h @@ -0,0 +1,63 @@ +#pragma once +#include +#include +#include + +struct TSqlEntry { + uint32_t QuerySize = 0; + char Query[0]; +}; + +struct TSqlLogEntry: public TLogEntry, public TSqlEntry +{ +}; + +struct TWriteSql: public TCommandRequest, public TSqlEntry +{ +}; + +struct TReadSql: public TCommandRequest, public TSqlEntry +{ +}; + +struct TRow { + std::vector> Values; +}; + +struct TResult { + std::vector Cols; + std::vector Rows; + + void Clear() { + Cols.clear(); + Rows.clear(); + } + + bool Empty() { + return Cols.empty(); + } +}; + +class TSql: public IRsm { +public: + void Validate(TMessageHolder message) const override; + TSql(const std::string& path, int serverId); + ~TSql(); + + // select + TMessageHolder Read(TMessageHolder message, uint64_t index) override; + // insert, update, create + TMessageHolder Write(TMessageHolder message, uint64_t index) override; + // convert request to log message + TMessageHolder Prepare(TMessageHolder message) override; + +private: + bool Execute(const std::string& q); + static int Process(void* self, int ncols, char** values, char** cols); + TMessageHolder Reply(const std::string& ans, uint64_t index); + + TResult Result; + std::string LastError; + sqlite3* Db = nullptr; +}; + diff --git a/miniraft/messages.h b/miniraft/messages.h index b8ee088..9e40d76 100644 --- a/miniraft/messages.h +++ b/miniraft/messages.h @@ -227,8 +227,8 @@ void ValidateMessageTree(const TMessageHolder& message, const TMessage template requires std::derived_from -T* NewMessage(uint32_t type, uint32_t len) { - if (len < sizeof(T)) throw std::invalid_argument("Message allocation is smaller than its type"); +T* NewMessage(uint32_t type, size_t len) { + if (len < sizeof(T) || len > UINT32_MAX) throw std::invalid_argument("Invalid message allocation size"); char* data = new char[len]; T* mes = new (data) T; mes->Type = type; @@ -238,7 +238,7 @@ T* NewMessage(uint32_t type, uint32_t len) { template requires std::derived_from -TMessageHolder NewHoldedMessage(uint32_t type, uint32_t len) +TMessageHolder NewHoldedMessage(uint32_t type, size_t len) { T* mes = NewMessage(type, len); return TMessageHolder(mes, std::shared_ptr(reinterpret_cast(mes))); @@ -250,7 +250,7 @@ TMessageHolder NewHoldedMessage() { } template -TMessageHolder NewHoldedMessage(uint32_t size) { +TMessageHolder NewHoldedMessage(size_t size) { return NewHoldedMessage(static_cast(T::MessageType), size); } diff --git a/miniraft/raft.cpp b/miniraft/raft.cpp index 36add82..6e1b802 100644 --- a/miniraft/raft.cpp +++ b/miniraft/raft.cpp @@ -25,13 +25,21 @@ static uint32_t rand_(uint32_t* seed) { } // namespace +void TDummyRsm::Validate(TMessageHolder message) const { + if (!message || message->Len < sizeof(TCommandRequest) || + (!(message->Flags & TCommandRequest::EWrite) && message->Len < sizeof(TCommandRequest) + sizeof(int64_t))) + throw TMessageError("Invalid dummy RSM request"); +} + TMessageHolder TDummyRsm::Read(TMessageHolder message, uint64_t index) { + if (!message || message->Len < sizeof(TCommandRequest) + sizeof(int64_t)) + throw TMessageError("Missing log read index"); int64_t readIndex; memcpy(&readIndex, message->Data, sizeof(readIndex)); if (readIndex > 0 && readIndex <= Log.size()) { auto log = Log[readIndex-1]; - auto size = log->Len - sizeof(TMessage); + auto size = log->Len - sizeof(TLogEntry); auto mes = NewHoldedMessage(sizeof(TCommandResponse) + size); mes->Index = index; memcpy(mes->Data, log->Data, size); @@ -52,6 +60,7 @@ TMessageHolder TDummyRsm::Write(TMessageHolder message, uin TMessageHolder TDummyRsm::Prepare(TMessageHolder command) { + if (!command || command->Len < sizeof(TCommandRequest)) throw TMessageError("Short command"); auto dataSize = command->Len - sizeof(TCommandRequest); auto entry = NewHoldedMessage(sizeof(TLogEntry)+dataSize); memcpy(entry->Data, command->Data, dataSize); @@ -755,6 +764,7 @@ void TRequestProcessor::OnWriteRequest(TMessageHolder command, } void TRequestProcessor::OnCommandRequest(TMessageHolder command, const std::shared_ptr& replyTo) { + Rsm->Validate(command); PruneRequests(); auto flags = command->Flags; @@ -796,7 +806,9 @@ void TRequestProcessor::ProcessCommitted() { if (entry->Flags == TLogEntry::EStub) { continue; } - auto reply = Rsm->Write(entry, i); + TMessageHolder reply; + try { reply = Rsm->Write(entry, i); } + catch (...) { std::terminate(); } auto pending = PendingWriteTerms.find(i); if (pending != PendingWriteTerms.end() && pending->second == entry->Term) { WriteAnswers.emplace(i, TAnswer { @@ -831,7 +843,10 @@ void TRequestProcessor::ProcessWaiting() { PendingWriteTerms.erase(w.Index); WriteAnswers.erase(w.Index); } else { - reply = Rsm->Read(std::move(w.Command), w.Index).Cast(); + try { reply = Rsm->Read(std::move(w.Command), w.Index).Cast(); } + catch (const std::exception&) { + reply = NewHoldedMessage(TCommandResponse{.Index = w.Index, .ErrorCode = 2}); + } } reply->Cookie = cookie; w.ReplyTo->Send(std::move(reply)); @@ -849,7 +864,10 @@ void TRequestProcessor::ProcessWaiting() { TMessageHolder reply; assert (!(w.Command->Flags & TCommandRequest::EWrite)); auto cookie = w.Command->Cookie; - reply = Rsm->Read(std::move(w.Command), w.Index).Cast(); + try { reply = Rsm->Read(std::move(w.Command), w.Index).Cast(); } + catch (const std::exception&) { + reply = NewHoldedMessage(TCommandResponse{.Index = w.Index, .ErrorCode = 2}); + } reply->Cookie = cookie; w.ReplyTo->Send(std::move(reply)); } diff --git a/miniraft/raft.h b/miniraft/raft.h index cb70861..31893fd 100644 --- a/miniraft/raft.h +++ b/miniraft/raft.h @@ -22,6 +22,8 @@ struct INode { // CommandRequest -> Read? -> CurrentIndex (fixate) >= CommittedIndex -> CommandResponse struct IRsm { virtual ~IRsm() = default; + // Check application wire shape before queueing/forwarding; must not mutate state. + virtual void Validate(TMessageHolder) const {} virtual TMessageHolder Read(TMessageHolder message, uint64_t index) = 0; virtual TMessageHolder Write(TMessageHolder message, uint64_t index) = 0; virtual TMessageHolder Prepare(TMessageHolder message) = 0; @@ -30,6 +32,7 @@ struct IRsm { }; struct TDummyRsm: public IRsm { + void Validate(TMessageHolder message) const override; TMessageHolder Read(TMessageHolder message, uint64_t index) override; TMessageHolder Write(TMessageHolder message, uint64_t index) override; TMessageHolder Prepare(TMessageHolder message) override; diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 48c87fe..7cff354 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -17,6 +17,12 @@ ut(test_correctness test_correctness.cpp miniraft) ut(test_forwarding test_forwarding.cpp miniraft) ut(test_message_safety test_message_safety.cpp miniraft) ut(test_rpc_validation test_rpc_validation.cpp miniraft) +ut(test_app_payloads "test_app_payloads.cpp;../examples/kv_rsm.cpp" miniraft) +find_package(SQLite3 QUIET) +if(SQLite3_FOUND) +ut(test_sql_payloads "test_sql_payloads.cpp;../examples/sql_rsm.cpp" miniraft) +target_link_libraries(test_sql_payloads PRIVATE SQLite::SQLite3) +endif() ut(test_persist_recovery test_persist_recovery.cpp miniraft) if(CMAKE_SYSTEM_NAME STREQUAL "Linux") ut(test_durable test_durable.cpp miniraft) diff --git a/test/test_app_payloads.cpp b/test/test_app_payloads.cpp new file mode 100644 index 0000000..7cc372e --- /dev/null +++ b/test/test_app_payloads.cpp @@ -0,0 +1,178 @@ +#include "../examples/kv.h" +#include "../examples/kv_messages.h" +#ifndef _WIN32 +#include +#include +#endif +#include +#include +#include +extern "C" { +#include +} +namespace { +template void Reject(F action) { + bool rejected = false; + try { action(); } catch (const TMessageError&) { rejected = true; } + assert_true(rejected); +} +void test_kv_prepare_rejects_inconsistent_lengths(void**) { + TKv kv; + auto command = NewHoldedMessage(); + command->KeySize = UINT16_MAX; + command->ValSize = UINT16_MAX; + Reject([&] { kv.Prepare(command); }); + assert_int_equal(kv.LastAppliedIndex, 0); +} +void test_kv_read_rejects_short_key(void**) { + TKv kv; + auto command = NewHoldedMessage(); + command->KeySize = UINT16_MAX; + Reject([&] { kv.Read(command, 0); }); +} +void test_kv_apply_rejects_before_state_mutation(void**) { + TKv kv; + auto entry = NewHoldedMessage(); + entry->KeySize = 1; + entry->ValSize = UINT16_MAX; + Reject([&] { kv.Write(entry, 1); }); + assert_int_equal(kv.LastAppliedIndex, 0); + auto list = NewHoldedMessage(); + assert_int_equal(kv.Read(list, 0)->Len, sizeof(TCommandResponse)); +} +void test_valid_kv_payload_roundtrip(void**) { + TKv kv; + auto command = NewHoldedMessage(sizeof(TWriteKv) + 4); + command->KeySize = 1; + command->ValSize = 3; + memcpy(command->TKvEntry::Data, "kabc", 4); + kv.Write(kv.Prepare(command), 1); + auto read = NewHoldedMessage(sizeof(TReadKv) + 1); + read->KeySize = 1; + read->Data[0] = 'k'; + auto reply = kv.Read(read, 1).Cast(); + assert_int_equal(reply->Len, sizeof(TCommandResponse) + 3); + assert_memory_equal(reply->Data, "abc", 3); + command->ValSize = 0; + kv.Write(kv.Prepare(command), 2); + assert_int_equal(kv.Read(read, 2)->Len, sizeof(TCommandResponse)); +} +void test_dummy_read_rejects_missing_index(void**) { + TDummyRsm rsm; + auto command = NewHoldedMessage(); + Reject([&] { rsm.Read(command, 0); }); +} +void test_dummy_read_copies_only_log_data(void**) { + TDummyRsm rsm; + auto entry = NewHoldedMessage(sizeof(TLogEntry) + 3); + memcpy(entry->Data, "abc", 3); + rsm.Write(entry, 1); + auto request = NewHoldedMessage(sizeof(TCommandRequest) + sizeof(int64_t)); + int64_t index = 1; + memcpy(request->Data, &index, sizeof(index)); + auto reply = rsm.Read(request, 1).Cast(); + assert_int_equal(reply->Len, sizeof(TCommandResponse) + 3); + assert_memory_equal(reply->Data, "abc", 3); +} +struct TEndpoint : INode { + void Send(TMessageHolder) override {} + void Drain() override {} +}; +void test_invalid_request_is_rejected_before_queue_or_forward(void**) { + for (bool write : {false, true}) { + auto state = std::make_shared(); + auto raft = std::make_shared(state, 1, TNodeDict{}); + if (!write) raft->Become(EState::LEADER); + auto kv = std::make_shared(); + TRequestProcessor processor(raft, kv, {}); + auto endpoint = std::make_shared(); + std::weak_ptr weak = endpoint; + auto command = NewHoldedMessage(); + command->Flags = write ? TCommandRequest::EWrite : TCommandRequest::EConsistent; + command->KeySize = UINT16_MAX; + Reject([&] { processor.OnCommandRequest(command, endpoint); }); + endpoint.reset(); + assert_true(weak.expired()); + assert_int_equal(state->LastLogIndex, 0); + } +} +void test_oversized_factory_argument_is_not_truncated(void**) { + if (sizeof(size_t) <= sizeof(uint32_t)) return; + bool rejected = false; + try { NewHoldedMessage(size_t(UINT32_MAX) + 1); } + catch (const std::invalid_argument&) { rejected = true; } + assert_true(rejected); +} +void test_read_application_error_does_not_stop_completion(void**) { + struct TFailRead : TKv { + TMessageHolder Read(TMessageHolder, uint64_t) override { + throw TMessageError("read output too large"); + } + }; + struct TSink : INode { + std::vector Cookies; + void Send(TMessageHolder message) override { + auto reply = message.Cast(); + assert_int_equal(reply->ErrorCode, 2); + Cookies.push_back(reply->Cookie); + } + void Drain() override {} + }; + auto state = std::make_shared(); + auto raft = std::make_shared(state, 1, TNodeDict{}); + raft->ProcessTimeout(ITimeSource::Time{}); + TRequestProcessor processor(raft, std::make_shared(), {}); + auto endpoint = std::make_shared(); + for (uint32_t flags : {0u, uint32_t(TCommandRequest::EConsistent)}) { + auto command = NewHoldedMessage(); + command->Flags = flags; + command->Cookie = flags + 7; + processor.OnCommandRequest(command, endpoint); + processor.ProcessCommitted(); + processor.ProcessWaiting(); + } + assert_int_equal(endpoint->Cookies.size(), 2); + assert_int_equal(endpoint->Cookies[0], 7); + assert_int_equal(endpoint->Cookies[1], 11); + assert_int_equal(raft->CurrentStateName(), EState::LEADER); +} +#ifndef _WIN32 +void test_invalid_committed_application_is_fatal(void**) { + const auto child = fork(); + assert_true(child >= 0); + if (child == 0) { + std::set_terminate([] { _exit(73); }); + auto state = std::make_shared(); + auto entry = NewHoldedMessage(); + entry->KeySize = UINT16_MAX; + state->Append(entry); + auto raft = std::make_shared(state, 1, TNodeDict{}); + const_cast(raft->GetVolatileState())->CommitIndex = 1; + TRequestProcessor processor(raft, std::make_shared(), {}); + try { processor.ProcessCommitted(); } catch (...) { _exit(74); } + _exit(75); + } + int status = 0; + assert_int_equal(waitpid(child, &status, 0), child); + assert_true(WIFEXITED(status)); + assert_int_equal(WEXITSTATUS(status), 73); +} +#endif +} +int main() { + const CMUnitTest tests[] = { + cmocka_unit_test(test_kv_prepare_rejects_inconsistent_lengths), + cmocka_unit_test(test_kv_read_rejects_short_key), + cmocka_unit_test(test_kv_apply_rejects_before_state_mutation), + cmocka_unit_test(test_valid_kv_payload_roundtrip), + cmocka_unit_test(test_dummy_read_rejects_missing_index), + cmocka_unit_test(test_dummy_read_copies_only_log_data), + cmocka_unit_test(test_invalid_request_is_rejected_before_queue_or_forward), + cmocka_unit_test(test_oversized_factory_argument_is_not_truncated), + cmocka_unit_test(test_read_application_error_does_not_stop_completion), +#ifndef _WIN32 + cmocka_unit_test(test_invalid_committed_application_is_fatal), +#endif + }; + return cmocka_run_group_tests(tests, nullptr, nullptr); +} diff --git a/test/test_sql_payloads.cpp b/test/test_sql_payloads.cpp new file mode 100644 index 0000000..ff716cb --- /dev/null +++ b/test/test_sql_payloads.cpp @@ -0,0 +1,73 @@ +#include "../examples/sql_rsm.h" +#include +#include +#include +extern "C" { +#include +} +namespace { +template void Reject(F action) { + bool rejected = false; + try { action(); } catch (const TMessageError&) { rejected = true; } + assert_true(rejected); +} +void test_sql_prepare_rejects_short_query(void**) { + TSql sql(":memory:", 1); + auto command = NewHoldedMessage(); + command->QuerySize = UINT32_MAX; + Reject([&] { sql.Prepare(command); }); +} +void test_sql_read_rejects_short_query(void**) { + TSql sql(":memory:", 1); + auto command = NewHoldedMessage(); + command->QuerySize = UINT32_MAX; + Reject([&] { sql.Read(command, 0); }); +} +void test_sql_apply_rejects_before_progress(void**) { + TSql sql(":memory:", 1); + auto entry = NewHoldedMessage(); + entry->QuerySize = UINT32_MAX; + Reject([&] { sql.Write(entry, 1); }); + assert_int_equal(sql.LastAppliedIndex, 0); +} +void test_valid_sql_payload_roundtrip(void**) { + TSql sql(":memory:", 1); + const std::string query = "CREATE TABLE test (value TEXT); INSERT INTO test VALUES ('ok');"; + auto command = NewHoldedMessage(sizeof(TWriteSql) + query.size()); + command->QuerySize = query.size(); + memcpy(command->Query, query.data(), query.size()); + sql.Write(sql.Prepare(command), 1); + assert_int_equal(sql.LastAppliedIndex, 1); + const std::string select = "SELECT value FROM test"; + auto read = NewHoldedMessage(sizeof(TReadSql) + select.size()); + read->QuerySize = select.size(); + memcpy(read->Query, select.data(), select.size()); + auto result = sql.Read(read, 1).Cast(); + std::string text(result->Data, result->Len - sizeof(TCommandResponse)); + assert_true(text.find("ok") != std::string::npos); +} +void test_sql_validation_runs_before_request_admission(void**) { + auto state = std::make_shared(); + auto raft = std::make_shared(state, 1, TNodeDict{}); + auto sql = std::make_shared(":memory:", 1); + TRequestProcessor processor(raft, sql, {}); + for (bool write : {false, true}) { + raft->Become(write ? EState::FOLLOWER : EState::LEADER); + auto command = NewHoldedMessage(); + command->Flags = write ? TCommandRequest::EWrite : TCommandRequest::EConsistent; + command->QuerySize = UINT32_MAX; + Reject([&] { processor.OnCommandRequest(command, {}); }); + assert_int_equal(state->LastLogIndex, 0); + } +} +} +int main() { + const CMUnitTest tests[] = { + cmocka_unit_test(test_sql_prepare_rejects_short_query), + cmocka_unit_test(test_sql_read_rejects_short_query), + cmocka_unit_test(test_sql_apply_rejects_before_progress), + cmocka_unit_test(test_valid_sql_payload_roundtrip), + cmocka_unit_test(test_sql_validation_runs_before_request_admission), + }; + return cmocka_run_group_tests(tests, nullptr, nullptr); +} From b5140880fe90841cfb5708c1e4a93824bac07e07 Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Sun, 8 Nov 2026 14:04:45 -0500 Subject: [PATCH 10/66] Enforce read-only SQL execution before stepping statements (#10) --- README.md | 2 + docs/sql-readonly.md | 39 ++++++++++++++++++ examples/sql_rsm.cpp | 81 +++++++++++++++++++++++++++++++------- examples/sql_rsm.h | 1 + test/test_sql_payloads.cpp | 76 +++++++++++++++++++++++++++++++++++ 5 files changed, 184 insertions(+), 15 deletions(-) create mode 100644 docs/sql-readonly.md diff --git a/README.md b/README.md index 77c65ca..9a6ee29 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,8 @@ The [consensus validation follow-up](docs/consensus-validation.md) checks RPC identities and indices, protects committed prefixes and correlates acknowledgements. The [application payload follow-up](docs/application-payloads.md) validates KV, SQL and dummy-RSM data before access and provides pre-admission validation hooks. +The [SQL read policy](docs/sql-readonly.md) prevents requests marked as reads +from executing unreplicated mutations or connection controls. [![Number of GitHub stars](https://img.shields.io/github/stars/resetius/miniraft-cpp)](https://github.com/resetius/miniraft-cpp/stargazers) ![GitHub commit activity](https://img.shields.io/github/commit-activity/m/resetius/miniraft-cpp) diff --git a/docs/sql-readonly.md b/docs/sql-readonly.md new file mode 100644 index 0000000..088f66d --- /dev/null +++ b/docs/sql-readonly.md @@ -0,0 +1,39 @@ +# SQL read policy: v0.1.0-alpha.10 + +The SQL example previously executed read requests through unrestricted +sqlite3_exec. A client could submit UPDATE, schema changes or multiple statements +as a read and mutate one replica without a Raft entry. Six new regression tests +failed against that implementation; the genuine-read positive test already passed. + +Reads now prepare exactly one statement under a deny-by-default SQLite authorizer. +The policy permits SELECT, column reads, recursive queries and an explicit list +of trusted built-in scalar/aggregate functions in sql_rsm.cpp. It rejects writes, +DDL, PRAGMA, transaction/savepoint controls, ATTACH/DETACH and unknown functions. +sqlite3_stmt_readonly provides an additional check. The entire trailing input is +prepared and checked before execution, so a second statement is rejected even +if it is another SELECT. Comments, trailing semicolons and semicolons within +strings work. Empty/comment-only input and embedded NUL bytes are rejected. + +The policy remains installed through stepping and finalization, including SQLite +automatic recompilation, and RAII removes it on success or failure. Read failures +use the existing ErrorCode=2 response path. Replicated writes retain their existing +execution path and continue working after rejected reads. + +SQLite documents why [stmt_readonly alone is insufficient](https://www.sqlite.org/c3ref/stmt_readonly.html) +and how [authorizers apply during preparation and recompilation](https://www.sqlite.org/c3ref/set_authorizer.html). +This policy assumes the example's private connection and trusted SQLite built-ins +and modules. It is not a sandbox for arbitrary registered extensions or virtual +tables. Functions outside the explicit allowlist require a deliberate policy +change; previously accepted read scripts with multiple statements now fail. + +Seven tests cover DML/schema rejection, multi-statement rejection before mutation, +connection controls, unsafe functions, real SELECT/CTE behavior, malformed text +and restoration before a subsequent write. All 152 tests pass locally across 11 +CTest targets under ASan/UBSan; the Release core suite also passes with SQLite. +Self-review checked policy lifetime, preparation before stepping, function-name +handling, statement cleanup and unchanged replicated-write routing. No independent +reviewer approval is claimed. + +SQL write transaction escape, metadata/progress semantics, deterministic execution, +result memory/CPU quotas, authentication and retry deduplication remain open. +The project remains experimental and does not yet provide NuRaft-level assurance. diff --git a/examples/sql_rsm.cpp b/examples/sql_rsm.cpp index 1ac73a4..089eaf5 100644 --- a/examples/sql_rsm.cpp +++ b/examples/sql_rsm.cpp @@ -1,6 +1,60 @@ #include "sql_rsm.h" #include "payload_validation.h" #include +#include + +namespace { +int ReadAuthorization(void*, int action, const char*, const char* second, const char*, const char*) { + if (action == SQLITE_SELECT || action == SQLITE_READ || action == SQLITE_RECURSIVE) return SQLITE_OK; + if (action == SQLITE_FUNCTION && second) { + // Keep this allowlist limited to trusted SQLite built-ins. In particular, + // readonly statements can still call extension functions with side effects. + for (const char* function : {"abs", "avg", "coalesce", "count", "glob", "hex", "ifnull", + "instr", "length", "like", "lower", "ltrim", "max", "min", "nullif", "quote", + "replace", "round", "rtrim", "substr", "substring", "sum", "total", "trim", + "typeof", "unicode", "upper", "group_concat"}) + if (sqlite3_stricmp(function, second) == 0) return SQLITE_OK; + } + return SQLITE_DENY; +} +struct TReadPolicy { + sqlite3* Db; + explicit TReadPolicy(sqlite3* db) : Db(db) { + if (sqlite3_set_authorizer(Db, ReadAuthorization, nullptr) != SQLITE_OK) + throw TMessageError("Cannot install SQL read policy"); + } + ~TReadPolicy() { sqlite3_set_authorizer(Db, nullptr, nullptr); } +}; +using TStatement = std::unique_ptr; +} + +void TSql::ExecuteRead(const std::string& q) { + if (q.size() > INT_MAX || q.find('\0') != std::string::npos) + throw TMessageError("Invalid SQL read text"); + TReadPolicy policy(Db); // retained through prepare, stepping and finalization + sqlite3_stmt* raw = nullptr; + const char* tail = nullptr; + int rc = sqlite3_prepare_v2(Db, q.data(), int(q.size()), &raw, &tail); + TStatement statement(raw, sqlite3_finalize); + if (rc != SQLITE_OK || !statement || !sqlite3_stmt_readonly(statement.get())) + throw TMessageError("SQL read must be one permitted read-only statement"); + raw = nullptr; + rc = sqlite3_prepare_v2(Db, tail, int(q.data() + q.size() - tail), &raw, nullptr); + TStatement extra(raw, sqlite3_finalize); + if (rc != SQLITE_OK || extra) throw TMessageError("Multiple SQL read statements are not permitted"); + Result.Clear(); + LastError.clear(); + while ((rc = sqlite3_step(statement.get())) == SQLITE_ROW) { + const int count = sqlite3_column_count(statement.get()); + std::vector values(count), columns(count); + for (int i = 0; i < count; ++i) { + values[i] = const_cast(reinterpret_cast(sqlite3_column_text(statement.get(), i))); + columns[i] = const_cast(sqlite3_column_name(statement.get(), i)); + } + Process(this, count, values.data(), columns.data()); + } + if (rc != SQLITE_DONE) throw TMessageError("SQL read execution failed"); +} void TSql::Validate(TMessageHolder message) const { auto query = message.Cast(); @@ -70,28 +124,25 @@ bool TSql::Execute(const std::string& q) { TMessageHolder TSql::Read(TMessageHolder message, uint64_t index) { auto readSql = message.Cast(); ValidatePayloadRange(message, readSql->Query, readSql->QuerySize); - if (!Execute(std::string(readSql->Query, readSql->QuerySize))) { - return Reply(LastError, index); - } else { - std::string text; + ExecuteRead(std::string(readSql->Query, readSql->QuerySize)); + std::string text; + for (int i = 0; i < Result.Cols.size(); i++) { + text += Result.Cols[i]; + if (i != Result.Cols.size() - 1) { + text += ","; + } + } + text += "\n"; + for (int j = 0; j < Result.Rows.size(); j++) { for (int i = 0; i < Result.Cols.size(); i++) { - text += Result.Cols[i]; + text += Result.Rows[j].Values[i] ? *Result.Rows[j].Values[i] : "null"; if (i != Result.Cols.size() - 1) { text += ","; } } text += "\n"; - for (int j = 0; j < Result.Rows.size(); j++) { - for (int i = 0; i < Result.Cols.size(); i++) { - text += Result.Rows[j].Values[i] ? *Result.Rows[j].Values[i] : "null"; - if (i != Result.Cols.size() - 1) { - text += ","; - } - } - text += "\n"; - } - return Reply(text, index); } + return Reply(text, index); } TMessageHolder TSql::Write(TMessageHolder message, uint64_t index) { diff --git a/examples/sql_rsm.h b/examples/sql_rsm.h index ded9e92..1812d00 100644 --- a/examples/sql_rsm.h +++ b/examples/sql_rsm.h @@ -53,6 +53,7 @@ class TSql: public IRsm { private: bool Execute(const std::string& q); + void ExecuteRead(const std::string& q); static int Process(void* self, int ncols, char** values, char** cols); TMessageHolder Reply(const std::string& ans, uint64_t index); diff --git a/test/test_sql_payloads.cpp b/test/test_sql_payloads.cpp index ff716cb..99ab6f3 100644 --- a/test/test_sql_payloads.cpp +++ b/test/test_sql_payloads.cpp @@ -6,6 +6,22 @@ extern "C" { #include } namespace { +TMessageHolder Query(const std::string& text) { + auto message = NewHoldedMessage(sizeof(TReadSql) + text.size()); + message->QuerySize = text.size(); + memcpy(message->Query, text.data(), text.size()); + return message; +} +std::string ReadText(TSql& sql, const std::string& query) { + auto reply = sql.Read(Query(query), sql.LastAppliedIndex).Cast(); + return std::string(reply->Data, reply->Len - sizeof(TCommandResponse)); +} +void Apply(TSql& sql, const std::string& query, uint64_t index) { + auto command = Query(query); + command->Flags = TCommandRequest::EWrite; + sql.Write(sql.Prepare(command), index); +} +void Seed(TSql& sql) { Apply(sql, "CREATE TABLE t (value TEXT); INSERT INTO t VALUES ('original');", 1); } template void Reject(F action) { bool rejected = false; try { action(); } catch (const TMessageError&) { rejected = true; } @@ -60,6 +76,59 @@ void test_sql_validation_runs_before_request_admission(void**) { assert_int_equal(state->LastLogIndex, 0); } } +void test_sql_read_cannot_insert_update_or_delete(void**) { + for (const std::string query : {"INSERT INTO t VALUES ('bad')", "UPDATE t SET value='bad'", "DELETE FROM t"}) { + TSql sql(":memory:", 1); + Seed(sql); + Reject([&] { sql.Read(Query(query), 1); }); + assert_true(ReadText(sql, "SELECT value FROM t").find("original") != std::string::npos); + assert_true(ReadText(sql, "SELECT count(*) FROM t").find("1") != std::string::npos); + assert_int_equal(sql.LastAppliedIndex, 1); + } +} +void test_sql_read_rejects_multi_statement_mutation_before_execution(void**) { + TSql sql(":memory:", 1); + Seed(sql); + Reject([&] { sql.Read(Query("SELECT value FROM t; UPDATE t SET value='bad';"), 1); }); + assert_true(ReadText(sql, "SELECT value FROM t").find("original") != std::string::npos); +} +void test_sql_read_cannot_change_schema(void**) { + for (const std::string query : {"DROP TABLE t", "CREATE TABLE rogue (x)", "ALTER TABLE t ADD COLUMN x"}) { + TSql sql(":memory:", 1); + Seed(sql); + Reject([&] { sql.Read(Query(query), 1); }); + assert_true(ReadText(sql, "SELECT value FROM t").find("original") != std::string::npos); + } +} +void test_sql_read_rejects_connection_control_and_unsafe_functions(void**) { + for (const std::string query : {"PRAGMA query_only=OFF", "BEGIN", "COMMIT", "SAVEPOINT x", + "ATTACH ':memory:' AS other", "VACUUM", "SELECT load_extension('nonexistent')", + "SELECT readfile('nonexistent')"}) { + TSql sql(":memory:", 1); + Reject([&] { sql.Read(Query(query), 0); }); + assert_int_equal(sql.LastAppliedIndex, 0); + } +} +void test_sql_read_supports_comments_ctes_and_safe_functions(void**) { + TSql sql(":memory:", 1); + Seed(sql); + assert_true(ReadText(sql, "-- leading comment\n SELECT upper(value), 'a;b' FROM t; -- tail").find("ORIGINAL") != std::string::npos); + assert_true(ReadText(sql, "WITH RECURSIVE n(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM n WHERE x<3) SELECT sum(x) FROM n").find("6") != std::string::npos); +} +void test_sql_read_rejects_nul_empty_and_second_select(void**) { + TSql sql(":memory:", 1); + const std::string embedded("SELECT 1\0;SELECT 2", 18); + for (const auto& query : {std::string{}, std::string("-- only comment"), std::string("SELECT 1; SELECT 2"), embedded}) + Reject([&] { sql.Read(Query(query), 0); }); +} +void test_sql_read_policy_is_restored_after_rejection(void**) { + TSql sql(":memory:", 1); + Seed(sql); + Reject([&] { sql.Read(Query("DELETE FROM t"), 1); }); + Apply(sql, "INSERT INTO t VALUES ('second')", 2); + assert_int_equal(sql.LastAppliedIndex, 2); + assert_true(ReadText(sql, "SELECT count(*) FROM t").find("2") != std::string::npos); +} } int main() { const CMUnitTest tests[] = { @@ -68,6 +137,13 @@ int main() { cmocka_unit_test(test_sql_apply_rejects_before_progress), cmocka_unit_test(test_valid_sql_payload_roundtrip), cmocka_unit_test(test_sql_validation_runs_before_request_admission), + cmocka_unit_test(test_sql_read_cannot_insert_update_or_delete), + cmocka_unit_test(test_sql_read_rejects_multi_statement_mutation_before_execution), + cmocka_unit_test(test_sql_read_cannot_change_schema), + cmocka_unit_test(test_sql_read_rejects_connection_control_and_unsafe_functions), + cmocka_unit_test(test_sql_read_supports_comments_ctes_and_safe_functions), + cmocka_unit_test(test_sql_read_rejects_nul_empty_and_second_select), + cmocka_unit_test(test_sql_read_policy_is_restored_after_rejection), }; return cmocka_run_group_tests(tests, nullptr, nullptr); } From 5849d85403c1aa718016dd3f3a37620f3d199631 Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Sun, 8 Nov 2026 14:12:34 -0500 Subject: [PATCH 11/66] Keep SQL application changes and progress within managed transactions (#11) --- README.md | 2 + docs/sql-application.md | 64 ++++++++++++++ examples/sql_rsm.cpp | 150 +++++++++++++++++++++----------- examples/sql_rsm.h | 3 +- test/test_sql_payloads.cpp | 173 ++++++++++++++++++++++++++++++++++++- 5 files changed, 339 insertions(+), 53 deletions(-) create mode 100644 docs/sql-application.md diff --git a/README.md b/README.md index 9a6ee29..afe9efa 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,8 @@ The [application payload follow-up](docs/application-payloads.md) validates KV, SQL and dummy-RSM data before access and provides pre-admission validation hooks. The [SQL read policy](docs/sql-readonly.md) prevents requests marked as reads from executing unreplicated mutations or connection controls. +The [SQL application follow-up](docs/sql-application.md) protects managed write +transactions and commits application progress together with successful changes. [![Number of GitHub stars](https://img.shields.io/github/stars/resetius/miniraft-cpp)](https://github.com/resetius/miniraft-cpp/stargazers) ![GitHub commit activity](https://img.shields.io/github/commit-activity/m/resetius/miniraft-cpp) diff --git a/docs/sql-application.md b/docs/sql-application.md new file mode 100644 index 0000000..ed102a1 --- /dev/null +++ b/docs/sql-application.md @@ -0,0 +1,64 @@ +# SQL transaction and progress handling: v0.1.0-alpha.11 + +The previous SQL write path concatenated client text, progress metadata and COMMIT +into one script. Client COMMIT/END could publish a prefix before a later statement +failed; metadata update errors were ignored on the failure path. Rejected SQL +also left LastAppliedIndex stale in memory even when its persisted marker advanced. +Restart used stoi, rejecting indices above INT_MAX and accepting malformed numeric +prefixes. Regression tests reproduced these failures before the fix; the old test +run aborted when the full-width-index restart threw out_of_range. + +Writes now begin an IMMEDIATE transaction and execute the client script separately +under a SQLite authorizer. The policy blocks transaction/savepoint controls, +PRAGMAs, ATTACH/DETACH, temporary objects, virtual-table creation/deletion and +ALTER TABLE. It permits +ordinary main-database DML, table/index/view/trigger creation and deletion, REINDEX, +ANALYZE and the same function allowlist as reads. This deliberately narrows the +previous unrestricted SQL interface. Scripts may still contain multiple ordinary +statements, and comments cannot swallow internal metadata SQL. + +Payload access to raft_metadata_ is denied, including through compiled views and +triggers. Payloads cannot create triggers or indexes on that table. The authorizer +is removed before the application's own metadata and transaction statements run. +The policy stays active throughout payload execution and automatic recompilation. +See SQLite's [authorizer contract](https://www.sqlite.org/c3ref/set_authorizer.html). + +Successful data changes and their progress marker commit in the same transaction. +SQLITE_ERROR, AUTH, CONSTRAINT and MISMATCH are treated as application rejections: +all payload effects are rolled back, then a checked transaction records that the +entry was consumed. The original SQL error text is returned using the existing +write reply format. Memory advances only after COMMIT succeeds. An already applied +index is skipped; original replies are not persisted for retry deduplication. + +Other execution errors, and any BEGIN, rollback, metadata or COMMIT failure, throw +without advancing memory. The committed-entry handler terminates on such an +exception, allowing restart/replay rather than acknowledging uncertain application. +A scope guard attempts rollback during unwinding. SQLite can itself roll back a +transaction, so cleanup checks [autocommit state](https://www.sqlite.org/c3ref/get_autocommit.html) +before issuing ROLLBACK. A crash between rolling back a rejected payload and +recording its marker can replay that rejection; no payload effects survived. + +Startup checks initialization and progress queries, closes the database on +constructor failure, and parses the complete stored text as uint64_t. NULL, empty, +negative, overflowing or partially numeric markers (including embedded NULs) are +rejected. An absent marker continues to mean zero for legacy empty databases; +this does not detect a marker deleted by external tampering or previous releases. + +Ten new tests cover transaction escape, direct/indirect metadata access, rollback +and restart, trailing comments, OR ROLLBACK, metadata-update failure, full-width +indices, malformed metadata, lock failures at BEGIN and COMMIT, and NUL rejection +before admission/preparation. All 162 tests across 11 CTest targets pass locally +with ASan/UBSan and SQLite; the Release core suite passes as well. + +Self-review checked authorizer arguments and lifetime, separate internal SQL, +rollback/automatic-rollback behavior, commit ordering, error classification, +constructor ownership and bounded full-width parsing. No independent reviewer +approval is claimed. These are deterministic logical/locking failure tests, +not new power-loss or filesystem fault-injection coverage for SQLite. + +This remains an experimental example using trusted SQLite builds/modules and a +private serialized connection. Existing unsafe schemas or legacy log scripts need +review before migration. Arbitrary extensions/virtual tables are not sandboxed. +Deterministic SQL across replicas, SQLite durability configuration, schema/metadata +migration and external modification, result/CPU quotas, authentication, retry +deduplication and independent review remain open production gates. diff --git a/examples/sql_rsm.cpp b/examples/sql_rsm.cpp index 089eaf5..b25d40f 100644 --- a/examples/sql_rsm.cpp +++ b/examples/sql_rsm.cpp @@ -2,6 +2,7 @@ #include "payload_validation.h" #include #include +#include namespace { int ReadAuthorization(void*, int action, const char*, const char* second, const char*, const char*) { @@ -17,13 +18,37 @@ int ReadAuthorization(void*, int action, const char*, const char* second, const } return SQLITE_DENY; } -struct TReadPolicy { +int WriteAuthorization(void*, int action, const char* first, const char* second, const char* database, const char* context) { + const auto metadata = [](const char* name) { + return name && sqlite3_stricmp(name, "raft_metadata_") == 0; + }; + if (metadata(first) || ((action == SQLITE_CREATE_TRIGGER || action == SQLITE_DROP_TRIGGER || + action == SQLITE_ALTER_TABLE || action == SQLITE_CREATE_INDEX || action == SQLITE_DROP_INDEX) && metadata(second))) return SQLITE_DENY; + if (database && sqlite3_stricmp(database, "main") != 0) return SQLITE_DENY; + switch (action) { + case SQLITE_INSERT: case SQLITE_UPDATE: case SQLITE_DELETE: + case SQLITE_CREATE_TABLE: case SQLITE_CREATE_INDEX: case SQLITE_CREATE_VIEW: case SQLITE_CREATE_TRIGGER: + case SQLITE_DROP_TABLE: case SQLITE_DROP_INDEX: case SQLITE_DROP_VIEW: case SQLITE_DROP_TRIGGER: + case SQLITE_REINDEX: case SQLITE_ANALYZE: + return SQLITE_OK; + default: + return ReadAuthorization(nullptr, action, first, second, database, context); + } +} +using TAuthorizer = int (*)(void*, int, const char*, const char*, const char*, const char*); +struct TSqlPolicy { + sqlite3* Db; + explicit TSqlPolicy(sqlite3* db, TAuthorizer authorize) : Db(db) { + if (sqlite3_set_authorizer(Db, authorize, nullptr) != SQLITE_OK) + throw TMessageError("Cannot install SQL policy"); + } + ~TSqlPolicy() { sqlite3_set_authorizer(Db, nullptr, nullptr); } +}; +struct TTransactionCleanup { sqlite3* Db; - explicit TReadPolicy(sqlite3* db) : Db(db) { - if (sqlite3_set_authorizer(Db, ReadAuthorization, nullptr) != SQLITE_OK) - throw TMessageError("Cannot install SQL read policy"); + ~TTransactionCleanup() { + if (!sqlite3_get_autocommit(Db)) sqlite3_exec(Db, "ROLLBACK", nullptr, nullptr, nullptr); } - ~TReadPolicy() { sqlite3_set_authorizer(Db, nullptr, nullptr); } }; using TStatement = std::unique_ptr; } @@ -31,7 +56,7 @@ using TStatement = std::unique_ptr; void TSql::ExecuteRead(const std::string& q) { if (q.size() > INT_MAX || q.find('\0') != std::string::npos) throw TMessageError("Invalid SQL read text"); - TReadPolicy policy(Db); // retained through prepare, stepping and finalization + TSqlPolicy policy(Db, ReadAuthorization); // retained through prepare, stepping and finalization sqlite3_stmt* raw = nullptr; const char* tail = nullptr; int rc = sqlite3_prepare_v2(Db, q.data(), int(q.size()), &raw, &tail); @@ -59,22 +84,38 @@ void TSql::ExecuteRead(const std::string& q) { void TSql::Validate(TMessageHolder message) const { auto query = message.Cast(); ValidatePayloadRange(message, query->Query, query->QuerySize); + if (memchr(query->Query, 0, query->QuerySize)) throw TMessageError("Invalid SQL query text"); } TSql::TSql(const std::string& path, int serverId) { std::string dbPath = path == ":memory:" ? path : path + "." + std::to_string(serverId); - if (sqlite3_open(dbPath.c_str(), &Db) != SQLITE_OK) { - std::cerr << "Cannot open db: `" << dbPath << "', " - << "error: " << sqlite3_errmsg(Db) - << std::endl; - throw std::runtime_error("Cannot open db"); - } - Execute(R"__(CREATE TABLE IF NOT EXISTS raft_metadata_ (key TEXT PRIMARY KEY, value TEXT))__"); - Execute(R"__(SELECT value FROM raft_metadata_ WHERE key = 'LastAppliedIndex')__"); - if (!Result.Empty()) { - LastAppliedIndex = std::stoi(*Result.Rows[0].Values[0]); + const int opened = sqlite3_open(dbPath.c_str(), &Db); + // Constructors that fail must also close the connection and release its locks. + std::unique_ptr cleanup(Db, sqlite3_close); + if (opened != SQLITE_OK) throw std::runtime_error("Cannot open SQL database"); + RequireExecute("CREATE TABLE IF NOT EXISTS raft_metadata_ (key TEXT PRIMARY KEY, value TEXT)"); + sqlite3_stmt* raw = nullptr; + const int prepared = sqlite3_prepare_v2(Db, + "SELECT value FROM raft_metadata_ WHERE key='LastAppliedIndex'", -1, &raw, nullptr); + TStatement statement(raw, sqlite3_finalize); + if (prepared != SQLITE_OK || !statement) throw std::runtime_error("Cannot read SQL application progress"); + int rc = sqlite3_step(statement.get()); + if (rc == SQLITE_ROW) { + if (sqlite3_column_type(statement.get(), 0) != SQLITE_TEXT) + throw std::runtime_error("Invalid SQL application progress"); + const auto* text = reinterpret_cast(sqlite3_column_text(statement.get(), 0)); + if (!text) throw std::runtime_error("Cannot read SQL application progress"); + const auto length = sqlite3_column_bytes(statement.get(), 0); + uint64_t index = 0; + const auto parsed = std::from_chars(text, text + length, index); + if (parsed.ec != std::errc{} || parsed.ptr != text + length) + throw std::runtime_error("Invalid SQL application progress"); + LastAppliedIndex = index; + rc = sqlite3_step(statement.get()); } + if (rc != SQLITE_DONE) throw std::runtime_error("Invalid SQL application progress"); + cleanup.release(); std::cerr << "LastAppliedIndex: " << LastAppliedIndex << std::endl; } @@ -106,19 +147,18 @@ int TSql::Process(void* self, int ncols, char** values, char** cols) { return 0; } -bool TSql::Execute(const std::string& q) { +int TSql::Execute(const std::string& q) { char* err = nullptr; - std::cerr << "Execute: " << q << std::endl; Result.Clear(); LastError.clear(); - if (sqlite3_exec(Db, q.c_str(), Process, this, &err) != SQLITE_OK) { - std::cerr << "Cannot apply changes, error: " << err << std::endl; - LastError = err; - sqlite3_free(err); - return false; - } - std::cerr << "OK" << std::endl; - return true; + const int rc = sqlite3_exec(Db, q.c_str(), nullptr, nullptr, &err); + if (rc != SQLITE_OK) LastError = err ? err : sqlite3_errmsg(Db); + sqlite3_free(err); + return rc; +} + +void TSql::RequireExecute(const std::string& q) { + if (Execute(q) != SQLITE_OK) throw std::runtime_error("SQL state update failed: " + LastError); } TMessageHolder TSql::Read(TMessageHolder message, uint64_t index) { @@ -146,32 +186,41 @@ TMessageHolder TSql::Read(TMessageHolder message, uin } TMessageHolder TSql::Write(TMessageHolder message, uint64_t index) { - // TODO: index + 1 == LastAppliedIndex - - std::string err; - if (LastAppliedIndex < index) { - auto entry = message.Cast(); - ValidatePayloadRange(message, entry->Query, entry->QuerySize); - std::cerr << "Execute write of size: " << entry->QuerySize << std::endl; - std::string updateLastApplied; - updateLastApplied += "INSERT INTO raft_metadata_ (key, value) VALUES ('LastAppliedIndex','" + std::to_string(index) + "')\n"; - updateLastApplied += "ON CONFLICT(key) DO UPDATE SET value = '" + std::to_string(index) + "';\n"; - std::string q = "BEGIN TRANSACTION;\n"; - q += std::string(entry->Query, entry->QuerySize); - if (q.back() != ';') { - q += ";\n"; - } - q += updateLastApplied; - q += "COMMIT;"; - if (Execute(q)) { - LastAppliedIndex = index; - } else { - err = LastError; - Execute("ROLLBACK;"); - Execute(updateLastApplied); // need to update LastAppliedIndex in order not to execute failed query aqain + if (index <= LastAppliedIndex) return Reply({}, index); + auto entry = message.Cast(); + ValidatePayloadRange(message, entry->Query, entry->QuerySize); + const std::string query(entry->Query, entry->QuerySize); + if (query.find('\0') != std::string::npos) throw TMessageError("Invalid SQL write text"); + if (!sqlite3_get_autocommit(Db)) throw std::runtime_error("SQL application transaction already active"); + TTransactionCleanup transaction{Db}; + RequireExecute("BEGIN IMMEDIATE"); + int rc; + std::string error; + { + // Only the payload runs under this policy. It cannot end the transaction + // or access the progress table, including through views and triggers. + TSqlPolicy policy(Db, WriteAuthorization); + rc = Execute(query); + error = LastError; + } + if (rc != SQLITE_OK) { + if (!sqlite3_get_autocommit(Db)) RequireExecute("ROLLBACK"); + switch (rc & 0xff) { + case SQLITE_ERROR: case SQLITE_AUTH: case SQLITE_CONSTRAINT: case SQLITE_MISMATCH: + break; // SQL/application rejection: consume this entry without its effects. + default: + throw std::runtime_error("SQL application execution failed: " + error); } + // Some errors (including OR ROLLBACK) already rolled back the transaction. + // Start a new one to record the rejected entry; never ignore failure here. + RequireExecute("BEGIN IMMEDIATE"); } - return Reply(err, index); + const auto progress = std::to_string(index); + RequireExecute("INSERT INTO raft_metadata_ (key, value) VALUES ('LastAppliedIndex','" + progress + + "') ON CONFLICT(key) DO UPDATE SET value='" + progress + "'"); + RequireExecute("COMMIT"); + LastAppliedIndex = index; + return Reply(error, index); } TMessageHolder TSql::Reply(const std::string& ans, uint64_t index) @@ -187,6 +236,7 @@ TMessageHolder TSql::Reply(const std::string& ans, uint64_t index) TMessageHolder TSql::Prepare(TMessageHolder command) { auto write = command.Cast(); ValidatePayloadRange(command, write->Query, write->QuerySize); + if (memchr(write->Query, 0, write->QuerySize)) throw TMessageError("Invalid SQL write text"); auto dataSize = command->Len - sizeof(TCommandRequest); auto entry = NewHoldedMessage(sizeof(TLogEntry)+dataSize); memcpy(entry->Data, command->Data, dataSize); diff --git a/examples/sql_rsm.h b/examples/sql_rsm.h index 1812d00..2c5fb29 100644 --- a/examples/sql_rsm.h +++ b/examples/sql_rsm.h @@ -52,7 +52,8 @@ class TSql: public IRsm { TMessageHolder Prepare(TMessageHolder message) override; private: - bool Execute(const std::string& q); + int Execute(const std::string& q); + void RequireExecute(const std::string& q); void ExecuteRead(const std::string& q); static int Process(void* self, int ncols, char** values, char** cols); TMessageHolder Reply(const std::string& ans, uint64_t index); diff --git a/test/test_sql_payloads.cpp b/test/test_sql_payloads.cpp index 99ab6f3..5e8446f 100644 --- a/test/test_sql_payloads.cpp +++ b/test/test_sql_payloads.cpp @@ -1,4 +1,6 @@ #include "../examples/sql_rsm.h" +#include +#include #include #include #include @@ -16,10 +18,11 @@ std::string ReadText(TSql& sql, const std::string& query) { auto reply = sql.Read(Query(query), sql.LastAppliedIndex).Cast(); return std::string(reply->Data, reply->Len - sizeof(TCommandResponse)); } -void Apply(TSql& sql, const std::string& query, uint64_t index) { +std::string Apply(TSql& sql, const std::string& query, uint64_t index) { auto command = Query(query); command->Flags = TCommandRequest::EWrite; - sql.Write(sql.Prepare(command), index); + auto reply = sql.Write(sql.Prepare(command), index).Cast(); + return std::string(reply->Data, reply->Len - sizeof(TCommandResponse)); } void Seed(TSql& sql) { Apply(sql, "CREATE TABLE t (value TEXT); INSERT INTO t VALUES ('original');", 1); } template void Reject(F action) { @@ -129,6 +132,162 @@ void test_sql_read_policy_is_restored_after_rejection(void**) { assert_int_equal(sql.LastAppliedIndex, 2); assert_true(ReadText(sql, "SELECT count(*) FROM t").find("2") != std::string::npos); } +struct TDatabase { + std::filesystem::path Directory; + std::string Path; + TDatabase() { + static unsigned counter = 0; + Directory = std::filesystem::temp_directory_path() / + ("miniraft-sql-" + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count()) + + "-" + std::to_string(counter++)); + assert_true(std::filesystem::create_directory(Directory)); + Path = (Directory / "db").string(); + } + ~TDatabase() { std::error_code ignored; std::filesystem::remove_all(Directory, ignored); } + void External(const std::string& query) { + sqlite3* db = nullptr; + assert_int_equal(sqlite3_open((Path + ".1").c_str(), &db), SQLITE_OK); + int rc = sqlite3_exec(db, query.c_str(), nullptr, nullptr, nullptr); + sqlite3_close(db); + assert_int_equal(rc, SQLITE_OK); + } +}; +void test_sql_write_cannot_escape_managed_transaction(void**) { + for (const std::string control : {"COMMIT", "END", "ROLLBACK", "BEGIN", "SAVEPOINT x", "RELEASE x", + "PRAGMA user_version=7", "ATTACH ':memory:' AS other"}) { + TSql sql(":memory:", 1); + Seed(sql); + auto error = Apply(sql, "INSERT INTO t VALUES ('partial'); " + control + "; INSERT INTO missing VALUES (1);", 2); + assert_false(error.empty()); + assert_true(ReadText(sql, "SELECT count(*) FROM t").find("\n1\n") != std::string::npos); + assert_int_equal(sql.LastAppliedIndex, 2); + } +} +void test_sql_write_cannot_modify_progress_metadata(void**) { + for (const std::string mutation : {"DELETE FROM raft_metadata_", "UPDATE RAFT_METADATA_ SET value='999'", + "DROP TABLE raft_metadata_", "ALTER TABLE raft_metadata_ RENAME TO stolen", + "CREATE INDEX tamper ON raft_metadata_ (value)", + "CREATE TRIGGER tamper AFTER UPDATE ON raft_metadata_ BEGIN SELECT 1; END", + "CREATE TRIGGER steal AFTER INSERT ON t BEGIN UPDATE raft_metadata_ SET value='999'; END; INSERT INTO t VALUES ('bad')"}) { + TSql sql(":memory:", 1); + Seed(sql); + auto error = Apply(sql, "INSERT INTO t VALUES ('partial'); " + mutation, 2); + assert_false(error.empty()); + assert_true(ReadText(sql, "SELECT count(*) FROM t").find("\n1\n") != std::string::npos); + assert_true(ReadText(sql, "SELECT value FROM raft_metadata_ WHERE key='LastAppliedIndex'").find("\n2\n") != std::string::npos); + assert_int_equal(sql.LastAppliedIndex, 2); + } +} +void test_sql_failed_write_progress_survives_restart(void**) { + TDatabase file; + { + TSql sql(file.Path, 1); + Seed(sql); + assert_false(Apply(sql, "INSERT INTO t VALUES ('partial'); INSERT INTO missing VALUES (1)", 2).empty()); + assert_int_equal(sql.LastAppliedIndex, 2); + assert_true(ReadText(sql, "SELECT count(*) FROM t").find("\n1\n") != std::string::npos); + } + { + TSql sql(file.Path, 1); + assert_int_equal(sql.LastAppliedIndex, 2); + Apply(sql, "INSERT INTO t VALUES ('must not replay')", 2); + assert_true(ReadText(sql, "SELECT count(*) FROM t").find("\n1\n") != std::string::npos); + assert_true(Apply(sql, "INSERT INTO t VALUES ('next')", 3).empty()); + } + TSql sql(file.Path, 1); + assert_int_equal(sql.LastAppliedIndex, 3); + assert_true(ReadText(sql, "SELECT count(*) FROM t").find("\n2\n") != std::string::npos); +} +void test_sql_write_handles_trailing_comment_without_losing_progress(void**) { + TDatabase file; + { + TSql sql(file.Path, 1); + Seed(sql); + assert_true(Apply(sql, "INSERT INTO t VALUES ('next'); -- trailing comment", 2).empty()); + assert_int_equal(sql.LastAppliedIndex, 2); + } + TSql sql(file.Path, 1); + assert_int_equal(sql.LastAppliedIndex, 2); + assert_true(ReadText(sql, "SELECT count(*) FROM t").find("\n2\n") != std::string::npos); +} +void test_sql_automatic_rollback_is_consumed_without_partial_data(void**) { + TSql sql(":memory:", 1); + assert_true(Apply(sql, "CREATE TABLE u (x INTEGER UNIQUE); INSERT INTO u VALUES (1)", 1).empty()); + assert_false(Apply(sql, "INSERT INTO u VALUES (2); INSERT OR ROLLBACK INTO u VALUES (1)", 2).empty()); + assert_int_equal(sql.LastAppliedIndex, 2); + assert_true(ReadText(sql, "SELECT count(*) FROM u").find("\n1\n") != std::string::npos); + assert_true(Apply(sql, "INSERT INTO u VALUES (3)", 3).empty()); +} +void test_sql_metadata_failure_rolls_back_data_and_does_not_advance(void**) { + TDatabase file; + { TSql sql(file.Path, 1); Seed(sql); } + file.External("CREATE TRIGGER fail_marker BEFORE UPDATE ON raft_metadata_ BEGIN SELECT RAISE(ABORT,'marker blocked'); END"); + { + TSql sql(file.Path, 1); + bool failed = false; + try { Apply(sql, "INSERT INTO t VALUES ('partial')", 2); } + catch (const std::runtime_error&) { failed = true; } + assert_true(failed); + assert_int_equal(sql.LastAppliedIndex, 1); + assert_true(ReadText(sql, "SELECT count(*) FROM t").find("\n1\n") != std::string::npos); + } + file.External("DROP TRIGGER fail_marker"); + TSql sql(file.Path, 1); + assert_int_equal(sql.LastAppliedIndex, 1); + assert_true(Apply(sql, "INSERT INTO t VALUES ('retry')", 2).empty()); +} +void test_sql_progress_restores_full_uint64_range(void**) { + TDatabase file; + { TSql sql(file.Path, 1); assert_true(Apply(sql, "CREATE TABLE t (x)", UINT64_MAX).empty()); } + TSql sql(file.Path, 1); + assert_true(sql.LastAppliedIndex == UINT64_MAX); +} +void test_sql_invalid_progress_fails_closed(void**) { + for (const std::string value : {"NULL", "'-1'", "'42junk'", "'18446744073709551616'", "''", "CAST(X'3432006a756e6b' AS TEXT)"}) { + TDatabase file; + { TSql sql(file.Path, 1); Seed(sql); } + file.External("UPDATE raft_metadata_ SET value=" + value); + bool failed = false; + try { TSql sql(file.Path, 1); } + catch (const std::exception&) { failed = true; } + assert_true(failed); + } +} + +void test_sql_busy_write_does_not_consume_entry(void**) { + for (const std::string lock : {"BEGIN IMMEDIATE", "BEGIN; SELECT * FROM t"}) { + TDatabase file; + TSql sql(file.Path, 1); + Seed(sql); + sqlite3* other = nullptr; + assert_int_equal(sqlite3_open((file.Path + ".1").c_str(), &other), SQLITE_OK); + assert_int_equal(sqlite3_exec(other, lock.c_str(), nullptr, nullptr, nullptr), SQLITE_OK); + bool failed = false; + try { Apply(sql, "INSERT INTO t VALUES ('busy')", 2); } + catch (const std::runtime_error&) { failed = true; } + sqlite3_exec(other, "ROLLBACK", nullptr, nullptr, nullptr); + sqlite3_close(other); + assert_true(failed); + assert_int_equal(sql.LastAppliedIndex, 1); + assert_true(ReadText(sql, "SELECT count(*) FROM t").find("\n1\n") != std::string::npos); + assert_true(Apply(sql, "INSERT INTO t VALUES ('retry')", 2).empty()); + assert_int_equal(sql.LastAppliedIndex, 2); + } +} +void test_sql_write_rejects_embedded_nul_before_admission(void**) { + TSql sql(":memory:", 1); + const std::string text = std::string("CREATE TABLE t (x);") + '\0' + "COMMIT"; + auto command = Query(text); + command->Flags = TCommandRequest::EWrite; + Reject([&] { sql.Validate(command); }); + Reject([&] { sql.Prepare(command); }); + auto entry = NewHoldedMessage(sizeof(TSqlLogEntry) + text.size()); + entry->QuerySize = text.size(); + memcpy(entry->Query, text.data(), text.size()); + Reject([&] { sql.Write(entry, 1); }); + assert_int_equal(sql.LastAppliedIndex, 0); +} + } int main() { const CMUnitTest tests[] = { @@ -144,6 +303,16 @@ int main() { cmocka_unit_test(test_sql_read_supports_comments_ctes_and_safe_functions), cmocka_unit_test(test_sql_read_rejects_nul_empty_and_second_select), cmocka_unit_test(test_sql_read_policy_is_restored_after_rejection), + cmocka_unit_test(test_sql_write_cannot_escape_managed_transaction), + cmocka_unit_test(test_sql_write_cannot_modify_progress_metadata), + cmocka_unit_test(test_sql_failed_write_progress_survives_restart), + cmocka_unit_test(test_sql_write_handles_trailing_comment_without_losing_progress), + cmocka_unit_test(test_sql_automatic_rollback_is_consumed_without_partial_data), + cmocka_unit_test(test_sql_metadata_failure_rolls_back_data_and_does_not_advance), + cmocka_unit_test(test_sql_progress_restores_full_uint64_range), + cmocka_unit_test(test_sql_invalid_progress_fails_closed), + cmocka_unit_test(test_sql_busy_write_does_not_consume_entry), + cmocka_unit_test(test_sql_write_rejects_embedded_nul_before_admission), }; return cmocka_run_group_tests(tests, nullptr, nullptr); } From f958c430b42e2668d9dab2c64e1acf90d4f3926e Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Sun, 8 Nov 2026 14:20:05 -0500 Subject: [PATCH 12/66] Bound SQL read execution and serialize replies within a byte budget (#12) --- README.md | 2 + docs/sql-read-budgets.md | 57 +++++++++++++++++++ examples/sql_rsm.cpp | 109 +++++++++++++++++++++---------------- examples/sql_rsm.h | 28 +++------- test/test_sql_payloads.cpp | 93 +++++++++++++++++++++++++++++++ 5 files changed, 220 insertions(+), 69 deletions(-) create mode 100644 docs/sql-read-budgets.md diff --git a/README.md b/README.md index afe9efa..e164c36 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,8 @@ The [SQL read policy](docs/sql-readonly.md) prevents requests marked as reads from executing unreplicated mutations or connection controls. The [SQL application follow-up](docs/sql-application.md) protects managed write transactions and commits application progress together with successful changes. +The [SQL read budget follow-up](docs/sql-read-budgets.md) interrupts expensive +reads and bounds serialized output before appending each field or separator. [![Number of GitHub stars](https://img.shields.io/github/stars/resetius/miniraft-cpp)](https://github.com/resetius/miniraft-cpp/stargazers) ![GitHub commit activity](https://img.shields.io/github/commit-activity/m/resetius/miniraft-cpp) diff --git a/docs/sql-read-budgets.md b/docs/sql-read-budgets.md new file mode 100644 index 0000000..dc9d951 --- /dev/null +++ b/docs/sql-read-budgets.md @@ -0,0 +1,57 @@ +# SQL read budgets: v0.1.0-alpha.12 + +SQL reads previously had no execution budget and collected all rows before +serializing and checking the response frame size. Even a small recursive query +could block the single event loop indefinitely; a large result could exhaust +memory before the final size check. Six regression tests failed with the previous +read implementation (with the new limit configuration accepted but unenforced). + +TSql now accepts an optional third constructor argument, TSqlReadLimits. Defaults +apply to the existing SQL server without CLI changes: + +* MaxVmSteps defaults to 1,000,000. SQLite progress callbacks interrupt long reads + during preparation or stepping. This is an approximate instruction budget, + checked at intervals of at most 1,000 VM instructions, not a wall-clock deadline + or an exact cross-version instruction counter. Zero is rejected. +* MaxResultBytes defaults to the existing 16 MiB frame cap minus the response + header. A single serialized buffer is built while stepping; headers, commas, + NULL markers, data bytes and newlines count toward the limit before each append. + Partial output is discarded on failure. A smaller nonnegative limit is allowed; + exceeding the response payload cap is rejected at construction. +* SQLITE_LIMIT_LENGTH is temporarily lowered to 16 MiB for reads, restricting + individual strings, blobs and encoded rows before application copying. The + previous connection limit is restored afterward. Oversized stored rows may + therefore fail reads even when the requested output is small. + +Prepared statements finalize before the authorizer, progress callback and length +limit are removed/restored, on both success and exceptions. Writes retain their +existing execution policy and length limit. Rejected reads do not change applied +progress. The existing request processor maps read exceptions to ErrorCode=2 and +preserves client cookies; subsequent requests continue normally. + +Removing the intermediate row collection also fixes silent truncation at embedded +NUL bytes: value serialization uses SQLite's byte length after text conversion. +The reply remains the example's simple comma/newline format, including a single +newline for an empty result. It does not add CSV escaping or typed blob encoding. + +Seven new tests cover finite expensive aggregate queries with explicit/default +budgets, exact output boundaries (including empty results), accumulated rows, +recovery through ordinary/strong request processing, binary text lengths, invalid +configuration and restoration of SQLite's value limit before writes. Tests use +SQL computation and fixed byte limits; they need no timing assumptions or sleeps. +All 169 tests across 11 CTest targets pass locally with ASan/UBSan and SQLite; +the Release core suite passes as well. + +Self-review checked callback state and lifetime, constructor validation, cleanup +order, subtraction before append, NULL versus conversion failure, SQLite text/byte +conversion order and the unchanged write path. No independent reviewer approval +is claimed. Relevant SQLite contracts are [progress callbacks](https://www.sqlite.org/c3ref/progress_handler.html), +[runtime limits](https://www.sqlite.org/c3ref/limit.html), and +[column conversion and byte lengths](https://www.sqlite.org/c3ref/column_blob.html). + +This closes unbounded read VM execution and application result accumulation, not +all resource-exhaustion risks. A VM instruction or built-in function can itself +be expensive; temporary sorting/storage, preparation memory, total SQLite heap, +database size and committed-write execution need separate controls. There is no +hard CPU-time or process-memory bound. Authentication, admission/backpressure, +deterministic SQL, SQLite durability policy and independent review remain open. diff --git a/examples/sql_rsm.cpp b/examples/sql_rsm.cpp index b25d40f..f6da187 100644 --- a/examples/sql_rsm.cpp +++ b/examples/sql_rsm.cpp @@ -3,6 +3,8 @@ #include #include #include +#include +#include namespace { int ReadAuthorization(void*, int action, const char*, const char* second, const char*, const char*) { @@ -50,12 +52,34 @@ struct TTransactionCleanup { if (!sqlite3_get_autocommit(Db)) sqlite3_exec(Db, "ROLLBACK", nullptr, nullptr, nullptr); } }; +struct TReadBudget { + sqlite3* Db; + uint32_t Remaining; + const int Interval; + const int PreviousLength; + TReadBudget(sqlite3* db, uint32_t maxSteps) + : Db(db), Remaining(maxSteps), Interval(std::min(maxSteps, 1000)), + PreviousLength(sqlite3_limit(db, SQLITE_LIMIT_LENGTH, TMessageLimits{}.MaxFrameBytes)) { + sqlite3_progress_handler(Db, Interval, Progress, this); + } + ~TReadBudget() { + sqlite3_progress_handler(Db, 0, nullptr, nullptr); + sqlite3_limit(Db, SQLITE_LIMIT_LENGTH, PreviousLength); + } + static int Progress(void* context) noexcept { + auto& budget = *static_cast(context); + if (budget.Remaining <= uint32_t(budget.Interval)) return 1; + budget.Remaining -= budget.Interval; + return 0; + } +}; using TStatement = std::unique_ptr; } -void TSql::ExecuteRead(const std::string& q) { +std::string TSql::ExecuteRead(const std::string& q) { if (q.size() > INT_MAX || q.find('\0') != std::string::npos) throw TMessageError("Invalid SQL read text"); + TReadBudget budget(Db, ReadLimits.MaxVmSteps); TSqlPolicy policy(Db, ReadAuthorization); // retained through prepare, stepping and finalization sqlite3_stmt* raw = nullptr; const char* tail = nullptr; @@ -67,18 +91,41 @@ void TSql::ExecuteRead(const std::string& q) { rc = sqlite3_prepare_v2(Db, tail, int(q.data() + q.size() - tail), &raw, nullptr); TStatement extra(raw, sqlite3_finalize); if (rc != SQLITE_OK || extra) throw TMessageError("Multiple SQL read statements are not permitted"); - Result.Clear(); - LastError.clear(); + std::string text; + const auto append = [&](std::string_view part) { + if (part.size() > ReadLimits.MaxResultBytes - text.size()) + throw TMessageError("SQL read output budget exceeded"); + text.append(part); + }; + bool firstRow = true; while ((rc = sqlite3_step(statement.get())) == SQLITE_ROW) { const int count = sqlite3_column_count(statement.get()); - std::vector values(count), columns(count); + if (firstRow) { + for (int i = 0; i < count; ++i) { + const char* name = sqlite3_column_name(statement.get(), i); + if (!name) throw TMessageError("Cannot read SQL column name"); + append(name); + if (i + 1 < count) append(","); + } + append("\n"); + firstRow = false; + } for (int i = 0; i < count; ++i) { - values[i] = const_cast(reinterpret_cast(sqlite3_column_text(statement.get(), i))); - columns[i] = const_cast(sqlite3_column_name(statement.get(), i)); + if (sqlite3_column_type(statement.get(), i) == SQLITE_NULL) { + append("null"); + } else { + const auto* value = reinterpret_cast(sqlite3_column_text(statement.get(), i)); + if (!value) throw TMessageError("Cannot read SQL column value"); + const auto length = sqlite3_column_bytes(statement.get(), i); + append(std::string_view(value, length)); + } + if (i + 1 < count) append(","); } - Process(this, count, values.data(), columns.data()); + append("\n"); } - if (rc != SQLITE_DONE) throw TMessageError("SQL read execution failed"); + if (rc != SQLITE_DONE) throw TMessageError("SQL read failed or exceeded its execution budget"); + if (firstRow) append("\n"); // Preserve the legacy empty-result format. + return text; } void TSql::Validate(TMessageHolder message) const { @@ -87,8 +134,12 @@ void TSql::Validate(TMessageHolder message) const { if (memchr(query->Query, 0, query->QuerySize)) throw TMessageError("Invalid SQL query text"); } -TSql::TSql(const std::string& path, int serverId) +TSql::TSql(const std::string& path, int serverId, TSqlReadLimits readLimits) + : ReadLimits(readLimits) { + if (!ReadLimits.MaxVmSteps || + ReadLimits.MaxResultBytes > TMessageLimits{}.MaxFrameBytes - sizeof(TCommandResponse)) + throw std::invalid_argument("Invalid SQL read limits"); std::string dbPath = path == ":memory:" ? path : path + "." + std::to_string(serverId); const int opened = sqlite3_open(dbPath.c_str(), &Db); // Constructors that fail must also close the connection and release its locks. @@ -128,28 +179,8 @@ TSql::~TSql() } } -int TSql::Process(void* self, int ncols, char** values, char** cols) { - TSql* this_ = (TSql*)self; - if (this_->Result.Cols.empty()) { - for (int i = 0; i < ncols; i++) { - this_->Result.Cols.emplace_back(cols[i]); - } - } - TRow row; - for (int i = 0; i < ncols; i++) { - if (values[i]) { - row.Values.emplace_back(values[i]); - } else { - row.Values.emplace_back(std::nullopt); - } - } - this_->Result.Rows.emplace_back(std::move(row)); - return 0; -} - int TSql::Execute(const std::string& q) { char* err = nullptr; - Result.Clear(); LastError.clear(); const int rc = sqlite3_exec(Db, q.c_str(), nullptr, nullptr, &err); if (rc != SQLITE_OK) LastError = err ? err : sqlite3_errmsg(Db); @@ -164,25 +195,7 @@ void TSql::RequireExecute(const std::string& q) { TMessageHolder TSql::Read(TMessageHolder message, uint64_t index) { auto readSql = message.Cast(); ValidatePayloadRange(message, readSql->Query, readSql->QuerySize); - ExecuteRead(std::string(readSql->Query, readSql->QuerySize)); - std::string text; - for (int i = 0; i < Result.Cols.size(); i++) { - text += Result.Cols[i]; - if (i != Result.Cols.size() - 1) { - text += ","; - } - } - text += "\n"; - for (int j = 0; j < Result.Rows.size(); j++) { - for (int i = 0; i < Result.Cols.size(); i++) { - text += Result.Rows[j].Values[i] ? *Result.Rows[j].Values[i] : "null"; - if (i != Result.Cols.size() - 1) { - text += ","; - } - } - text += "\n"; - } - return Reply(text, index); + return Reply(ExecuteRead(std::string(readSql->Query, readSql->QuerySize)), index); } TMessageHolder TSql::Write(TMessageHolder message, uint64_t index) { diff --git a/examples/sql_rsm.h b/examples/sql_rsm.h index 2c5fb29..e7f72b6 100644 --- a/examples/sql_rsm.h +++ b/examples/sql_rsm.h @@ -1,6 +1,5 @@ #pragma once #include -#include #include struct TSqlEntry { @@ -20,28 +19,16 @@ struct TReadSql: public TCommandRequest, public TSqlEntry { }; -struct TRow { - std::vector> Values; -}; - -struct TResult { - std::vector Cols; - std::vector Rows; - - void Clear() { - Cols.clear(); - Rows.clear(); - } - - bool Empty() { - return Cols.empty(); - } +struct TSqlReadLimits { + // Approximate SQLite VM instruction budget, checked at intervals <= 1000. + uint32_t MaxVmSteps = 1'000'000; + size_t MaxResultBytes = TMessageLimits{}.MaxFrameBytes - sizeof(TCommandResponse); }; class TSql: public IRsm { public: void Validate(TMessageHolder message) const override; - TSql(const std::string& path, int serverId); + TSql(const std::string& path, int serverId, TSqlReadLimits readLimits = {}); ~TSql(); // select @@ -54,11 +41,10 @@ class TSql: public IRsm { private: int Execute(const std::string& q); void RequireExecute(const std::string& q); - void ExecuteRead(const std::string& q); - static int Process(void* self, int ncols, char** values, char** cols); + std::string ExecuteRead(const std::string& q); TMessageHolder Reply(const std::string& ans, uint64_t index); - TResult Result; + TSqlReadLimits ReadLimits; std::string LastError; sqlite3* Db = nullptr; }; diff --git a/test/test_sql_payloads.cpp b/test/test_sql_payloads.cpp index 5e8446f..fb767a0 100644 --- a/test/test_sql_payloads.cpp +++ b/test/test_sql_payloads.cpp @@ -288,6 +288,92 @@ void test_sql_write_rejects_embedded_nul_before_admission(void**) { assert_int_equal(sql.LastAppliedIndex, 0); } +const std::string ExpensiveRead = "WITH RECURSIVE n(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM n WHERE x<100000) SELECT sum(x) FROM n"; +void test_sql_read_stops_at_work_budget(void**) { + TSql sql(":memory:", 1, {.MaxVmSteps = 1000}); + Reject([&] { sql.Read(Query(ExpensiveRead), 0); }); + assert_true(ReadText(sql, "SELECT 7 AS v") == "v\n7\n"); + assert_int_equal(sql.LastAppliedIndex, 0); + TSql defaults(":memory:", 1); + Reject([&] { defaults.Read(Query(ExpensiveRead), 0); }); +} +void test_sql_read_output_budget_counts_headers_separators_and_null(void**) { + for (const auto& query : {std::string("SELECT 'abc' AS v"), std::string("SELECT NULL AS v"), + std::string("SELECT 1 AS a, 2 AS b"), std::string("SELECT 1 AS v WHERE 0")}) { + TSql reference(":memory:", 1); + const auto expected = ReadText(reference, query); + TSql exact(":memory:", 1, {.MaxResultBytes = expected.size()}); + assert_true(ReadText(exact, query) == expected); + TSql shortByOne(":memory:", 1, {.MaxResultBytes = expected.size() - 1}); + Reject([&] { shortByOne.Read(Query(query), 0); }); + } +} +void test_sql_read_bounds_accumulated_rows_and_cleans_up(void**) { + TSql sql(":memory:", 1, {.MaxResultBytes = 64}); + Reject([&] { sql.Read(Query("WITH RECURSIVE n(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM n WHERE x<10000) SELECT x FROM n"), 0); }); + assert_true(ReadText(sql, "SELECT 8 AS v") == "v\n8\n"); + assert_true(Apply(sql, "CREATE TABLE t (x); INSERT INTO t VALUES (9)", 1).empty()); + assert_true(ReadText(sql, "SELECT x FROM t") == "x\n9\n"); +} +void test_sql_read_budget_failures_recover_through_request_processor(void**) { + struct TSink : INode { + std::vector> Replies; + void Send(TMessageHolder message) override { + auto reply = message.Cast(); + Replies.emplace_back(reply->Cookie, reply->ErrorCode); + } + void Drain() override {} + }; + auto raft = std::make_shared(std::make_shared(), 1, TNodeDict{}); + raft->ProcessTimeout(ITimeSource::Time{}); + auto sql = std::make_shared(":memory:", 1, TSqlReadLimits{.MaxVmSteps = 1000, .MaxResultBytes = 64}); + TRequestProcessor processor(raft, sql, {}); + auto client = std::make_shared(); + uint32_t cookie = 1; + for (uint32_t flags : {0u, uint32_t(TCommandRequest::EConsistent)}) { + for (const auto& query : {ExpensiveRead, std::string("SELECT 'abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz' AS v"), std::string("SELECT 1 AS v")}) { + auto command = Query(query); + command->Flags = flags; + command->Cookie = cookie++; + processor.OnCommandRequest(command, client); + processor.ProcessCommitted(); + processor.ProcessWaiting(); + } + } + assert_int_equal(client->Replies.size(), 6); + for (size_t i = 0; i < client->Replies.size(); ++i) { + assert_int_equal(client->Replies[i].first, i + 1); + assert_int_equal(client->Replies[i].second, i % 3 == 2 ? 0 : 2); + } + assert_int_equal(raft->CurrentStateName(), EState::LEADER); +} +void test_sql_read_preserves_nul_bytes_and_counts_their_length(void**) { + TSql sql(":memory:", 1, {.MaxResultBytes = 6}); + assert_true(ReadText(sql, "SELECT CAST(X'610062' AS TEXT) AS v") == std::string("v\na\0b\n", 6)); + Reject([&] { sql.Read(Query("SELECT CAST(X'61006263' AS TEXT) AS v"), 0); }); +} +void test_sql_read_rejects_invalid_limits(void**) { + for (const auto limits : {TSqlReadLimits{.MaxVmSteps = 0}, + TSqlReadLimits{.MaxResultBytes = TMessageLimits{}.MaxFrameBytes}}) { + bool rejected = false; + try { TSql sql(":memory:", 1, limits); } + catch (const std::invalid_argument&) { rejected = true; } + assert_true(rejected); + } +} + +void test_sql_read_value_limit_is_restored_before_writes(void**) { + TSql sql(":memory:", 1); + const std::string value(9 * 1024 * 1024, 'x'); + assert_true(Apply(sql, "CREATE TABLE big (v TEXT); INSERT INTO big VALUES ('" + value + "')", 1).empty()); + Reject([&] { sql.Read(Query("SELECT hex(v) FROM big"), 1); }); + // Writes retain their previous SQLite length limit, even after a failed read. + assert_true(Apply(sql, "UPDATE big SET v=v||v", 2).empty()); + assert_true(Apply(sql, "UPDATE big SET v=substr(v,1,1)", 3).empty()); + assert_true(ReadText(sql, "SELECT v FROM big") == "v\nx\n"); + assert_int_equal(sql.LastAppliedIndex, 3); +} + } int main() { const CMUnitTest tests[] = { @@ -313,6 +399,13 @@ int main() { cmocka_unit_test(test_sql_invalid_progress_fails_closed), cmocka_unit_test(test_sql_busy_write_does_not_consume_entry), cmocka_unit_test(test_sql_write_rejects_embedded_nul_before_admission), + cmocka_unit_test(test_sql_read_stops_at_work_budget), + cmocka_unit_test(test_sql_read_output_budget_counts_headers_separators_and_null), + cmocka_unit_test(test_sql_read_bounds_accumulated_rows_and_cleans_up), + cmocka_unit_test(test_sql_read_budget_failures_recover_through_request_processor), + cmocka_unit_test(test_sql_read_preserves_nul_bytes_and_counts_their_length), + cmocka_unit_test(test_sql_read_rejects_invalid_limits), + cmocka_unit_test(test_sql_read_value_limit_is_restored_before_writes), }; return cmocka_run_group_tests(tests, nullptr, nullptr); } From a0c593de77879ede1c422bf0f6ac22ffedc0a54c Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Sun, 8 Nov 2026 14:26:56 -0500 Subject: [PATCH 13/66] Preserve recovered application progress and protect its committed prefix (#13) --- README.md | 2 + docs/application-recovery.md | 64 +++++++++++++++++ docs/correctness-baseline.md | 5 +- miniraft/raft.cpp | 17 +++++ miniraft/raft.h | 11 +-- test/test_correctness.cpp | 136 +++++++++++++++++++++++++++++++++++ test/test_sql_payloads.cpp | 40 +++++++++++ 7 files changed, 269 insertions(+), 6 deletions(-) create mode 100644 docs/application-recovery.md diff --git a/README.md b/README.md index e164c36..868ab2e 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,8 @@ The [SQL application follow-up](docs/sql-application.md) protects managed write transactions and commits application progress together with successful changes. The [SQL read budget follow-up](docs/sql-read-budgets.md) interrupts expensive reads and bounds serialized output before appending each field or separator. +The [application recovery follow-up](docs/application-recovery.md) preserves +recovered progress and prevents committed commands from being applied again. [![Number of GitHub stars](https://img.shields.io/github/stars/resetius/miniraft-cpp)](https://github.com/resetius/miniraft-cpp/stargazers) ![GitHub commit activity](https://img.shields.io/github/commit-activity/m/resetius/miniraft-cpp) diff --git a/docs/application-recovery.md b/docs/application-recovery.md new file mode 100644 index 0000000..2751cb7 --- /dev/null +++ b/docs/application-recovery.md @@ -0,0 +1,64 @@ +# Recovered application progress: v0.1.0-alpha.13 + +After restarting a durable application, Raft's volatile commit index began at +zero. ProcessCommitted unconditionally copied that zero into LastAppliedIndex, +discarding the application's recovered marker. Re-establishing commitment then +reapplied old commands. A file-backed SQL/TDiskState regression reproduced an +INSERT being applied twice after a clean restart. A separate restart scenario +also exposed the loss of committed-prefix protection before leader contact. + +TRequestProcessor construction now initializes a commit floor from the recovered +application marker. The marker must not exceed the recovered local log or the +signed log-index range; invalid pairings throw before changing the commit index. +The floor preserves any higher commitment already known to Raft. The existing +AppendEntries validation then protects this applied prefix immediately, while +allowing replacement of an uncommitted suffix. The restoration helper is private +to Raft and the request processor. + +ProcessCommitted returns when commitment is at or below application progress. +It never rewinds the marker or computes LastAppliedIndex+1 in that case. New +commitment applies only the remaining entries. Framework progress still advances +over internal no-op entries; an application need only persist progress alongside +its own mutations, so harmless no-ops can be revisited after restart. + +Recovery contract for embedders: + +* Restore application state and LastAppliedIndex atomically before constructing + TRequestProcessor, and construct the processor before accepting network traffic. +* A nonzero marker must represent genuinely committed progress in the same local + Raft log. Do not derive it from the log's last index: a suffix may be uncommitted. +* A volatile application starts at zero and rebuilds after commitment is known. +* Keep application and Raft storage paired. An application ahead of the local log + now fails startup instead of silently forgetting or skipping progress. + +An applied marker is trusted evidence of commitment; a range check cannot detect +a forged marker, mismatched data at the same index or application state persisted +without its marker. Snapshot/compaction recovery is not supported. Strong reads +still require a current-term commit and a fresh majority; the recovered floor +does not replace those checks. This follows the monotonic application-progress +and state-machine safety rules in [Raft's Figure 2 and Figure 3](https://raft.github.io/raft.pdf). + +Six new tests failed against the prior implementation and now pass: + +1. Restarted application progress survives an early timer and is not replayed. +2. Lower commitment input cannot rewind application progress. +3. A recovered applied prefix resists conflicting RPCs while its suffix can change. +4. Application progress ahead of the recovered log is rejected without mutation. +5. A three-node restart preserves durable application histories, rebuilds a + volatile application, replaces an uncommitted old-leader suffix and tolerates + delayed, reversed and duplicate traffic after a new election. It also checks + that recovered commitment cannot authorize new-term reads by itself. +6. Real SQL and legacy Raft files reopen without repeating a committed INSERT. + +All 175 tests across 11 CTest targets pass locally under ASan/UBSan with SQLite; +the Release core suite also passes. Self-review covered initialization ordering, +range checks before mutation, monotonicity, no-op handling, conflict protection, +current-term read gating and repeat application calls. No independent reviewer +approval is claimed. + +The simulator retains stable term/vote/log objects and reconstructs Raft volatile +state; it is a deterministic process-restart model, not torn-write simulation. +The SQL integration test performs clean file close/reopen. These additions do not +establish power-loss durability for TDiskState or SQLite, authenticated pairing +of stores, exhaustive fault schedules, snapshots or retry deduplication. Those +and independent review remain production gates. diff --git a/docs/correctness-baseline.md b/docs/correctness-baseline.md index 2fbae1b..1c968ba 100644 --- a/docs/correctness-baseline.md +++ b/docs/correctness-baseline.md @@ -32,7 +32,10 @@ The eight tests cover: Cluster scenarios check commit progress and equality of committed log prefixes. These are selected deterministic schedules, not exhaustive state-space exploration. -Restart and storage crash injection are not covered by the simulator yet. +The original baseline did not cover restarts or storage crash injection. +The [application recovery follow-up](application-recovery.md) adds deterministic +node restarts and committed application-history checks; torn-write simulation +and broader generated schedules remain open. ## Reproduce diff --git a/miniraft/raft.cpp b/miniraft/raft.cpp index 6e1b802..d53c60d 100644 --- a/miniraft/raft.cpp +++ b/miniraft/raft.cpp @@ -607,6 +607,21 @@ uint64_t TRaft::GetLastIndex() const { return State->LastLogIndex; } +void TRaft::RestoreAppliedIndex(uint64_t index) { + if (index > uint64_t(INT64_MAX) || index > State->LastLogIndex) + throw std::invalid_argument("Application progress exceeds recovered Raft log"); + // A correctly recovered application marker is evidence of commitment. + // Protect its prefix immediately, before any AppendEntries can be handled. + VolatileState->CommitIndex = std::max(VolatileState->CommitIndex, index); +} + +TRequestProcessor::TRequestProcessor(std::shared_ptr raft, std::shared_ptr rsm, const TNodeDict& nodes) + : Raft(std::move(raft)), Rsm(std::move(rsm)), Nodes(nodes) +{ + if (!Raft || !Rsm) throw std::invalid_argument("Request processor requires Raft and application state"); + Raft->RestoreAppliedIndex(Rsm->LastAppliedIndex); +} + void TRequestProcessor::CheckStateChange() { PruneRequests(); if (WaitingStateChange.empty()) { @@ -800,6 +815,8 @@ void TRequestProcessor::CleanUp(const std::shared_ptr& replyTo) { void TRequestProcessor::ProcessCommitted() { PruneRequests(); auto commitIndex = Raft->GetVolatileState()->CommitIndex; + // Never rewind application progress, or overflow LastAppliedIndex + 1. + if (commitIndex <= Rsm->LastAppliedIndex) return; auto& state = Raft->GetState(); for (auto i = Rsm->LastAppliedIndex+1; i <= commitIndex; i++) { auto entry = state->Get(i); diff --git a/miniraft/raft.h b/miniraft/raft.h index 31893fd..a8738d4 100644 --- a/miniraft/raft.h +++ b/miniraft/raft.h @@ -28,6 +28,8 @@ struct IRsm { virtual TMessageHolder Write(TMessageHolder message, uint64_t index) = 0; virtual TMessageHolder Prepare(TMessageHolder message) = 0; + // Durable RSMs restore this atomically with application data before constructing + // TRequestProcessor. It must identify committed progress in the same Raft log. uint64_t LastAppliedIndex = 0; }; @@ -136,6 +138,8 @@ class TRaft { } private: + friend class TRequestProcessor; + void RestoreAppliedIndex(uint64_t index); void Candidate(ITimeSource::Time now, TMessageHolder message); void Follower(ITimeSource::Time now, TMessageHolder message); void Leader(ITimeSource::Time now, TMessageHolder message); @@ -170,11 +174,8 @@ class TRaft { class TRequestProcessor { public: - TRequestProcessor(std::shared_ptr raft, std::shared_ptr rsm, const TNodeDict& nodes) - : Raft(raft) - , Rsm(rsm) - , Nodes(nodes) - { } + // Construct after restoring the RSM and before accepting network traffic. + TRequestProcessor(std::shared_ptr raft, std::shared_ptr rsm, const TNodeDict& nodes); void CheckStateChange(); void OnCommandRequest(TMessageHolder message, const std::shared_ptr& replyTo); diff --git a/test/test_correctness.cpp b/test/test_correctness.cpp index 8a3dfce..a984da3 100644 --- a/test/test_correctness.cpp +++ b/test/test_correctness.cpp @@ -34,6 +34,17 @@ struct TCluster { Nodes.push_back(std::make_shared(std::make_shared(), id, peers)); } } + void Restart(uint32_t id) { + auto state = Nodes.at(id - 1)->GetState(); + TNodeDict peers; + for (uint32_t dst = 1; dst <= Nodes.size(); ++dst) { + if (dst != id) peers[dst] = std::make_shared([this](auto m) { + Packets.push_back(std::move(m)); + }); + } + // Retain stable term/vote/log; discard all Raft volatile state. + Nodes.at(id - 1) = std::make_shared(state, id, peers); + } void Drain(bool reverse = false, bool duplicate = false) { size_t budget = 10000; while (!Packets.empty()) { @@ -696,10 +707,135 @@ void test_even_cluster_partition_requires_majority(void**) { for (auto& node : cluster.Nodes) assert_int_equal(node->GetVolatileState()->CommitIndex, index); cluster.AssertCommittedPrefix(); } +std::shared_ptr RecoverApplication(const TEchoRsm& old) { + auto recovered = std::make_shared(); + recovered->Applied = old.Applied; + recovered->LastAppliedIndex = old.LastAppliedIndex; + return recovered; +} +void test_recovered_application_progress_is_not_rewound(void**) { + TCluster cluster(1); + cluster.Elect(1); + cluster.Nodes[0]->Append(LogValue("once")); + cluster.Replicate(1); + auto original = std::make_shared(); + { TRequestProcessor processor(cluster.Nodes[0], original, {}); processor.ProcessCommitted(); } + auto recovered = RecoverApplication(*original); + cluster.Restart(1); + TRequestProcessor processor(cluster.Nodes[0], recovered, {}); + processor.ProcessCommitted(); // timer before the first post-restart election + assert_int_equal(recovered->LastAppliedIndex, original->LastAppliedIndex); + cluster.Elect(1); + processor.ProcessCommitted(); + processor.ProcessCommitted(); + assert_true(recovered->Applied == std::vector{"once"}); +} +void test_application_progress_never_decreases_on_lower_commit_input(void**) { + TCluster cluster(1); + cluster.Elect(1); + auto recovered = std::make_shared(); + recovered->LastAppliedIndex = 1; + TRequestProcessor processor(cluster.Nodes[0], recovered, {}); + // Defensive check even if an embedder violates commit-index monotonicity. + const_cast(cluster.Nodes[0]->GetVolatileState())->CommitIndex = 0; + processor.ProcessCommitted(); + assert_int_equal(recovered->LastAppliedIndex, 1); +} +void test_recovered_applied_prefix_is_protected_and_suffix_can_change(void**) { + TCluster cluster(2); + auto state = cluster.Nodes[0]->GetState(); + state->CurrentTerm = 2; + for (const std::string value : {"applied", "uncommitted"}) { + auto entry = LogValue(value); + entry->Term = 2; + state->Append(entry); + } + auto recovered = std::make_shared(); + recovered->Applied = {"applied"}; + recovered->LastAppliedIndex = 1; + TRequestProcessor processor(cluster.Nodes[0], recovered, {}); + auto request = [&](uint64_t prev, uint64_t prevTerm, const std::string& value) { + auto message = NewHoldedMessage(TMessageEx{.Src = 2, .Dst = 1, .Term = 3}, + TAppendEntriesRequest{.PrevLogIndex = prev, .PrevLogTerm = prevTerm, + .LeaderCommit = 2, .LeaderId = 2, .Nentries = 1}); + message.InitPayload(1); + auto entry = LogValue(value); + entry->Term = 3; + message.Payload[0] = entry; + return message; + }; + cluster.Nodes[0]->Process(cluster.Now, request(0, 0, "overwrite-applied")); + assert_int_equal(state->LogTerm(1), 2); + assert_int_equal(cluster.Nodes[0]->GetVolatileState()->CommitIndex, 1); + cluster.Nodes[0]->Process(cluster.Now, request(1, 2, "replacement")); + processor.ProcessCommitted(); + assert_true(recovered->Applied == (std::vector{"applied", "replacement"})); + assert_int_equal(recovered->LastAppliedIndex, 2); +} +void test_application_ahead_of_recovered_log_is_rejected(void**) { + TCluster cluster(1); + for (const auto index : {uint64_t(1), UINT64_MAX}) { + auto recovered = std::make_shared(); + recovered->LastAppliedIndex = index; + bool rejected = false; + try { TRequestProcessor processor(cluster.Nodes[0], recovered, {}); } + catch (const std::invalid_argument&) { rejected = true; } + assert_true(rejected); + assert_true(recovered->LastAppliedIndex == index); + assert_int_equal(cluster.Nodes[0]->GetVolatileState()->CommitIndex, 0); + assert_int_equal(cluster.Nodes[0]->GetLastIndex(), 0); + } +} +void test_cluster_restart_preserves_applied_history_and_rebuilds_volatile_application(void**) { + TCluster cluster(3); + cluster.Elect(1); + cluster.Nodes[0]->Append(LogValue("first")); + cluster.Replicate(1); + std::vector> recovered; + for (auto& node : cluster.Nodes) { + auto rsm = std::make_shared(); + TRequestProcessor processor(node, rsm, {}); + processor.ProcessCommitted(); + recovered.push_back(RecoverApplication(*rsm)); + } + cluster.Nodes[0]->Append(LogValue("old-uncommitted-tail")); + // Keep delayed old-term traffic across all node restarts. + cluster.Nodes[0]->ProcessTimeout(cluster.Now + std::chrono::seconds(1)); + auto delayed = std::move(cluster.Packets); + cluster.Packets.clear(); + for (uint32_t id = 1; id <= 3; ++id) cluster.Restart(id); + // Node 3 instead uses a volatile application and must rebuild from index zero. + recovered[2] = std::make_shared(); + std::vector> processors; + for (size_t i = 0; i < recovered.size(); ++i) { + processors.push_back(std::make_unique(cluster.Nodes[i], recovered[i], TNodeDict{})); + processors.back()->ProcessCommitted(); + } + cluster.Elect(2); + // Recovered old-term commitment cannot replace a fresh current-term read quorum. + assert_int_equal(cluster.Nodes[1]->CommitSeqno(), 0); + cluster.Nodes[1]->Append(LogValue("second")); + cluster.Replicate(2, true, true); + cluster.Packets = std::move(delayed); + cluster.Drain(true, true); + cluster.Replicate(2, true, true); + for (auto& processor : processors) { processor->ProcessCommitted(); processor->ProcessCommitted(); } + for (const auto& rsm : recovered) { + assert_true(rsm->Applied == (std::vector{"first", "second"})); + assert_int_equal(rsm->LastAppliedIndex, cluster.Nodes[1]->GetVolatileState()->CommitIndex); + } + cluster.AssertCommittedPrefix(); +} + } int main() { const CMUnitTest tests[] = { + cmocka_unit_test(test_recovered_application_progress_is_not_rewound), + cmocka_unit_test(test_application_progress_never_decreases_on_lower_commit_input), + cmocka_unit_test(test_recovered_applied_prefix_is_protected_and_suffix_can_change), + cmocka_unit_test(test_application_ahead_of_recovered_log_is_rejected), + cmocka_unit_test(test_cluster_restart_preserves_applied_history_and_rebuilds_volatile_application), cmocka_unit_test(test_majority_all_cluster_sizes), cmocka_unit_test(test_read_fifo), cmocka_unit_test(test_strong_read_fifo), diff --git a/test/test_sql_payloads.cpp b/test/test_sql_payloads.cpp index fb767a0..a724ea3 100644 --- a/test/test_sql_payloads.cpp +++ b/test/test_sql_payloads.cpp @@ -1,4 +1,5 @@ #include "../examples/sql_rsm.h" +#include #include #include #include @@ -374,6 +375,44 @@ void test_sql_read_value_limit_is_restored_before_writes(void**) { assert_int_equal(sql.LastAppliedIndex, 3); } +void test_sql_and_raft_restart_do_not_reapply_committed_insert(void**) { + TDatabase file; + const auto logPath = file.Path + "-raft"; + { + auto state = std::make_shared(logPath, 1); + auto raft = std::make_shared(state, 1, TNodeDict{}); + auto sql = std::make_shared(file.Path, 1); + TRequestProcessor processor(raft, sql, {}); + auto now = ITimeSource::Time{}; + raft->ProcessTimeout(now); + for (const auto query : {"CREATE TABLE t (value TEXT)", "INSERT INTO t VALUES ('once')"}) { + auto command = Query(query); + command->Flags = TCommandRequest::EWrite; + processor.OnCommandRequest(command, {}); + now += std::chrono::milliseconds(500); + raft->ProcessTimeout(now); + processor.ProcessCommitted(); + } + assert_int_equal(sql->LastAppliedIndex, 3); + assert_true(ReadText(*sql, "SELECT count(*) AS n FROM t") == "n\n1\n"); + } + { + auto state = std::make_shared(logPath, 1); + auto raft = std::make_shared(state, 1, TNodeDict{}); + auto sql = std::make_shared(file.Path, 1); + assert_int_equal(sql->LastAppliedIndex, 3); + TRequestProcessor processor(raft, sql, {}); + processor.ProcessCommitted(); // server timer can fire before any leader is known + const auto beforeElection = sql->LastAppliedIndex; + raft->ProcessTimeout(ITimeSource::Time{}); + processor.ProcessCommitted(); + processor.ProcessCommitted(); + assert_true(ReadText(*sql, "SELECT count(*) AS n FROM t") == "n\n1\n"); + assert_int_equal(beforeElection, 3); + assert_int_equal(sql->LastAppliedIndex, 4); // new-term stub advanced framework progress + } +} + } int main() { const CMUnitTest tests[] = { @@ -406,6 +445,7 @@ int main() { cmocka_unit_test(test_sql_read_preserves_nul_bytes_and_counts_their_length), cmocka_unit_test(test_sql_read_rejects_invalid_limits), cmocka_unit_test(test_sql_read_value_limit_is_restored_before_writes), + cmocka_unit_test(test_sql_and_raft_restart_do_not_reapply_committed_insert), }; return cmocka_run_group_tests(tests, nullptr, nullptr); } From 93f83323877816e0055cbe1a93e8c41ac8cba34a Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Sun, 8 Nov 2026 19:18:58 -0500 Subject: [PATCH 14/66] Reject impossible log terms in legacy storage recovery and reads (#14) --- README.md | 2 + docs/legacy-log-terms.md | 51 ++++++++++++++++++ docs/persistence-recovery.md | 2 + miniraft/persist.cpp | 7 +++ test/test_persist_recovery.cpp | 95 ++++++++++++++++++++++++++++++++++ test/test_raft.cpp | 6 ++- 6 files changed, 162 insertions(+), 1 deletion(-) create mode 100644 docs/legacy-log-terms.md diff --git a/README.md b/README.md index 868ab2e..76c2e2d 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,8 @@ The [SQL read budget follow-up](docs/sql-read-budgets.md) interrupts expensive reads and bounds serialized output before appending each field or separator. The [application recovery follow-up](docs/application-recovery.md) preserves recovered progress and prevents committed commands from being applied again. +The [legacy log-term follow-up](docs/legacy-log-terms.md) rejects impossible +stored term sequences before they influence elections or log matching. [![Number of GitHub stars](https://img.shields.io/github/stars/resetius/miniraft-cpp)](https://github.com/resetius/miniraft-cpp/stargazers) ![GitHub commit activity](https://img.shields.io/github/commit-activity/m/resetius/miniraft-cpp) diff --git a/docs/legacy-log-terms.md b/docs/legacy-log-terms.md new file mode 100644 index 0000000..bc94536 --- /dev/null +++ b/docs/legacy-log-terms.md @@ -0,0 +1,51 @@ +# Legacy log-term validation: v0.1.0-alpha.14 + +TDiskState recovery checked file sizes, offsets and entry framing but did not +validate the selected entries' term values. It could accept zero terms, a log +newer than CurrentTerm, or a term sequence that went backward. Those invalid +values then fed Raft's vote freshness and log-matching decisions. The opt-in +TDurableState backend already rejected these term sequences on recovery. + +Legacy recovery now validates every selected entry's term: it must be nonzero, +no greater than the stored CurrentTerm and no less than the preceding term. +Startup throws std::runtime_error without modifying any stored bytes. Equal +terms and gaps between terms are valid, including full-width uint64_t values. + +The basic range check is also applied when reading an entry after startup. If a +live read detects a zero term or a term newer than the in-memory CurrentTerm, the +existing Fail path terminates the node. An ordinary exception would be caught by +the network layer and allow a node with detected storage damage to keep serving. +Full sequence ordering is checked at startup, not by scanning all neighbors on +each live Get. + +Only the prefix selected by LastLogIndex is validated. Obsolete suffix bytes left +by logical truncation remain ignored and can be overwritten by a valid append. +There is no file-format or storage-interface change. Previously accepted stores +with impossible term metadata now require verified recovery or safe rebuilding; +the code does not guess corrections or reset the node's term/vote. + +Six new tests cover zero terms at both selected entry positions, future terms or +stale current-term metadata, decreasing terms, valid repeated/skipped/full-width +terms, an invalid obsolete suffix followed by replacement, and fatal live-read +range violations. Four tests failed against the previous implementation; two +positive compatibility tests already passed. Rejection tests compare snapshots +of every storage file before and after startup to verify byte-for-byte preservation. +The live-read subprocess test is excluded on Windows. + +Two old test fixtures appended terms newer than CurrentTerm and only updated the +current term afterward (or never did). They now persist CurrentTerm first, +matching TRaft's existing term-update ordering before AppendEntries application. +Direct storage users remain responsible for consistent fields and valid append +inputs; this change does not add a proposal validator to the legacy writer. + +All 181 tests across 11 CTest targets pass locally with SQLite under ASan/UBSan; +the Release core suite passes as well. Self-review checked full-prefix traversal, +unsigned comparisons, validation before exposing an entry, catchable startup +errors versus fatal live errors, ignored suffixes and fixture ordering. No +independent reviewer approval is claimed. + +This detects impossible metadata rather than all corruption. Valid-looking term +or payload changes remain undetectable without checksums. Legacy TDiskState still +requires one writer and lacks power-loss durability, atomic publication and +crash-safe replacement. It is not upgraded to TDurableState by this validation +change. Snapshots, safe migration and independent review remain production gates. diff --git a/docs/persistence-recovery.md b/docs/persistence-recovery.md index 6c27856..11b96e2 100644 --- a/docs/persistence-recovery.md +++ b/docs/persistence-recovery.md @@ -17,6 +17,8 @@ nonzero current term. Recovery validates the complete selected log prefix: index extent, contiguous offsets, entry types, minimum lengths and file bounds. Unreferenced trailing bytes remain allowed because log truncation is logical. The file format and public storage interface are unchanged. +The [log-term follow-up](legacy-log-terms.md) additionally validates nonzero, +nondecreasing selected entry terms bounded by CurrentTerm. RemoveLast now updates LastLogTerm, including when the log becomes empty. Append explicitly seeks the write position, so appending after truncation to diff --git a/miniraft/persist.cpp b/miniraft/persist.cpp index a02f68b..dce7979 100644 --- a/miniraft/persist.cpp +++ b/miniraft/persist.cpp @@ -53,6 +53,9 @@ TDiskState::TDiskState(const std::string& name, uint32_t id) throw std::runtime_error("Invalid Raft log offset: " + indexPath); } auto entry = Get(i); + if (entry->Term < LastLogTerm) { + throw std::runtime_error("Decreasing Raft log terms: " + entriesPath); + } expectedOffset += entry->Len; LastLogTerm = entry->Term; } @@ -102,6 +105,10 @@ TMessageHolder TDiskState::Read() const if (!Entries.read(reinterpret_cast(message->Value), len - sizeof(TMessage))) { Fail("Cannot read Raft log entry"); } + const auto term = message.Cast()->Term; + if (!term || term > CurrentTerm) { + Fail("Raft log term outside current term range"); + } return message; } diff --git a/test/test_persist_recovery.cpp b/test/test_persist_recovery.cpp index 38b8b4b..449018e 100644 --- a/test/test_persist_recovery.cpp +++ b/test/test_persist_recovery.cpp @@ -178,6 +178,73 @@ void test_truncate_updates_last_term_and_restarts(void**) { assert_int_equal(recovered.Get(1)->Term, 6); assert_int_equal(recovered.LastLogTerm, 6); } +void test_zero_log_entry_terms_are_rejected_without_mutation(void**) { + for (uint64_t index : {0, 1}) { + TFiles files; + files.Seed(); + files.Overwrite("entries", index * sizeof(TLogEntry) + sizeof(TMessage), uint64_t(0)); + files.RejectRecovery(); + } +} +void test_log_terms_newer_than_current_term_are_rejected(void**) { + for (uint64_t term : {uint64_t(8), UINT64_MAX}) { + TFiles files; + files.Seed(); + files.Overwrite("entries", sizeof(TLogEntry) + sizeof(TMessage), term); + files.RejectRecovery(); + } + TFiles files; + files.Seed(); + files.Overwrite("state", sizeof(uint64_t), uint64_t(4)); + files.RejectRecovery(); +} +void test_decreasing_log_terms_are_rejected(void**) { + for (uint64_t index : {0, 1}) { + TFiles files; + files.Seed(); + files.Overwrite("entries", index * sizeof(TLogEntry) + sizeof(TMessage), uint64_t(index == 0 ? 6 : 1)); + files.RejectRecovery(); + } +} +void test_equal_and_skipped_terms_and_full_width_current_term_recover(void**) { + TFiles files; + { + TDiskState state(files.Name, 1); + state.CurrentTerm = UINT64_MAX; + state.VotedFor = 2; + for (uint64_t term : {uint64_t(1), uint64_t(1), uint64_t(5), UINT64_MAX, UINT64_MAX}) { + auto entry = NewHoldedMessage(); + entry->Term = term; + state.Append(entry); + } + } + TDiskState recovered(files.Name, 1); + assert_true(recovered.CurrentTerm == UINT64_MAX); + assert_true(recovered.LastLogTerm == UINT64_MAX); + assert_int_equal(recovered.LastLogIndex, 5); + assert_int_equal(recovered.VotedFor, 2); + assert_int_equal(recovered.Get(1)->Term, 1); + assert_int_equal(recovered.Get(2)->Term, 1); + assert_int_equal(recovered.Get(3)->Term, 5); +} +void test_invalid_terms_in_logically_removed_suffix_are_ignored(void**) { + TFiles files; + files.Seed(); + { + TDiskState state(files.Name, 1); + state.RemoveLast(); + } + files.Overwrite("entries", sizeof(TLogEntry) + sizeof(TMessage), uint64_t(0)); + TDiskState recovered(files.Name, 1); + assert_int_equal(recovered.LastLogIndex, 1); + assert_int_equal(recovered.LastLogTerm, 2); + auto entry = NewHoldedMessage(); + entry->Term = 7; + recovered.Append(entry); + assert_int_equal(recovered.LastLogIndex, 2); + assert_int_equal(recovered.Get(2)->Term, 7); +} + #ifndef _WIN32 void test_live_storage_failure_is_fatal(void**) { TFiles files; @@ -198,6 +265,28 @@ void test_live_storage_failure_is_fatal(void**) { assert_true(WIFEXITED(status)); assert_int_equal(WEXITSTATUS(status), 73); } +void test_live_invalid_log_term_is_fatal(void**) { + for (uint64_t term : {uint64_t(0), uint64_t(8)}) { + TFiles files; + files.Seed(); + const auto child = fork(); + assert_true(child >= 0); + if (child == 0) { + std::set_terminate([] { _exit(73); }); + try { + TDiskState state(files.Name, 1); + files.Overwrite("entries", sizeof(TMessage), term); + state.Get(1); + } catch (...) { _exit(74); } + _exit(75); + } + int status = 0; + assert_int_equal(waitpid(child, &status, 0), child); + assert_true(WIFEXITED(status)); + assert_int_equal(WEXITSTATUS(status), 73); + } +} + #endif } @@ -213,8 +302,14 @@ int main() { cmocka_unit_test(test_invalid_term_is_rejected), cmocka_unit_test(test_invalid_log_count_is_rejected), cmocka_unit_test(test_truncate_updates_last_term_and_restarts), + cmocka_unit_test(test_zero_log_entry_terms_are_rejected_without_mutation), + cmocka_unit_test(test_log_terms_newer_than_current_term_are_rejected), + cmocka_unit_test(test_decreasing_log_terms_are_rejected), + cmocka_unit_test(test_equal_and_skipped_terms_and_full_width_current_term_recover), + cmocka_unit_test(test_invalid_terms_in_logically_removed_suffix_are_ignored), #ifndef _WIN32 cmocka_unit_test(test_live_storage_failure_is_fatal), + cmocka_unit_test(test_live_invalid_log_term_is_fatal), #endif }; return cmocka_run_group_tests(tests, nullptr, nullptr); diff --git a/test/test_raft.cpp b/test/test_raft.cpp index 8808490..0756b42 100644 --- a/test/test_raft.cpp +++ b/test/test_raft.cpp @@ -675,6 +675,8 @@ void test_disk_state(void**) { assert_int_equal(state.LastLogIndex, 0); assert_int_equal(state.CurrentTerm, 1); assert_int_equal(state.VotedFor, 0); + state.CurrentTerm = 6; + state.Commit(); auto log = MakeLog({1,1,1,4,4,5,5,6,6}); for (auto entry : log) { state.Append(entry); @@ -730,13 +732,15 @@ void test_disk_state_restore(void**) { assert_int_equal(state->LastLogIndex, 0); assert_int_equal(state->CurrentTerm, 1); assert_int_equal(state->VotedFor, 0); + // Raft persists a new current term before appending entries from that term. + state->CurrentTerm = 10; + state->Commit(); auto log = MakeLog({1,1,1,4,4,5,5,6,6}); for (auto entry : log) { state->Append(entry); } assert_int_equal(state->LastLogIndex, 9); state->VotedFor = 2; - state->CurrentTerm = 10; state->Commit(); state.reset(); From 4b76d8b5290387706ec1d436b5b8b64d5aec656d Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Sun, 8 Nov 2026 19:32:09 -0500 Subject: [PATCH 15/66] Add durable KV checkpoints with validated snapshot recovery (#15) --- README.md | 3 + docs/snapshots.md | 119 ++++++++++++++ examples/kv.cpp | 41 ++++- examples/kv.h | 2 + examples/kv_rsm.cpp | 46 ++++++ miniraft/CMakeLists.txt | 1 + miniraft/raft.cpp | 7 + miniraft/raft.h | 6 + miniraft/snapshot.cpp | 200 +++++++++++++++++++++++ miniraft/snapshot.h | 45 ++++++ miniraft/snapshot_codec.h | 40 +++++ test/CMakeLists.txt | 9 ++ test/smoke_kv_checkpoints.py | 120 ++++++++++++++ test/test_snapshots.cpp | 304 +++++++++++++++++++++++++++++++++++ 14 files changed, 942 insertions(+), 1 deletion(-) create mode 100644 docs/snapshots.md create mode 100644 miniraft/snapshot.cpp create mode 100644 miniraft/snapshot.h create mode 100644 miniraft/snapshot_codec.h create mode 100644 test/smoke_kv_checkpoints.py create mode 100644 test/test_snapshots.cpp diff --git a/README.md b/README.md index 76c2e2d..8c0f8ce 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,9 @@ The [application recovery follow-up](docs/application-recovery.md) preserves recovered progress and prevents committed commands from being applied again. The [legacy log-term follow-up](docs/legacy-log-terms.md) rejects impossible stored term sequences before they influence elections or log matching. +The [snapshot foundation](docs/snapshots.md) adds opt-in durable KV checkpoints +and tail-only application recovery with `--durable --snapshots N`. Logs remain +fully retained; compaction, snapshot transfer and membership changes are next. [![Number of GitHub stars](https://img.shields.io/github/stars/resetius/miniraft-cpp)](https://github.com/resetius/miniraft-cpp/stargazers) ![GitHub commit activity](https://img.shields.io/github/commit-activity/m/resetius/miniraft-cpp) diff --git a/docs/snapshots.md b/docs/snapshots.md new file mode 100644 index 0000000..7f6bc1a --- /dev/null +++ b/docs/snapshots.md @@ -0,0 +1,119 @@ +# Snapshot foundation: durable KV checkpoints, v0.1.0-alpha.15 + +This milestone adds local application checkpoint creation and restart recovery. +It retains the entire Raft log. InstallSnapshot RPCs and prefix compaction remain +unimplemented, so checkpoints improve application recovery but do not bound log +storage or bootstrap a node whose local log is missing. + +## Use + +On Linux, start a KV server with its existing fixed node list and add: + +```sh +./build/full/examples/kv --server --id 1 --node 127.0.0.1:8001:1 \ + --durable --snapshots 1000 +``` + +Use the complete, identical voter-ID configuration on each node as before. +`--snapshots N` requires a positive interval and `--server --durable`; it is off +by default. A one-second polling coroutine captures a checkpoint after at least +N additional applied log indices, including internal no-ops. Creation, encoding +and publication execute synchronously on the serialized event loop. Large maps +and slow storage can pause processing; asynchronous/COW capture is future work. + +The server writes `state.snapshot.` beside `state.durable.`, with separate +`.lock` and `.tmp` paths. Startup restores an existing checkpoint before creating +the request processor or accepting traffic. The recovered application marker +establishes the existing commit floor, and later commitment replays only the tail. +Without a checkpoint, the application rebuilds from the retained log. Disabling +the option also rebuilds from that log, leaving checkpoint files unused. + +## Contracts and storage + +IRsm gains optional SaveSnapshot/LoadSnapshot callbacks; the defaults throw an +unsupported-operation error. KV implements canonical, sorted, binary-safe state +encoding. Restore validates the entire image into a temporary map before swapping +state and progress. Key/value lengths retain the KV wire format's 65,535-byte +limits. SQL and the dummy application do not implement snapshots in this release. + +CaptureSnapshot requires a nonzero applied index no greater than the known commit +index or retained log end. Metadata records its index, term and sorted fixed voter +IDs. RestoreSnapshot requires a fresh application and an exact voter-set match; +the boundary index must exist in the local log with the same term, not newer than +CurrentTerm. Restore does not overwrite current term or votedFor. These APIs must +run serialized, with restore only during startup before request processing. + +The versioned image contains little-endian fields in this order: + +| Field | Bytes | +| --- | ---: | +| Magic/version `MRFTS001` | 8 | +| Last included index and term | 8 each | +| Voter count | 4 | +| Application byte length | 8 | +| Sorted unique positive voter IDs | 4 each | +| Application bytes | declared length | +| CRC32 covering every preceding byte | 4 | + +The complete image is capped at 64 MiB and the voter count at 4,096. Decoding +checks lengths/counts against remaining input before allocation/copying, rejects +unknown versions and trailing bytes, and validates metadata after CRC verification. +The KV payload has its own `MRKV0001` version, count and length-prefixed entries. + +TSnapshotStore takes an exclusive advisory writer lock and publishes through temp +creation, complete write, file fsync, atomic rename and directory fsync. Startup +stabilizes the selected image/name with fsync. Temp files are never recovery +candidates. A missing published image is safe because the complete log remains; +an existing corrupt or mismatched image fails startup without speculative repair. +The directory and local writers are trusted; CRC32 detects damage, not forgery or +wrong-store copies with otherwise matching boundary metadata and voter IDs. + +Snapshot-save failures are catchable and logged/retried by the example. No Raft +entry is discarded and the selected checkpoint is a complete committed image, +so consensus can continue using its retained log. A failure around rename can +leave either complete checkpoint selected after a crash. Never delete Raft term, +vote or log files to work around a checkpoint recovery error. + +## Validation and review + +Eleven new unit tests cover codec round trips, every-byte corruption/truncation, +semantic corruption with a valid checksum, bounded file reads, temp symlink and +concurrent-writer rejection, canonical binary KV encoding, atomic malformed KV +restore rejection, capture before commitment, log/configuration mismatch, +publication errors, crashes at all five publication stages on first/replacement +save, and durable Raft/KV restart with tail-only application. Crash tests use +explicit stage hooks and subprocess exit, with no timing-dependent schedules. + +A separate real-process CLI smoke test covers option validation, publication, +restart and strong read, changed voter IDs, corrupt checkpoint rejection and +unchanged Raft storage. It uses bounded waits/sockets and is distinct from the +deterministic fault tests. Linux full CTest runs it when Python 3 is available. +All 192 unit tests and the CLI test pass locally across 13 CTest targets under +ASan/UBSan; the Release core suite passes too. + +Self-review checked serialized capture boundaries, metadata and allocation limits, +checksum coverage, version rejection, atomic map replacement, restore before +commit-floor initialization, fsync/rename order, publication failure semantics, +and retention of all logs. No independent reviewer approval is claimed. Process +crash injection does not substitute for hardware power-cut/storage fault testing. + +## Next snapshot and membership milestones + +1. Introduce snapshot-boundary-aware log indexing and atomic durable publication + of snapshot plus retained suffix. Validate votes, AppendEntries consistency, + current-term commitment and strong reads at the compacted boundary. +2. Add bounded, resumable InstallSnapshot transfer. Test interrupted/reordered/ + duplicate chunks, leadership changes, concurrent tail replication, stale + snapshots and follower restart. Enable prefix deletion only once lagging + followers have a verified recovery path. +3. Add non-voting learners and catch-up gates, then replicated configuration + entries with joint consensus. Elections, commits and strong-read confirmation + must honor both old and new majorities during transition. Persist configuration + state in logs/snapshots and test rollback, restart and leader removal. + +These follow the snapshot metadata/state-transfer requirements in +[Raft section 7](https://raft.github.io/raft.pdf) and the overlapping-quorum +configuration protocol in section 6. Editing the static `--node` list is not a +membership-change protocol. Snapshots do not make such edits safe. This release +keeps fixed membership and leaves full compaction, transfer, dynamic membership, +background capture, SQL snapshots and independent review as open release gates. diff --git a/examples/kv.cpp b/examples/kv.cpp index 227a7d4..6aedd94 100644 --- a/examples/kv.cpp +++ b/examples/kv.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include "kv.h" @@ -94,8 +95,25 @@ NNet::TFuture Client(TPoller& poller, TSocket socket, uint32_t flags) { co_return; } +template +NNet::TVoidTask CheckpointLoop(TPoller& poller, std::shared_ptr raft, + std::shared_ptr rsm, std::shared_ptr store, uint64_t interval, uint64_t saved) { + while (true) { + co_await poller.Sleep(std::chrono::seconds(1)); + if (rsm->LastAppliedIndex <= saved || rsm->LastAppliedIndex - saved < interval) continue; + try { + auto snapshot = CaptureSnapshot(*raft, *rsm); + store->Save(snapshot); + saved = snapshot.Index; + } catch (const std::exception& error) { + // Checkpoints are optional while every log entry is retained. + std::cerr << "Checkpoint failed: " << error.what() << "\n"; + } + } +} + void usage(const char* prog) { - std::cerr << prog << "--client|--server --id myid --node ip:port:id [--node ip:port:id ...] [--persist|--durable] [--stale] [--consistent]" << "\n"; + std::cerr << prog << "--client|--server --id myid --node ip:port:id [--node ip:port:id ...] [--persist|--durable] [--snapshots entries] [--stale] [--consistent]" << "\n"; exit(0); } @@ -108,6 +126,7 @@ int main(int argc, char** argv) { bool server = false; bool persist = false; bool durable = false; + uint64_t snapshotInterval = 0; uint32_t flags = 0; for (int i = 1; i < argc; i++) { if (!strcmp(argv[i], "--server")) { @@ -122,6 +141,13 @@ int main(int argc, char** argv) { } else if (!strcmp(argv[i], "--durable")) { persist = true; durable = true; + } else if (!strcmp(argv[i], "--snapshots")) { + if (i + 1 >= argc) { std::cerr << "--snapshots requires a positive entry interval\n"; return 1; } + const std::string_view value(argv[++i]); + const auto result = std::from_chars(value.data(), value.data() + value.size(), snapshotInterval); + if (result.ec != std::errc{} || result.ptr != value.data() + value.size() || !snapshotInterval) { + std::cerr << "--snapshots requires a positive entry interval\n"; return 1; + } } else if (!strcmp(argv[i], "--stale")) { flags |= TCommandRequest::EStale; } else if (!strcmp(argv[i], "--consistent")) { @@ -131,6 +157,9 @@ int main(int argc, char** argv) { } } + if (snapshotInterval && (!server || !durable)) { + std::cerr << "--snapshots requires --server --durable\n"; return 1; + } using TPoller = NNet::TDefaultPoller; std::shared_ptr timeSource = std::make_shared(); NNet::TLoop loop; @@ -162,11 +191,21 @@ int main(int argc, char** argv) { else state = std::make_shared("state", myHost.Id); } auto raft = std::make_shared(state, myHost.Id, nodes); + std::shared_ptr snapshots; + uint64_t savedSnapshot = 0; + if (snapshotInterval) { + snapshots = std::make_shared("state", myHost.Id); + if (auto snapshot = snapshots->Load()) { + RestoreSnapshot(*snapshot, *raft, *rsm); + savedSnapshot = snapshot->Index; + } + } TPoller::TSocket socket(NNet::TAddress{myHost.Address, myHost.Port}, loop.Poller()); socket.Bind(); socket.Listen(); TRaftServer server(loop.Poller(), std::move(socket), raft, rsm, nodes, timeSource); server.Serve(); + if (snapshots) CheckpointLoop(loop.Poller(), raft, rsm, snapshots, snapshotInterval, savedSnapshot); loop.Loop(); } else { NNet::TAddress addr{hosts[0].Address, hosts[0].Port}; diff --git a/examples/kv.h b/examples/kv.h index c584ee4..6309fa9 100644 --- a/examples/kv.h +++ b/examples/kv.h @@ -11,6 +11,8 @@ class TKv: public IRsm { TMessageHolder Read(TMessageHolder message, uint64_t index) override; TMessageHolder Write(TMessageHolder message, uint64_t index) override; TMessageHolder Prepare(TMessageHolder message) override; + std::string SaveSnapshot() const override; + void LoadSnapshot(std::string_view bytes, uint64_t index) override; private: std::unordered_map H; diff --git a/examples/kv_rsm.cpp b/examples/kv_rsm.cpp index 061232f..85cfbf3 100644 --- a/examples/kv_rsm.cpp +++ b/examples/kv_rsm.cpp @@ -2,6 +2,9 @@ #include "kv_messages.h" #include "payload_validation.h" #include +#include +#include +#include void TKv::Validate(TMessageHolder message) const { if (!message || message->Len < sizeof(TCommandRequest)) throw TMessageError("Short KV command"); @@ -69,3 +72,46 @@ TMessageHolder TKv::Prepare(TMessageHolder command) return entry; } +std::string TKv::SaveSnapshot() const { + using namespace NSnapshotCodec; + std::vector keys; + for (const auto& [key, _] : H) keys.push_back(key); + std::sort(keys.begin(), keys.end()); + std::string bytes = "MRKV0001"; + Put(bytes, H.size(), 8); + for (auto key : keys) { + const auto& value = H.at(std::string(key)); + if (key.size() > UINT16_MAX || value.size() > UINT16_MAX) + throw std::runtime_error("KV snapshot field too large"); + Put(bytes, key.size(), 4); + Put(bytes, value.size(), 4); + Append(bytes, key); + Append(bytes, value); + } + return bytes; +} + +void TKv::LoadSnapshot(std::string_view bytes, uint64_t index) { + using namespace NSnapshotCodec; + if (!index || index > uint64_t(INT64_MAX) || index < LastAppliedIndex || bytes.size() > TSnapshot::MaxBytes) + throw std::runtime_error("Invalid KV snapshot progress/size"); + TReader reader{bytes}; + if (reader.Read(8) != "MRKV0001") throw std::runtime_error("Unknown KV snapshot format"); + const auto count = reader.Take(8); + if (count > reader.Input.size() / 8) throw std::runtime_error("Invalid KV snapshot count"); + std::unordered_map candidate; + std::string_view previous; + for (uint64_t i = 0; i < count; ++i) { + const auto keyLength = reader.Take(4), valueLength = reader.Take(4); + if (keyLength > UINT16_MAX || valueLength > UINT16_MAX) + throw std::runtime_error("KV snapshot field exceeds application limits"); + const auto key = reader.Read(keyLength), value = reader.Read(valueLength); + if (!valueLength || (i && key <= previous)) throw std::runtime_error("Invalid KV snapshot order/value"); + candidate.emplace(std::string(key), std::string(value)); + previous = key; + } + if (!reader.Input.empty()) throw std::runtime_error("Trailing KV snapshot bytes"); + H.swap(candidate); + LastAppliedIndex = index; +} + diff --git a/miniraft/CMakeLists.txt b/miniraft/CMakeLists.txt index 6345728..e2a1ab5 100644 --- a/miniraft/CMakeLists.txt +++ b/miniraft/CMakeLists.txt @@ -3,6 +3,7 @@ add_library(miniraft raft.cpp persist.cpp durable.cpp + snapshot.cpp ) target_include_directories(miniraft PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/..) diff --git a/miniraft/raft.cpp b/miniraft/raft.cpp index d53c60d..a52a2ea 100644 --- a/miniraft/raft.cpp +++ b/miniraft/raft.cpp @@ -607,6 +607,13 @@ uint64_t TRaft::GetLastIndex() const { return State->LastLogIndex; } +std::vector TRaft::GetVoterIds() const { + std::vector ids{Id}; + for (const auto& [id, _] : Nodes) ids.push_back(id); + std::sort(ids.begin(), ids.end()); + return ids; +} + void TRaft::RestoreAppliedIndex(uint64_t index) { if (index > uint64_t(INT64_MAX) || index > State->LastLogIndex) throw std::invalid_argument("Application progress exceeds recovered Raft log"); diff --git a/miniraft/raft.h b/miniraft/raft.h index a8738d4..2e5880c 100644 --- a/miniraft/raft.h +++ b/miniraft/raft.h @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -27,6 +28,10 @@ struct IRsm { virtual TMessageHolder Read(TMessageHolder message, uint64_t index) = 0; virtual TMessageHolder Write(TMessageHolder message, uint64_t index) = 0; virtual TMessageHolder Prepare(TMessageHolder message) = 0; + virtual std::string SaveSnapshot() const { throw std::logic_error("Application snapshots are unsupported"); } + // Implementations must validate completely before atomically replacing state + // and LastAppliedIndex. Invoked during startup before request processing. + virtual void LoadSnapshot(std::string_view, uint64_t) { throw std::logic_error("Application snapshots are unsupported"); } // Durable RSMs restore this atomically with application data before constructing // TRequestProcessor. It must identify committed progress in the same Raft log. @@ -101,6 +106,7 @@ class TRaft { uint64_t Append(TMessageHolder entry); uint32_t GetLeaderId() const; uint64_t GetLastIndex() const; + std::vector GetVoterIds() const; uint64_t ApproveRead(); // Highest barrier confirmed by a fresh majority in this leadership term. // Zero means no confirmation (including before a current-term commit). diff --git a/miniraft/snapshot.cpp b/miniraft/snapshot.cpp new file mode 100644 index 0000000..b2d1a10 --- /dev/null +++ b/miniraft/snapshot.cpp @@ -0,0 +1,200 @@ +#include "snapshot.h" +#include "snapshot_codec.h" +#include "raft.h" +#include +#include +#include +#include +#ifdef __linux__ +#include +#include +#include +#include +#endif + +namespace { +using namespace NSnapshotCodec; +constexpr std::string_view Magic = "MRFTS001"; +void Validate(const TSnapshot& snapshot) { + if (!snapshot.Index || snapshot.Index > uint64_t(INT64_MAX) || !snapshot.Term || + snapshot.Voters.empty() || snapshot.Voters.size() > TSnapshot::MaxVoters || + snapshot.Data.size() > TSnapshot::MaxBytes - 40 - 4 * snapshot.Voters.size()) + throw std::runtime_error("Invalid snapshot metadata/size"); + uint32_t previous = 0; + for (auto voter : snapshot.Voters) { + if (voter <= previous) throw std::runtime_error("Snapshot voters must be positive, sorted and unique"); + previous = voter; + } +} +void CheckLocalLog(const TSnapshot& snapshot, const TRaft& raft) { + Validate(snapshot); + if (snapshot.Voters != raft.GetVoterIds() || snapshot.Index > raft.GetLastIndex() || + snapshot.Term > raft.GetState()->CurrentTerm || + raft.GetState()->LogTerm(snapshot.Index) != snapshot.Term) + throw std::runtime_error("Snapshot does not match retained log and fixed configuration"); +} +#ifdef __linux__ +struct TFd { + int Value = -1; + explicit TFd(int fd = -1) : Value(fd) {} + ~TFd() { if (Value >= 0) close(Value); } + TFd(const TFd&) = delete; + TFd& operator=(const TFd&) = delete; +}; +void Require(bool ok, const char* message) { + if (!ok) throw std::system_error(errno, std::generic_category(), message); +} +void Sync(int fd) { + int rc; + do { rc = fsync(fd); } while (rc < 0 && errno == EINTR); + Require(rc == 0, "sync snapshot"); +} +#endif +} + +std::string EncodeSnapshot(const TSnapshot& snapshot) { + Validate(snapshot); + std::string out(Magic); + Put(out, snapshot.Index, 8); + Put(out, snapshot.Term, 8); + Put(out, snapshot.Voters.size(), 4); + Put(out, snapshot.Data.size(), 8); + for (auto voter : snapshot.Voters) Put(out, voter, 4); + Append(out, snapshot.Data); + Put(out, Checksum(out), 4); + return out; +} + +TSnapshot DecodeSnapshot(std::string_view bytes) { + if (bytes.size() < 44 || bytes.size() > TSnapshot::MaxBytes) + throw std::runtime_error("Invalid snapshot size"); + TReader trailer{bytes.substr(bytes.size() - 4)}; + if (Checksum(bytes.substr(0, bytes.size() - 4)) != trailer.Take(4)) + throw std::runtime_error("Snapshot checksum mismatch"); + TReader reader{bytes.substr(0, bytes.size() - 4)}; + if (reader.Read(Magic.size()) != Magic) throw std::runtime_error("Unknown snapshot format"); + TSnapshot snapshot; + snapshot.Index = reader.Take(8); + snapshot.Term = reader.Take(8); + const auto count = reader.Take(4); + const auto length = reader.Take(8); + if (count > TSnapshot::MaxVoters || count > reader.Input.size() / 4) + throw std::runtime_error("Invalid snapshot voter count"); + for (uint32_t i = 0; i < count; ++i) snapshot.Voters.push_back(reader.Take(4)); + if (length > reader.Input.size()) throw std::runtime_error("Invalid snapshot data length"); + snapshot.Data = reader.Read(size_t(length)); + if (!reader.Input.empty()) throw std::runtime_error("Trailing snapshot data"); + Validate(snapshot); + return snapshot; +} + +TSnapshot CaptureSnapshot(const TRaft& raft, const IRsm& rsm) { + const auto index = rsm.LastAppliedIndex; + if (!index || index > raft.GetVolatileState()->CommitIndex || index > raft.GetLastIndex()) + throw std::runtime_error("Snapshot requires applied committed progress"); + TSnapshot snapshot{index, raft.GetState()->LogTerm(index), raft.GetVoterIds(), {}}; + CheckLocalLog(snapshot, raft); + snapshot.Data = rsm.SaveSnapshot(); + Validate(snapshot); + return snapshot; +} + +void RestoreSnapshot(const TSnapshot& snapshot, const TRaft& raft, IRsm& rsm) { + CheckLocalLog(snapshot, raft); + if (rsm.LastAppliedIndex != 0) throw std::runtime_error("Snapshot restore requires a fresh application"); + rsm.LoadSnapshot(snapshot.Data, snapshot.Index); + if (rsm.LastAppliedIndex != snapshot.Index) throw std::runtime_error("Application failed to restore snapshot progress"); +} + +struct TSnapshotStore::TImpl { + THook Hook; + std::string File, Temp; +#ifdef __linux__ + TFd Directory, Lock; +#endif + void Stage(EStage stage) { if (Hook) Hook(stage); } +}; + +TSnapshotStore::TSnapshotStore(const std::string& name, uint32_t id, THook hook) + : Impl(std::make_unique()) +{ +#ifndef __linux__ + throw std::runtime_error("Snapshot store currently requires Linux"); +#else + if (!id) throw std::invalid_argument("Snapshot store requires a positive node ID"); + const auto base = std::filesystem::path(name); + auto directory = base.parent_path(); + if (directory.empty()) directory = "."; + Impl->File = base.filename().string() + ".snapshot." + std::to_string(id); + Impl->Temp = Impl->File + ".tmp"; + Impl->Hook = std::move(hook); + Impl->Directory.Value = open(directory.c_str(), O_RDONLY | O_DIRECTORY | O_CLOEXEC); + Require(Impl->Directory.Value >= 0, "open snapshot directory"); + const auto lock = Impl->File + ".lock"; + Impl->Lock.Value = openat(Impl->Directory.Value, lock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC | O_NOFOLLOW | O_NONBLOCK, 0600); + Require(Impl->Lock.Value >= 0, "open snapshot lock"); + struct stat lockInfo; + Require(fstat(Impl->Lock.Value, &lockInfo) == 0, "stat snapshot lock"); + if (!S_ISREG(lockInfo.st_mode)) throw std::runtime_error("Invalid snapshot lock file type"); + Require(flock(Impl->Lock.Value, LOCK_EX | LOCK_NB) == 0, "lock snapshot store"); +#endif +} + +TSnapshotStore::~TSnapshotStore() = default; + +std::optional TSnapshotStore::Load() { +#ifndef __linux__ + throw std::runtime_error("Snapshot store currently requires Linux"); +#else + TFd file(openat(Impl->Directory.Value, Impl->File.c_str(), O_RDONLY | O_CLOEXEC | O_NOFOLLOW | O_NONBLOCK)); + if (file.Value < 0 && errno == ENOENT) return {}; + Require(file.Value >= 0, "open snapshot"); + struct stat info; + Require(fstat(file.Value, &info) == 0, "stat snapshot"); + if (!S_ISREG(info.st_mode) || info.st_size < 44 || uint64_t(info.st_size) > TSnapshot::MaxBytes) + throw std::runtime_error("Invalid snapshot file type/size"); + std::string bytes(size_t(info.st_size), '\0'); + size_t done = 0; + while (done < bytes.size()) { + const auto n = read(file.Value, bytes.data() + done, bytes.size() - done); + if (n < 0 && errno == EINTR) continue; + if (!n) throw std::runtime_error("Truncated snapshot file"); + Require(n > 0, "read snapshot"); + done += size_t(n); + } + auto snapshot = DecodeSnapshot(bytes); + Sync(file.Value); + Sync(Impl->Directory.Value); + return snapshot; +#endif +} + +void TSnapshotStore::Save(const TSnapshot& snapshot) { + const auto bytes = EncodeSnapshot(snapshot); +#ifndef __linux__ + throw std::runtime_error("Snapshot store currently requires Linux"); +#else + TFd file(openat(Impl->Directory.Value, Impl->Temp.c_str(), + O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC | O_NOFOLLOW | O_NONBLOCK, 0600)); + Require(file.Value >= 0, "create snapshot temporary image"); + struct stat info; + Require(fstat(file.Value, &info) == 0, "stat snapshot temporary image"); + if (!S_ISREG(info.st_mode)) throw std::runtime_error("Invalid snapshot temporary image type"); + Impl->Stage(EStage::TempCreated); + size_t done = 0; + while (done < bytes.size()) { + const auto n = write(file.Value, bytes.data() + done, bytes.size() - done); + if (n < 0 && errno == EINTR) continue; + Require(n > 0, "write snapshot"); + done += size_t(n); + } + Impl->Stage(EStage::Written); + Sync(file.Value); + Impl->Stage(EStage::FileSynced); + Require(renameat(Impl->Directory.Value, Impl->Temp.c_str(), Impl->Directory.Value, Impl->File.c_str()) == 0, + "publish snapshot"); + Impl->Stage(EStage::Renamed); + Sync(Impl->Directory.Value); + Impl->Stage(EStage::DirectorySynced); +#endif +} diff --git a/miniraft/snapshot.h b/miniraft/snapshot.h new file mode 100644 index 0000000..ffdb31d --- /dev/null +++ b/miniraft/snapshot.h @@ -0,0 +1,45 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include + +class TRaft; +struct IRsm; + +// Local application checkpoint. Full Raft logs are retained in this milestone. +struct TSnapshot { + static constexpr size_t MaxBytes = 64 * 1024 * 1024; + static constexpr uint32_t MaxVoters = 4096; + uint64_t Index = 0, Term = 0; + std::vector Voters; + std::string Data; + bool operator==(const TSnapshot&) const = default; +}; +std::string EncodeSnapshot(const TSnapshot& snapshot); +TSnapshot DecodeSnapshot(std::string_view bytes); +TSnapshot CaptureSnapshot(const TRaft& raft, const IRsm& rsm); +// Startup only, with a fresh application and the same fully retained local log. +void RestoreSnapshot(const TSnapshot& snapshot, const TRaft& raft, IRsm& rsm); + +class TSnapshotStore { +public: + enum class EStage { TempCreated, Written, FileSynced, Renamed, DirectorySynced }; + using THook = std::function; + // Linux atomic publication, exclusive writer lock, bounded reads and CRC32. + TSnapshotStore(const std::string& name, uint32_t id, THook hook = {}); + ~TSnapshotStore(); + TSnapshotStore(const TSnapshotStore&) = delete; + TSnapshotStore& operator=(const TSnapshotStore&) = delete; + std::optional Load(); + // Failure is catchable: no log is discarded and every published image is a + // complete committed checkpoint. Crash recovery may select the old or new one. + void Save(const TSnapshot& snapshot); +private: + struct TImpl; + std::unique_ptr Impl; +}; diff --git a/miniraft/snapshot_codec.h b/miniraft/snapshot_codec.h new file mode 100644 index 0000000..b819609 --- /dev/null +++ b/miniraft/snapshot_codec.h @@ -0,0 +1,40 @@ +#pragma once +#include "snapshot.h" +#include + +namespace NSnapshotCodec { +inline void Append(std::string& out, std::string_view part) { + if (out.size() > TSnapshot::MaxBytes || part.size() > TSnapshot::MaxBytes - out.size()) + throw std::runtime_error("Snapshot exceeds size limit"); + out.append(part); +} +inline void Put(std::string& out, uint64_t value, unsigned width) { + for (unsigned i = 0; i < width; ++i) { + const char byte = char(value >> (8 * i)); + Append(out, std::string_view(&byte, 1)); + } +} +struct TReader { + std::string_view Input; + std::string_view Read(size_t length) { + if (length > Input.size()) throw std::runtime_error("Truncated snapshot"); + const auto result = Input.substr(0, length); + Input.remove_prefix(length); + return result; + } + uint64_t Take(unsigned width) { + const auto bytes = Read(width); + uint64_t value = 0; + for (unsigned i = 0; i < width; ++i) value |= uint64_t(uint8_t(bytes[i])) << (8 * i); + return value; + } +}; +inline uint32_t Checksum(std::string_view bytes) { + uint32_t crc = UINT32_MAX; + for (uint8_t byte : bytes) { + crc ^= byte; + for (int i = 0; i < 8; ++i) crc = (crc >> 1) ^ (0xedb88320u & (0u - (crc & 1u))); + } + return ~crc; +} +} diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 7cff354..ad4fe69 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -18,6 +18,7 @@ ut(test_forwarding test_forwarding.cpp miniraft) ut(test_message_safety test_message_safety.cpp miniraft) ut(test_rpc_validation test_rpc_validation.cpp miniraft) ut(test_app_payloads "test_app_payloads.cpp;../examples/kv_rsm.cpp" miniraft) +ut(test_snapshots "test_snapshots.cpp;../examples/kv_rsm.cpp" miniraft) find_package(SQLite3 QUIET) if(SQLite3_FOUND) ut(test_sql_payloads "test_sql_payloads.cpp;../examples/sql_rsm.cpp" miniraft) @@ -30,5 +31,13 @@ endif() if(MINIRAFT_BUILD_NETWORK) ut(test_read_write test_read_write.cpp miniraft.net) ut(test_network_frames test_network_frames.cpp miniraft.net) +if(CMAKE_SYSTEM_NAME STREQUAL "Linux") +find_package(Python3 COMPONENTS Interpreter QUIET) +if(Python3_Interpreter_FOUND) +add_test(NAME test_kv_checkpoints_cli COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/smoke_kv_checkpoints.py $) +set_tests_properties(test_kv_checkpoints_cli PROPERTIES TIMEOUT 35) +endif() +endif() endif() endif () diff --git a/test/smoke_kv_checkpoints.py b/test/smoke_kv_checkpoints.py new file mode 100644 index 0000000..94c4a7c --- /dev/null +++ b/test/smoke_kv_checkpoints.py @@ -0,0 +1,120 @@ +"""Real CLI restart smoke test; deterministic fault schedules live in test_snapshots.""" +import pathlib +import resource +import socket +import struct +import subprocess +import sys +import tempfile +import time + + +def no_core(): + resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) + + +def port(): + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + return sock.getsockname()[1] + + +def receive(sock, size): + data = bytearray() + while len(data) < size: + chunk = sock.recv(size - len(data)) + assert chunk, "server closed the connection" + data.extend(chunk) + return bytes(data) + + +def request(sock, key, value=None): + if value is None: + frame = struct.pack("= index: + break + time.sleep(0.02) + else: + raise AssertionError("checkpoint was not published") + stop(process) + process, endpoint = launch() + with connect(process, endpoint) as sock: + _, value = request(sock, b"a") + assert value == b"persisted", value + stop(process) + process, _ = launch(["--node", f"127.0.0.1:{port()}:2"]) + assert process.wait(timeout=5) != 0, "changed membership must reject checkpoint" + before = (root / "state.durable.1").read_bytes() + damaged = bytearray(image.read_bytes()) + damaged[-1] ^= 1 + image.write_bytes(damaged) + process, _ = launch() + assert process.wait(timeout=5) != 0, "corrupt checkpoint must fail startup" + assert (root / "state.durable.1").read_bytes() == before + log.flush() + log.seek(0) + output = log.read() + assert "ERROR: AddressSanitizer" not in output and "runtime error:" not in output, output + print("KV checkpoint CLI: save, restart/read, configuration and corruption rejection passed") + except Exception: + log.flush() + log.seek(0) + print(log.read(), file=sys.stderr) + raise + finally: + if process is not None and process.poll() is None: + process.kill() + process.wait(timeout=5) + + +if __name__ == "__main__": + main() diff --git a/test/test_snapshots.cpp b/test/test_snapshots.cpp new file mode 100644 index 0000000..e875cac --- /dev/null +++ b/test/test_snapshots.cpp @@ -0,0 +1,304 @@ +#include +#include +#include +#include +#include "../examples/kv.h" +#include "../examples/kv_messages.h" +#include +#include +#include +#include +#ifdef __linux__ +#include +#include +#endif +#include +#include +#include +extern "C" { +#include +} + +namespace { +template void Reject(F action) { + bool rejected = false; + try { action(); } catch (const std::exception&) { rejected = true; } + assert_true(rejected); +} +struct TSink : INode { void Send(TMessageHolder) override {} void Drain() override {} }; +TMessageHolder Command(const std::string& key, const std::string& value) { + auto command = NewHoldedMessage(sizeof(TWriteKv) + key.size() + value.size()); + command->Flags = TCommandRequest::EWrite; + command->KeySize = key.size(); + command->ValSize = value.size(); + memcpy(command->TKvEntry::Data, key.data(), key.size()); + memcpy(command->TKvEntry::Data + key.size(), value.data(), value.size()); + return command; +} +std::string Read(TKv& kv, const std::string& key) { + auto command = NewHoldedMessage(sizeof(TReadKv) + key.size()); + command->KeySize = key.size(); + memcpy(command->Data, key.data(), key.size()); + auto reply = kv.Read(command, kv.LastAppliedIndex).Cast(); + return std::string(reply->Data, reply->Len - sizeof(TCommandResponse)); +} +void Apply(TKv& kv, const std::string& key, const std::string& value, uint64_t index) { + kv.Write(kv.Prepare(Command(key, value)), index); +} +TSnapshot Sample() { return {2, 1, {1, 2, 3}, std::string("a\0b", 3)}; } +void FixChecksum(std::string& bytes) { + bytes.resize(bytes.size() - 4); + NSnapshotCodec::Put(bytes, NSnapshotCodec::Checksum(bytes), 4); +} +struct TFiles { + std::filesystem::path Directory; + std::string Name; + TFiles() { + static unsigned next = 0; + Directory = std::filesystem::temp_directory_path() / ("miniraft-snapshot-" + + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count()) + "-" + std::to_string(next++)); + assert_true(std::filesystem::create_directory(Directory)); + Name = (Directory / "raft").string(); + } + ~TFiles() { std::error_code ignored; std::filesystem::remove_all(Directory, ignored); } + std::string Image() const { return Name + ".snapshot.1"; } +}; +void test_snapshot_codec_roundtrip_and_all_byte_corruption(void**) { + const auto original = Sample(); + const auto encoded = EncodeSnapshot(original); + assert_true(DecodeSnapshot(encoded) == original); + for (size_t i = 0; i < encoded.size(); ++i) { + auto damaged = encoded; + damaged[i] ^= 1; + Reject([&] { DecodeSnapshot(damaged); }); + Reject([&] { DecodeSnapshot(std::string_view(encoded).substr(0, i)); }); + } +} +void test_snapshot_metadata_and_declared_lengths_are_bounded(void**) { + auto snapshot = Sample(); + for (uint64_t index : {uint64_t(0), UINT64_MAX}) { + auto bad = snapshot; bad.Index = index; + Reject([&] { EncodeSnapshot(bad); }); + } + for (auto voters : {std::vector{}, std::vector{0, 1}, + std::vector{1, 1}, std::vector{2, 1}}) { + auto bad = snapshot; bad.Voters = voters; + Reject([&] { EncodeSnapshot(bad); }); + } + auto bad = snapshot; bad.Term = 0; + Reject([&] { EncodeSnapshot(bad); }); + for (size_t offset : {size_t(0), size_t(8), size_t(16), size_t(24), size_t(28)}) { + auto damaged = EncodeSnapshot(snapshot); + for (size_t i = offset; i < offset + (offset == 24 ? 4 : 8); ++i) damaged[i] = char(0xff); + FixChecksum(damaged); // Exercise semantic validation, not just CRC rejection. + if (offset == 16) { + // Terms are full-width; UINT64_MAX is valid in the codec. + assert_true(DecodeSnapshot(damaged).Term == UINT64_MAX); + } else Reject([&] { DecodeSnapshot(damaged); }); + } +} +void test_kv_snapshot_is_canonical_and_binary_safe(void**) { + TKv a, b; + const std::string binary("a\0z", 3); + Apply(a, "z", "last", 1); Apply(a, "a", binary, 2); Apply(a, "gone", "remove", 3); Apply(a, "gone", {}, 4); + Apply(b, "a", binary, 1); Apply(b, "z", "last", 2); + assert_true(a.SaveSnapshot() == b.SaveSnapshot()); + TKv restored; + restored.LoadSnapshot(a.SaveSnapshot(), 4); + assert_int_equal(restored.LastAppliedIndex, 4); + assert_true(Read(restored, "a") == binary); + assert_true(Read(restored, "z") == "last"); + assert_true(Read(restored, "gone").empty()); +} +void test_kv_invalid_snapshot_preserves_existing_state_and_progress(void**) { + TKv source, target; + Apply(source, "a", "one", 1); Apply(source, "b", "two", 2); + Apply(target, "old", "preserved", 1); + const auto bytes = source.SaveSnapshot(); + for (size_t size = 0; size < bytes.size(); ++size) { + Reject([&] { target.LoadSnapshot(std::string_view(bytes).substr(0, size), 2); }); + assert_true(Read(target, "old") == "preserved"); + assert_int_equal(target.LastAppliedIndex, 1); + } + auto trailing = bytes + "x"; + Reject([&] { target.LoadSnapshot(trailing, 2); }); + auto duplicate = bytes; + duplicate[36] = 'a'; // second record's key; duplicate canonical key must fail + Reject([&] { target.LoadSnapshot(duplicate, 2); }); + std::string oversized = "MRKV0001"; + NSnapshotCodec::Put(oversized, 1, 8); + NSnapshotCodec::Put(oversized, 1, 4); + NSnapshotCodec::Put(oversized, uint64_t(UINT16_MAX) + 1, 4); + NSnapshotCodec::Append(oversized, "a"); + NSnapshotCodec::Append(oversized, std::string(size_t(UINT16_MAX) + 1, 'x')); + Reject([&] { target.LoadSnapshot(oversized, 2); }); + assert_true(Read(target, "old") == "preserved"); + assert_int_equal(target.LastAppliedIndex, 1); +} +void test_capture_requires_committed_applied_progress(void**) { + auto state = std::make_shared(); + TRaft raft(state, 1, {}); + TKv kv; + Reject([&] { CaptureSnapshot(raft, kv); }); + auto entry = kv.Prepare(Command("a", "one")); + state->Append(entry); + kv.Write(entry, 1); + Reject([&] { CaptureSnapshot(raft, kv); }); + const_cast(raft.GetVolatileState())->CommitIndex = 1; + const auto snapshot = CaptureSnapshot(raft, kv); + assert_int_equal(snapshot.Index, 1); + assert_int_equal(snapshot.Term, 1); + assert_true(snapshot.Voters == std::vector{1}); +} +void test_restore_rejects_mismatched_log_configuration_and_used_application(void**) { + auto state = std::make_shared(); + TKv source; + auto entry = source.Prepare(Command("a", "one")); + state->Append(entry); source.Write(entry, 1); + TRaft raft(state, 1, {{2, std::make_shared()}, {3, std::make_shared()}}); + const_cast(raft.GetVolatileState())->CommitIndex = 1; + auto snapshot = CaptureSnapshot(raft, source); + for (int mode = 0; mode < 4; ++mode) { + auto bad = snapshot; + if (mode == 0) bad.Index = 2; + if (mode == 1) bad.Term = 2; + if (mode == 2) bad.Voters = {1, 2}; + if (mode == 3) bad.Data = "invalid KV data"; + TKv restored; + Reject([&] { RestoreSnapshot(bad, raft, restored); }); + assert_int_equal(restored.LastAppliedIndex, 0); + assert_true(Read(restored, "a").empty()); + } + TKv restored; + RestoreSnapshot(snapshot, raft, restored); + Reject([&] { RestoreSnapshot(snapshot, raft, restored); }); + assert_true(Read(restored, "a") == "one"); +} +#ifdef __linux__ +void test_snapshot_store_rejects_oversized_files_and_temp_symlinks(void**) { + TFiles files; + TSnapshotStore store(files.Name, 1); + store.Save(Sample()); + const auto saved = store.Load(); + std::filesystem::create_symlink(files.Image(), files.Image() + ".tmp"); + Reject([&] { store.Save(Sample()); }); + assert_true(store.Load() == saved); + std::filesystem::resize_file(files.Image(), TSnapshot::MaxBytes + 1); + Reject([&] { store.Load(); }); + assert_true(std::filesystem::file_size(files.Image()) == TSnapshot::MaxBytes + 1); +} +void test_snapshot_store_roundtrip_lock_and_corrupt_recovery(void**) { + TFiles files; + { + TSnapshotStore store(files.Name, 1); + assert_false(store.Load().has_value()); + Reject([&] { TSnapshotStore second(files.Name, 1); }); + store.Save(Sample()); + assert_true(store.Load() == std::optional{Sample()}); + } + std::ifstream input(files.Image(), std::ios::binary); + std::string bytes(std::istreambuf_iterator(input), {}); + input.close(); + bytes.back() ^= 1; + { std::ofstream out(files.Image(), std::ios::binary); out.write(bytes.data(), bytes.size()); } + TSnapshotStore store(files.Name, 1); + Reject([&] { store.Load(); }); + std::ifstream after(files.Image(), std::ios::binary); + assert_true(std::string(std::istreambuf_iterator(after), {}) == bytes); +} +void test_snapshot_crash_at_each_publication_boundary(void**) { + using TStage = TSnapshotStore::EStage; + for (bool initial : {false, true}) for (auto stage : {TStage::TempCreated, TStage::Written, + TStage::FileSynced, TStage::Renamed, TStage::DirectorySynced}) { + TFiles files; + if (!initial) { TSnapshotStore store(files.Name, 1); store.Save(Sample()); } + auto updated = Sample(); updated.Index = 3; updated.Data = "new committed state"; + const auto child = fork(); + assert_true(child >= 0); + if (child == 0) { + try { + TSnapshotStore store(files.Name, 1, [&](auto point) { if (point == stage) _exit(90); }); + store.Save(updated); + } catch (...) { _exit(91); } + _exit(92); + } + int status = 0; + assert_int_equal(waitpid(child, &status, 0), child); + assert_true(WIFEXITED(status)); assert_int_equal(WEXITSTATUS(status), 90); + TSnapshotStore store(files.Name, 1); + auto selected = store.Load(); + if (stage >= TStage::Renamed) assert_true(selected == std::optional{updated}); + else if (initial) assert_false(selected.has_value()); + else assert_true(selected == std::optional{Sample()}); + } +} +void test_snapshot_publication_error_preserves_old_checkpoint(void**) { + TFiles files; + { TSnapshotStore store(files.Name, 1); store.Save(Sample()); } + TSnapshotStore store(files.Name, 1, [](auto stage) { + if (stage == TSnapshotStore::EStage::Written) throw std::runtime_error("injected checkpoint failure"); + }); + auto updated = Sample(); updated.Index = 3; + Reject([&] { store.Save(updated); }); + assert_true(store.Load() == std::optional{Sample()}); +} +void test_durable_kv_restart_restores_checkpoint_and_replays_only_tail(void**) { + struct TCountingKv : TKv { + size_t Applied = 0; + TMessageHolder Write(TMessageHolder entry, uint64_t index) override { + ++Applied; return TKv::Write(entry, index); + } + }; + TFiles files; + { + auto state = std::make_shared(files.Name, 1); + auto raft = std::make_shared(state, 1, TNodeDict{}); + auto kv = std::make_shared(); + TRequestProcessor processor(raft, kv, {}); + auto now = ITimeSource::Time{}; + raft->ProcessTimeout(now); + auto write = [&](const std::string& key, const std::string& value) { + processor.OnCommandRequest(Command(key, value), {}); + now += std::chrono::milliseconds(500); + raft->ProcessTimeout(now); processor.ProcessCommitted(); + }; + write("a", "one"); + { TSnapshotStore store(files.Name, 1); store.Save(CaptureSnapshot(*raft, *kv)); } + write("b", "tail"); + assert_int_equal(state->LastLogIndex, 3); + } + auto state = std::make_shared(files.Name, 1); + auto raft = std::make_shared(state, 1, TNodeDict{}); + auto kv = std::make_shared(); + TSnapshotStore store(files.Name, 1); + auto snapshot = store.Load(); assert_true(snapshot.has_value()); + RestoreSnapshot(*snapshot, *raft, *kv); + assert_true(Read(*kv, "a") == "one"); assert_true(Read(*kv, "b").empty()); + TRequestProcessor processor(raft, kv, {}); + processor.ProcessCommitted(); assert_int_equal(kv->Applied, 0); + raft->ProcessTimeout(ITimeSource::Time{}); processor.ProcessCommitted(); + assert_int_equal(kv->Applied, 1); + assert_true(Read(*kv, "a") == "one"); assert_true(Read(*kv, "b") == "tail"); + assert_int_equal(kv->LastAppliedIndex, 4); + assert_int_equal(state->LastLogIndex, 4); // no compaction: every entry is retained +} +#endif +} +int main() { + const CMUnitTest tests[] = { + cmocka_unit_test(test_snapshot_codec_roundtrip_and_all_byte_corruption), + cmocka_unit_test(test_snapshot_metadata_and_declared_lengths_are_bounded), + cmocka_unit_test(test_kv_snapshot_is_canonical_and_binary_safe), + cmocka_unit_test(test_kv_invalid_snapshot_preserves_existing_state_and_progress), + cmocka_unit_test(test_capture_requires_committed_applied_progress), + cmocka_unit_test(test_restore_rejects_mismatched_log_configuration_and_used_application), +#ifdef __linux__ + cmocka_unit_test(test_snapshot_store_rejects_oversized_files_and_temp_symlinks), + cmocka_unit_test(test_snapshot_store_roundtrip_lock_and_corrupt_recovery), + cmocka_unit_test(test_snapshot_crash_at_each_publication_boundary), + cmocka_unit_test(test_snapshot_publication_error_preserves_old_checkpoint), + cmocka_unit_test(test_durable_kv_restart_restores_checkpoint_and_replays_only_tail), +#endif + }; + return cmocka_run_group_tests(tests, nullptr, nullptr); +} From edea4290162fc051f2535b680e359a501f611351 Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Mon, 9 Nov 2026 06:14:02 -0500 Subject: [PATCH 16/66] Compact durable logs and install chunked snapshots on lagging followers (#16) --- README.md | 4 +- docs/application-recovery.md | 3 + docs/compaction.md | 132 +++++++++++ docs/snapshots.md | 4 + examples/kv.cpp | 13 +- examples/kv.h | 1 + examples/kv_rsm.cpp | 5 + miniraft/durable.cpp | 83 +++++-- miniraft/durable.h | 7 +- miniraft/messages.cpp | 6 + miniraft/messages.h | 23 +- miniraft/raft.cpp | 178 +++++++++++++- miniraft/raft.h | 24 +- miniraft/snapshot.h | 8 +- miniraft/state.h | 41 +++- test/CMakeLists.txt | 4 + test/smoke_install_snapshot.py | 120 ++++++++++ test/smoke_kv_checkpoints.py | 11 +- test/test_compaction.cpp | 418 +++++++++++++++++++++++++++++++++ test/test_network_frames.cpp | 28 ++- 20 files changed, 1058 insertions(+), 55 deletions(-) create mode 100644 docs/compaction.md create mode 100644 test/smoke_install_snapshot.py create mode 100644 test/test_compaction.cpp diff --git a/README.md b/README.md index 8c0f8ce..5db9c06 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,9 @@ The [legacy log-term follow-up](docs/legacy-log-terms.md) rejects impossible stored term sequences before they influence elections or log matching. The [snapshot foundation](docs/snapshots.md) adds opt-in durable KV checkpoints and tail-only application recovery with `--durable --snapshots N`. Logs remain -fully retained; compaction, snapshot transfer and membership changes are next. +fully retained in that baseline. The [compaction follow-up](docs/compaction.md) +enables prefix deletion and chunked InstallSnapshot catch-up; fixed membership +remains required. Upgrade all voters before enabling the new compaction behavior. [![Number of GitHub stars](https://img.shields.io/github/stars/resetius/miniraft-cpp)](https://github.com/resetius/miniraft-cpp/stargazers) ![GitHub commit activity](https://img.shields.io/github/commit-activity/m/resetius/miniraft-cpp) diff --git a/docs/application-recovery.md b/docs/application-recovery.md index 2751cb7..3b87004 100644 --- a/docs/application-recovery.md +++ b/docs/application-recovery.md @@ -1,5 +1,8 @@ # Recovered application progress: v0.1.0-alpha.13 +The [compaction follow-up](compaction.md) extends this recovery contract to +embedded snapshots and absolute log boundaries. + After restarting a durable application, Raft's volatile commit index began at zero. ProcessCommitted unconditionally copied that zero into LastAppliedIndex, discarding the application's recovered marker. Re-establishing commitment then diff --git a/docs/compaction.md b/docs/compaction.md new file mode 100644 index 0000000..8ad63bc --- /dev/null +++ b/docs/compaction.md @@ -0,0 +1,132 @@ +# Compaction and InstallSnapshot: v0.1.0-alpha.16 + +This milestone adds prefix deletion and follower state transfer to the alpha.15 +KV checkpoint foundation. TState and Linux TDurableState support compacted logs; +legacy TDiskState does not. KV implements snapshot validation/restoration. SQL and +dummy state machines remain unsupported for compaction and installation. + +## Operation and compatibility + +The existing Linux KV option `--durable --snapshots N` now compacts after N more +applied indices. Raft::Compact captures committed applied state, validates its +application image and atomically stores the snapshot plus retained suffix. The +separate `state.snapshot.` remains a checkpoint export for compatibility; the +embedded snapshot in `state.durable.` is authoritative after compaction. + +Stop and upgrade every voter before enabling this release in an existing cluster. +Old binaries cannot read compacted durable images or process InstallSnapshot RPCs. +The meaning of `--snapshots` changes from checkpoint-only to actual compaction. +Mixed-version operation is unsupported. Existing v1 durable files and alpha.15 +checkpoints can be read without destructive migration. The first compaction or +remote installation writes v2; downgrading that store is unsupported. Fixed voter +IDs must match the embedded snapshot. Static `--node` edits remain unsafe as a +membership-change mechanism. + +Startup restores a fresh application from the embedded snapshot before request +processing, even when periodic snapshots are disabled. A newer standalone +checkpoint can restore progress within the retained suffix; an older one never +replaces embedded state. With periodic snapshots enabled, an existing corrupt +standalone file is still rejected conservatively. Snapshot recovery never resets +the persisted current term or vote. + +## Boundaries and atomic persistence + +SnapshotIndex/SnapshotTerm identify the last included entry. Get only returns +entries after SnapshotIndex, while LogTerm at the boundary returns SnapshotTerm. +LastLogIndex and all RPC/application indices remain absolute. Appends, truncation, +votes, current-term commitment and strong reads use these absolute boundaries. +Truncation cannot remove a snapshot. Election no-op creation also checks signed +index exhaustion, now reachable without allocating an enormous prefix. + +An installation retains the suffix only when the local entry at the included +index has the same term. Otherwise its uncommitted log is discarded. A follower +does not install an image at or below known commitment, so stale snapshots cannot +rewind application state. Retained suffix entries are applied only after a leader +establishes their commitment. + +TDurableState v2 (`MRFTD002`) publishes hard state, the complete encoded snapshot +and suffix in one atomic image, using the existing temp-write/file-fsync/rename/ +directory-fsync protocol. Its header remains magic[8], payload length[8], CRC[4]. +The payload is CurrentTerm[8], VotedFor[4], retained count[8], snapshot length[8], +the existing MRFTS001 image, then the existing term/flags/length/data records. +The v2 CRC covers the version magic plus payload; v1's checksum rules are retained +for backward reading. Recovery bounds the embedded length, validates both CRCs, +the absolute count range and nondecreasing terms beginning at SnapshotTerm. +Malformed recovery does not rewrite the image. Live publication failure is fatal. + +## Transfer protocol + +The previously reserved message types 6/7 now carry InstallSnapshot requests and +responses. A request identifies leader/term, boundary index/term, total image size, +offset and final-chunk flag. Responses carry the boundary, next offset, success +and an explicit Installed flag. Seqno correlates acknowledgements to the active +per-peer RPC. Frames are capped at a 64 KiB body before allocation; complete +images remain bounded at 64 MiB with at most 4,096 fixed voters. + +Leaders use stop-and-wait chunks when NextIndex is at or below their compacted +boundary. Offset and RPC context survive retries within the leadership term; +delayed/duplicate responses cannot confirm another image. Followers hold one +bounded partial image, validate duplicate bytes, reject gaps and bind assembly +to leader, term, index, boundary term and total length. A restarted follower asks +for offset zero; partial state is not persisted. Transfers restart on newer terms +or newer images, and valid chunks refresh the election timer. + +Partial chunks never change application progress or MatchIndex. Completion first +checks CRC, metadata, fixed voters and the application payload without changing +live application state. The follower durably installs snapshot plus suffix, then +replaces application state, then acknowledges Installed. Failure after durable +installation terminates the process; startup restores the selected image. A lost +completion ACK can be retried without applying the image again. + +Only Installed completion advances leader MatchIndex. Neither partial nor final +snapshot acknowledgements count as fresh-read confirmations: subsequent +AppendEntries must establish read barriers and replicate any suffix. A compacted +follower can return its boundary in a failed AppendEntries MatchIndex field as a +probe hint. That hint advances only NextIndex; a matching AppendEntries probe is +required before counting the index as replicated or acknowledging reads. + +This implements the boundary/suffix rules of [Raft section 7 and Figure 13](https://raft.github.io/raft.pdf). +Transfer is deliberately serialized and bounded; it is not disk-streaming or a +parallel multi-image protocol. + +## Validation and review + +Fourteen new deterministic core/storage tests cover lagging follower catch-up and +tail replication; restart mid-transfer; delayed, duplicate and impossible ACKs; +leadership changes; strong-read gating; compacted-prefix probing; corrupt images +and invalid headers; voter and application mismatch; retained but uncommitted +suffixes; memory/durable boundary behavior; index exhaustion; v2 semantic +corruption; disk-space reclamation; and first/replacement image selection at all +five publication boundaries for retained/discarded suffixes. + +One new transport test checks fragmented request/response round trips and rejects +empty/oversized chunks before body allocation. The existing CLI smoke now checks +v2 publication and restoration with periodic snapshotting disabled. A new real +three-process test compacts a two-voter majority, restarts both to clear queued +old log traffic, starts an empty third voter, verifies local reads after snapshot +installation, replicates a tail, then restarts that voter and checks recovery. +It uses bounded waits, separately from deterministic virtual-time fault schedules. + +That integration test exposed a pre-existing election tie: all nodes used the +same RNG seed and campaigned in lockstep. Node-specific nonzero deterministic +seeds now provide distinct election schedules; a simultaneous-startup regression +verifies election progress after the initial split vote. This is scheduling +diversification, not a security/entropy source. + +Local validation: 207 unit tests and two CLI tests pass under ASan/UBSan across +15 CTest targets; Release core checks pass. Self-review covered boundary arithmetic, +suffix selection, persistence/application/ACK ordering, parser allocation limits, +transfer context and retries, stale snapshots, fresh-read separation, recovery +ordering and backward v1 decoding. No independent reviewer approval is claimed. +Publication-stage process crashes do not replace hardware power-cut testing. + +## Remaining work + +Membership remains fixed. Learners, catch-up gates, replicated joint-consensus +configuration entries, leader removal and corresponding quorum/read changes are +next. All participating applications/backends must support snapshots before +compaction is enabled; no capability negotiation is implemented. SQL snapshots, +asynchronous/COW capture, disk-streamed transfer, bounded transport queues, total +heap quotas, cluster/store authentication and independent review remain open. +The conservative backend still rewrites the entire snapshot and retained suffix +on each update, and capture/publication runs synchronously on the event loop. diff --git a/docs/snapshots.md b/docs/snapshots.md index 7f6bc1a..e1ac824 100644 --- a/docs/snapshots.md +++ b/docs/snapshots.md @@ -1,5 +1,9 @@ # Snapshot foundation: durable KV checkpoints, v0.1.0-alpha.15 +The [alpha.16 follow-up](compaction.md) adds compaction and InstallSnapshot and +changes the behavior of the same `--snapshots` option. This document describes +the original checkpoint-only baseline. + This milestone adds local application checkpoint creation and restart recovery. It retains the entire Raft log. InstallSnapshot RPCs and prefix compaction remain unimplemented, so checkpoints improve application recovery but do not bound log diff --git a/examples/kv.cpp b/examples/kv.cpp index 6aedd94..d3a4085 100644 --- a/examples/kv.cpp +++ b/examples/kv.cpp @@ -102,11 +102,12 @@ NNet::TVoidTask CheckpointLoop(TPoller& poller, std::shared_ptr raft, co_await poller.Sleep(std::chrono::seconds(1)); if (rsm->LastAppliedIndex <= saved || rsm->LastAppliedIndex - saved < interval) continue; try { - auto snapshot = CaptureSnapshot(*raft, *rsm); + auto snapshot = raft->Compact(); store->Save(snapshot); saved = snapshot.Index; } catch (const std::exception& error) { - // Checkpoints are optional while every log entry is retained. + // The durable state image already contains the authoritative snapshot. + // A failed standalone checkpoint export can be retried safely. std::cerr << "Checkpoint failed: " << error.what() << "\n"; } } @@ -192,12 +193,14 @@ int main(int argc, char** argv) { } auto raft = std::make_shared(state, myHost.Id, nodes); std::shared_ptr snapshots; - uint64_t savedSnapshot = 0; + uint64_t savedSnapshot = state->SnapshotIndex; if (snapshotInterval) { snapshots = std::make_shared("state", myHost.Id); if (auto snapshot = snapshots->Load()) { - RestoreSnapshot(*snapshot, *raft, *rsm); - savedSnapshot = snapshot->Index; + if (snapshot->Index > state->SnapshotIndex) { + RestoreSnapshot(*snapshot, *raft, *rsm); + savedSnapshot = snapshot->Index; + } } } TPoller::TSocket socket(NNet::TAddress{myHost.Address, myHost.Port}, loop.Poller()); diff --git a/examples/kv.h b/examples/kv.h index 6309fa9..8597ed2 100644 --- a/examples/kv.h +++ b/examples/kv.h @@ -12,6 +12,7 @@ class TKv: public IRsm { TMessageHolder Write(TMessageHolder message, uint64_t index) override; TMessageHolder Prepare(TMessageHolder message) override; std::string SaveSnapshot() const override; + void ValidateSnapshot(std::string_view bytes, uint64_t index) const override; void LoadSnapshot(std::string_view bytes, uint64_t index) override; private: diff --git a/examples/kv_rsm.cpp b/examples/kv_rsm.cpp index 85cfbf3..ff03837 100644 --- a/examples/kv_rsm.cpp +++ b/examples/kv_rsm.cpp @@ -115,3 +115,8 @@ void TKv::LoadSnapshot(std::string_view bytes, uint64_t index) { LastAppliedIndex = index; } +void TKv::ValidateSnapshot(std::string_view bytes, uint64_t index) const { + TKv candidate; + candidate.LoadSnapshot(bytes, index); +} + diff --git a/miniraft/durable.cpp b/miniraft/durable.cpp index f9196e2..9741bf7 100644 --- a/miniraft/durable.cpp +++ b/miniraft/durable.cpp @@ -1,5 +1,6 @@ #include "durable.h" #include +#include #include #include #include @@ -17,6 +18,7 @@ namespace { using TBytes = std::vector; constexpr std::array Magic{'M','R','F','T','D','0','0','1'}; +constexpr std::array SnapshotMagic{'M','R','F','T','D','0','0','2'}; void Put(TBytes& bytes, uint64_t value, unsigned width) { for (unsigned i = 0; i < width; ++i) bytes.push_back(uint8_t(value >> (8 * i))); } @@ -27,12 +29,14 @@ uint64_t Take(const TBytes& bytes, size_t& position, unsigned width) { for (unsigned i = 0; i < width; ++i) value |= uint64_t(bytes[position++]) << (8 * i); return value; } -uint32_t Checksum(const TBytes& bytes, size_t start) { +uint32_t Checksum(const TBytes& bytes, size_t start, bool versioned = false) { uint32_t crc = UINT32_MAX; - for (size_t i = start; i < bytes.size(); ++i) { - crc ^= bytes[i]; + const auto update = [&](uint8_t byte) { + crc ^= byte; for (int bit = 0; bit < 8; ++bit) crc = (crc >> 1) ^ (0xedb88320u & (0u - (crc & 1u))); - } + }; + if (versioned) for (auto byte : SnapshotMagic) update(byte); + for (size_t i = start; i < bytes.size(); ++i) update(bytes[i]); return ~crc; } TMessageHolder Clone(TMessageHolder entry) { @@ -89,6 +93,7 @@ TBytes ReadAll(int fd) { struct TDurableState::TImpl { std::vector> Log; + std::shared_ptr Snapshot; THook Hook; std::string File, Temp; #ifdef __linux__ @@ -132,17 +137,30 @@ TDurableState::TDurableState(const std::string& name, uint32_t id, THook hook) return; } const auto bytes = ReadAll(file.Value); - if (!std::equal(Magic.begin(), Magic.end(), bytes.begin())) + const bool compacted = std::equal(SnapshotMagic.begin(), SnapshotMagic.end(), bytes.begin()); + if (!compacted && !std::equal(Magic.begin(), Magic.end(), bytes.begin())) throw std::runtime_error("Unknown durable state format"); size_t position = Magic.size(); const auto length = Take(bytes, position, 8); const auto crc = Take(bytes, position, 4); - if (length != bytes.size() - position || crc != Checksum(bytes, position)) + if (length != bytes.size() - position || crc != Checksum(bytes, position, compacted)) throw std::runtime_error("Durable state length/checksum mismatch"); CurrentTerm = Take(bytes, position, 8); VotedFor = Take(bytes, position, 4); const auto count = Take(bytes, position, 8); - if (!CurrentTerm || count > (bytes.size() - position) / 20) + if (compacted) { + const auto length = Take(bytes, position, 8); + if (length > TSnapshot::MaxBytes || length > bytes.size() - position) + throw std::runtime_error("Invalid embedded snapshot size"); + auto snapshot = DecodeSnapshot(std::string_view(reinterpret_cast(bytes.data() + position), size_t(length))); + if (snapshot.Term > CurrentTerm) throw std::runtime_error("Future durable snapshot term"); + position += size_t(length); + SnapshotIndex = snapshot.Index; + SnapshotTerm = snapshot.Term; + LastLogTerm = SnapshotTerm; + Impl->Snapshot = std::make_shared(std::move(snapshot)); + } + if (!CurrentTerm || count > uint64_t(INT64_MAX) - SnapshotIndex || count > (bytes.size() - position) / 20) throw std::runtime_error("Invalid durable state metadata"); for (uint64_t i = 0; i < count; ++i) { const auto term = Take(bytes, position, 8); @@ -160,7 +178,7 @@ TDurableState::TDurableState(const std::string& name, uint32_t id, THook hook) LastLogTerm = term; } if (position != bytes.size()) throw std::runtime_error("Trailing durable state data"); - LastLogIndex = count; + LastLogIndex = SnapshotIndex + count; // A crash after rename but before directory fsync may leave a visible new // image. Stabilize the selected image and its name before serving requests. Sync(file.Value); @@ -176,14 +194,21 @@ void TDurableState::Publish() #ifndef __linux__ throw std::runtime_error("TDurableState currently requires Linux"); #else - if (!CurrentTerm || LastLogIndex != Impl->Log.size() || - LastLogTerm != (Impl->Log.empty() ? 0 : Impl->Log.back()->Term)) + if (!CurrentTerm || LastLogIndex > uint64_t(INT64_MAX) || LastLogIndex != SnapshotIndex + Impl->Log.size() || + LastLogTerm != (Impl->Log.empty() ? SnapshotTerm : Impl->Log.back()->Term) || + (Impl->Snapshot ? (Impl->Snapshot->Index != SnapshotIndex || Impl->Snapshot->Term != SnapshotTerm || + SnapshotTerm > CurrentTerm) : (SnapshotIndex != 0 || SnapshotTerm != 0))) throw std::runtime_error("Invalid durable state fields"); TBytes payload; Put(payload, CurrentTerm, 8); Put(payload, VotedFor, 4); - Put(payload, LastLogIndex, 8); - uint64_t previousTerm = 0; + Put(payload, Impl->Log.size(), 8); + if (Impl->Snapshot) { + const auto image = EncodeSnapshot(*Impl->Snapshot); + Put(payload, image.size(), 8); + payload.insert(payload.end(), image.begin(), image.end()); + } + uint64_t previousTerm = SnapshotTerm; for (auto entry : Impl->Log) { if (!entry->Term || entry->Term < previousTerm || entry->Term > CurrentTerm) throw std::runtime_error("Invalid durable log term"); @@ -193,9 +218,10 @@ void TDurableState::Publish() Put(payload, entry->Len - sizeof(TLogEntry), 4); payload.insert(payload.end(), entry->Data, entry->Data + entry->Len - sizeof(TLogEntry)); } - TBytes bytes(Magic.begin(), Magic.end()); + const auto& magic = Impl->Snapshot ? SnapshotMagic : Magic; + TBytes bytes(magic.begin(), magic.end()); Put(bytes, payload.size(), 8); - Put(bytes, Checksum(payload, 0), 4); + Put(bytes, Checksum(payload, 0, bool(Impl->Snapshot)), 4); bytes.insert(bytes.end(), payload.begin(), payload.end()); TFd temp(openat(Impl->Directory.Value, Impl->Temp.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC | O_NOFOLLOW, 0600)); @@ -225,7 +251,7 @@ void TDurableState::Commit() void TDurableState::Append(TMessageHolder entry) { Impl->Log.push_back(Clone(entry)); - LastLogIndex = Impl->Log.size(); + LastLogIndex = SnapshotIndex + Impl->Log.size(); LastLogTerm = Impl->Log.back()->Term; Commit(); } @@ -234,13 +260,32 @@ void TDurableState::RemoveLast() { if (Impl->Log.empty()) return; Impl->Log.pop_back(); - LastLogIndex = Impl->Log.size(); - LastLogTerm = Impl->Log.empty() ? 0 : Impl->Log.back()->Term; + LastLogIndex = SnapshotIndex + Impl->Log.size(); + LastLogTerm = Impl->Log.empty() ? SnapshotTerm : Impl->Log.back()->Term; Commit(); } TMessageHolder TDurableState::Get(int64_t index) const { - if (index < 1 || uint64_t(index) > Impl->Log.size()) return {}; - return Clone(Impl->Log[index - 1]); + if (index < 1 || uint64_t(index) <= SnapshotIndex || uint64_t(index) > LastLogIndex) return {}; + return Clone(Impl->Log[index - SnapshotIndex - 1]); +} + +std::shared_ptr TDurableState::GetSnapshot() const { return Impl->Snapshot; } + +void TDurableState::InstallSnapshot(const TSnapshot& snapshot) { + EncodeSnapshot(snapshot); + if (snapshot.Index <= SnapshotIndex || snapshot.Term > CurrentTerm) + throw std::invalid_argument("Obsolete or future durable snapshot"); + auto image = std::make_shared(snapshot); + std::vector> suffix; + if (snapshot.Index <= LastLogIndex && LogTerm(snapshot.Index) == snapshot.Term) + suffix.assign(Impl->Log.begin() + (snapshot.Index - SnapshotIndex), Impl->Log.end()); + Impl->Log.swap(suffix); + Impl->Snapshot = std::move(image); + SnapshotIndex = snapshot.Index; + SnapshotTerm = snapshot.Term; + LastLogIndex = SnapshotIndex + Impl->Log.size(); + LastLogTerm = Impl->Log.empty() ? SnapshotTerm : Impl->Log.back()->Term; + Commit(); // Atomically publishes hard state, snapshot and the selected suffix. } diff --git a/miniraft/durable.h b/miniraft/durable.h index 4c5cad7..4a28601 100644 --- a/miniraft/durable.h +++ b/miniraft/durable.h @@ -4,8 +4,8 @@ #include #include -// Linux: atomic full-image replacement, with file and directory fsync. -// O(log size) per update; intended as a conservative durability baseline. +// Linux: atomic hard-state + snapshot + suffix replacement, with file/directory +// fsync. O(snapshot + retained log size) per update: a durability baseline. class TDurableState : public IState { public: enum class EStage { TempCreated, Written, FileSynced, Renamed, DirectorySynced }; @@ -20,6 +20,9 @@ class TDurableState : public IState { void Append(TMessageHolder entry) override; TMessageHolder Get(int64_t index) const override; void Commit() override; + bool SupportsSnapshots() const override { return true; } + std::shared_ptr GetSnapshot() const override; + void InstallSnapshot(const TSnapshot& snapshot) override; private: struct TImpl; std::unique_ptr Impl; diff --git a/miniraft/messages.cpp b/miniraft/messages.cpp index c8751f1..7d0e662 100644 --- a/miniraft/messages.cpp +++ b/miniraft/messages.cpp @@ -14,6 +14,12 @@ void ValidateMessageHeader(uint32_t type, uint32_t length, const TMessageLimits& case EMessageType::REQUEST_VOTE_RESPONSE: minimum = sizeof(TRequestVoteResponse); break; case EMessageType::APPEND_ENTRIES_REQUEST: minimum = sizeof(TAppendEntriesRequest); break; case EMessageType::APPEND_ENTRIES_RESPONSE: minimum = sizeof(TAppendEntriesResponse); break; + case EMessageType::INSTALL_SNAPSHOT_REQUEST: + minimum = sizeof(TInstallSnapshotRequest) + 1; + if (length > sizeof(TInstallSnapshotRequest) + TInstallSnapshotRequest::MaxChunkBytes) + throw TMessageError("Snapshot chunk exceeds byte limit"); + fixed = false; break; + case EMessageType::INSTALL_SNAPSHOT_RESPONSE: minimum = sizeof(TInstallSnapshotResponse); break; case EMessageType::COMMAND_REQUEST: minimum = sizeof(TCommandRequest); fixed = false; break; case EMessageType::COMMAND_RESPONSE: minimum = sizeof(TCommandResponse); fixed = false; break; default: throw TMessageError("Unknown or unsupported message type"); diff --git a/miniraft/messages.h b/miniraft/messages.h index 9e40d76..ee49738 100644 --- a/miniraft/messages.h +++ b/miniraft/messages.h @@ -27,8 +27,8 @@ enum class EMessageType : uint32_t { REQUEST_VOTE_RESPONSE = 3, APPEND_ENTRIES_REQUEST = 4, APPEND_ENTRIES_RESPONSE = 5, - INSTALL_SNAPSHOT_REQUEST = 6, // TODO: not implemented it - INSTALL_SNAPSHOT_RESPONSE = 7, // TODO: not implemented it + INSTALL_SNAPSHOT_REQUEST = 6, + INSTALL_SNAPSHOT_RESPONSE = 7, COMMAND_REQUEST = 8, COMMAND_RESPONSE = 9, }; @@ -93,6 +93,8 @@ static_assert(sizeof(TAppendEntriesRequest) == sizeof(TMessageEx) + 32); struct TAppendEntriesResponse: public TMessageEx { static constexpr EMessageType MessageType = EMessageType::APPEND_ENTRIES_RESPONSE; + // Success: matched index. Failure: zero or a compacted-boundary probe hint; + // a hint never advances MatchIndex or confirms a read. uint64_t MatchIndex; uint32_t Success; uint32_t Padding = 0; @@ -100,6 +102,21 @@ struct TAppendEntriesResponse: public TMessageEx { static_assert(sizeof(TAppendEntriesResponse) == sizeof(TMessageEx) + 16); +struct TInstallSnapshotRequest: public TMessageEx { + static constexpr EMessageType MessageType = EMessageType::INSTALL_SNAPSHOT_REQUEST; + static constexpr uint32_t MaxChunkBytes = 64 * 1024; + uint64_t SnapshotIndex = 0, SnapshotTerm = 0, TotalBytes = 0, Offset = 0; + uint32_t LeaderId = 0, Done = 0; + char Data[0]; +}; +static_assert(sizeof(TInstallSnapshotRequest) == sizeof(TMessageEx) + 40); +struct TInstallSnapshotResponse: public TMessageEx { + static constexpr EMessageType MessageType = EMessageType::INSTALL_SNAPSHOT_RESPONSE; + uint64_t SnapshotIndex = 0, NextOffset = 0; + uint32_t Success = 0, Installed = 0; +}; +static_assert(sizeof(TInstallSnapshotResponse) == sizeof(TMessageEx) + 24); + struct TCommandRequest: public TMessage { static constexpr EMessageType MessageType = EMessageType::COMMAND_REQUEST; enum EFlags { @@ -186,7 +203,7 @@ struct TMessageHolder { } bool IsEx() const { - return Mes && Mes->Len >= sizeof(TMessageEx) && (2 <= Mes->Type && Mes->Type <= 5); + return Mes && Mes->Len >= sizeof(TMessageEx) && (2 <= Mes->Type && Mes->Type <= 7); } template diff --git a/miniraft/raft.cpp b/miniraft/raft.cpp index a52a2ea..156b565 100644 --- a/miniraft/raft.cpp +++ b/miniraft/raft.cpp @@ -156,11 +156,18 @@ TRaft::TRaft(std::shared_ptr state, int node, const TNodeDict& nodes) , StateName(EState::FOLLOWER) { if (node <= 0 || !State) throw std::invalid_argument("Raft requires a positive ID and state"); + // Equal seeds make simultaneously started voters campaign in lockstep. + // An odd multiplier gives every positive node ID a distinct nonzero stream. + Seed = uint32_t(node) * 0x9e3779b9u; for (const auto& [id, peer] : Nodes) { if (id == 0 || id == Id || !peer) throw std::invalid_argument("Invalid Raft peer configuration"); VolatileState->NextIndex[id] = 1; } + if (auto snapshot = State->GetSnapshot()) { + if (snapshot->Voters != GetVoterIds()) throw std::invalid_argument("Compacted state voter configuration mismatch"); + VolatileState->CommitIndex = State->SnapshotIndex; + } } void TRaft::OnRequestVote(ITimeSource::Time now, TMessageHolder message) { @@ -223,9 +230,9 @@ void TRaft::OnAppendEntries(ITimeSource::Time now, TMessageHolderCommitIndex; bool success = false; - if (message->PrevLogIndex == 0 || + if (message->PrevLogIndex >= State->SnapshotIndex && (message->PrevLogIndex == 0 || (message->PrevLogIndex <= State->LastLogIndex - && State->LogTerm(message->PrevLogIndex) == message->PrevLogTerm)) + && State->LogTerm(message->PrevLogIndex) == message->PrevLogTerm))) { success = true; auto index = message->PrevLogIndex; @@ -246,6 +253,7 @@ void TRaft::OnAppendEntries(ITimeSource::Time now, TMessageHolderLeaderCommit, matchIndex)); } + if (!success && message->PrevLogIndex < State->SnapshotIndex) matchIndex = State->SnapshotIndex; auto reply = NewHoldedMessage( TMessageEx {.Src = Id, .Dst = message->Src, .Term = State->CurrentTerm, .Seqno = message->Seqno}, TAppendEntriesResponse {.MatchIndex = matchIndex, .Success = success}); @@ -290,6 +298,11 @@ void TRaft::OnAppendEntries(TMessageHolder message) { .SetBatchSize(nodeId, 1024) .SetBackOff(nodeId, 1) .CommitAdvance(Nservers, *State); + } else if (message->MatchIndex > 0) { + // A compacted-prefix hint is not replication evidence. Probe its boundary + // with AppendEntries before advancing MatchIndex or confirming reads. + (*VolatileState).SetNextIndex(nodeId, std::max(VolatileState->MatchIndex[nodeId] + 1, message->MatchIndex + 1)) + .SetRpcDue(nodeId, ITimeSource::Time{}).SetBatchSize(nodeId, 1).SetBackOff(nodeId, 1); } else { auto backOff = std::max(VolatileState->BackOff[nodeId], 1); auto nextIndex = VolatileState->NextIndex[nodeId] > backOff @@ -360,6 +373,8 @@ void TRaft::Follower(ITimeSource::Time now, TMessageHolder message) { OnRequestVote(now, std::move(maybeRequestVote.Cast())); } else if (auto maybeAppendEntries = message.Maybe()) { OnAppendEntries(now, std::move(maybeAppendEntries.Cast())); + } else if (auto snapshot = message.Maybe()) { + OnInstallSnapshot(now, snapshot.Cast()); } } @@ -370,16 +385,22 @@ void TRaft::Candidate(ITimeSource::Time now, TMessageHolder message) { OnRequestVote(now, std::move(maybeRequestVote.Cast())); } else if (auto maybeAppendEntries = message.Maybe()) { OnAppendEntries(now, std::move(maybeAppendEntries.Cast())); + } else if (auto snapshot = message.Maybe()) { + OnInstallSnapshot(now, snapshot.Cast()); } } void TRaft::Leader(ITimeSource::Time now, TMessageHolder message) { - if (auto maybeAppendEntries = message.Maybe()) { + if (auto snapshot = message.Maybe()) { + OnInstallSnapshot(snapshot.Cast()); + } else if (auto maybeAppendEntries = message.Maybe()) { OnAppendEntries(std::move(maybeAppendEntries.Cast())); } else if (auto maybeVoteRequest = message.Maybe()) { OnRequestVote(now, std::move(maybeVoteRequest.Cast())); } else if (auto maybeAppendEntries = message.Maybe()) { OnAppendEntries(now, std::move(maybeAppendEntries.Cast())); + } else if (auto snapshot = message.Maybe()) { + OnInstallSnapshot(now, snapshot.Cast()); } } @@ -398,6 +419,8 @@ void TRaft::Process(ITimeSource::Time now, TMessageHolder message, con State->VotedFor = 0; State->Commit(); StateName = EState::FOLLOWER; + IncomingSnapshot.reset(); + VolatileState->SnapshotRpcs.clear(); VolatileState->LeaderId = 0; if (VolatileState->ElectionDue <= now || VolatileState->ElectionDue == ITimeSource::Max) { VolatileState->ElectionDue = MakeElection(now); @@ -438,7 +461,23 @@ bool TRaft::ValidateRpc(TMessageHolder message) const { case EMessageType::APPEND_ENTRIES_RESPONSE: { auto reply = message.Cast(); return reply->Success <= 1 && reply->MatchIndex <= maxIndex && - (reply->Success ? reply->MatchIndex <= State->LastLogIndex : reply->MatchIndex == 0); + reply->MatchIndex <= State->LastLogIndex; + } + case EMessageType::INSTALL_SNAPSHOT_REQUEST: { + auto request = message.Cast(); + const auto length = request->Len - sizeof(TInstallSnapshotRequest); + return request->LeaderId == header->Src && request->SnapshotIndex > 0 && request->SnapshotIndex <= maxIndex && + request->SnapshotTerm > 0 && request->SnapshotTerm <= header->Term && request->Done <= 1 && + request->TotalBytes >= 44 && request->TotalBytes <= TSnapshot::MaxBytes && + request->Offset <= request->TotalBytes && length > 0 && length <= TInstallSnapshotRequest::MaxChunkBytes && + length <= request->TotalBytes - request->Offset && + bool(request->Done) == (request->Offset + length == request->TotalBytes); + } + case EMessageType::INSTALL_SNAPSHOT_RESPONSE: { + auto response = message.Cast(); + return response->SnapshotIndex > 0 && response->SnapshotIndex <= maxIndex && + response->NextOffset <= TSnapshot::MaxBytes && response->Success <= 1 && response->Installed <= 1 && + (!response->Installed || response->Success) && (response->Success || response->NextOffset == 0); } case EMessageType::APPEND_ENTRIES_REQUEST: { auto append = message.Cast(); @@ -452,7 +491,7 @@ bool TRaft::ValidateRpc(TMessageHolder message) const { if (!entry->Term || entry->Term < previousTerm || entry->Term > header->Term) return false; previousTerm = entry->Term; const auto index = append->PrevLogIndex + i + 1; - if (index <= VolatileState->CommitIndex && State->LogTerm(index) != entry->Term) return false; + if (index >= State->SnapshotIndex && index <= VolatileState->CommitIndex && State->LogTerm(index) != entry->Term) return false; } return true; } @@ -483,7 +522,8 @@ void TRaft::LeaderTimeout(ITimeSource::Time now) { { VolatileState->HeartbeatDue[id] = now + TTimeout::Election / 2; VolatileState->RpcDue[id] = now + TTimeout::Rpc; - node->Send(CreateAppendEntries(id)); + if (VolatileState->NextIndex[id] <= State->SnapshotIndex) node->Send(CreateInstallSnapshot(id)); + else node->Send(CreateAppendEntries(id)); } } @@ -498,7 +538,8 @@ uint64_t TRaft::ApproveRead() { // emitted below is newer than this barrier; old replies cannot satisfy it. const uint64_t seqno = NextSeqno(); for (auto& [id, node] : Nodes) { - node->Send(CreateAppendEntries(id)); + if (VolatileState->NextIndex[id] <= State->SnapshotIndex) node->Send(CreateInstallSnapshot(id)); + else node->Send(CreateAppendEntries(id)); } return seqno; } @@ -525,6 +566,7 @@ void TRaft::ProcessTimeout(ITimeSource::Time now) { if (StateName == EState::CANDIDATE || StateName == EState::FOLLOWER) { if (VolatileState->ElectionDue <= now) { if (State->CurrentTerm == UINT64_MAX) throw std::overflow_error("Raft term exhausted"); + IncomingSnapshot.reset(); auto nextVolatileState = std::make_unique(); for (auto [id, _] : Nodes) { nextVolatileState->NextIndex[id] = 1; @@ -542,6 +584,7 @@ void TRaft::ProcessTimeout(ITimeSource::Time now) { if (StateName == EState::CANDIDATE) { int nvotes = VolatileState->Votes.size()+1; if (nvotes >= MinVotes) { + if (State->LastLogIndex >= uint64_t(INT64_MAX)) throw std::overflow_error("Raft log index exhausted"); auto value = State->LastLogIndex+1; decltype(VolatileState->NextIndex) nextIndex; decltype(VolatileState->RpcDue) rpcDue; @@ -614,6 +657,122 @@ std::vector TRaft::GetVoterIds() const { return ids; } +TSnapshot TRaft::Compact() { + if (!State->SupportsSnapshots()) throw std::logic_error("Storage cannot compact snapshots"); + auto application = SnapshotApplication.lock(); + if (!application) throw std::logic_error("Compaction requires a bound application"); + if (application->LastAppliedIndex == State->SnapshotIndex && State->GetSnapshot()) return *State->GetSnapshot(); + auto snapshot = CaptureSnapshot(*this, *application); + application->ValidateSnapshot(snapshot.Data, snapshot.Index); + State->InstallSnapshot(snapshot); + return snapshot; +} + +TMessageHolder TRaft::CreateInstallSnapshot(uint32_t nodeId) { + auto snapshot = State->GetSnapshot(); + if (!snapshot) throw std::logic_error("Missing snapshot for compacted log"); + if (EncodedSnapshotIndex != snapshot->Index) { + EncodedSnapshot = std::make_shared(EncodeSnapshot(*snapshot)); + EncodedSnapshotIndex = snapshot->Index; + } + auto& rpc = VolatileState->SnapshotRpcs[nodeId]; + if (!rpc.Snapshot || rpc.Snapshot->Index != snapshot->Index) { + rpc = TVolatileState::TSnapshotRpc{snapshot, EncodedSnapshot}; + VolatileState->PendingAppendRpcs[nodeId].clear(); + } + const auto count = std::min(TInstallSnapshotRequest::MaxChunkBytes, rpc.Bytes->size() - rpc.Offset); + auto request = NewHoldedMessage(sizeof(TInstallSnapshotRequest) + count); + request->Src = Id; request->Dst = nodeId; request->Term = State->CurrentTerm; + request->Seqno = rpc.Seqno = NextSeqno(); + request->SnapshotIndex = snapshot->Index; request->SnapshotTerm = snapshot->Term; + request->Offset = rpc.Offset; request->TotalBytes = rpc.Bytes->size(); + rpc.SentEnd = rpc.Offset + count; + request->Done = rpc.SentEnd == rpc.Bytes->size(); request->LeaderId = Id; + memcpy(request->Data, rpc.Bytes->data() + rpc.Offset, count); + return request; +} + +void TRaft::OnInstallSnapshot(TMessageHolder response) { + if (response->Term != State->CurrentTerm) return; + auto found = VolatileState->SnapshotRpcs.find(response->Src); + if (found == VolatileState->SnapshotRpcs.end()) return; + auto& rpc = found->second; + if (response->Seqno != rpc.Seqno || response->SnapshotIndex != rpc.Snapshot->Index) return; + if (!response->Success) { + rpc.Offset = 0; + } else if (response->Installed) { + if (response->NextOffset != rpc.Bytes->size()) return; + const auto index = std::max(VolatileState->MatchIndex[response->Src], response->SnapshotIndex); + (*VolatileState).SetMatchIndex(response->Src, index).SetNextIndex(response->Src, index + 1) + .SetBatchSize(response->Src, 1024).SetBackOff(response->Src, 1).CommitAdvance(Nservers, *State); + VolatileState->SnapshotRpcs.erase(found); + // Snapshot completion is not a fresh-read acknowledgement. AppendEntries + // must still establish the next read barrier and replicate the suffix. + } else { + if (response->NextOffset < rpc.SentEnd || response->NextOffset >= rpc.Bytes->size()) return; + rpc.Offset = response->NextOffset; + } + VolatileState->RpcDue[response->Src] = ITimeSource::Time{}; +} + +void TRaft::OnInstallSnapshot(ITimeSource::Time now, TMessageHolder request) { + auto reply = [&](bool success, bool installed, uint64_t offset) { + Nodes.at(request->Src)->Send(NewHoldedMessage( + TMessageEx{.Src = Id, .Dst = request->Src, .Term = State->CurrentTerm, .Seqno = request->Seqno}, + TInstallSnapshotResponse{.SnapshotIndex = request->SnapshotIndex, .NextOffset = offset, + .Success = uint32_t(success), .Installed = uint32_t(installed)})); + }; + if (request->Term < State->CurrentTerm) { reply(false, false, 0); return; } + Become(EState::FOLLOWER); + VolatileState->LeaderId = request->LeaderId; + VolatileState->ElectionDue = MakeElection(now); + if (request->SnapshotIndex <= VolatileState->CommitIndex) { + const auto term = State->LogTerm(request->SnapshotIndex); + const bool covered = request->SnapshotIndex < State->SnapshotIndex || term == request->SnapshotTerm; + if (IncomingSnapshot && IncomingSnapshot->Index <= VolatileState->CommitIndex) IncomingSnapshot.reset(); + reply(covered, covered, covered ? request->TotalBytes : 0); + return; + } + auto application = SnapshotApplication.lock(); + if (!State->SupportsSnapshots() || !application) { reply(false, false, 0); return; } + const bool same = IncomingSnapshot && IncomingSnapshot->Term == request->Term && + IncomingSnapshot->Leader == request->LeaderId && IncomingSnapshot->Index == request->SnapshotIndex && + IncomingSnapshot->BoundaryTerm == request->SnapshotTerm && IncomingSnapshot->Total == request->TotalBytes; + if (!same) { + if (request->Offset != 0 || (IncomingSnapshot && IncomingSnapshot->Term == request->Term && + IncomingSnapshot->Index > request->SnapshotIndex)) { reply(false, false, 0); return; } + IncomingSnapshot = TIncomingSnapshot{request->Term, request->SnapshotIndex, request->SnapshotTerm, + request->TotalBytes, request->LeaderId, {}}; + } + auto& bytes = IncomingSnapshot->Bytes; + const auto count = request->Len - sizeof(TInstallSnapshotRequest); + if (request->Offset > bytes.size()) { reply(false, false, 0); return; } + if (request->Offset < bytes.size()) { + if (count > bytes.size() - request->Offset || + std::string_view(bytes).substr(request->Offset, count) != std::string_view(request->Data, count)) { + IncomingSnapshot.reset(); reply(false, false, 0); return; + } + } else bytes.append(request->Data, count); + if (bytes.size() < request->TotalBytes) { reply(true, false, bytes.size()); return; } + TSnapshot snapshot; + try { + snapshot = DecodeSnapshot(bytes); + if (snapshot.Index != request->SnapshotIndex || snapshot.Term != request->SnapshotTerm || snapshot.Voters != GetVoterIds()) + throw TMessageError("Snapshot metadata mismatch"); + application->ValidateSnapshot(snapshot.Data, snapshot.Index); + } catch (const std::exception&) { + IncomingSnapshot.reset(); reply(false, false, 0); return; + } + try { + State->InstallSnapshot(snapshot); // durable snapshot and suffix before application replacement or ACK + application->LoadSnapshot(snapshot.Data, snapshot.Index); + if (application->LastAppliedIndex != snapshot.Index) std::terminate(); + } catch (...) { std::terminate(); } + VolatileState->CommitIndex = snapshot.Index; + IncomingSnapshot.reset(); + reply(true, true, request->TotalBytes); +} + void TRaft::RestoreAppliedIndex(uint64_t index) { if (index > uint64_t(INT64_MAX) || index > State->LastLogIndex) throw std::invalid_argument("Application progress exceeds recovered Raft log"); @@ -626,7 +785,12 @@ TRequestProcessor::TRequestProcessor(std::shared_ptr raft, std::shared_pt : Raft(std::move(raft)), Rsm(std::move(rsm)), Nodes(nodes) { if (!Raft || !Rsm) throw std::invalid_argument("Request processor requires Raft and application state"); + if (auto snapshot = Raft->GetState()->GetSnapshot(); snapshot && Rsm->LastAppliedIndex == 0) + RestoreSnapshot(*snapshot, *Raft, *Rsm); + if (Rsm->LastAppliedIndex < Raft->GetState()->SnapshotIndex) + throw std::invalid_argument("Application progress precedes compacted log"); Raft->RestoreAppliedIndex(Rsm->LastAppliedIndex); + Raft->SnapshotApplication = Rsm; } void TRequestProcessor::CheckStateChange() { diff --git a/miniraft/raft.h b/miniraft/raft.h index 2e5880c..ff1ae85 100644 --- a/miniraft/raft.h +++ b/miniraft/raft.h @@ -29,8 +29,10 @@ struct IRsm { virtual TMessageHolder Write(TMessageHolder message, uint64_t index) = 0; virtual TMessageHolder Prepare(TMessageHolder message) = 0; virtual std::string SaveSnapshot() const { throw std::logic_error("Application snapshots are unsupported"); } + virtual void ValidateSnapshot(std::string_view, uint64_t) const { throw std::logic_error("Application snapshots are unsupported"); } // Implementations must validate completely before atomically replacing state - // and LastAppliedIndex. Invoked during startup before request processing. + // and LastAppliedIndex. Invoked at startup or during serialized installation; + // a failure after durable installation is fatal and requires restart. virtual void LoadSnapshot(std::string_view, uint64_t) { throw std::logic_error("Application snapshots are unsupported"); } // Durable RSMs restore this atomically with application data before constructing @@ -58,6 +60,12 @@ struct TVolatileState { std::unordered_map ReadAckSeqno; std::unordered_map LastSentSeqno; std::unordered_map> PendingAppendRpcs; + struct TSnapshotRpc { + std::shared_ptr Snapshot; + std::shared_ptr Bytes; + uint64_t Offset = 0, Seqno = 0, SentEnd = 0; + }; + std::unordered_map SnapshotRpcs; uint32_t LeaderId = 0; std::unordered_map NextIndex; std::unordered_map MatchIndex; @@ -107,6 +115,8 @@ class TRaft { uint32_t GetLeaderId() const; uint64_t GetLastIndex() const; std::vector GetVoterIds() const; + // Capture applied state and atomically replace its prefix with a snapshot. + TSnapshot Compact(); uint64_t ApproveRead(); // Highest barrier confirmed by a fresh majority in this leadership term. // Zero means no confirmation (including before a current-term commit). @@ -146,6 +156,9 @@ class TRaft { private: friend class TRequestProcessor; void RestoreAppliedIndex(uint64_t index); + void OnInstallSnapshot(ITimeSource::Time now, TMessageHolder message); + void OnInstallSnapshot(TMessageHolder message); + TMessageHolder CreateInstallSnapshot(uint32_t nodeId); void Candidate(ITimeSource::Time now, TMessageHolder message); void Follower(ITimeSource::Time now, TMessageHolder message); void Leader(ITimeSource::Time now, TMessageHolder message); @@ -173,6 +186,15 @@ class TRaft { std::shared_ptr State; std::unique_ptr VolatileState; uint64_t Seqno = 1; // zero is reserved for an unconfirmed read + std::weak_ptr SnapshotApplication; + struct TIncomingSnapshot { + uint64_t Term, Index, BoundaryTerm, Total; + uint32_t Leader; + std::string Bytes; + }; + std::optional IncomingSnapshot; + std::shared_ptr EncodedSnapshot; + uint64_t EncodedSnapshotIndex = 0; EState StateName; uint32_t Seed = 31337; diff --git a/miniraft/snapshot.h b/miniraft/snapshot.h index ffdb31d..710fda0 100644 --- a/miniraft/snapshot.h +++ b/miniraft/snapshot.h @@ -11,7 +11,7 @@ class TRaft; struct IRsm; -// Local application checkpoint. Full Raft logs are retained in this milestone. +// Versioned application snapshot, shared by local compaction and state transfer. struct TSnapshot { static constexpr size_t MaxBytes = 64 * 1024 * 1024; static constexpr uint32_t MaxVoters = 4096; @@ -23,7 +23,7 @@ struct TSnapshot { std::string EncodeSnapshot(const TSnapshot& snapshot); TSnapshot DecodeSnapshot(std::string_view bytes); TSnapshot CaptureSnapshot(const TRaft& raft, const IRsm& rsm); -// Startup only, with a fresh application and the same fully retained local log. +// Startup only, with a fresh application and a matching local boundary/log. void RestoreSnapshot(const TSnapshot& snapshot, const TRaft& raft, IRsm& rsm); class TSnapshotStore { @@ -36,8 +36,8 @@ class TSnapshotStore { TSnapshotStore(const TSnapshotStore&) = delete; TSnapshotStore& operator=(const TSnapshotStore&) = delete; std::optional Load(); - // Failure is catchable: no log is discarded and every published image is a - // complete committed checkpoint. Crash recovery may select the old or new one. + // Standalone checkpoint export, not the compaction commit point. Failure is + // catchable when the authoritative snapshot/log remains in the Raft state. void Save(const TSnapshot& snapshot); private: struct TImpl; diff --git a/miniraft/state.h b/miniraft/state.h index cb6c488..dc0c0a9 100644 --- a/miniraft/state.h +++ b/miniraft/state.h @@ -1,12 +1,15 @@ #pragma once #include "messages.h" +#include "snapshot.h" struct TStateFields { uint64_t CurrentTerm = 1; uint32_t VotedFor = 0; uint64_t LastLogIndex = 0; uint64_t LastLogTerm = 0; + uint64_t SnapshotIndex = 0; + uint64_t SnapshotTerm = 0; bool operator==(const TStateFields& other) const = default; }; @@ -14,16 +17,21 @@ struct TStateFields { struct IState: TStateFields { virtual void RemoveLast() = 0; virtual void Append(TMessageHolder) = 0; - // index starts from 1 + // Absolute indices; only the suffix after SnapshotIndex has log entries. + // The boundary term remains available through LogTerm(SnapshotIndex). virtual TMessageHolder Get(int64_t index) const = 0; virtual void Commit() = 0; + virtual bool SupportsSnapshots() const { return false; } + virtual std::shared_ptr GetSnapshot() const { return {}; } + virtual void InstallSnapshot(const TSnapshot&) { throw std::logic_error("Storage snapshots are unsupported"); } virtual ~IState() = default; uint64_t LogTerm(int64_t index = -1) const { if (index < 0) { index = LastLogIndex; } - if (index < 1 || index > LastLogIndex) { + if (index > 0 && uint64_t(index) == SnapshotIndex) return SnapshotTerm; + if (index < 1 || uint64_t(index) < SnapshotIndex || index > LastLogIndex) { return 0; } else { return Get(index)->Term; @@ -33,6 +41,7 @@ struct IState: TStateFields { struct TState: IState { std::vector> Log; + std::shared_ptr Snapshot; TState() = default; TState(uint64_t currentTerm, uint32_t votedFor, const std::vector>& log) @@ -47,19 +56,39 @@ struct TState: IState { } void RemoveLast() override { + if (Log.empty()) return; Log.pop_back(); - LastLogIndex = Log.size(); - LastLogTerm = Log.empty() ? 0 : Log.back()->Term; + LastLogIndex = SnapshotIndex + Log.size(); + LastLogTerm = Log.empty() ? SnapshotTerm : Log.back()->Term; } void Append(TMessageHolder entry) override { Log.emplace_back(std::move(entry)); - LastLogIndex = Log.size(); + LastLogIndex = SnapshotIndex + Log.size(); LastLogTerm = Log.back()->Term; } TMessageHolder Get(int64_t index) const override { - return Log[index-1]; + if (index < 1 || uint64_t(index) <= SnapshotIndex || uint64_t(index) > LastLogIndex) return {}; + return Log[index - SnapshotIndex - 1]; + } + + bool SupportsSnapshots() const override { return true; } + std::shared_ptr GetSnapshot() const override { return Snapshot; } + void InstallSnapshot(const TSnapshot& snapshot) override { + EncodeSnapshot(snapshot); // Validate before changing any state. + if (snapshot.Index <= SnapshotIndex || snapshot.Term > CurrentTerm) + throw std::invalid_argument("Obsolete or future snapshot"); + auto image = std::make_shared(snapshot); + std::vector> suffix; + if (snapshot.Index <= LastLogIndex && LogTerm(snapshot.Index) == snapshot.Term) + suffix.assign(Log.begin() + (snapshot.Index - SnapshotIndex), Log.end()); + Log.swap(suffix); + Snapshot = std::move(image); + SnapshotIndex = snapshot.Index; + SnapshotTerm = snapshot.Term; + LastLogIndex = SnapshotIndex + Log.size(); + LastLogTerm = Log.empty() ? SnapshotTerm : Log.back()->Term; } void Commit() override { } diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index ad4fe69..5fff9f7 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -19,6 +19,7 @@ ut(test_message_safety test_message_safety.cpp miniraft) ut(test_rpc_validation test_rpc_validation.cpp miniraft) ut(test_app_payloads "test_app_payloads.cpp;../examples/kv_rsm.cpp" miniraft) ut(test_snapshots "test_snapshots.cpp;../examples/kv_rsm.cpp" miniraft) +ut(test_compaction "test_compaction.cpp;../examples/kv_rsm.cpp" miniraft) find_package(SQLite3 QUIET) if(SQLite3_FOUND) ut(test_sql_payloads "test_sql_payloads.cpp;../examples/sql_rsm.cpp" miniraft) @@ -37,6 +38,9 @@ if(Python3_Interpreter_FOUND) add_test(NAME test_kv_checkpoints_cli COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/smoke_kv_checkpoints.py $) set_tests_properties(test_kv_checkpoints_cli PROPERTIES TIMEOUT 35) +add_test(NAME test_install_snapshot_cli COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/smoke_install_snapshot.py $) +set_tests_properties(test_install_snapshot_cli PROPERTIES TIMEOUT 120) endif() endif() endif() diff --git a/test/smoke_install_snapshot.py b/test/smoke_install_snapshot.py new file mode 100644 index 0000000..ecf96f1 --- /dev/null +++ b/test/smoke_install_snapshot.py @@ -0,0 +1,120 @@ +"""Real three-process durable catch-up smoke, separate from deterministic schedules.""" +import pathlib +import socket +import struct +import subprocess +import sys +import tempfile +import time +sys.dont_write_bytecode = True +from smoke_kv_checkpoints import no_core, port, request + + +def main(): + binary = str(pathlib.Path(sys.argv[1]).resolve()) + with tempfile.TemporaryDirectory(prefix="miniraft-install-snapshot-") as temporary: + root = pathlib.Path(temporary) + ports = [] + while len(ports) < 3: + candidate = port() + if candidate not in ports: + ports.append(candidate) + logs = [] + directories = [] + for index in range(3): + directory = root / str(index + 1) + directory.mkdir() + directories.append(directory) + logs.append((directory / "output.log").open("w+")) + processes = [None] * 3 + + def start(index): + args = [binary, "--server", "--id", str(index + 1), "--durable"] + for node, endpoint in enumerate(ports): + args += ["--node", f"127.0.0.1:{endpoint}:{node + 1}"] + if index < 2: + args += ["--snapshots", "1"] + processes[index] = subprocess.Popen(args, cwd=directories[index], stdout=logs[index], stderr=logs[index], preexec_fn=no_core) + + def call(index, key, value=None): + child = processes[index] + assert child is not None and child.poll() is None, f"node {index + 1} exited" + with socket.create_connection(("127.0.0.1", ports[index]), timeout=1) as sock: + sock.settimeout(2) + return request(sock, key, value, flags=2) # local stale read, never forwarded + + def wait_for(action, timeout): + end = time.monotonic() + timeout + while time.monotonic() < end: + try: + if action(): + return + except (OSError, AssertionError): + pass + time.sleep(0.05) + raise AssertionError("timed out waiting for cluster progress") + + try: + start(0) + start(1) # two voters form a majority; node 3 starts with no log later + values = {b"a": b"a" * 60000, b"b": b"b" * 60000, b"c": b"c" * 60000} + last = [0] + for key, value in values.items(): + def write(): + last[0], _ = call(0, key, value) + return last[0] > 0 + wait_for(write, 30) + + def compacted(): + for index in (0, 1): + image = directories[index] / f"state.snapshot.{index + 1}" + durable = directories[index] / f"state.durable.{index + 1}" + if not image.exists() or struct.unpack(" last[0], 15) + wait_for(lambda: call(2, b"tail")[1] == b"after install", 15) + processes[2].terminate() + processes[2].wait(timeout=5) + start(2) + wait_for(lambda: caught_up() and call(2, b"tail")[1] == b"after install", 25) + for log in logs: + log.flush(); log.seek(0) + output = log.read() + assert "ERROR: AddressSanitizer" not in output and "runtime error:" not in output, output + print("InstallSnapshot CLI: compacted majority, empty follower catch-up, tail replication and durable restart passed") + except Exception: + for index, log in enumerate(logs): + log.flush(); log.seek(0) + print(f"node {index + 1}:\n{log.read()}", file=sys.stderr) + raise + finally: + for child in processes: + if child is not None and child.poll() is None: + child.kill(); child.wait(timeout=5) + for log in logs: + log.close() + + +if __name__ == "__main__": + main() diff --git a/test/smoke_kv_checkpoints.py b/test/smoke_kv_checkpoints.py index 94c4a7c..dce9d4d 100644 --- a/test/smoke_kv_checkpoints.py +++ b/test/smoke_kv_checkpoints.py @@ -28,9 +28,9 @@ def receive(sock, size): return bytes(data) -def request(sock, key, value=None): +def request(sock, key, value=None, flags=None): if value is None: - frame = struct.pack(" +#include +#include +#include "../examples/kv.h" +#include "../examples/kv_messages.h" +#include +#include +#include +#include +#include +#ifdef __linux__ +#include +#include +#endif +#include +#include +#include +extern "C" { +#include +} +namespace { +template void Reject(F action) { + bool rejected = false; + try { action(); } catch (const std::exception&) { rejected = true; } + assert_true(rejected); +} +struct TEndpoint : INode { + std::function)> SendTo; + explicit TEndpoint(std::function)> send) : SendTo(std::move(send)) {} + void Send(TMessageHolder message) override { SendTo(std::move(message)); } + void Drain() override {} +}; +TMessageHolder Command(const std::string& key, const std::string& value) { + auto message = NewHoldedMessage(sizeof(TWriteKv) + key.size() + value.size()); + message->Flags = TCommandRequest::EWrite; + message->KeySize = key.size(); message->ValSize = value.size(); + memcpy(message->TKvEntry::Data, key.data(), key.size()); + memcpy(message->TKvEntry::Data + key.size(), value.data(), value.size()); + return message; +} +std::string Read(TKv& kv, const std::string& key) { + auto message = NewHoldedMessage(sizeof(TReadKv) + key.size()); + message->KeySize = key.size(); memcpy(message->Data, key.data(), key.size()); + auto response = kv.Read(message, kv.LastAppliedIndex).Cast(); + return std::string(response->Data, response->Len - sizeof(TCommandResponse)); +} +struct TApplication : TKv { + size_t Loads = 0; + void LoadSnapshot(std::string_view bytes, uint64_t index) override { TKv::LoadSnapshot(bytes, index); ++Loads; } +}; +struct TCluster { + ITimeSource::Time Now{}; + std::vector> Nodes{3}; + std::vector> Apps{3}; + std::vector> Processors{3}; + std::deque> Packets; + std::set Isolated; + TCluster() { for (uint32_t id = 1; id <= 3; ++id) Build(id, std::make_shared()); } + void Build(uint32_t id, std::shared_ptr state) { + TNodeDict peers; + for (uint32_t dst = 1; dst <= 3; ++dst) if (dst != id) + peers[dst] = std::make_shared([this](auto message) { Packets.push_back(std::move(message)); }); + Processors[id - 1].reset(); + Nodes[id - 1] = std::make_shared(state, id, peers); + Apps[id - 1] = std::make_shared(); + Processors[id - 1] = std::make_unique(Nodes[id - 1], Apps[id - 1], peers); + } + void Restart(uint32_t id) { auto state = Nodes[id - 1]->GetState(); Build(id, state); } + void Deliver(TMessageHolder message, bool duplicate = false) { + const auto dst = message.Cast()->Dst; + Nodes[dst - 1]->Process(Now, message); Processors[dst - 1]->ProcessCommitted(); + if (duplicate) { Nodes[dst - 1]->Process(Now, message); Processors[dst - 1]->ProcessCommitted(); } + } + void Drain(bool duplicate = false) { + size_t budget = 10000; + while (!Packets.empty()) { + assert_true(budget-- > 0); + auto message = Packets.front(); Packets.pop_front(); + auto header = message.Cast(); + if (!Isolated.contains(header->Src) && !Isolated.contains(header->Dst)) Deliver(message, duplicate); + } + } + void Elect(uint32_t id) { + Now += std::chrono::seconds(10); + Nodes[id - 1]->ProcessTimeout(Now); Drain(); + Nodes[id - 1]->ProcessTimeout(Now); Drain(); + assert_int_equal(Nodes[id - 1]->CurrentStateName(), EState::LEADER); + } + void Pump(uint32_t leader, int rounds = 40, bool duplicate = false) { + for (int i = 0; i < rounds; ++i) { + Now += std::chrono::milliseconds(100); + Nodes[leader - 1]->ProcessTimeout(Now); Drain(duplicate); + for (auto& processor : Processors) { processor->ProcessCommitted(); processor->ProcessWaiting(); } + } + } + void Write(uint32_t leader, const std::string& key, const std::string& value) { + Processors[leader - 1]->OnCommandRequest(Command(key, value), {}); Pump(leader); + } + void Prepare() { + Elect(1); Pump(1); Isolated = {3}; + for (const std::string key : {"a", "b", "c"}) Write(1, key, std::string(60000, key[0])); + Nodes[0]->Compact(); Nodes[1]->Compact(); + assert_true(Nodes[0]->GetState()->SnapshotIndex > 1); + Packets.clear(); Isolated.clear(); + } + TMessageHolder Take(uint32_t src, uint32_t dst, EMessageType type) { + for (auto it = Packets.begin(); it != Packets.end(); ++it) { + auto header = it->Cast(); + if (header->Src == src && header->Dst == dst && header->Type == uint32_t(type)) { + auto message = *it; Packets.erase(it); return message; + } + } + fail_msg("Missing packet %u -> %u type %u", src, dst, uint32_t(type)); return {}; + } + TMessageHolder SendChunk() { + Now += std::chrono::seconds(3); + Nodes[0]->ProcessTimeout(Now); + return Take(1, 3, EMessageType::INSTALL_SNAPSHOT_REQUEST).Cast(); + } + TMessageHolder AckChunk(TMessageHolder request) { + Deliver(request); + auto response = Take(3, 1, EMessageType::INSTALL_SNAPSHOT_RESPONSE).Cast(); + Deliver(response); return response; + } +}; +void test_compaction_lagging_follower_installs_snapshot_then_tail(void**) { + TCluster cluster; cluster.Prepare(); + const auto boundary = cluster.Nodes[0]->GetState()->SnapshotIndex; + cluster.Isolated = {3}; cluster.Write(1, "tail", "after snapshot"); cluster.Isolated.clear(); + cluster.Pump(1, 80, true); + assert_int_equal(cluster.Apps[2]->Loads, 1); + assert_int_equal(cluster.Nodes[2]->GetState()->SnapshotIndex, boundary); + for (auto& app : cluster.Apps) { + assert_true(Read(*app, "a") == std::string(60000, 'a')); + assert_true(Read(*app, "tail") == "after snapshot"); + assert_int_equal(app->LastAppliedIndex, cluster.Nodes[0]->GetLastIndex()); + } + assert_false(cluster.Nodes[0]->GetState()->Get(1)); +} +void test_simultaneous_startup_candidates_use_distinct_election_deadlines(void**) { + TCluster cluster; + for (auto& node : cluster.Nodes) node->ProcessTimeout(cluster.Now); + cluster.Drain(); // all voted for themselves in the first term + std::set deadlines; + uint32_t first = 1; + for (uint32_t id = 1; id <= 3; ++id) { + const auto due = cluster.Nodes[id - 1]->GetVolatileState()->ElectionDue; + deadlines.insert(due); + if (due < cluster.Nodes[first - 1]->GetVolatileState()->ElectionDue) first = id; + } + assert_int_equal(deadlines.size(), 3); + cluster.Now = *deadlines.begin(); + for (auto& node : cluster.Nodes) node->ProcessTimeout(cluster.Now); + cluster.Drain(); + cluster.Nodes[first - 1]->ProcessTimeout(cluster.Now); cluster.Drain(); + assert_int_equal(cluster.Nodes[first - 1]->CurrentStateName(), EState::LEADER); +} +void test_partial_transfer_restart_and_delayed_ack_are_safe(void**) { + TCluster cluster; cluster.Prepare(); + auto first = cluster.SendChunk(); + auto delayed = cluster.AckChunk(first); + assert_false(delayed->Installed); + assert_int_equal(cluster.Apps[2]->Loads, 0); + auto invalid = NewHoldedMessage(TMessageEx{.Src = 3, .Dst = 1, .Term = first->Term, .Seqno = first->Seqno}, + TInstallSnapshotResponse{.SnapshotIndex = first->SnapshotIndex, .NextOffset = 1, .Success = 1, .Installed = 1}); + cluster.Deliver(invalid); + assert_true(cluster.Nodes[0]->GetVolatileState()->MatchIndex.at(3) < first->SnapshotIndex); + cluster.Restart(3); // all partial bytes disappear, stable log/vote remain + auto second = cluster.SendChunk(); assert_true(second->Offset > 0); + auto failure = cluster.AckChunk(second); assert_false(failure->Success); + cluster.Deliver(delayed); // stale sequence cannot restore old progress + auto restarted = cluster.SendChunk(); assert_int_equal(restarted->Offset, 0); + cluster.AckChunk(restarted); cluster.Pump(1, 80, true); + assert_int_equal(cluster.Apps[2]->Loads, 1); + assert_true(Read(*cluster.Apps[2], "c") == std::string(60000, 'c')); +} +void test_leader_change_mid_snapshot_rejects_old_chunks(void**) { + TCluster cluster; cluster.Prepare(); + auto old = cluster.SendChunk(); cluster.AckChunk(old); + cluster.Packets.clear(); cluster.Isolated = {1}; + cluster.Elect(2); cluster.Pump(2, 80, true); + assert_true(cluster.Nodes[2]->GetState()->CurrentTerm > old->Term); + const auto loads = cluster.Apps[2]->Loads; + cluster.Deliver(old); + assert_int_equal(cluster.Apps[2]->Loads, loads); + cluster.Write(2, "new leader", "tail"); + assert_true(Read(*cluster.Apps[2], "new leader") == "tail"); +} +void test_partial_and_installed_snapshot_acks_do_not_confirm_strong_reads(void**) { + TCluster cluster; cluster.Prepare(); cluster.Isolated = {2}; + size_t replies = 0; + auto client = std::make_shared([&](auto message) { + assert_int_equal(message.template Cast()->ErrorCode, 0); ++replies; + }); + auto read = NewHoldedMessage(sizeof(TReadKv) + 1); + read->KeySize = 1; read->Data[0] = 'a'; read->Flags = TCommandRequest::EConsistent; + cluster.Processors[0]->OnCommandRequest(read, client); + auto chunk = cluster.Take(1, 3, EMessageType::INSTALL_SNAPSHOT_REQUEST).Cast(); + while (true) { + auto response = cluster.AckChunk(chunk); + cluster.Processors[0]->ProcessWaiting(); assert_int_equal(replies, 0); + if (response->Installed) break; + chunk = cluster.SendChunk(); + } + cluster.Pump(1); assert_int_equal(replies, 1); +} +void test_compacted_follower_hint_requires_boundary_probe_before_match(void**) { + TCluster cluster; cluster.Elect(1); cluster.Write(1, "key", "value"); + cluster.Nodes[1]->Compact(); + const auto boundary = cluster.Nodes[1]->GetState()->SnapshotIndex; + auto* progress = const_cast(cluster.Nodes[0]->GetVolatileState()); + progress->NextIndex[2] = 1; progress->MatchIndex[2] = 0; progress->RpcDue[2] = {}; + cluster.Now += std::chrono::seconds(3); cluster.Nodes[0]->ProcessTimeout(cluster.Now); + cluster.Deliver(cluster.Take(1, 2, EMessageType::APPEND_ENTRIES_REQUEST)); + auto hint = cluster.Take(2, 1, EMessageType::APPEND_ENTRIES_RESPONSE).Cast(); + assert_false(hint->Success); assert_int_equal(hint->MatchIndex, boundary); + cluster.Deliver(hint); + assert_int_equal(progress->MatchIndex[2], 0); assert_int_equal(progress->NextIndex[2], boundary + 1); + cluster.Pump(1); assert_int_equal(progress->MatchIndex[2], cluster.Nodes[0]->GetLastIndex()); +} + +TMessageHolder Chunk(const TSnapshot& snapshot, std::string_view bytes, uint64_t offset, uint32_t count) { + auto message = NewHoldedMessage(sizeof(TInstallSnapshotRequest) + count); + message->Src = 1; message->Dst = 2; message->Term = 3; message->Seqno = offset + 1; + message->SnapshotIndex = snapshot.Index; message->SnapshotTerm = snapshot.Term; + message->Offset = offset; message->TotalBytes = bytes.size(); message->Done = offset + count == bytes.size(); + message->LeaderId = 1; memcpy(message->Data, bytes.data() + offset, count); return message; +} +void test_corrupt_snapshot_and_invalid_headers_cannot_mutate_application(void**) { + TKv source; source.Write(source.Prepare(Command("a", "new")), 5); + TSnapshot snapshot{5, 1, {1, 2}, source.SaveSnapshot()}; + const auto encoded = EncodeSnapshot(snapshot); + std::vector> replies; + auto peer = std::make_shared([&](auto message) { replies.push_back(message.template Cast()); }); + auto state = std::make_shared(); + auto raft = std::make_shared(state, 2, TNodeDict{{1, peer}}); + auto app = std::make_shared(); TRequestProcessor processor(raft, app, {}); + for (int invalid = 0; invalid < 6; ++invalid) { + auto request = Chunk(snapshot, encoded, 0, encoded.size()); request->Term = 99; + if (invalid == 0) request->Offset = UINT64_MAX; + if (invalid == 1) request->TotalBytes = TSnapshot::MaxBytes + 1; + if (invalid == 2) request->Done = 0; + if (invalid == 3) request->SnapshotTerm = 0; + if (invalid == 4) request->LeaderId = 9; + if (invalid == 5) request->SnapshotIndex = UINT64_MAX; + raft->Process({}, request); assert_int_equal(state->CurrentTerm, 1); + } + auto corrupt = Chunk(snapshot, encoded, 0, encoded.size()); corrupt->Data[encoded.size() - 1] ^= 1; + raft->Process({}, corrupt); assert_false(replies.back()->Success); + assert_int_equal(app->LastAppliedIndex, 0); assert_int_equal(state->SnapshotIndex, 0); + for (int invalid = 0; invalid < 2; ++invalid) { + auto wrong = snapshot; + if (invalid == 0) wrong.Voters = {1, 2, 3}; else wrong.Data = "invalid application image"; + const auto bytes = EncodeSnapshot(wrong); + raft->Process({}, Chunk(wrong, bytes, 0, bytes.size())); + assert_false(replies.back()->Success); assert_int_equal(state->SnapshotIndex, 0); + assert_int_equal(app->LastAppliedIndex, 0); + } + auto valid = Chunk(snapshot, encoded, 0, encoded.size()); raft->Process({}, valid); + assert_true(replies.back()->Installed); assert_int_equal(app->Loads, 1); + assert_true(Read(*app, "a") == "new"); + raft->Process({}, valid); assert_int_equal(app->Loads, 1); // duplicate completion is idempotent + snapshot.Index = 3; auto stale = EncodeSnapshot(snapshot); + raft->Process({}, Chunk(snapshot, stale, 0, stale.size())); + assert_int_equal(app->LastAppliedIndex, 5); assert_int_equal(app->Loads, 1); +} + +void test_remote_snapshot_retains_suffix_without_applying_it_before_commit(void**) { + auto state = std::make_shared(); + TKv prefix; + for (const std::string key : {"a", "b", "c", "d"}) { + auto entry = prefix.Prepare(Command(key, "value")); + state->Append(entry); + if (state->LastLogIndex <= 2) prefix.Write(entry, state->LastLogIndex); + } + auto peer = std::make_shared([](auto) {}); + auto raft = std::make_shared(state, 2, TNodeDict{{1, peer}}); + auto app = std::make_shared(); TRequestProcessor processor(raft, app, {}); + TSnapshot snapshot{2, 1, {1, 2}, prefix.SaveSnapshot()}; const auto bytes = EncodeSnapshot(snapshot); + raft->Process({}, Chunk(snapshot, bytes, 0, bytes.size())); processor.ProcessCommitted(); + assert_int_equal(state->LastLogIndex, 4); assert_int_equal(app->LastAppliedIndex, 2); + assert_true(Read(*app, "b") == "value"); assert_true(Read(*app, "c").empty()); + auto heartbeat = NewHoldedMessage(TMessageEx{.Src = 1, .Dst = 2, .Term = 3}, + TAppendEntriesRequest{.PrevLogIndex = 4, .PrevLogTerm = 1, .LeaderCommit = 4, .LeaderId = 1}); + raft->Process({}, heartbeat); processor.ProcessCommitted(); + assert_true(Read(*app, "d") == "value"); assert_int_equal(app->LastAppliedIndex, 4); + assert_int_equal(app->Loads, 1); +} +void test_snapshot_boundary_index_exhaustion_does_not_append_wrapped_noop(void**) { + auto state = std::make_shared(); + TKv image; + state->InstallSnapshot({uint64_t(INT64_MAX), 1, {1}, image.SaveSnapshot()}); + auto raft = std::make_shared(state, 1, TNodeDict{}); + auto app = std::make_shared(); TRequestProcessor processor(raft, app, {}); + Reject([&] { raft->ProcessTimeout({}); }); + assert_true(state->LastLogIndex == uint64_t(INT64_MAX)); + assert_int_equal(state->LastLogTerm, 1); +} + +void ExerciseBoundary(IState& state) { + state.CurrentTerm = 3; state.VotedFor = 2; state.Commit(); + for (uint64_t term : {1, 2, 3}) { auto entry = NewHoldedMessage(); entry->Term = term; state.Append(entry); } + state.InstallSnapshot({2, 2, {1, 2}, "application"}); + assert_int_equal(state.LastLogIndex, 3); assert_int_equal(state.SnapshotIndex, 2); + assert_int_equal(state.LogTerm(1), 0); assert_int_equal(state.LogTerm(2), 2); assert_int_equal(state.Get(3)->Term, 3); + assert_false(state.Get(1)); assert_false(state.Get(2)); + state.RemoveLast(); state.RemoveLast(); + assert_int_equal(state.LastLogIndex, 2); assert_int_equal(state.LastLogTerm, 2); + auto entry = NewHoldedMessage(); entry->Term = 3; state.Append(entry); + assert_int_equal(state.LastLogIndex, 3); assert_int_equal(state.Get(3)->Term, 3); + state.InstallSnapshot({3, 2, {1, 2}, "different boundary"}); + assert_int_equal(state.LastLogIndex, 3); assert_int_equal(state.LastLogTerm, 2); assert_false(state.Get(3)); + Reject([&] { state.InstallSnapshot({2, 2, {1, 2}, "obsolete"}); }); + assert_int_equal(state.SnapshotIndex, 3); +} +void test_memory_snapshot_boundary_retains_matching_suffix_and_replaces_conflict(void**) { TState state; ExerciseBoundary(state); } +#ifdef __linux__ +struct TFiles { + std::filesystem::path Directory; + std::string Name; + TFiles() { + static unsigned next = 0; + Directory = std::filesystem::temp_directory_path() / ("miniraft-compaction-" + std::to_string(getpid()) + "-" + std::to_string(next++)); + assert_true(std::filesystem::create_directory(Directory)); Name = (Directory / "raft").string(); + } + ~TFiles() { std::error_code ignored; std::filesystem::remove_all(Directory, ignored); } +}; +void test_durable_compacted_boundary_survives_restart(void**) { + TFiles files; + { TDurableState state(files.Name, 1); ExerciseBoundary(state); } + TDurableState state(files.Name, 1); + assert_int_equal(state.SnapshotIndex, 3); assert_int_equal(state.LogTerm(3), 2); + assert_int_equal(state.CurrentTerm, 3); assert_int_equal(state.VotedFor, 2); + assert_true(state.GetSnapshot()->Data == "different boundary"); + auto entry = NewHoldedMessage(); entry->Term = 3; state.Append(entry); + assert_int_equal(state.LastLogIndex, 4); assert_int_equal(state.Get(4)->Term, 3); +} +void test_snapshot_and_suffix_are_atomic_at_every_publication_stage(void**) { + using TStage = TDurableState::EStage; + for (bool priorSnapshot : {false, true}) for (bool retain : {false, true}) for (auto stage : {TStage::TempCreated, TStage::Written, TStage::FileSynced, + TStage::Renamed, TStage::DirectorySynced}) { + TFiles files; + { TDurableState state(files.Name, 1); state.CurrentTerm = 3; state.VotedFor = 2; + for (uint64_t term : {1, 2, 3}) { auto entry = NewHoldedMessage(); entry->Term = term; state.Append(entry); } + if (priorSnapshot) state.InstallSnapshot({1, 1, {1, 2}, "previous application"}); } + const auto child = fork(); assert_true(child >= 0); + if (child == 0) { + try { TDurableState state(files.Name, 1, [&](auto point) { if (point == stage) _exit(90); }); + state.InstallSnapshot({retain ? 2u : 4u, retain ? 2u : 3u, {1, 2}, "durable application"}); + } catch (...) { _exit(91); } _exit(92); + } + int status = 0; assert_int_equal(waitpid(child, &status, 0), child); + assert_true(WIFEXITED(status)); assert_int_equal(WEXITSTATUS(status), 90); + TDurableState state(files.Name, 1); + const bool updated = stage >= TStage::Renamed; + assert_int_equal(state.CurrentTerm, 3); assert_int_equal(state.VotedFor, 2); + assert_int_equal(state.SnapshotIndex, updated ? retain ? 2 : 4 : priorSnapshot ? 1 : 0); + assert_int_equal(state.LastLogIndex, updated && !retain ? 4 : 3); + if (updated) assert_true(state.GetSnapshot()->Data == "durable application"); + else if (priorSnapshot) assert_true(state.GetSnapshot()->Data == "previous application"); + if (retain || !updated) assert_int_equal(state.Get(3)->Term, 3); + } +} +void test_compaction_reclaims_durable_prefix_bytes(void**) { + TFiles files; TDurableState state(files.Name, 1); + for (int i = 0; i < 30; ++i) { + auto entry = NewHoldedMessage(sizeof(TLogEntry) + 4096); + memset(entry->Data, 0, 4096); state.Append(entry); + } + const auto before = std::filesystem::file_size(files.Name + ".durable.1"); + state.InstallSnapshot({29, 1, {1}, "small current application"}); + const auto after = std::filesystem::file_size(files.Name + ".durable.1"); + assert_true(after < before / 10); assert_int_equal(state.LastLogIndex, 30); + assert_true(bool(state.Get(30))); assert_false(state.Get(29)); +} +void test_durable_v2_semantic_corruption_rejects_without_rewriting(void**) { + for (int invalid = 0; invalid < 3; ++invalid) { + TFiles files; + { TDurableState state(files.Name, 1); ExerciseBoundary(state); } + const auto path = files.Name + ".durable.1"; + std::ifstream input(path, std::ios::binary); + std::string bytes(std::istreambuf_iterator(input), {}); input.close(); + assert_true(bytes.substr(0, 8) == "MRFTD002"); + if (invalid < 2) { + const auto offset = invalid == 0 ? 40 : 32; // snapshot length or retained entry count + for (size_t i = offset; i < size_t(offset + 8); ++i) bytes[i] = char(0xff); + } else bytes.back() ^= 1; // inner snapshot checksum with a valid outer checksum + const auto crc = NSnapshotCodec::Checksum(bytes.substr(0, 8) + bytes.substr(20)); + for (int i = 0; i < 4; ++i) bytes[16 + i] = char(crc >> (8 * i)); + { std::ofstream out(path, std::ios::binary); out.write(bytes.data(), bytes.size()); } + Reject([&] { TDurableState state(files.Name, 1); }); + std::ifstream after(path, std::ios::binary); + assert_true(std::string(std::istreambuf_iterator(after), {}) == bytes); + } +} +#endif +} +int main() { + const CMUnitTest tests[] = { + cmocka_unit_test(test_compaction_lagging_follower_installs_snapshot_then_tail), + cmocka_unit_test(test_simultaneous_startup_candidates_use_distinct_election_deadlines), + cmocka_unit_test(test_partial_transfer_restart_and_delayed_ack_are_safe), + cmocka_unit_test(test_leader_change_mid_snapshot_rejects_old_chunks), + cmocka_unit_test(test_partial_and_installed_snapshot_acks_do_not_confirm_strong_reads), + cmocka_unit_test(test_compacted_follower_hint_requires_boundary_probe_before_match), + cmocka_unit_test(test_corrupt_snapshot_and_invalid_headers_cannot_mutate_application), + cmocka_unit_test(test_remote_snapshot_retains_suffix_without_applying_it_before_commit), + cmocka_unit_test(test_snapshot_boundary_index_exhaustion_does_not_append_wrapped_noop), + cmocka_unit_test(test_memory_snapshot_boundary_retains_matching_suffix_and_replaces_conflict), +#ifdef __linux__ + cmocka_unit_test(test_durable_compacted_boundary_survives_restart), + cmocka_unit_test(test_snapshot_and_suffix_are_atomic_at_every_publication_stage), + cmocka_unit_test(test_compaction_reclaims_durable_prefix_bytes), + cmocka_unit_test(test_durable_v2_semantic_corruption_rejects_without_rewriting), +#endif + }; + return cmocka_run_group_tests(tests, nullptr, nullptr); +} diff --git a/test/test_network_frames.cpp b/test/test_network_frames.cpp index 40b43a7..558f0c7 100644 --- a/test/test_network_frames.cpp +++ b/test/test_network_frames.cpp @@ -71,7 +71,7 @@ void test_short_fixed_frame_is_rejected(void**) { stream.Add(uint32_t(EMessageType::REQUEST_VOTE_REQUEST), sizeof(TMessage)); assert_true(Rejected(stream)); } -void test_unknown_and_unimplemented_types_are_rejected(void**) { +void test_unknown_types_and_short_snapshot_headers_are_rejected(void**) { for (auto type : {0u, 6u, 7u, 10u, UINT32_MAX}) { TStream stream; stream.Add(type, sizeof(TMessage)); @@ -218,13 +218,36 @@ void test_fragmented_roundtrip_keeps_consecutive_frame_boundaries(void**) { assert_int_equal(next->Len, sizeof(TCommandRequest)); assert_int_equal(stream.Position, stream.Bytes.size()); } +void test_fragmented_snapshot_request_and_response_roundtrip(void**) { + for (uint32_t length : {uint32_t(sizeof(TInstallSnapshotRequest)), + uint32_t(sizeof(TInstallSnapshotRequest) + TInstallSnapshotRequest::MaxChunkBytes + 1)}) { + TStream invalid; invalid.Add(uint32_t(EMessageType::INSTALL_SNAPSHOT_REQUEST), length); + assert_true(Rejected(invalid)); assert_int_equal(invalid.Position, sizeof(TMessage)); + } + auto request = NewHoldedMessage(sizeof(TInstallSnapshotRequest) + 1024); + request->Src = 1; request->Dst = 2; request->Term = 3; request->Seqno = 7; + request->SnapshotIndex = 100; request->SnapshotTerm = 2; + request->TotalBytes = 1024; request->Offset = 0; request->Done = 1; request->LeaderId = 1; + memset(request->Data, 's', 1024); + auto response = NewHoldedMessage(TMessageEx{.Src = 2, .Dst = 1, .Term = 3, .Seqno = 7}, + TInstallSnapshotResponse{.SnapshotIndex = 100, .NextOffset = 1024, .Success = 1, .Installed = 1}); + TStream stream; stream.Chunk = 3; + auto send = TMessageWriter(stream).Write(request); send.await_resume(); + auto reply = TMessageWriter(stream).Write(response); reply.await_resume(); + stream.Bytes = stream.Output; + auto received = Read(stream); assert_true(received.IsEx()); + auto snapshot = received.Cast(); + assert_int_equal(snapshot->SnapshotIndex, 100); assert_memory_equal(snapshot->Data, request->Data, 1024); + auto ack = Read(stream).Cast(); assert_true(ack->Installed); + assert_int_equal(ack->NextOffset, 1024); assert_int_equal(stream.Position, stream.Bytes.size()); +} } int main() { const CMUnitTest tests[] = { cmocka_unit_test(test_fragmented_header_and_body_are_read_exactly), cmocka_unit_test(test_short_fixed_frame_is_rejected), - cmocka_unit_test(test_unknown_and_unimplemented_types_are_rejected), + cmocka_unit_test(test_unknown_types_and_short_snapshot_headers_are_rejected), cmocka_unit_test(test_append_entries_rejects_non_log_child), cmocka_unit_test(test_append_entries_rejects_nested_append), cmocka_unit_test(test_oversized_frame_is_rejected_before_body_read), @@ -236,6 +259,7 @@ int main() { cmocka_unit_test(test_failed_reader_cannot_resume_at_another_frame), cmocka_unit_test(test_writer_validates_entire_batch_before_first_byte), cmocka_unit_test(test_fragmented_roundtrip_keeps_consecutive_frame_boundaries), + cmocka_unit_test(test_fragmented_snapshot_request_and_response_roundtrip), }; return cmocka_run_group_tests(tests, nullptr, nullptr); } From 9a5752e9d1357827732753e252d69557088e4220 Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Mon, 9 Nov 2026 07:03:41 -0500 Subject: [PATCH 17/66] Add non-voting learners and persist fixed voter configuration (#17) --- README.md | 3 + docs/compaction.md | 3 + docs/learners.md | 130 ++++++++++++ examples/kv.cpp | 19 +- miniraft/durable.cpp | 66 ++++-- miniraft/durable.h | 3 + miniraft/raft.cpp | 79 ++++++-- miniraft/raft.h | 12 +- miniraft/state.h | 34 ++++ test/CMakeLists.txt | 4 + test/smoke_install_snapshot.py | 20 +- test/smoke_kv_checkpoints.py | 23 ++- test/test_learners.cpp | 355 +++++++++++++++++++++++++++++++++ 13 files changed, 713 insertions(+), 38 deletions(-) create mode 100644 docs/learners.md create mode 100644 test/test_learners.cpp diff --git a/README.md b/README.md index 5db9c06..c2b5f60 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,9 @@ and tail-only application recovery with `--durable --snapshots N`. Logs remain fully retained in that baseline. The [compaction follow-up](docs/compaction.md) enables prefix deletion and chunked InstallSnapshot catch-up; fixed membership remains required. Upgrade all voters before enabling the new compaction behavior. +The [learner follow-up](docs/learners.md) adds non-voting catch-up and durable +voter binding. KV uses `--learner id` on every replica's configuration. Promotion +and joint-consensus voter changes remain disabled. [![Number of GitHub stars](https://img.shields.io/github/stars/resetius/miniraft-cpp)](https://github.com/resetius/miniraft-cpp/stargazers) ![GitHub commit activity](https://img.shields.io/github/commit-activity/m/resetius/miniraft-cpp) diff --git a/docs/compaction.md b/docs/compaction.md index 8ad63bc..8ee2731 100644 --- a/docs/compaction.md +++ b/docs/compaction.md @@ -1,5 +1,8 @@ # Compaction and InstallSnapshot: v0.1.0-alpha.16 +The [learner follow-up](learners.md) adds non-voting replication and v3 durable +voter binding while retaining backward decoding of the v1/v2 formats below. + This milestone adds prefix deletion and follower state transfer to the alpha.15 KV checkpoint foundation. TState and Linux TDurableState support compacted logs; legacy TDiskState does not. KV implements snapshot validation/restoration. SQL and diff --git a/docs/learners.md b/docs/learners.md new file mode 100644 index 0000000..dda2e0e --- /dev/null +++ b/docs/learners.md @@ -0,0 +1,130 @@ +# Non-voting learners: v0.1.0-alpha.17 + +This milestone separates replication targets from the fixed voting set. Learners +receive log entries, install snapshots and apply committed commands, but never +campaign, vote, lead, propose entries or contribute to election, commit or strong +read quorums. It implements the catch-up phase described in +[Raft section 6](https://raft.github.io/raft.pdf). Promotion, voter removal and +joint-consensus configuration entries remain the next milestone. + +## Configure a learner + +The KV CLI accepts repeatable `--learner id` options, requiring `--server --durable`. +All replicas must agree on the same voter IDs and role list. Include every voter +and learner endpoint in `--node`; mark each learner by ID on every process. For +example, with voters 1/2/3 and learner 4: + +```sh +./build/full/examples/kv --server --id 4 --durable \ + --node 127.0.0.1:8001:1 --node 127.0.0.1:8002:2 \ + --node 127.0.0.1:8003:3 --node 127.0.0.1:8004:4 --learner 4 +``` + +Use the same endpoints and `--learner 4` on voter processes, with their own IDs. +Learners can catch up from a compacted prefix even without local periodic +snapshotting. Clients can connect through them: writes and strong/default reads +are forwarded to the known voter leader; explicit stale reads use local applied +state. A learner does not elect a replacement leader when disconnected. + +The constructor gains an optional fourth argument, the learner-ID vector. +Omitting it preserves all-voter interpretation. IDs must be positive, unique and +present among the local/peer IDs. At least one voter is required. Transport peer +count still includes learners; GetNservers/GetMinVotes describe voters/quorums. +Memory TState supports role binding for library tests/volatile deployments, but +only the durable backend preserves that binding across actual process loss. +Legacy TDiskState does not support learner mode. + +Transport discovery is still static. Adding/removing learner endpoints requires +updating replica startup configuration and restarting those transports while +keeping the bound voter set unchanged. There is no online enrollment API yet. +Deleting `--learner 4` to turn it into a voter is rejected for bound stores; it is +not a membership-change procedure. Voter-set changes remain disabled. + +## Quorum and term isolation + +Leaders replicate to every configured peer, tracking normal RPC-correlated +MatchIndex/NextIndex and snapshot progress. Commit advancement selects indices +only for explicit voter IDs. Election RPCs are sent only to voters, learner vote +responses are ignored, and strong-read confirmation collects only voter ACKs. +One voter with multiple learners can still commit/read alone; a voter minority +cannot make decisions merely because learners are available. + +A learner acknowledges vote requests with a denial and never records a vote. +It follows legitimate voter terms but cannot advance its own term by election +timeout. Voters discard learner-origin vote/leader requests before term mutation. +Replication ACKs from learners are accepted only in the voter's current term, so +an outlier learner term cannot depose a healthy voter leader. A learner with an +incompatible future term may remain unable to catch up; this does not force the +voting quorum to adopt that term. This is a trusted-member protocol, not network +authentication against forged voter IDs. + +IsLearnerCaughtUp(id) is a leader-only, point-in-time log watermark: the configured +learner must match the leader's current log end and the leader must already know +a current-term commit. It does not change roles, authorize promotion, attest to +application health or supply an application-applied watermark. Persistence of +ACKed entries depends on the selected storage backend. Unknown IDs and voter IDs +return false. A new unreplicated tail makes the status false again. + +## Durable voter binding and upgrade + +Fresh binding-capable stores record the voter IDs before accepting any vote or +replication traffic, including all-voter deployments. A snapshot-backed v2 store +can adopt exactly the voter set already recorded in its embedded snapshot. Bound +stores reject different voter IDs on restart and reject snapshots that would +replace that set. Learner local stores must not contain a vote. Ordinary role +validation failure does not modify term, vote, log or application data. + +A nonfresh v1 store without an embedded snapshot has no trustworthy record of its +old voter IDs. Learner mode refuses to guess. First run it under its unchanged, +verified all-voter configuration and create an embedded snapshot with compaction; +then enable learner-aware discovery with the same voter IDs. Unbound legacy +all-voter operation retains its previous startup contract until that proof exists. +A standalone alpha.15 checkpoint alone is not an embedded configuration witness. +Never delete voter state or edit votes/terms to work around this validation. + +The new durable format `MRFTD003` retains the 20-byte header and stores: +CurrentTerm[8], VotedFor[4], retained-log count[8], voter count[4], sorted voter +IDs[4 each], snapshot length[8], optional MRFTS001 image, then retained log records. +An absent snapshot uses length zero. The CRC covers version magic plus payload. +Recovery bounds and validates the voter list and requires any snapshot to name +the same voters. Both v1 and v2 remain readable; the first binding writes v3 using +the existing atomic write/fsync/rename/directory-fsync protocol. A crash before +first binding leaves either a fresh unbound store with no votes/log, or a complete +bound store. Publication failure is fatal before the node can serve traffic. + +Upgrade all voters before configuring any learners. Older binaries treat every +peer as a voter and cannot read v3 stores; mixed-role/mixed-version operation is +unsupported. All fresh bootstrap configurations must agree. Persistent binding +does not authenticate initial bootstrap choices or wrong-store copies. No cluster +UUID, role negotiation or automatic migration from unknown membership is added. + +## Validation and review + +Fifteen new deterministic unit tests cover malformed role lists, no learner +campaign/vote/proposal, term isolation, voter-majority arithmetic for sizes 1–8 +with arbitrary learner matches, a single voter with several learners, ignored +learner grants, commit/read isolation during voter partitions, ordinary and +snapshot replication, log catch-up status, restart role preservation, rejected +manual promotion, learner-only transport expansion, unbound legacy rejection, +v2 snapshot-backed adoption, v3 corruption with valid checksums, unsupported +legacy storage, and crashes at all five first-binding publication stages. + +A new real-process learner CLI test starts an offline learner after a compacted +two-voter majority, verifies local catch-up and tail application, restarts the +learner, confirms its persisted vote is zero, and checks that removing the role +flag fails without rewriting its store. The existing checkpoint/InstallSnapshot +smokes also run against v3 stores. Fault schedules are deterministic core tests; +CLI smokes use separately bounded socket/process waits. + +Local validation passes 222 unit tests and three CLI tests under ASan/UBSan across +17 CTest targets; Release core checks pass. Self-review checked every quorum +site, pre-term RPC filtering, local-role guards, immutable voter binding, adoption +proofs, snapshot voter metadata, v1/v2/v3 decoding, publication ordering and the +distinction between log catch-up and promotion. No independent reviewer approval +is claimed. Process crashes do not substitute for hardware power-cut testing. + +This closes the non-voting catch-up phase, not dynamic voting membership. Next +are replicated joint configurations, old/new election/commit/read majorities, +promotion gates, leader removal and configuration history in snapshots. Online +transport discovery, authenticated bootstrap, SQL snapshots, resource quotas and +independent review remain open production gates. diff --git a/examples/kv.cpp b/examples/kv.cpp index d3a4085..dfc79f6 100644 --- a/examples/kv.cpp +++ b/examples/kv.cpp @@ -114,7 +114,7 @@ NNet::TVoidTask CheckpointLoop(TPoller& poller, std::shared_ptr raft, } void usage(const char* prog) { - std::cerr << prog << "--client|--server --id myid --node ip:port:id [--node ip:port:id ...] [--persist|--durable] [--snapshots entries] [--stale] [--consistent]" << "\n"; + std::cerr << prog << "--client|--server --id myid --node ip:port:id [--node ip:port:id ...] [--persist|--durable] [--snapshots entries] [--learner id ...] [--stale] [--consistent]" << "\n"; exit(0); } @@ -123,6 +123,7 @@ int main(int argc, char** argv) { std::vector hosts; THost myHost; TNodeDict nodes; + std::vector learners; uint32_t id = 0; bool server = false; bool persist = false; @@ -149,6 +150,15 @@ int main(int argc, char** argv) { if (result.ec != std::errc{} || result.ptr != value.data() + value.size() || !snapshotInterval) { std::cerr << "--snapshots requires a positive entry interval\n"; return 1; } + } else if (!strcmp(argv[i], "--learner")) { + if (i + 1 >= argc) { std::cerr << "--learner requires a positive node ID\n"; return 1; } + const std::string_view value(argv[++i]); + uint32_t learner = 0; + const auto result = std::from_chars(value.data(), value.data() + value.size(), learner); + if (result.ec != std::errc{} || result.ptr != value.data() + value.size() || !learner) { + std::cerr << "--learner requires a positive node ID\n"; return 1; + } + learners.push_back(learner); } else if (!strcmp(argv[i], "--stale")) { flags |= TCommandRequest::EStale; } else if (!strcmp(argv[i], "--consistent")) { @@ -158,6 +168,9 @@ int main(int argc, char** argv) { } } + if (!learners.empty() && (!server || !durable)) { + std::cerr << "--learner requires --server --durable\n"; return 1; + } if (snapshotInterval && (!server || !durable)) { std::cerr << "--snapshots requires --server --durable\n"; return 1; } @@ -166,7 +179,9 @@ int main(int argc, char** argv) { NNet::TLoop loop; if (server) { + std::unordered_set hostIds; for (auto& host : hosts) { + if (!hostIds.insert(host.Id).second) { std::cerr << "Duplicate node ID\n"; return 1; } if (!host) { std::cerr << "Empty host\n"; return 1; } @@ -191,7 +206,7 @@ int main(int argc, char** argv) { if (durable) state = std::make_shared("state", myHost.Id); else state = std::make_shared("state", myHost.Id); } - auto raft = std::make_shared(state, myHost.Id, nodes); + auto raft = std::make_shared(state, myHost.Id, nodes, learners); std::shared_ptr snapshots; uint64_t savedSnapshot = state->SnapshotIndex; if (snapshotInterval) { diff --git a/miniraft/durable.cpp b/miniraft/durable.cpp index 9741bf7..7e7f779 100644 --- a/miniraft/durable.cpp +++ b/miniraft/durable.cpp @@ -19,6 +19,7 @@ namespace { using TBytes = std::vector; constexpr std::array Magic{'M','R','F','T','D','0','0','1'}; constexpr std::array SnapshotMagic{'M','R','F','T','D','0','0','2'}; +constexpr std::array ConfigurationMagic{'M','R','F','T','D','0','0','3'}; void Put(TBytes& bytes, uint64_t value, unsigned width) { for (unsigned i = 0; i < width; ++i) bytes.push_back(uint8_t(value >> (8 * i))); } @@ -29,13 +30,13 @@ uint64_t Take(const TBytes& bytes, size_t& position, unsigned width) { for (unsigned i = 0; i < width; ++i) value |= uint64_t(bytes[position++]) << (8 * i); return value; } -uint32_t Checksum(const TBytes& bytes, size_t start, bool versioned = false) { +uint32_t Checksum(const TBytes& bytes, size_t start, const std::array* version = nullptr) { uint32_t crc = UINT32_MAX; const auto update = [&](uint8_t byte) { crc ^= byte; for (int bit = 0; bit < 8; ++bit) crc = (crc >> 1) ^ (0xedb88320u & (0u - (crc & 1u))); }; - if (versioned) for (auto byte : SnapshotMagic) update(byte); + if (version) for (auto byte : *version) update(byte); for (size_t i = start; i < bytes.size(); ++i) update(bytes[i]); return ~crc; } @@ -94,6 +95,7 @@ TBytes ReadAll(int fd) { struct TDurableState::TImpl { std::vector> Log; std::shared_ptr Snapshot; + std::vector Voters; THook Hook; std::string File, Temp; #ifdef __linux__ @@ -137,28 +139,42 @@ TDurableState::TDurableState(const std::string& name, uint32_t id, THook hook) return; } const auto bytes = ReadAll(file.Value); + const bool configured = std::equal(ConfigurationMagic.begin(), ConfigurationMagic.end(), bytes.begin()); const bool compacted = std::equal(SnapshotMagic.begin(), SnapshotMagic.end(), bytes.begin()); - if (!compacted && !std::equal(Magic.begin(), Magic.end(), bytes.begin())) + const auto* version = configured ? &ConfigurationMagic : compacted ? &SnapshotMagic : nullptr; + if (!version && !std::equal(Magic.begin(), Magic.end(), bytes.begin())) throw std::runtime_error("Unknown durable state format"); size_t position = Magic.size(); const auto length = Take(bytes, position, 8); const auto crc = Take(bytes, position, 4); - if (length != bytes.size() - position || crc != Checksum(bytes, position, compacted)) + if (length != bytes.size() - position || crc != Checksum(bytes, position, version)) throw std::runtime_error("Durable state length/checksum mismatch"); CurrentTerm = Take(bytes, position, 8); VotedFor = Take(bytes, position, 4); const auto count = Take(bytes, position, 8); - if (compacted) { + if (configured) { + const auto voters = Take(bytes, position, 4); + if (!voters || voters > TSnapshot::MaxVoters || voters > (bytes.size() - position) / 4) + throw std::runtime_error("Invalid durable voter count"); + for (uint32_t i = 0; i < voters; ++i) Impl->Voters.push_back(Take(bytes, position, 4)); + try { ValidateFixedVoters(Impl->Voters); } + catch (const std::invalid_argument&) { throw std::runtime_error("Invalid durable voter configuration"); } + } + if (compacted || configured) { const auto length = Take(bytes, position, 8); if (length > TSnapshot::MaxBytes || length > bytes.size() - position) throw std::runtime_error("Invalid embedded snapshot size"); - auto snapshot = DecodeSnapshot(std::string_view(reinterpret_cast(bytes.data() + position), size_t(length))); - if (snapshot.Term > CurrentTerm) throw std::runtime_error("Future durable snapshot term"); - position += size_t(length); - SnapshotIndex = snapshot.Index; - SnapshotTerm = snapshot.Term; - LastLogTerm = SnapshotTerm; - Impl->Snapshot = std::make_shared(std::move(snapshot)); + if (!length && compacted) throw std::runtime_error("Missing embedded snapshot"); + if (length) { + auto snapshot = DecodeSnapshot(std::string_view(reinterpret_cast(bytes.data() + position), size_t(length))); + if (snapshot.Term > CurrentTerm) throw std::runtime_error("Future durable snapshot term"); + if (configured && snapshot.Voters != Impl->Voters) throw std::runtime_error("Durable snapshot/voter mismatch"); + position += size_t(length); + SnapshotIndex = snapshot.Index; + SnapshotTerm = snapshot.Term; + LastLogTerm = SnapshotTerm; + Impl->Snapshot = std::make_shared(std::move(snapshot)); + } } if (!CurrentTerm || count > uint64_t(INT64_MAX) - SnapshotIndex || count > (bytes.size() - position) / 20) throw std::runtime_error("Invalid durable state metadata"); @@ -203,8 +219,15 @@ void TDurableState::Publish() Put(payload, CurrentTerm, 8); Put(payload, VotedFor, 4); Put(payload, Impl->Log.size(), 8); - if (Impl->Snapshot) { - const auto image = EncodeSnapshot(*Impl->Snapshot); + if (!Impl->Voters.empty()) { + ValidateFixedVoters(Impl->Voters); + if (Impl->Snapshot && Impl->Snapshot->Voters != Impl->Voters) + throw std::runtime_error("Snapshot would change bound voters"); + Put(payload, Impl->Voters.size(), 4); + for (auto id : Impl->Voters) Put(payload, id, 4); + } + if (Impl->Snapshot || !Impl->Voters.empty()) { + const auto image = Impl->Snapshot ? EncodeSnapshot(*Impl->Snapshot) : std::string{}; Put(payload, image.size(), 8); payload.insert(payload.end(), image.begin(), image.end()); } @@ -218,10 +241,10 @@ void TDurableState::Publish() Put(payload, entry->Len - sizeof(TLogEntry), 4); payload.insert(payload.end(), entry->Data, entry->Data + entry->Len - sizeof(TLogEntry)); } - const auto& magic = Impl->Snapshot ? SnapshotMagic : Magic; + const auto& magic = !Impl->Voters.empty() ? ConfigurationMagic : Impl->Snapshot ? SnapshotMagic : Magic; TBytes bytes(magic.begin(), magic.end()); Put(bytes, payload.size(), 8); - Put(bytes, Checksum(payload, 0, bool(Impl->Snapshot)), 4); + Put(bytes, Checksum(payload, 0, !Impl->Voters.empty() ? &ConfigurationMagic : Impl->Snapshot ? &SnapshotMagic : nullptr), 4); bytes.insert(bytes.end(), payload.begin(), payload.end()); TFd temp(openat(Impl->Directory.Value, Impl->Temp.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC | O_NOFOLLOW, 0600)); @@ -275,6 +298,8 @@ std::shared_ptr TDurableState::GetSnapshot() const { return Imp void TDurableState::InstallSnapshot(const TSnapshot& snapshot) { EncodeSnapshot(snapshot); + if (!Impl->Voters.empty() && snapshot.Voters != Impl->Voters) + throw std::invalid_argument("Snapshot would change bound voters"); if (snapshot.Index <= SnapshotIndex || snapshot.Term > CurrentTerm) throw std::invalid_argument("Obsolete or future durable snapshot"); auto image = std::make_shared(snapshot); @@ -289,3 +314,12 @@ void TDurableState::InstallSnapshot(const TSnapshot& snapshot) { LastLogTerm = Impl->Log.empty() ? SnapshotTerm : Impl->Log.back()->Term; Commit(); // Atomically publishes hard state, snapshot and the selected suffix. } + +std::vector TDurableState::GetBoundVoters() const { return Impl->Voters; } + +void TDurableState::BindVoters(const std::vector& voters) { + ValidateVoterBinding(*this, Impl->Voters, voters); + if (Impl->Voters == voters) return; + Impl->Voters = voters; + Commit(); // Bind before acknowledging any vote/replication traffic. +} diff --git a/miniraft/durable.h b/miniraft/durable.h index 4a28601..4e10a9d 100644 --- a/miniraft/durable.h +++ b/miniraft/durable.h @@ -23,6 +23,9 @@ class TDurableState : public IState { bool SupportsSnapshots() const override { return true; } std::shared_ptr GetSnapshot() const override; void InstallSnapshot(const TSnapshot& snapshot) override; + bool SupportsVoterBinding() const override { return true; } + std::vector GetBoundVoters() const override; + void BindVoters(const std::vector& voters) override; private: struct TImpl; std::unique_ptr Impl; diff --git a/miniraft/raft.cpp b/miniraft/raft.cpp index 156b565..819d310 100644 --- a/miniraft/raft.cpp +++ b/miniraft/raft.cpp @@ -99,6 +99,19 @@ TVolatileState& TVolatileState::CommitAdvance(int nservers, const IState& state) return *this; } +TVolatileState& TVolatileState::CommitAdvance(const std::vector& voters, uint32_t self, const IState& state) { + ValidateFixedVoters(voters); + Indices.clear(); + for (auto id : voters) { + auto match = MatchIndex.find(id); + Indices.push_back(id == self ? state.LastLogIndex : match == MatchIndex.end() ? 0 : match->second); + } + std::sort(Indices.begin(), Indices.end()); + const auto index = std::max(CommitIndex, Indices[(voters.size() - 1) / 2]); + if (state.LogTerm(index) == state.CurrentTerm) CommitIndex = index; + return *this; +} + TVolatileState& TVolatileState::SetNextIndex(uint32_t id, uint64_t nextIndex) { NextIndex[id] = nextIndex; @@ -145,12 +158,12 @@ TVolatileState& TVolatileState::SetCommitIndex(uint64_t index) return *this; } -TRaft::TRaft(std::shared_ptr state, int node, const TNodeDict& nodes) +TRaft::TRaft(std::shared_ptr state, int node, const TNodeDict& nodes, std::vector learners) : Id(node) , Nodes(nodes) - , MinVotes((nodes.size()+2+nodes.size()%2)/2) + , MinVotes(0) // derived from voting IDs after role validation , Npeers(nodes.size()) - , Nservers(nodes.size()+1) + , Nservers(0) , State(std::move(state)) , VolatileState(std::make_unique()) , StateName(EState::FOLLOWER) @@ -164,6 +177,28 @@ TRaft::TRaft(std::shared_ptr state, int node, const TNodeDict& nodes) throw std::invalid_argument("Invalid Raft peer configuration"); VolatileState->NextIndex[id] = 1; } + Learners = std::move(learners); + std::sort(Learners.begin(), Learners.end()); + uint32_t previous = 0; + for (auto id : Learners) { + if (id <= previous || (id != Id && !Nodes.contains(id))) + throw std::invalid_argument("Invalid or duplicate learner ID"); + previous = id; + } + if (!std::binary_search(Learners.begin(), Learners.end(), Id)) Voters.push_back(Id); + for (const auto& [id, _] : Nodes) + if (!std::binary_search(Learners.begin(), Learners.end(), id)) Voters.push_back(id); + std::sort(Voters.begin(), Voters.end()); + ValidateFixedVoters(Voters); + if (!IsVoter(Id) && State->VotedFor) + throw std::invalid_argument("A learner store must not contain a vote"); + Nservers = Voters.size(); MinVotes = Nservers / 2 + 1; + if (!Learners.empty() && !State->SupportsVoterBinding()) + throw std::invalid_argument("Learners require voter-binding-capable storage"); + const auto bound = State->GetBoundVoters(); + const bool fresh = State->CurrentTerm == 1 && !State->VotedFor && !State->LastLogIndex; + if (State->SupportsVoterBinding() && (!bound.empty() || fresh || State->GetSnapshot() || !Learners.empty())) + State->BindVoters(Voters); if (auto snapshot = State->GetSnapshot()) { if (snapshot->Voters != GetVoterIds()) throw std::invalid_argument("Compacted state voter configuration mismatch"); VolatileState->CommitIndex = State->SnapshotIndex; @@ -171,6 +206,11 @@ TRaft::TRaft(std::shared_ptr state, int node, const TNodeDict& nodes) } void TRaft::OnRequestVote(ITimeSource::Time now, TMessageHolder message) { + if (!IsVoter(Id)) { + Nodes.at(message->Src)->Send(NewHoldedMessage( + TMessageEx{.Src = Id, .Dst = message->Src, .Term = State->CurrentTerm}, TRequestVoteResponse{.VoteGranted = false})); + return; + } if (message->Term < State->CurrentTerm) { auto reply = NewHoldedMessage( TMessageEx {.Src = Id, .Dst = message->Src, .Term = State->CurrentTerm}, @@ -297,7 +337,7 @@ void TRaft::OnAppendEntries(TMessageHolder message) { .SetRpcDue(nodeId, ITimeSource::Time{}) .SetBatchSize(nodeId, 1024) .SetBackOff(nodeId, 1) - .CommitAdvance(Nservers, *State); + .CommitAdvance(Voters, Id, *State); } else if (message->MatchIndex > 0) { // A compacted-prefix hint is not replication evidence. Probe its boundary // with AppendEntries before advancing MatchIndex or confirming reads. @@ -405,6 +445,8 @@ void TRaft::Leader(ITimeSource::Time now, TMessageHolder message) { } void TRaft::Become(EState newStateName) { + if (!IsVoter(Id) && (newStateName == EState::CANDIDATE || newStateName == EState::LEADER)) + throw std::logic_error("A learner cannot campaign or lead"); if (StateName != newStateName) { StateName = newStateName; } @@ -448,6 +490,11 @@ bool TRaft::ValidateRpc(TMessageHolder message) const { try { ValidateMessageTree(message); } catch (const TMessageError&) { return false; } auto header = message.Cast(); if (header->Dst != Id || !Nodes.contains(header->Src)) return false; + if (!IsVoter(header->Src)) { + if ((header->Type != uint32_t(EMessageType::APPEND_ENTRIES_RESPONSE) && + header->Type != uint32_t(EMessageType::INSTALL_SNAPSHOT_RESPONSE)) || header->Term != State->CurrentTerm) + return false; // Nonvoters cannot campaign, lead or force a term change. + } constexpr uint64_t maxIndex = INT64_MAX; switch (EMessageType(message->Type)) { case EMessageType::REQUEST_VOTE_REQUEST: { @@ -507,6 +554,7 @@ void TRaft::FollowerTimeout(ITimeSource::Time now) { void TRaft::CandidateTimeout(ITimeSource::Time now) { for (auto& [id, node] : Nodes) { + if (!IsVoter(id)) continue; if (VolatileState->RpcDue[id] <= now) { VolatileState->RpcDue[id] = now + TTimeout::Rpc; node->Send(CreateVote(id)); @@ -528,7 +576,7 @@ void TRaft::LeaderTimeout(ITimeSource::Time now) { } if (Nservers == 1) { - VolatileState->CommitAdvance(Nservers, *State); + VolatileState->CommitAdvance(Voters, Id, *State); } } @@ -549,7 +597,8 @@ uint64_t TRaft::CommitSeqno() const { State->LogTerm(VolatileState->CommitIndex) != State->CurrentTerm) return 0; std::vector acknowledgements{Seqno - 1}; // leader's own confirmation acknowledgements.reserve(Nservers); - for (const auto& [id, _] : Nodes) { + for (auto id : Voters) { + if (id == Id) continue; auto it = VolatileState->ReadAckSeqno.find(id); acknowledgements.push_back(it == VolatileState->ReadAckSeqno.end() ? 0 : it->second); } @@ -563,6 +612,7 @@ uint64_t TRaft::NextSeqno() { } void TRaft::ProcessTimeout(ITimeSource::Time now) { + if (!IsVoter(Id)) return; if (StateName == EState::CANDIDATE || StateName == EState::FOLLOWER) { if (VolatileState->ElectionDue <= now) { if (State->CurrentTerm == UINT64_MAX) throw std::overflow_error("Raft term exhausted"); @@ -633,6 +683,7 @@ ITimeSource::Time TRaft::MakeElection(ITimeSource::Time now) { } uint64_t TRaft::Append(TMessageHolder entry) { + if (!IsVoter(Id)) throw std::logic_error("A learner cannot propose log entries"); if (State->LastLogIndex >= uint64_t(INT64_MAX)) throw std::overflow_error("Raft log index exhausted"); if (!entry || entry->Type != uint32_t(EMessageType::LOG_ENTRY)) throw TMessageError("Invalid proposed log entry"); @@ -650,11 +701,15 @@ uint64_t TRaft::GetLastIndex() const { return State->LastLogIndex; } -std::vector TRaft::GetVoterIds() const { - std::vector ids{Id}; - for (const auto& [id, _] : Nodes) ids.push_back(id); - std::sort(ids.begin(), ids.end()); - return ids; +std::vector TRaft::GetVoterIds() const { return Voters; } + +bool TRaft::IsVoter(uint32_t id) const { return std::binary_search(Voters.begin(), Voters.end(), id); } + +bool TRaft::IsLearnerCaughtUp(uint32_t id) const { + if (StateName != EState::LEADER || !std::binary_search(Learners.begin(), Learners.end(), id) || + VolatileState->CommitIndex == 0 || State->LogTerm(VolatileState->CommitIndex) != State->CurrentTerm) return false; + auto match = VolatileState->MatchIndex.find(id); + return match != VolatileState->MatchIndex.end() && match->second >= State->LastLogIndex; } TSnapshot TRaft::Compact() { @@ -704,7 +759,7 @@ void TRaft::OnInstallSnapshot(TMessageHolder response) if (response->NextOffset != rpc.Bytes->size()) return; const auto index = std::max(VolatileState->MatchIndex[response->Src], response->SnapshotIndex); (*VolatileState).SetMatchIndex(response->Src, index).SetNextIndex(response->Src, index + 1) - .SetBatchSize(response->Src, 1024).SetBackOff(response->Src, 1).CommitAdvance(Nservers, *State); + .SetBatchSize(response->Src, 1024).SetBackOff(response->Src, 1).CommitAdvance(Voters, Id, *State); VolatileState->SnapshotRpcs.erase(found); // Snapshot completion is not a fresh-read acknowledgement. AppendEntries // must still establish the next read barrier and replicate the suffix. diff --git a/miniraft/raft.h b/miniraft/raft.h index ff1ae85..3768bc5 100644 --- a/miniraft/raft.h +++ b/miniraft/raft.h @@ -79,7 +79,9 @@ struct TVolatileState { std::vector Indices; TVolatileState& Vote(uint32_t id); + // Compatibility helper for all-voter fixtures; mixed roles require IDs below. TVolatileState& CommitAdvance(int nservers, const IState& state); + TVolatileState& CommitAdvance(const std::vector& voters, uint32_t self, const IState& state); TVolatileState& SetCommitIndex(uint64_t index); TVolatileState& SetElectionDue(ITimeSource::Time); TVolatileState& SetNextIndex(uint32_t id, uint64_t nextIndex); @@ -106,7 +108,9 @@ enum class EState: int { class TRaft { public: - TRaft(std::shared_ptr state, int node, const TNodeDict& nodes); + // Nodes contains every replication peer; learners may include the local ID. + // Roles are fixed in this milestone: no voter promotion/reconfiguration API. + TRaft(std::shared_ptr state, int node, const TNodeDict& nodes, std::vector learners = {}); void Process(ITimeSource::Time now, TMessageHolder message, const std::shared_ptr& replyTo = {}); void ProcessTimeout(ITimeSource::Time now); @@ -115,6 +119,10 @@ class TRaft { uint32_t GetLeaderId() const; uint64_t GetLastIndex() const; std::vector GetVoterIds() const; + std::vector GetLearnerIds() const { return Learners; } + bool IsVoter(uint32_t id) const; + // Leader-only, point-in-time log catch-up; not permission to promote. + bool IsLearnerCaughtUp(uint32_t id) const; // Capture applied state and atomically replace its prefix with a snapshot. TSnapshot Compact(); uint64_t ApproveRead(); @@ -150,6 +158,7 @@ class TRaft { } int GetNservers() const { + // Voting members only; GetNpeers counts all remote replication targets. return Nservers; } @@ -180,6 +189,7 @@ class TRaft { uint32_t Id; TNodeDict Nodes; + std::vector Voters, Learners; int MinVotes; int Npeers; int Nservers; diff --git a/miniraft/state.h b/miniraft/state.h index dc0c0a9..c69f432 100644 --- a/miniraft/state.h +++ b/miniraft/state.h @@ -2,6 +2,7 @@ #include "messages.h" #include "snapshot.h" +#include struct TStateFields { uint64_t CurrentTerm = 1; @@ -24,6 +25,9 @@ struct IState: TStateFields { virtual bool SupportsSnapshots() const { return false; } virtual std::shared_ptr GetSnapshot() const { return {}; } virtual void InstallSnapshot(const TSnapshot&) { throw std::logic_error("Storage snapshots are unsupported"); } + virtual bool SupportsVoterBinding() const { return false; } + virtual std::vector GetBoundVoters() const { return {}; } + virtual void BindVoters(const std::vector&) { throw std::logic_error("Storage voter binding is unsupported"); } virtual ~IState() = default; uint64_t LogTerm(int64_t index = -1) const { @@ -39,9 +43,31 @@ struct IState: TStateFields { } }; +inline void ValidateFixedVoters(const std::vector& voters) { + if (voters.empty() || voters.size() > TSnapshot::MaxVoters) throw std::invalid_argument("Invalid voter count"); + uint32_t previous = 0; + for (auto id : voters) { + if (id <= previous) throw std::invalid_argument("Voters must be positive, sorted and unique"); + previous = id; + } +} +inline void ValidateVoterBinding(const IState& state, const std::vector& old, const std::vector& voters) { + ValidateFixedVoters(voters); + if (state.VotedFor && !std::binary_search(voters.begin(), voters.end(), state.VotedFor)) + throw std::invalid_argument("Stored vote does not name a configured voter"); + if (!old.empty()) { + if (old != voters) throw std::invalid_argument("Persisted voter configuration cannot be changed by restart"); + } else if (auto snapshot = state.GetSnapshot()) { + if (snapshot->Voters != voters) throw std::invalid_argument("Snapshot voter configuration mismatch"); + } else if (state.CurrentTerm != 1 || state.VotedFor || state.LastLogIndex) { + throw std::invalid_argument("Unbound nonfresh store needs a verified snapshot before enabling learners"); + } +} + struct TState: IState { std::vector> Log; std::shared_ptr Snapshot; + std::vector BoundVoters; TState() = default; TState(uint64_t currentTerm, uint32_t votedFor, const std::vector>& log) @@ -74,9 +100,17 @@ struct TState: IState { } bool SupportsSnapshots() const override { return true; } + bool SupportsVoterBinding() const override { return true; } + std::vector GetBoundVoters() const override { return BoundVoters; } + void BindVoters(const std::vector& voters) override { + ValidateVoterBinding(*this, BoundVoters, voters); + BoundVoters = voters; + } std::shared_ptr GetSnapshot() const override { return Snapshot; } void InstallSnapshot(const TSnapshot& snapshot) override { EncodeSnapshot(snapshot); // Validate before changing any state. + if (!BoundVoters.empty() && snapshot.Voters != BoundVoters) + throw std::invalid_argument("Snapshot would change bound voters"); if (snapshot.Index <= SnapshotIndex || snapshot.Term > CurrentTerm) throw std::invalid_argument("Obsolete or future snapshot"); auto image = std::make_shared(snapshot); diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 5fff9f7..91647f1 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -20,6 +20,7 @@ ut(test_rpc_validation test_rpc_validation.cpp miniraft) ut(test_app_payloads "test_app_payloads.cpp;../examples/kv_rsm.cpp" miniraft) ut(test_snapshots "test_snapshots.cpp;../examples/kv_rsm.cpp" miniraft) ut(test_compaction "test_compaction.cpp;../examples/kv_rsm.cpp" miniraft) +ut(test_learners "test_learners.cpp;../examples/kv_rsm.cpp" miniraft) find_package(SQLite3 QUIET) if(SQLite3_FOUND) ut(test_sql_payloads "test_sql_payloads.cpp;../examples/sql_rsm.cpp" miniraft) @@ -41,6 +42,9 @@ set_tests_properties(test_kv_checkpoints_cli PROPERTIES TIMEOUT 35) add_test(NAME test_install_snapshot_cli COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/smoke_install_snapshot.py $) set_tests_properties(test_install_snapshot_cli PROPERTIES TIMEOUT 120) +add_test(NAME test_learner_cli COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/smoke_install_snapshot.py $ learner) +set_tests_properties(test_learner_cli PROPERTIES TIMEOUT 120) endif() endif() endif() diff --git a/test/smoke_install_snapshot.py b/test/smoke_install_snapshot.py index ecf96f1..4a8ca7d 100644 --- a/test/smoke_install_snapshot.py +++ b/test/smoke_install_snapshot.py @@ -7,11 +7,12 @@ import tempfile import time sys.dont_write_bytecode = True -from smoke_kv_checkpoints import no_core, port, request +from smoke_kv_checkpoints import no_core, port, request, snapshot_boundary def main(): binary = str(pathlib.Path(sys.argv[1]).resolve()) + learner_mode = len(sys.argv) > 2 and sys.argv[2] == "learner" with tempfile.TemporaryDirectory(prefix="miniraft-install-snapshot-") as temporary: root = pathlib.Path(temporary) ports = [] @@ -28,10 +29,12 @@ def main(): logs.append((directory / "output.log").open("w+")) processes = [None] * 3 - def start(index): + def start(index, use_role=True): args = [binary, "--server", "--id", str(index + 1), "--durable"] for node, endpoint in enumerate(ports): args += ["--node", f"127.0.0.1:{endpoint}:{node + 1}"] + if learner_mode and use_role: + args += ["--learner", "3"] if index < 2: args += ["--snapshots", "1"] processes[index] = subprocess.Popen(args, cwd=directories[index], stdout=logs[index], stderr=logs[index], preexec_fn=no_core) @@ -71,7 +74,7 @@ def compacted(): durable = directories[index] / f"state.durable.{index + 1}" if not image.exists() or struct.unpack(" 0 and all(call(2, key)[1] == value for key, value in values.items()) wait_for(caught_up, 25) # Node 3 has no periodic snapshotting: its compacted image necessarily # came from the InstallSnapshot RPC, not a local checkpoint timer. @@ -96,13 +99,20 @@ def caught_up(): wait_for(lambda: call(2, b"tail")[1] == b"after install", 15) processes[2].terminate() processes[2].wait(timeout=5) + if learner_mode: + durable = directories[2] / "state.durable.3" + before = durable.read_bytes() + assert struct.unpack("= index, "compacted durable image was not published" stop(process) process, endpoint = launch(periodic=False) with connect(process, endpoint) as sock: diff --git a/test/test_learners.cpp b/test/test_learners.cpp new file mode 100644 index 0000000..5bd42b6 --- /dev/null +++ b/test/test_learners.cpp @@ -0,0 +1,355 @@ +#include +#include +#include +#include +#include "../examples/kv.h" +#include "../examples/kv_messages.h" +#include +#include +#include +#include +#include +#ifdef __linux__ +#include +#include +#endif +#include +#include +#include +extern "C" { +#include +} +namespace { +template void Reject(F action) { + bool rejected = false; + try { action(); } catch (const std::exception&) { rejected = true; } + assert_true(rejected); +} +struct TEndpoint : INode { + std::function)> Callback; + explicit TEndpoint(std::function)> callback = {}) : Callback(std::move(callback)) {} + void Send(TMessageHolder message) override { if (Callback) Callback(std::move(message)); } + void Drain() override {} +}; +TNodeDict Peers(uint32_t self, uint32_t count = 4) { + TNodeDict peers; + for (uint32_t id = 1; id <= count; ++id) if (id != self) peers[id] = std::make_shared(); + return peers; +} +TMessageHolder Command(const std::string& key, const std::string& value) { + auto command = NewHoldedMessage(sizeof(TWriteKv) + key.size() + value.size()); + command->Flags = TCommandRequest::EWrite; command->KeySize = key.size(); command->ValSize = value.size(); + memcpy(command->TKvEntry::Data, key.data(), key.size()); + memcpy(command->TKvEntry::Data + key.size(), value.data(), value.size()); + return command; +} +std::string Read(TKv& kv, const std::string& key) { + auto command = NewHoldedMessage(sizeof(TReadKv) + key.size()); + command->KeySize = key.size(); memcpy(command->Data, key.data(), key.size()); + auto result = kv.Read(command, kv.LastAppliedIndex).Cast(); + return std::string(result->Data, result->Len - sizeof(TCommandResponse)); +} +struct TApp : TKv { + size_t Loads = 0; + void LoadSnapshot(std::string_view bytes, uint64_t index) override { TKv::LoadSnapshot(bytes, index); ++Loads; } +}; +struct TCluster { + ITimeSource::Time Now{}; + std::vector> Nodes{4}; + std::vector> Apps{4}; + std::vector> Processors{4}; + std::deque> Packets; + std::vector VoteTargets; + std::set Isolated; + TCluster() { for (uint32_t id = 1; id <= 4; ++id) Build(id, std::make_shared()); } + void Build(uint32_t id, std::shared_ptr state) { + auto peers = Peers(id); + for (auto& [_, peer] : peers) peer = std::make_shared([this](auto message) { + if (message->Type == uint32_t(EMessageType::REQUEST_VOTE_REQUEST)) VoteTargets.push_back(message.template Cast()->Dst); + Packets.push_back(std::move(message)); + }); + Processors[id - 1].reset(); + Nodes[id - 1] = std::make_shared(state, id, peers, std::vector{4}); + Apps[id - 1] = std::make_shared(); + Processors[id - 1] = std::make_unique(Nodes[id - 1], Apps[id - 1], peers); + } + void Drain(bool duplicate = false) { + size_t budget = 10000; + while (!Packets.empty()) { + assert_true(budget-- > 0); + auto message = Packets.front(); Packets.pop_front(); auto header = message.Cast(); + if (Isolated.contains(header->Src) || Isolated.contains(header->Dst)) continue; + Nodes[header->Dst - 1]->Process(Now, message); + if (duplicate) Nodes[header->Dst - 1]->Process(Now, message); + Processors[header->Dst - 1]->ProcessCommitted(); + } + } + void Elect(uint32_t id) { + Now += std::chrono::seconds(10); Nodes[id - 1]->ProcessTimeout(Now); Drain(); + Nodes[id - 1]->ProcessTimeout(Now); Drain(); + assert_int_equal(Nodes[id - 1]->CurrentStateName(), EState::LEADER); + } + void Pump(uint32_t leader = 1, int rounds = 40, bool duplicate = false) { + for (int i = 0; i < rounds; ++i) { + Now += std::chrono::milliseconds(100); Nodes[leader - 1]->ProcessTimeout(Now); Drain(duplicate); + for (auto& processor : Processors) { processor->ProcessCommitted(); processor->ProcessWaiting(); } + } + } + void Write(const std::string& key, const std::string& value) { Processors[0]->OnCommandRequest(Command(key, value), {}); Pump(); } +}; +void test_invalid_learner_role_configurations_fail_before_binding(void**) { + for (auto learners : {std::vector{0}, std::vector{4, 4}, + std::vector{5}, std::vector{1, 2, 3, 4}}) { + auto state = std::make_shared(); + Reject([&] { TRaft raft(state, 1, Peers(1), learners); }); + assert_true(state->GetBoundVoters().empty()); assert_int_equal(state->CurrentTerm, 1); + } +} +void test_learner_never_campaigns_votes_or_proposes(void**) { + std::vector> replies; + auto peers = Peers(4); + peers[1] = std::make_shared([&](auto message) { replies.push_back(message.template Cast()); }); + auto state = std::make_shared(); TRaft learner(state, 4, peers, {4}); + for (int i = 0; i < 5; ++i) learner.ProcessTimeout(ITimeSource::Time{} + std::chrono::hours(i + 1)); + assert_int_equal(state->CurrentTerm, 1); assert_int_equal(state->VotedFor, 0); + Reject([&] { learner.Become(EState::CANDIDATE); }); Reject([&] { learner.Become(EState::LEADER); }); + Reject([&] { learner.Append(NewHoldedMessage()); }); + auto request = NewHoldedMessage(TMessageEx{.Src = 1, .Dst = 4, .Term = 5}, + TRequestVoteRequest{.LastLogIndex = 0, .LastLogTerm = 0, .CandidateId = 1}); + learner.Process({}, request); + assert_int_equal(replies.size(), 1); assert_false(replies[0]->VoteGranted); + assert_int_equal(state->CurrentTerm, 5); assert_int_equal(state->VotedFor, 0); + assert_int_equal(learner.CurrentStateName(), EState::FOLLOWER); +} +void test_learner_messages_cannot_change_voter_term_or_leadership(void**) { + TCluster cluster; cluster.Elect(1); cluster.Pump(); + const auto term = cluster.Nodes[0]->GetState()->CurrentTerm; + std::vector> messages; + const TMessageEx header{.Src = 4, .Dst = 1, .Term = 99}; + messages.push_back(NewHoldedMessage(header, TRequestVoteRequest{.LastLogIndex = 0, .LastLogTerm = 0, .CandidateId = 4})); + messages.push_back(NewHoldedMessage(header, TRequestVoteResponse{.VoteGranted = 1})); + messages.push_back(NewHoldedMessage(header, TAppendEntriesRequest{.LeaderId = 4})); + messages.push_back(NewHoldedMessage(header, TAppendEntriesResponse{.MatchIndex = 0, .Success = 0})); + auto install = NewHoldedMessage(sizeof(TInstallSnapshotRequest) + 1); + install->Src = 4; install->Dst = 1; install->Term = 99; install->SnapshotIndex = 1; install->SnapshotTerm = 1; + install->TotalBytes = 44; install->LeaderId = 4; install->Data[0] = 'x'; messages.push_back(install); + messages.push_back(NewHoldedMessage(header, TInstallSnapshotResponse{.SnapshotIndex = 1})); + for (auto message : messages) { + cluster.Nodes[0]->Process(cluster.Now, message); + assert_int_equal(cluster.Nodes[0]->GetState()->CurrentTerm, term); + assert_int_equal(cluster.Nodes[0]->CurrentStateName(), EState::LEADER); + } +} +void test_voter_commit_quorums_ignore_arbitrary_learner_matches(void**) { + TState state; state.Append(NewHoldedMessage()); + for (uint32_t count = 1; count <= 8; ++count) for (uint32_t acknowledgements = 0; acknowledgements < count; ++acknowledgements) { + std::vector voters; + for (uint32_t id = 1; id <= count; ++id) voters.push_back(id); + TVolatileState progress; + for (uint32_t id = 2; id <= acknowledgements + 1; ++id) progress.MatchIndex[id] = 1; + for (uint32_t id = count + 1; id <= count + 5; ++id) progress.MatchIndex[id] = 1; + progress.CommitAdvance(voters, 1, state); + assert_int_equal(progress.CommitIndex, acknowledgements + 1 > count / 2 ? 1 : 0); + } +} +void test_single_voter_commits_and_reads_without_learner_acknowledgements(void**) { + auto state = std::make_shared(); + auto raft = std::make_shared(state, 1, Peers(1), std::vector{2, 3, 4}); + auto app = std::make_shared(); TRequestProcessor processor(raft, app, {}); + raft->ProcessTimeout({}); assert_int_equal(raft->CurrentStateName(), EState::LEADER); + assert_int_equal(raft->GetNservers(), 1); assert_int_equal(raft->GetNpeers(), 3); + processor.OnCommandRequest(Command("a", "committed alone"), {}); + raft->ProcessTimeout({}); processor.ProcessCommitted(); + assert_true(Read(*app, "a") == "committed alone"); + size_t replies = 0; + auto client = std::make_shared([&](auto message) { + assert_int_equal(message.template Cast()->ErrorCode, 0); ++replies; + }); + auto read = NewHoldedMessage(sizeof(TReadKv) + 1); + read->KeySize = 1; read->Data[0] = 'a'; read->Flags = TCommandRequest::EConsistent; + processor.OnCommandRequest(read, client); processor.ProcessWaiting(); assert_int_equal(replies, 1); +} +void test_learner_grants_cannot_elect_candidate_and_no_votes_are_solicited(void**) { + TCluster cluster; cluster.Isolated = {2, 3}; + cluster.Nodes[0]->ProcessTimeout(cluster.Now); cluster.Drain(); + auto grant = NewHoldedMessage(TMessageEx{.Src = 4, .Dst = 1, .Term = cluster.Nodes[0]->GetState()->CurrentTerm}, + TRequestVoteResponse{.VoteGranted = 1}); + cluster.Nodes[0]->Process(cluster.Now, grant); cluster.Nodes[0]->ProcessTimeout(cluster.Now); + assert_int_equal(cluster.Nodes[0]->CurrentStateName(), EState::CANDIDATE); + assert_int_equal(cluster.Nodes[0]->GetMinVotes(), 2); + for (auto id : cluster.VoteTargets) assert_true(id != 4); + cluster.Isolated = {3}; cluster.Elect(1); +} +void test_learner_replication_cannot_commit_or_satisfy_strong_read(void**) { + TCluster cluster; cluster.Elect(1); cluster.Pump(); + const auto before = cluster.Nodes[0]->GetVolatileState()->CommitIndex; + cluster.Isolated = {2, 3}; cluster.Write("x", "new"); + assert_true(cluster.Nodes[0]->GetVolatileState()->MatchIndex.at(4) > before); + assert_int_equal(cluster.Nodes[0]->GetVolatileState()->CommitIndex, before); + assert_true(Read(*cluster.Apps[0], "x").empty()); assert_true(Read(*cluster.Apps[3], "x").empty()); + size_t replies = 0; + auto client = std::make_shared([&](auto message) { + auto reply = message.template Cast(); + assert_int_equal(reply->Cookie, 77); assert_int_equal(reply->ErrorCode, 0); ++replies; + }); + auto read = NewHoldedMessage(sizeof(TReadKv) + 1); + read->Flags = TCommandRequest::EConsistent; read->Cookie = 77; read->KeySize = 1; read->Data[0] = 'x'; + cluster.Processors[0]->OnCommandRequest(read, client); cluster.Pump(); assert_int_equal(replies, 0); + cluster.Isolated = {3}; cluster.Pump(1, 80); + assert_int_equal(replies, 1); assert_true(Read(*cluster.Apps[3], "x") == "new"); +} +void test_learner_snapshot_catchup_status_and_restart_preserve_role(void**) { + TCluster cluster; cluster.Isolated = {4}; cluster.Elect(1); + for (const std::string key : {"a", "b", "c"}) cluster.Write(key, std::string(60000, key[0])); + auto snapshot = cluster.Nodes[0]->Compact(); + assert_true(snapshot.Voters == (std::vector{1, 2, 3})); + assert_false(cluster.Nodes[0]->IsLearnerCaughtUp(4)); cluster.Isolated.clear(); cluster.Pump(1, 100, true); + assert_int_equal(cluster.Apps[3]->Loads, 1); assert_true(cluster.Nodes[0]->IsLearnerCaughtUp(4)); + assert_false(cluster.Nodes[0]->IsLearnerCaughtUp(2)); assert_false(cluster.Nodes[0]->IsLearnerCaughtUp(99)); + assert_true(Read(*cluster.Apps[3], "c") == std::string(60000, 'c')); + cluster.Nodes[0]->Append(cluster.Apps[0]->Prepare(Command("tail", "later"))); + assert_false(cluster.Nodes[0]->IsLearnerCaughtUp(4)); cluster.Pump(); + assert_true(cluster.Nodes[0]->IsLearnerCaughtUp(4)); + auto state = cluster.Nodes[3]->GetState(); cluster.Build(4, state); + cluster.Nodes[3]->ProcessTimeout(cluster.Now + std::chrono::hours(1)); cluster.Pump(); + assert_true(Read(*cluster.Apps[3], "tail") == "later"); + assert_false(cluster.Nodes[3]->IsVoter(4)); assert_int_equal(state->VotedFor, 0); + assert_int_equal(cluster.Nodes[3]->CurrentStateName(), EState::FOLLOWER); +} +void test_restart_cannot_promote_learner_or_change_voters_but_can_add_transport(void**) { + auto state = std::make_shared(); TRaft learner(state, 4, Peers(4), {4}); + const auto fields = static_cast(*state); + Reject([&] { TRaft promoted(state, 4, Peers(4)); }); + assert_true(static_cast(*state) == fields); + TRaft expanded(state, 4, Peers(4, 5), {4, 5}); + assert_true(expanded.GetVoterIds() == (std::vector{1, 2, 3})); + assert_true(expanded.GetLearnerIds() == (std::vector{4, 5})); + Reject([&] { state->BindVoters({1, 2, 3, 4}); }); + Reject([&] { state->InstallSnapshot({1, 1, {1, 2, 3, 4}, "changed voters"}); }); +} +void test_nonfresh_unbound_store_requires_snapshot_proof_for_learners(void**) { + auto state = std::make_shared(); + state->CurrentTerm = 2; state->VotedFor = 1; state->Append(NewHoldedMessage()); + Reject([&] { TRaft learnerAware(state, 1, Peers(1), {4}); }); + assert_true(state->GetBoundVoters().empty()); + TKv image; state->InstallSnapshot({1, 1, {1, 2, 3}, image.SaveSnapshot()}); + TRaft adopted(state, 1, Peers(1), {4}); + assert_true(state->GetBoundVoters() == (std::vector{1, 2, 3})); +} +#ifdef __linux__ +struct TFiles { + std::filesystem::path Directory; std::string Name; + TFiles() { + static unsigned next = 0; + Directory = std::filesystem::temp_directory_path() / ("miniraft-learners-" + std::to_string(getpid()) + "-" + std::to_string(next++)); + assert_true(std::filesystem::create_directory(Directory)); Name = (Directory / "raft").string(); + } + ~TFiles() { std::error_code ignored; std::filesystem::remove_all(Directory, ignored); } +}; +void test_durable_voter_binding_is_atomic_before_first_vote(void**) { + using TStage = TDurableState::EStage; + for (auto stage : {TStage::TempCreated, TStage::Written, TStage::FileSynced, TStage::Renamed, TStage::DirectorySynced}) { + TFiles files; { TDurableState initial(files.Name, 4); } + auto child = fork(); assert_true(child >= 0); + if (!child) { + try { auto state = std::make_shared(files.Name, 4, [&](auto point) { if (point == stage) _exit(90); }); + TRaft raft(state, 4, Peers(4), {4}); + } catch (...) { _exit(91); } _exit(92); + } + int status = 0; assert_int_equal(waitpid(child, &status, 0), child); + assert_true(WIFEXITED(status)); assert_int_equal(WEXITSTATUS(status), 90); + TDurableState state(files.Name, 4); + if (stage >= TStage::Renamed) assert_true(state.GetBoundVoters() == (std::vector{1, 2, 3})); + else assert_true(state.GetBoundVoters().empty()); + assert_int_equal(state.CurrentTerm, 1); assert_int_equal(state.VotedFor, 0); assert_int_equal(state.LastLogIndex, 0); + } +} +void test_durable_learner_snapshot_suffix_and_role_survive_restart(void**) { + TFiles files; + { + auto state = std::make_shared(files.Name, 4); TRaft raft(state, 4, Peers(4), {4}); + TKv image; auto entry = image.Prepare(Command("a", "snapshot")); state->Append(entry); image.Write(entry, 1); + state->InstallSnapshot({1, 1, {1, 2, 3}, image.SaveSnapshot()}); + state->Append(image.Prepare(Command("b", "tail"))); + } + auto state = std::make_shared(files.Name, 4); + Reject([&] { TRaft promoted(state, 4, Peers(4)); }); + auto raft = std::make_shared(state, 4, Peers(4), std::vector{4}); + auto app = std::make_shared(); TRequestProcessor processor(raft, app, {}); + assert_true(Read(*app, "a") == "snapshot"); assert_true(Read(*app, "b").empty()); + auto heartbeat = NewHoldedMessage(TMessageEx{.Src = 1, .Dst = 4, .Term = 1}, + TAppendEntriesRequest{.PrevLogIndex = 2, .PrevLogTerm = 1, .LeaderCommit = 2, .LeaderId = 1}); + raft->Process({}, heartbeat); processor.ProcessCommitted(); + assert_true(Read(*app, "b") == "tail"); assert_int_equal(state->VotedFor, 0); + assert_true(state->GetBoundVoters() == (std::vector{1, 2, 3})); +} +void test_durable_v3_corrupt_roles_fail_without_rewriting(void**) { + for (size_t offset : {size_t(40), size_t(44), size_t(48), size_t(56)}) { + TFiles files; + { auto state = std::make_shared(files.Name, 4); TRaft raft(state, 4, Peers(4), {4}); } + const auto path = files.Name + ".durable.4"; + std::ifstream input(path, std::ios::binary); std::string bytes(std::istreambuf_iterator(input), {}); input.close(); + assert_true(bytes.substr(0, 8) == "MRFTD003"); + const auto width = offset == 56 ? 8 : 4; + for (size_t i = offset; i < offset + width; ++i) bytes[i] = offset == 44 || offset == 48 ? 0 : char(0xff); + const auto crc = NSnapshotCodec::Checksum(bytes.substr(0, 8) + bytes.substr(20)); + for (int i = 0; i < 4; ++i) bytes[16 + i] = char(crc >> (8 * i)); + { std::ofstream out(path, std::ios::binary); out.write(bytes.data(), bytes.size()); } + Reject([&] { TDurableState invalid(files.Name, 4); }); + std::ifstream after(path, std::ios::binary); assert_true(std::string(std::istreambuf_iterator(after), {}) == bytes); + } +} +void test_durable_v2_snapshot_adoption_preserves_vote_log_and_application(void**) { + TFiles files; TStateFields before; std::string application; + { + TDurableState state(files.Name, 1); TKv app; + state.CurrentTerm = 3; state.VotedFor = 1; + auto entry = app.Prepare(Command("a", "snapshot")); state.Append(entry); app.Write(entry, 1); + application = app.SaveSnapshot(); state.InstallSnapshot({1, 1, {1, 2, 3}, application}); + state.Append(app.Prepare(Command("b", "tail"))); before = state; + } + const auto path = files.Name + ".durable.1"; + { std::ifstream input(path, std::ios::binary); std::string magic(8, '\0'); input.read(magic.data(), 8); assert_true(magic == "MRFTD002"); } + { + auto state = std::make_shared(files.Name, 1); + auto raft = std::make_shared(state, 1, Peers(1), std::vector{4}); + assert_true(static_cast(*state) == before); + assert_true(state->GetSnapshot()->Data == application); assert_true(bool(state->Get(2))); + auto app = std::make_shared(); TRequestProcessor processor(raft, app, {}); + assert_true(Read(*app, "a") == "snapshot"); assert_true(Read(*app, "b").empty()); + Reject([&] { TRaft promoted(state, 1, Peers(1)); }); + } + TDurableState state(files.Name, 1); + assert_true(static_cast(state) == before); + assert_true(state.GetSnapshot()->Data == application); + assert_true(state.GetBoundVoters() == (std::vector{1, 2, 3})); +} +void test_legacy_disk_backend_cannot_enable_learner_roles(void**) { + TFiles files; auto state = std::make_shared(files.Name, 4); + Reject([&] { TRaft raft(state, 4, Peers(4), {4}); }); + assert_int_equal(state->CurrentTerm, 1); assert_int_equal(state->VotedFor, 0); +} +#endif +} +int main() { + const CMUnitTest tests[] = { + cmocka_unit_test(test_invalid_learner_role_configurations_fail_before_binding), + cmocka_unit_test(test_learner_never_campaigns_votes_or_proposes), + cmocka_unit_test(test_learner_messages_cannot_change_voter_term_or_leadership), + cmocka_unit_test(test_voter_commit_quorums_ignore_arbitrary_learner_matches), + cmocka_unit_test(test_single_voter_commits_and_reads_without_learner_acknowledgements), + cmocka_unit_test(test_learner_grants_cannot_elect_candidate_and_no_votes_are_solicited), + cmocka_unit_test(test_learner_replication_cannot_commit_or_satisfy_strong_read), + cmocka_unit_test(test_learner_snapshot_catchup_status_and_restart_preserve_role), + cmocka_unit_test(test_restart_cannot_promote_learner_or_change_voters_but_can_add_transport), + cmocka_unit_test(test_nonfresh_unbound_store_requires_snapshot_proof_for_learners), +#ifdef __linux__ + cmocka_unit_test(test_durable_voter_binding_is_atomic_before_first_vote), + cmocka_unit_test(test_durable_learner_snapshot_suffix_and_role_survive_restart), + cmocka_unit_test(test_durable_v3_corrupt_roles_fail_without_rewriting), + cmocka_unit_test(test_durable_v2_snapshot_adoption_preserves_vote_log_and_application), + cmocka_unit_test(test_legacy_disk_backend_cannot_enable_learner_roles), +#endif + }; + return cmocka_run_group_tests(tests, nullptr, nullptr); +} From 049907d1fcee5bf3a80015e901ec4d39524a5580 Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Mon, 9 Nov 2026 07:25:14 -0500 Subject: [PATCH 18/66] Fix snapshot catch-up starvation and reconnect use-after-free (#18) * Preserve snapshot chunk identity across retries to avoid ACK starvation * Protect active reconnects from stale reader failures --- README.md | 7 ++- docs/snapshot-retries.md | 55 +++++++++++++++++++++++ miniraft/net/node_impl.h | 84 ++++++++++++++++++++++++++++++++++++ miniraft/net/server.cpp | 78 +-------------------------------- miniraft/net/server.h | 9 +++- miniraft/raft.cpp | 11 ++++- miniraft/raft.h | 2 + test/CMakeLists.txt | 1 + test/test_compaction.cpp | 49 ++++++++++++++++++++- test/test_node_lifecycle.cpp | 67 ++++++++++++++++++++++++++++ 10 files changed, 279 insertions(+), 84 deletions(-) create mode 100644 docs/snapshot-retries.md create mode 100644 miniraft/net/node_impl.h create mode 100644 test/test_node_lifecycle.cpp diff --git a/README.md b/README.md index c2b5f60..d438e1e 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,9 @@ remains required. Upgrade all voters before enabling the new compaction behavior The [learner follow-up](docs/learners.md) adds non-voting catch-up and durable voter binding. KV uses `--learner id` on every replica's configuration. Promotion and joint-consensus voter changes remain disabled. +The [snapshot retry follow-up](docs/snapshot-retries.md) prevents timeout retries +and strong-read traffic from starving delayed snapshot acknowledgements, and +prevents stale reader failures from destroying newer reconnect attempts. [![Number of GitHub stars](https://img.shields.io/github/stars/resetius/miniraft-cpp)](https://github.com/resetius/miniraft-cpp/stargazers) ![GitHub commit activity](https://img.shields.io/github/commit-activity/m/resetius/miniraft-cpp) @@ -72,8 +75,8 @@ log replication, and fault tolerance in a distributed system. - **Leader Election** (supported) - **Log Replication** (supported) - **Persistence** (supported) -- **Snapshots** (not supported yet) -- **Membership Change** (not supported yet) +- **Snapshots** (opt-in KV compaction and InstallSnapshot; Linux durable backend) +- **Membership Change** (non-voting learners; voting changes not supported yet) The project focuses on a clear, modular design that leverages C++20 coroutines to simplify asynchronous flows. diff --git a/docs/snapshot-retries.md b/docs/snapshot-retries.md new file mode 100644 index 0000000..706372f --- /dev/null +++ b/docs/snapshot-retries.md @@ -0,0 +1,55 @@ +# Snapshot retry progress: v0.1.0-alpha.18 + +Timeout retries and strong-read requests could indefinitely stall InstallSnapshot +catch-up for a healthy follower or learner. Each send replaced the outstanding +chunk's sequence number. If an ACK arrived after another send, the leader rejected +it, retransmitted the same offset with another sequence number, and repeated the +cycle. This could happen without any corrupt messages or failed storage. + +Retries now preserve the identity of the same immutable chunk. A valid partial +ACK consumes that identity; the next chunk receives a fresh sequence number. A +failure also consumes it before restarting from offset zero. Duplicate ACKs stay +stale both before and after the next send. A changed snapshot or leadership term +continues to discard the old transfer context. Invalid replies do not consume the +outstanding identity. + +This reuse applies only to snapshot chunks. Snapshot ACKs still cannot confirm a +strong-read barrier; a subsequent fresh AppendEntries ACK is required. No wire +layout, durable format, role configuration or migration procedure changes. + +Two deterministic regressions run against both voter and learner receivers. Each +ACK is held while the leader emits three retries, driven by either timeouts or +repeated ApproveRead calls; retry packets are dropped and the original ACK is +then delivered. Before the fix both tests failed at the first 64 KiB boundary: +the leader retransmitted offset zero instead of advancing to 65536. After the fix +the multi-chunk image completes, the application loads once, duplicate ACKs do +not consume subsequent chunks, and a new committed tail reaches the receiver. + +The release also fixes a reconnect use-after-free discovered by the sanitizer CI +snapshot CLI test. A writer could start reconnecting before the old reader +reported its failure. Disconnecting that reader destroyed the pending connector, +but its socket connect timer still held the coroutine handle. When the timer +fired, it resumed freed memory. Disconnects now identify the reader's connection +generation and preserve active connectors; an old reader cannot destroy a newer +pending or established connection. This is not a general asynchronous shutdown +or cancellation API. + +Two deterministic node lifecycle tests use the real node wrapper and poller with +a controllable socket awaiter. They cover disconnect during a pending connect +timer and an old reader failure racing a writer-triggered reconnect, both before +and after replacement establishment. The first reproduced the same ASan +use-after-free before the fix. Node template definitions are now in node_impl.h +so alternate sockets can exercise the production implementation without real +network timing. The coroio dependency is unchanged. + +The full validation set is 226 unit tests plus three CLI tests across 18 CTest +targets under ASan/UBSan, with 12 Release core targets. Existing tests cover +partial-transfer restart, stale replies after a reset, leader replacement, +duplicate installation, corruption, durable publication crashes and strong-read +isolation. Self-review checked connection generation ownership and outstanding +connector lifetime as well as chunk identity lifetime. No independent approval +is claimed. + +Dynamic promotion/removal and replicated joint-consensus configurations remain +open. Snapshot catch-up is a prerequisite for promotion, not authorization to +change the fixed voter set. diff --git a/miniraft/net/node_impl.h b/miniraft/net/node_impl.h new file mode 100644 index 0000000..e8f5921 --- /dev/null +++ b/miniraft/net/node_impl.h @@ -0,0 +1,84 @@ +#pragma once + +// Template definitions kept visible for alternative socket implementations. +template +void TNode::Send(TMessageHolder message) { + Messages.emplace_back(std::move(message)); +} + +template +void TNode::Drain() { + if (!Connected) { + Connect(); + return; + } + if (!Drainer || Drainer.done()) { + if (Drainer && Drainer.done()) { + Drainer.destroy(); + } + Drainer = DoDrain(); + } +} + +template +NNet::TVoidSuspendedTask TNode::DoDrain() { + try { + while (!Messages.empty()) { + auto tosend = std::move(Messages); Messages.clear(); + for (auto&& m : tosend) { + co_await TMessageWriter(Socket).Write(std::move(m)); + } + } + } catch (const std::exception& ex) { + std::cout << "Error on write: " << ex.what() << "\n"; + Connect(); + } + co_return; +} + +template +void TNode::Disconnect(uint64_t generation) { + if (generation != ConnectionGeneration || (Connector && !Connector.done())) return; + // DoConnect may already be retrying after a write failure. Its awaiter can + // still be registered with the poller; never destroy that live coroutine + // because an older reader failed. Established connectors are done here. + Connected = false; + if (Drainer) { Drainer.destroy(); Drainer = {}; } + if (Connector) { Connector.destroy(); Connector = {}; } + std::destroy_at(&Socket); + std::construct_at(&Socket); +} + +template +void TNode::Connect() { + if (Address && (!Connector || Connector.done())) { + if (ConnectionGeneration == UINT64_MAX) throw std::overflow_error("Connection generation exhausted"); + if (Connector && Connector.done()) { + Connector.destroy(); + } + ++ConnectionGeneration; + Connector = DoConnect(); + } +} + +template +NNet::TVoidSuspendedTask TNode::DoConnect() { + std::cout << "Connecting " << Name << "\n"; + Connected = false; + while (!Connected) { + try { + auto deadline = NNet::TClock::now() + std::chrono::milliseconds(100); // TODO: broken timeout in coroio + Socket = SocketFactory(*Address); + co_await Socket.Connect(deadline); + std::cout << "Connected " << Name << "\n"; + Connected = true; + } catch (const std::exception& ex) { + std::cout << "Error on connect: " << Name << " " << ex.what() << "\n"; + } + if (!Connected) { + co_await Socket.Poller()->Sleep(std::chrono::milliseconds(1000)); + } + } + co_return; +} + diff --git a/miniraft/net/server.cpp b/miniraft/net/server.cpp index 8e76913..bc31336 100644 --- a/miniraft/net/server.cpp +++ b/miniraft/net/server.cpp @@ -9,81 +9,6 @@ #include "server.h" -template -void TNode::Send(TMessageHolder message) { - Messages.emplace_back(std::move(message)); -} - -template -void TNode::Drain() { - if (!Connected) { - Connect(); - return; - } - if (!Drainer || Drainer.done()) { - if (Drainer && Drainer.done()) { - Drainer.destroy(); - } - Drainer = DoDrain(); - } -} - -template -NNet::TVoidSuspendedTask TNode::DoDrain() { - try { - while (!Messages.empty()) { - auto tosend = std::move(Messages); Messages.clear(); - for (auto&& m : tosend) { - co_await TMessageWriter(Socket).Write(std::move(m)); - } - } - } catch (const std::exception& ex) { - std::cout << "Error on write: " << ex.what() << "\n"; - Connect(); - } - co_return; -} - -template -void TNode::Disconnect() { - Connected = false; - if (Drainer) { Drainer.destroy(); Drainer = {}; } - if (Connector) { Connector.destroy(); Connector = {}; } - std::destroy_at(&Socket); - std::construct_at(&Socket); -} - -template -void TNode::Connect() { - if (Address && (!Connector || Connector.done())) { - if (Connector && Connector.done()) { - Connector.destroy(); - } - Connector = DoConnect(); - } -} - -template -NNet::TVoidSuspendedTask TNode::DoConnect() { - std::cout << "Connecting " << Name << "\n"; - Connected = false; - while (!Connected) { - try { - auto deadline = NNet::TClock::now() + std::chrono::milliseconds(100); // TODO: broken timeout in coroio - Socket = SocketFactory(*Address); - co_await Socket.Connect(deadline); - std::cout << "Connected " << Name << "\n"; - Connected = true; - } catch (const std::exception& ex) { - std::cout << "Error on connect: " << Name << " " << ex.what() << "\n"; - } - if (!Connected) { - co_await Socket.Poller()->Sleep(std::chrono::milliseconds(1000)); - } - } - co_return; -} - template NNet::TVoidTask TRaftServer::InboundConnection(TSocket socket) { std::shared_ptr> client; @@ -142,6 +67,7 @@ NNet::TVoidTask TRaftServer::OutboundServe(std::shared_ptrIsConnected(); + const auto generation = node->GetConnectionGeneration(); try { if (!node->IsConnected()) { throw std::runtime_error("Not connected"); @@ -160,7 +86,7 @@ NNet::TVoidTask TRaftServer::OutboundServe(std::shared_ptrOnPeerDisconnected(node); - if (wasConnected) node->Disconnect(); + if (wasConnected) node->Disconnect(generation); DrainNodes(); co_await Poller.Sleep(std::chrono::milliseconds(1000)); } diff --git a/miniraft/net/server.h b/miniraft/net/server.h index 4f02dda..dd9af49 100644 --- a/miniraft/net/server.h +++ b/miniraft/net/server.h @@ -121,8 +121,11 @@ class TNode: public INode { void Send(TMessageHolder message) override; void Drain() override; - // Called after a failed outbound read, once its reader has unwound. - void Disconnect(); + // Called after a failed outbound read, once its reader has unwound. A stale + // reader cannot disconnect a newer connection or its pending connector. + void Disconnect() { Disconnect(ConnectionGeneration); } + void Disconnect(uint64_t generation); + uint64_t GetConnectionGeneration() const { return ConnectionGeneration; } bool IsConnected() { return Connected; } TSocket& Sock() { return Socket; @@ -140,6 +143,7 @@ class TNode: public INode { std::function SocketFactory; TSocket Socket; bool Connected = false; + uint64_t ConnectionGeneration = 0; std::coroutine_handle<> Drainer; std::coroutine_handle<> Connector; @@ -190,3 +194,4 @@ class TRaftServer { TVolatileState VolatileFields; }; +#include "node_impl.h" diff --git a/miniraft/raft.cpp b/miniraft/raft.cpp index 819d310..96c0907 100644 --- a/miniraft/raft.cpp +++ b/miniraft/raft.cpp @@ -738,7 +738,12 @@ TMessageHolder TRaft::CreateInstallSnapshot(uint32_t no const auto count = std::min(TInstallSnapshotRequest::MaxChunkBytes, rpc.Bytes->size() - rpc.Offset); auto request = NewHoldedMessage(sizeof(TInstallSnapshotRequest) + count); request->Src = Id; request->Dst = nodeId; request->Term = State->CurrentTerm; - request->Seqno = rpc.Seqno = NextSeqno(); + // A retry sends the same immutable chunk. Keep its identity until a reply + // consumes it, otherwise traffic faster than the RTT invalidates every ACK. + // Snapshot ACKs never confirm read barriers, so reuse cannot confirm a read + // that arrived after the first transmission. + if (!rpc.Seqno) rpc.Seqno = NextSeqno(); + request->Seqno = rpc.Seqno; request->SnapshotIndex = snapshot->Index; request->SnapshotTerm = snapshot->Term; request->Offset = rpc.Offset; request->TotalBytes = rpc.Bytes->size(); rpc.SentEnd = rpc.Offset + count; @@ -752,9 +757,10 @@ void TRaft::OnInstallSnapshot(TMessageHolder response) auto found = VolatileState->SnapshotRpcs.find(response->Src); if (found == VolatileState->SnapshotRpcs.end()) return; auto& rpc = found->second; - if (response->Seqno != rpc.Seqno || response->SnapshotIndex != rpc.Snapshot->Index) return; + if (!rpc.Seqno || response->Seqno != rpc.Seqno || response->SnapshotIndex != rpc.Snapshot->Index) return; if (!response->Success) { rpc.Offset = 0; + rpc.Seqno = 0; } else if (response->Installed) { if (response->NextOffset != rpc.Bytes->size()) return; const auto index = std::max(VolatileState->MatchIndex[response->Src], response->SnapshotIndex); @@ -766,6 +772,7 @@ void TRaft::OnInstallSnapshot(TMessageHolder response) } else { if (response->NextOffset < rpc.SentEnd || response->NextOffset >= rpc.Bytes->size()) return; rpc.Offset = response->NextOffset; + rpc.Seqno = 0; // Next chunk gets a new identity; old replies stay stale. } VolatileState->RpcDue[response->Src] = ITimeSource::Time{}; } diff --git a/miniraft/raft.h b/miniraft/raft.h index 3768bc5..039343d 100644 --- a/miniraft/raft.h +++ b/miniraft/raft.h @@ -63,6 +63,8 @@ struct TVolatileState { struct TSnapshotRpc { std::shared_ptr Snapshot; std::shared_ptr Bytes; + // Seqno identifies the outstanding chunk across retries; zero means + // its reply was consumed and the next send needs a fresh identity. uint64_t Offset = 0, Seqno = 0, SentEnd = 0; }; std::unordered_map SnapshotRpcs; diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 91647f1..d94c824 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -33,6 +33,7 @@ endif() if(MINIRAFT_BUILD_NETWORK) ut(test_read_write test_read_write.cpp miniraft.net) ut(test_network_frames test_network_frames.cpp miniraft.net) +ut(test_node_lifecycle test_node_lifecycle.cpp miniraft.net) if(CMAKE_SYSTEM_NAME STREQUAL "Linux") find_package(Python3 COMPONENTS Interpreter QUIET) if(Python3_Interpreter_FOUND) diff --git a/test/test_compaction.cpp b/test/test_compaction.cpp index 00d2116..9f92471 100644 --- a/test/test_compaction.cpp +++ b/test/test_compaction.cpp @@ -55,13 +55,16 @@ struct TCluster { std::vector> Processors{3}; std::deque> Packets; std::set Isolated; - TCluster() { for (uint32_t id = 1; id <= 3; ++id) Build(id, std::make_shared()); } + std::vector Learners; + explicit TCluster(std::vector learners = {}) : Learners(std::move(learners)) { + for (uint32_t id = 1; id <= 3; ++id) Build(id, std::make_shared()); + } void Build(uint32_t id, std::shared_ptr state) { TNodeDict peers; for (uint32_t dst = 1; dst <= 3; ++dst) if (dst != id) peers[dst] = std::make_shared([this](auto message) { Packets.push_back(std::move(message)); }); Processors[id - 1].reset(); - Nodes[id - 1] = std::make_shared(state, id, peers); + Nodes[id - 1] = std::make_shared(state, id, peers, Learners); Apps[id - 1] = std::make_shared(); Processors[id - 1] = std::make_unique(Nodes[id - 1], Apps[id - 1], peers); } @@ -174,6 +177,46 @@ void test_partial_transfer_restart_and_delayed_ack_are_safe(void**) { assert_int_equal(cluster.Apps[2]->Loads, 1); assert_true(Read(*cluster.Apps[2], "c") == std::string(60000, 'c')); } +void ExerciseDelayedSnapshotAcks(bool readTraffic) { + for (auto learners : {std::vector{}, std::vector{3}}) { + TCluster cluster(learners); cluster.Prepare(); + const auto boundary = cluster.Nodes[0]->GetState()->SnapshotIndex; + auto chunk = cluster.SendChunk(); + size_t chunks = 0; + while (true) { + assert_true(++chunks < 10); + cluster.Deliver(chunk); + auto ack = cluster.Take(3, 1, EMessageType::INSTALL_SNAPSHOT_RESPONSE).Cast(); + // Every ACK arrives after several retries. Drop retry packets while + // preserving the original successful reply, a valid lossy schedule. + for (int retry = 0; retry < 3; ++retry) { + if (readTraffic) cluster.Nodes[0]->ApproveRead(); + else { cluster.Now += std::chrono::seconds(3); cluster.Nodes[0]->ProcessTimeout(cluster.Now); } + auto retransmit = cluster.Take(1, 3, EMessageType::INSTALL_SNAPSHOT_REQUEST).Cast(); + assert_int_equal(retransmit->Offset, chunk->Offset); + cluster.Packets.clear(); + } + cluster.Deliver(ack); + if (ack->Installed) break; + cluster.Deliver(ack); // Also stale between consuming an ACK and the next send. + auto next = cluster.SendChunk(); + assert_int_equal(next->Offset, ack->NextOffset); + // A duplicate ACK for the previous chunk must not consume the new + // chunk's correlation or reset its progress. + cluster.Deliver(ack); + chunk = next; + } + assert_true(chunks > 1); + assert_int_equal(cluster.Nodes[0]->GetVolatileState()->MatchIndex.at(3), boundary); + assert_int_equal(cluster.Apps[2]->Loads, 1); + cluster.Write(1, "tail", "after delayed ACKs"); + cluster.Pump(1, 80); + assert_true(Read(*cluster.Apps[2], "tail") == "after delayed ACKs"); + if (!learners.empty()) assert_true(cluster.Nodes[0]->IsLearnerCaughtUp(3)); + } +} +void test_snapshot_timeout_retries_do_not_starve_delayed_acks(void**) { ExerciseDelayedSnapshotAcks(false); } +void test_snapshot_read_traffic_does_not_starve_delayed_acks(void**) { ExerciseDelayedSnapshotAcks(true); } void test_leader_change_mid_snapshot_rejects_old_chunks(void**) { TCluster cluster; cluster.Prepare(); auto old = cluster.SendChunk(); cluster.AckChunk(old); @@ -400,6 +443,8 @@ int main() { cmocka_unit_test(test_compaction_lagging_follower_installs_snapshot_then_tail), cmocka_unit_test(test_simultaneous_startup_candidates_use_distinct_election_deadlines), cmocka_unit_test(test_partial_transfer_restart_and_delayed_ack_are_safe), + cmocka_unit_test(test_snapshot_timeout_retries_do_not_starve_delayed_acks), + cmocka_unit_test(test_snapshot_read_traffic_does_not_starve_delayed_acks), cmocka_unit_test(test_leader_change_mid_snapshot_rejects_old_chunks), cmocka_unit_test(test_partial_and_installed_snapshot_acks_do_not_confirm_strong_reads), cmocka_unit_test(test_compacted_follower_hint_requires_boundary_probe_before_match), diff --git a/test/test_node_lifecycle.cpp b/test/test_node_lifecycle.cpp new file mode 100644 index 0000000..8a8a8d7 --- /dev/null +++ b/test/test_node_lifecycle.cpp @@ -0,0 +1,67 @@ +#include +#include +#include +#include +extern "C" { +#include +} + +namespace { +struct TPoller : NNet::TPollerBase { + void Tick() { ProcessTimers(); } +}; +// Model the dependency's connect awaiter: a pending timer retains the coroutine +// handle, and destroying the awaiter does not unregister that timer. The timer +// is already due, so the schedule needs neither sockets nor wall-clock waits. +struct TSocket { + TPoller* Owner = nullptr; + bool FailWrite = false; + auto Connect(NNet::TTime) { + struct TAwaiter { + TPoller* Owner; + bool await_ready() { return false; } + void await_suspend(std::coroutine_handle<> h) { Owner->AddTimer({}, h); } + void await_resume() {} + }; + return TAwaiter{Owner}; + } + TPoller* Poller() { return Owner; } + NNet::TValueTask WriteSome(const void*, size_t length) { + if (FailWrite) throw std::runtime_error("injected write failure"); + co_return length; + } +}; +void test_disconnect_during_pending_connect_does_not_destroy_timer_target(void**) { + TPoller poller; + TNode node([&](const auto&) { return TSocket{&poller}; }, "peer", + NNet::TAddress("127.0.0.1", 1), std::make_shared()); + node.Drain(); assert_false(node.IsConnected()); + node.Disconnect(); // Must preserve the active reconnect until it completes. + poller.Tick(); + assert_true(node.IsConnected()); + node.Disconnect(); assert_false(node.IsConnected()); +} +void test_old_reader_failure_cannot_disconnect_writer_reconnect(void**) { + TPoller poller; + TNode node([&](const auto&) { return TSocket{&poller}; }, "peer", + NNet::TAddress("127.0.0.1", 1), std::make_shared()); + node.Drain(); poller.Tick(); assert_true(node.IsConnected()); + const auto old = node.GetConnectionGeneration(); + node.Sock().FailWrite = true; + node.Send(NewHoldedMessage()); node.Drain(); + assert_false(node.IsConnected()); assert_true(node.GetConnectionGeneration() != old); + node.Disconnect(old); // Old reader unwinds while the new connect is pending. + poller.Tick(); assert_true(node.IsConnected()); + node.Disconnect(old); // It is stale even after the replacement connected. + assert_true(node.IsConnected()); + node.Send(NewHoldedMessage()); node.Drain(); + node.Disconnect(node.GetConnectionGeneration()); assert_false(node.IsConnected()); +} +} +int main() { + const CMUnitTest tests[] = { + cmocka_unit_test(test_disconnect_during_pending_connect_does_not_destroy_timer_target), + cmocka_unit_test(test_old_reader_failure_cannot_disconnect_writer_reconnect), + }; + return cmocka_run_group_tests(tests, nullptr, nullptr); +} From eb16ab657b1542802132b5c7bd243b22c25027e2 Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Mon, 9 Nov 2026 12:56:16 -0500 Subject: [PATCH 19/66] Fix snapshot compaction starvation and unsafe connect deadlines (#19) * Pin active snapshot transfers across ongoing compaction * Avoid competing connect deadline wakeups after completion --- README.md | 4 +++ docs/snapshot-pinning.md | 61 ++++++++++++++++++++++++++++++++++++ docs/snapshot-retries.md | 3 ++ miniraft/net/node_impl.h | 7 +++-- miniraft/raft.cpp | 18 ++++++----- miniraft/raft.h | 1 + test/test_compaction.cpp | 46 +++++++++++++++++++++++++++ test/test_node_lifecycle.cpp | 30 ++++++++++++++++-- 8 files changed, 158 insertions(+), 12 deletions(-) create mode 100644 docs/snapshot-pinning.md diff --git a/README.md b/README.md index d438e1e..12d7b54 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,10 @@ and joint-consensus voter changes remain disabled. The [snapshot retry follow-up](docs/snapshot-retries.md) prevents timeout retries and strong-read traffic from starving delayed snapshot acknowledgements, and prevents stale reader failures from destroying newer reconnect attempts. +The [snapshot pinning follow-up](docs/snapshot-pinning.md) lets an active transfer +finish while the leader continues compacting newer application state, and avoids +the dependency's unsafe connect-deadline path. TCP connect failure timing follows +the OS; a safe bounded application deadline remains open. [![Number of GitHub stars](https://img.shields.io/github/stars/resetius/miniraft-cpp)](https://github.com/resetius/miniraft-cpp/stargazers) ![GitHub commit activity](https://img.shields.io/github/commit-activity/m/resetius/miniraft-cpp) diff --git a/docs/snapshot-pinning.md b/docs/snapshot-pinning.md new file mode 100644 index 0000000..c7e3638 --- /dev/null +++ b/docs/snapshot-pinning.md @@ -0,0 +1,61 @@ +# Snapshot transfer pinning: v0.1.0-alpha.19 + +Periodic compaction could starve snapshot catch-up even after alpha.18 fixed +retry correlation. Each send selected the leader's newest snapshot and reset the +peer's transfer if its index changed. When compaction outpaced a multi-chunk +transfer, the receiver repeatedly started new images without completing one. +This affected both voters and non-voting learners. + +Each peer now pins the immutable image and encoded bytes selected at the start +of its transfer. New local snapshots do not change its index, offset, content or +outstanding chunk identity. A completed transfer releases that image. If the +leader compacted past the receiver's next index, the next transfer starts from +the newest available image; otherwise ordinary AppendEntries supplies the tail. +An ACK for the older image proves only that image's boundary, never the newer +prefix. Snapshot ACKs still cannot confirm strong-read barriers. + +Failed chunks restart the pinned image from offset zero. Leadership changes still +discard outgoing transfer contexts. One image/encoding pair is retained per +active peer transfer, with sharing between transfers of the same image. A stalled +peer may keep an older image alive while the store compacts; total snapshot memory +is not independent of the configured peer count. This does not introduce a +resource quota or general transfer cancellation API. + +A deterministic regression commits and compacts new data before every chunk ACK, +including the final ACK, for both voter and learner receivers. The old code fails +on the first replacement (snapshot 5 instead of 4). The fixed code completes the +original image without resetting offsets, releases its snapshot object, then +installs the newer image and replicates a committed tail. Existing tests cover +timeouts, read traffic, duplicate/stale replies, receiver restart, leader change, +corruption, atomic durable publication and strong-read isolation. + +Sanitizer CI also exposed a second reconnect deadline race after alpha.18's +generation checks. The dependency's finite-deadline connect registers both I/O +and timer wakeups. After timeout, a late I/O event can resume the retry sleep; +that sleep's timer may remain queued even after the connector finishes. Destroying +the completed connector then leaves a timer pointing to freed memory. Preserving +active connectors alone cannot prevent this case. + +MiniRaft now calls the dependency's default connect operation, which registers no +connect deadline timer. TCP connection failure timing follows the operating +system, so a failed or blackholed connection may take longer than the former +100 ms application deadline. A safe bounded application-level connect deadline, +including cancellation of both wakeup sources, remains open. The coroio dependency +is unchanged. This is a targeted avoidance of the unsafe deadline path, not a +general asynchronous cancellation or shutdown implementation. + +A deterministic lifecycle regression models a second deadline wakeup surviving +completion and disconnect; it reproduced the ASan use-after-free with the former +finite-deadline call and passes with the default connect operation. + +Validation: 228 unit tests and three CLI tests across 18 CTest targets under +ASan/UBSan, plus 12 Release core targets. Self-review checked image ownership, +encoding reuse, completed-boundary accounting, follow-up transfer selection and +term replacement, connection wakeup sources and deadline behavior. No independent +reviewer approval is claimed. Wire messages and +durable formats are unchanged; no additional migration is required. + +This guarantees progress of the selected finite image under eventual chunk +delivery and ACKs. Reaching the live log end still requires enough replication +capacity relative to writes/compaction and sufficiently stable leadership. +Dynamic promotion/removal and joint consensus remain open. diff --git a/docs/snapshot-retries.md b/docs/snapshot-retries.md index 706372f..14b4e98 100644 --- a/docs/snapshot-retries.md +++ b/docs/snapshot-retries.md @@ -1,5 +1,8 @@ # Snapshot retry progress: v0.1.0-alpha.18 +The [alpha.19 follow-up](snapshot-pinning.md) also pins an active transfer across +local compaction, replacing the image-switching behavior described below. + Timeout retries and strong-read requests could indefinitely stall InstallSnapshot catch-up for a healthy follower or learner. Each send replaced the outstanding chunk's sequence number. If an ACK arrived after another send, the leader rejected diff --git a/miniraft/net/node_impl.h b/miniraft/net/node_impl.h index e8f5921..61f08c1 100644 --- a/miniraft/net/node_impl.h +++ b/miniraft/net/node_impl.h @@ -67,9 +67,12 @@ NNet::TVoidSuspendedTask TNode::DoConnect() { Connected = false; while (!Connected) { try { - auto deadline = NNet::TClock::now() + std::chrono::milliseconds(100); // TODO: broken timeout in coroio Socket = SocketFactory(*Address); - co_await Socket.Connect(deadline); + // The dependency's finite deadline can leave I/O and timer wakeups + // registered for the same coroutine after a timeout. Use its default + // I/O-only connect until cancellation of both sources is supported. + // TCP connect failure timing is therefore controlled by the OS. + co_await Socket.Connect(); std::cout << "Connected " << Name << "\n"; Connected = true; } catch (const std::exception& ex) { diff --git a/miniraft/raft.cpp b/miniraft/raft.cpp index 96c0907..015429f 100644 --- a/miniraft/raft.cpp +++ b/miniraft/raft.cpp @@ -724,17 +724,21 @@ TSnapshot TRaft::Compact() { } TMessageHolder TRaft::CreateInstallSnapshot(uint32_t nodeId) { - auto snapshot = State->GetSnapshot(); - if (!snapshot) throw std::logic_error("Missing snapshot for compacted log"); - if (EncodedSnapshotIndex != snapshot->Index) { - EncodedSnapshot = std::make_shared(EncodeSnapshot(*snapshot)); - EncodedSnapshotIndex = snapshot->Index; - } auto& rpc = VolatileState->SnapshotRpcs[nodeId]; - if (!rpc.Snapshot || rpc.Snapshot->Index != snapshot->Index) { + if (!rpc.Snapshot) { + auto snapshot = State->GetSnapshot(); + if (!snapshot) throw std::logic_error("Missing snapshot for compacted log"); + if (EncodedSnapshotIndex != snapshot->Index) { + EncodedSnapshot = std::make_shared(EncodeSnapshot(*snapshot)); + EncodedSnapshotIndex = snapshot->Index; + } rpc = TVolatileState::TSnapshotRpc{snapshot, EncodedSnapshot}; VolatileState->PendingAppendRpcs[nodeId].clear(); } + // Pin one immutable image until this peer finishes. Periodic compaction + // must not restart every chunk; after completion a still-lagging peer can + // start a separate transfer of the newest image. + const auto& snapshot = rpc.Snapshot; const auto count = std::min(TInstallSnapshotRequest::MaxChunkBytes, rpc.Bytes->size() - rpc.Offset); auto request = NewHoldedMessage(sizeof(TInstallSnapshotRequest) + count); request->Src = Id; request->Dst = nodeId; request->Term = State->CurrentTerm; diff --git a/miniraft/raft.h b/miniraft/raft.h index 039343d..d15d7ab 100644 --- a/miniraft/raft.h +++ b/miniraft/raft.h @@ -61,6 +61,7 @@ struct TVolatileState { std::unordered_map LastSentSeqno; std::unordered_map> PendingAppendRpcs; struct TSnapshotRpc { + // Pinned for this transfer even if the local store compacts again. std::shared_ptr Snapshot; std::shared_ptr Bytes; // Seqno identifies the outstanding chunk across retries; zero means diff --git a/test/test_compaction.cpp b/test/test_compaction.cpp index 9f92471..ef4766f 100644 --- a/test/test_compaction.cpp +++ b/test/test_compaction.cpp @@ -217,6 +217,51 @@ void ExerciseDelayedSnapshotAcks(bool readTraffic) { } void test_snapshot_timeout_retries_do_not_starve_delayed_acks(void**) { ExerciseDelayedSnapshotAcks(false); } void test_snapshot_read_traffic_does_not_starve_delayed_acks(void**) { ExerciseDelayedSnapshotAcks(true); } +void test_compaction_during_each_chunk_cannot_restart_snapshot_catchup(void**) { + for (auto learners : {std::vector{}, std::vector{3}}) { + TCluster cluster(learners); cluster.Prepare(); + auto chunk = cluster.SendChunk(); + const auto boundary = chunk->SnapshotIndex; + std::weak_ptr original = cluster.Nodes[0]->GetState()->GetSnapshot(); + size_t chunks = 0; + while (true) { + assert_true(++chunks < 10); + cluster.Deliver(chunk); + auto ack = cluster.Take(3, 1, EMessageType::INSTALL_SNAPSHOT_RESPONSE).Cast(); + // Commit and compact again before every ACK arrives, including the + // final one. The majority remains healthy while the receiver lags. + cluster.Isolated = {3}; cluster.Packets.clear(); + cluster.Write(1, "latest", std::to_string(chunks)); + cluster.Nodes[0]->Compact(); + assert_true(cluster.Nodes[0]->GetState()->SnapshotIndex > boundary); + assert_false(original.expired()); // Owned by the active transfer. + cluster.Isolated.clear(); + auto retry = cluster.SendChunk(); + assert_int_equal(retry->SnapshotIndex, boundary); + assert_int_equal(retry->Offset, chunk->Offset); + cluster.Packets.clear(); cluster.Deliver(ack); + if (ack->Installed) break; + chunk = cluster.SendChunk(); + assert_int_equal(chunk->SnapshotIndex, boundary); + assert_int_equal(chunk->Offset, ack->NextOffset); + } + assert_true(chunks > 1); + assert_true(original.expired()); // Completion releases the pinned image. + assert_int_equal(cluster.Apps[2]->Loads, 1); + assert_int_equal(cluster.Nodes[2]->GetState()->SnapshotIndex, boundary); + // The older completed image does not claim the leader's newer prefix. + assert_int_equal(cluster.Nodes[0]->GetVolatileState()->MatchIndex.at(3), boundary); + assert_true(Read(*cluster.Apps[2], "latest").empty()); + cluster.Packets.clear(); + auto newer = cluster.SendChunk(); + assert_true(newer->SnapshotIndex > boundary); assert_int_equal(newer->Offset, 0); + cluster.AckChunk(newer); cluster.Pump(1, 80, true); + assert_true(Read(*cluster.Apps[2], "latest") == std::to_string(chunks)); + cluster.Write(1, "tail", "after compaction during transfer"); cluster.Pump(1, 80); + assert_true(Read(*cluster.Apps[2], "tail") == "after compaction during transfer"); + if (!learners.empty()) assert_true(cluster.Nodes[0]->IsLearnerCaughtUp(3)); + } +} void test_leader_change_mid_snapshot_rejects_old_chunks(void**) { TCluster cluster; cluster.Prepare(); auto old = cluster.SendChunk(); cluster.AckChunk(old); @@ -445,6 +490,7 @@ int main() { cmocka_unit_test(test_partial_transfer_restart_and_delayed_ack_are_safe), cmocka_unit_test(test_snapshot_timeout_retries_do_not_starve_delayed_acks), cmocka_unit_test(test_snapshot_read_traffic_does_not_starve_delayed_acks), + cmocka_unit_test(test_compaction_during_each_chunk_cannot_restart_snapshot_catchup), cmocka_unit_test(test_leader_change_mid_snapshot_rejects_old_chunks), cmocka_unit_test(test_partial_and_installed_snapshot_acks_do_not_confirm_strong_reads), cmocka_unit_test(test_compacted_follower_hint_requires_boundary_probe_before_match), diff --git a/test/test_node_lifecycle.cpp b/test/test_node_lifecycle.cpp index 8a8a8d7..7d0405d 100644 --- a/test/test_node_lifecycle.cpp +++ b/test/test_node_lifecycle.cpp @@ -9,6 +9,12 @@ extern "C" { namespace { struct TPoller : NNet::TPollerBase { void Tick() { ProcessTimers(); } + void TickOne() { + assert_false(Timers_.empty()); + auto timer = Timers_.top(); Timers_.pop(); + LastFiredTimer_ = timer.Id; + if (timer.Handle) timer.Handle.resume(); + } }; // Model the dependency's connect awaiter: a pending timer retains the coroutine // handle, and destroying the awaiter does not unregister that timer. The timer @@ -16,14 +22,19 @@ struct TPoller : NNet::TPollerBase { struct TSocket { TPoller* Owner = nullptr; bool FailWrite = false; - auto Connect(NNet::TTime) { + bool DuplicateDeadline = false; + auto Connect(NNet::TTime deadline = NNet::TTime::max()) { struct TAwaiter { TPoller* Owner; + bool DuplicateDeadline; bool await_ready() { return false; } - void await_suspend(std::coroutine_handle<> h) { Owner->AddTimer({}, h); } + void await_suspend(std::coroutine_handle<> h) { + Owner->AddTimer({}, h); // Controllable stand-in for I/O readiness. + if (DuplicateDeadline) Owner->AddTimer({}, h); + } void await_resume() {} }; - return TAwaiter{Owner}; + return TAwaiter{Owner, DuplicateDeadline && deadline != NNet::TTime::max()}; } TPoller* Poller() { return Owner; } NNet::TValueTask WriteSome(const void*, size_t length) { @@ -57,11 +68,24 @@ void test_old_reader_failure_cannot_disconnect_writer_reconnect(void**) { node.Send(NewHoldedMessage()); node.Drain(); node.Disconnect(node.GetConnectionGeneration()); assert_false(node.IsConnected()); } +void test_connect_cannot_leave_a_second_deadline_wakeup_after_completion(void**) { + TPoller poller; + // The dependency's finite-deadline connect registers the same coroutine for + // both I/O and a timer. Model a second wakeup surviving the first, as seen + // in CI when timeout/I/O completion races with the retry sleep. + TNode node([&](const auto&) { return TSocket{&poller, false, true}; }, "peer", + NNet::TAddress("127.0.0.1", 1), std::make_shared()); + node.Drain(); poller.TickOne(); assert_true(node.IsConnected()); + node.Disconnect(); + poller.Tick(); // A leftover deadline must not resume the destroyed connector. + assert_false(node.IsConnected()); +} } int main() { const CMUnitTest tests[] = { cmocka_unit_test(test_disconnect_during_pending_connect_does_not_destroy_timer_target), cmocka_unit_test(test_old_reader_failure_cannot_disconnect_writer_reconnect), + cmocka_unit_test(test_connect_cannot_leave_a_second_deadline_wakeup_after_completion), }; return cmocka_run_group_tests(tests, nullptr, nullptr); } From e0a4cb995d15e87494687e54c236f33b81ea38d7 Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Fri, 13 Nov 2026 09:42:10 -0500 Subject: [PATCH 20/66] Reject nonleader proposals before mutating the Raft log (#20) --- README.md | 3 +++ docs/proposal-admission.md | 43 ++++++++++++++++++++++++++++++++++ miniraft/raft.cpp | 1 + miniraft/raft.h | 2 ++ test/test_correctness.cpp | 47 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 96 insertions(+) create mode 100644 docs/proposal-admission.md diff --git a/README.md b/README.md index 12d7b54..dd9ef15 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,9 @@ The [snapshot pinning follow-up](docs/snapshot-pinning.md) lets an active transf finish while the leader continues compacting newer application state, and avoids the dependency's unsafe connect-deadline path. TCP connect failure timing follows the OS; a safe bounded application deadline remains open. +The [proposal admission follow-up](docs/proposal-admission.md) restricts direct +Append calls to leaders, preventing nonleaders from creating conflicting commands +at the same log index and term. [![Number of GitHub stars](https://img.shields.io/github/stars/resetius/miniraft-cpp)](https://github.com/resetius/miniraft-cpp/stargazers) ![GitHub commit activity](https://img.shields.io/github/commit-activity/m/resetius/miniraft-cpp) diff --git a/docs/proposal-admission.md b/docs/proposal-admission.md new file mode 100644 index 0000000..5976053 --- /dev/null +++ b/docs/proposal-admission.md @@ -0,0 +1,43 @@ +# Leader-only proposal admission: v0.1.0-alpha.20 + +TRaft::Append previously rejected learners but accepted proposals on voter +followers and candidates. This violated the log-matching assumption that a term +and index identify one command. A follower could append a local command in the +current term at the same index where its leader subsequently proposed another. +AppendEntries compares entry terms, so the follower retained its different +command and acknowledged it. Both commands could then be applied as committed. + +The deterministic three-node reproducer showed the leader applying `valid` and +a follower applying `rogue` at the same committed index/term. Ordinary client +requests through TRequestProcessor already route proposals through the leader; +the missing check affected callers using the public low-level Append API. + +Append now requires the locally known LEADER state as well as a voter role. +Nonleaders receive std::logic_error before entry validation, term assignment or +storage mutation. This includes a former leader after a higher-term message has +made it step down. A candidate's self-vote does not permit proposals. Existing +leader-side frame validation remains in place, and incoming replication still +uses the storage backend through the validated AppendEntries path. + +Two regressions cover both state preservation and the observed consensus failure. +The first rejects initial-follower, candidate and stepped-down-leader calls, +checking all stored fields and the supplied entry term remain unchanged. The +second attempts the conflicting follower proposal, commits a valid leader write +under duplicate/reordered delivery, and checks application histories, successful +client completion and identical committed prefixes on all three nodes. Both +failed on the old code; the convergence test observed `rogue` instead of `valid`. + +Validation: 230 unit tests and three CLI tests under ASan/UBSan across 18 CTest +targets, plus all 12 Release core targets. Self-review checked the guard's position +before mutation, all production Append call sites, learner admission, higher-term +stepdown and the distinction between local proposals and follower replication. +No independent reviewer approval is claimed. There are no wire or durable format +changes and no migration requirement. +Previously divergent logs are not detected or repaired by this admission guard. + +The role check is not a quorum lease: an isolated leader may still accept entries +that remain uncommitted, and a successful Append return is only a local log index. +Callers must serialize Raft access and wait for committed application/client +completion as appropriate. Directly editing IState or forcing states through the +test-facing Become API is outside proposal admission. Joint consensus and voter +promotion/removal remain open. diff --git a/miniraft/raft.cpp b/miniraft/raft.cpp index 015429f..bb72989 100644 --- a/miniraft/raft.cpp +++ b/miniraft/raft.cpp @@ -684,6 +684,7 @@ ITimeSource::Time TRaft::MakeElection(ITimeSource::Time now) { uint64_t TRaft::Append(TMessageHolder entry) { if (!IsVoter(Id)) throw std::logic_error("A learner cannot propose log entries"); + if (StateName != EState::LEADER) throw std::logic_error("Only a leader can propose log entries"); if (State->LastLogIndex >= uint64_t(INT64_MAX)) throw std::overflow_error("Raft log index exhausted"); if (!entry || entry->Type != uint32_t(EMessageType::LOG_ENTRY)) throw TMessageError("Invalid proposed log entry"); diff --git a/miniraft/raft.h b/miniraft/raft.h index d15d7ab..97e89d4 100644 --- a/miniraft/raft.h +++ b/miniraft/raft.h @@ -118,6 +118,8 @@ class TRaft { void Process(ITimeSource::Time now, TMessageHolder message, const std::shared_ptr& replyTo = {}); void ProcessTimeout(ITimeSource::Time now); + // Local proposals require leader state. Role rejection leaves storage and + // the supplied entry unchanged; followers receive entries through Process. uint64_t Append(TMessageHolder entry); uint32_t GetLeaderId() const; uint64_t GetLastIndex() const; diff --git a/test/test_correctness.cpp b/test/test_correctness.cpp index a984da3..c183b0c 100644 --- a/test/test_correctness.cpp +++ b/test/test_correctness.cpp @@ -183,6 +183,51 @@ TMessageHolder LogValue(const std::string& value) { return entry; } +void test_nonleader_proposal_rejection_preserves_store_and_input(void**) { + for (int phase = 0; phase < 3; ++phase) { + TCluster cluster(3); + auto raft = cluster.Nodes[0]; + if (phase == 1) { + raft->ProcessTimeout(cluster.Now); + assert_int_equal(raft->CurrentStateName(), EState::CANDIDATE); + } else if (phase == 2) { + cluster.Elect(1); cluster.Replicate(1); + auto vote = NewHoldedMessage( + TMessageEx{.Src = 2, .Dst = 1, .Term = raft->GetState()->CurrentTerm + 1}, + TRequestVoteRequest{.LastLogIndex = raft->GetLastIndex(), + .LastLogTerm = raft->GetState()->LastLogTerm, .CandidateId = 2}); + raft->Process(cluster.Now, vote); + assert_int_equal(raft->CurrentStateName(), EState::FOLLOWER); + } + const auto before = static_cast(*raft->GetState()); + auto entry = LogValue("must not be proposed"); entry->Term = 77; + bool rejected = false; + try { raft->Append(entry); } catch (const std::logic_error&) { rejected = true; } + assert_true(rejected); + assert_true(static_cast(*raft->GetState()) == before); + assert_int_equal(entry->Term, 77); + } +} + +void test_follower_proposal_cannot_diverge_same_term_committed_command(void**) { + TCluster cluster(3); cluster.Elect(1); cluster.Replicate(1); + TWriteClient leader(cluster.Nodes[0]), follower(cluster.Nodes[1]), other(cluster.Nodes[2]); + bool rejected = false; + try { cluster.Nodes[1]->Append(LogValue("rogue")); } + catch (const std::logic_error&) { rejected = true; } + leader.Write(77, "valid"); // Same index/term as the unauthorized follower entry on old code. + cluster.Replicate(1, true, true); + leader.Poll(); follower.Poll(); other.Poll(); + for (auto* client : {&leader, &follower, &other}) { + assert_int_equal(client->Rsm->Applied.size(), 1); + assert_string_equal(client->Rsm->Applied[0].c_str(), "valid"); + } + cluster.AssertCommittedPrefix(); + assert_true(rejected); + assert_int_equal(leader.Replies.size(), 1); + assert_int_equal(leader.Replies[0].Error, 0); +} + void test_forwarded_write_and_strong_read_complete_through_real_raft(void**) { TCluster cluster(3); cluster.Elect(1); @@ -831,6 +876,8 @@ void test_cluster_restart_preserves_applied_history_and_rebuilds_volatile_applic int main() { const CMUnitTest tests[] = { + cmocka_unit_test(test_nonleader_proposal_rejection_preserves_store_and_input), + cmocka_unit_test(test_follower_proposal_cannot_diverge_same_term_committed_command), cmocka_unit_test(test_recovered_application_progress_is_not_rewound), cmocka_unit_test(test_application_progress_never_decreases_on_lower_commit_input), cmocka_unit_test(test_recovered_applied_prefix_is_protected_and_suffix_can_change), From 704ae08d0e2ec6059eb9c30b4601424324b481a8 Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Fri, 13 Nov 2026 09:47:02 -0500 Subject: [PATCH 21/66] Isolate in-memory log entries from mutable callers (#21) --- README.md | 2 ++ docs/memory-log-ownership.md | 23 +++++++++++++++++++++++ miniraft/state.h | 24 +++++++++++++++++++++--- test/test_correctness.cpp | 16 ++++++++++++++++ test/test_message_safety.cpp | 31 +++++++++++++++++++++++++++++++ 5 files changed, 93 insertions(+), 3 deletions(-) create mode 100644 docs/memory-log-ownership.md diff --git a/README.md b/README.md index dd9ef15..2fa418b 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,8 @@ the OS; a safe bounded application deadline remains open. The [proposal admission follow-up](docs/proposal-admission.md) restricts direct Append calls to leaders, preventing nonleaders from creating conflicting commands at the same log index and term. +The [memory log ownership follow-up](docs/memory-log-ownership.md) prevents +caller-owned entry buffers from changing entries already stored in TState. [![Number of GitHub stars](https://img.shields.io/github/stars/resetius/miniraft-cpp)](https://github.com/resetius/miniraft-cpp/stargazers) ![GitHub commit activity](https://img.shields.io/github/commit-activity/m/resetius/miniraft-cpp) diff --git a/docs/memory-log-ownership.md b/docs/memory-log-ownership.md new file mode 100644 index 0000000..f8d9c7c --- /dev/null +++ b/docs/memory-log-ownership.md @@ -0,0 +1,23 @@ +# In-memory log ownership: v0.1.0-alpha.21 + +The in-memory TState backend retained the mutable buffer supplied to Append and +returned the same buffer from Get. A caller or application could therefore change +an existing log entry, including its term or command, after Raft appended it. +That could break log matching and change a committed application command. The +durable backend already copies entries on append and read. + +TState now copies entry bytes on append, construction and readback. Copies of a +TState also receive independent log entries. This preserves stored terms and data +when the original proposal or a returned entry is modified. Copies cost additional +memory and CPU, especially for large entries. TState is still volatile and does +not provide crash durability. Its public Log member and IState mutation methods +remain test/backend interfaces: modifying them while Raft runs bypasses its safety +rules, and serialized access is still required. + +Deterministic tests cover mutation of the appended buffer, returned buffer, +constructor input and copied states, plus a three-node replication schedule that +changes the proposal after Append and verifies the original command commits on +all nodes. The first two ownership tests failed on the previous implementation. +Self-review covers snapshot suffix retention and the durable backend's existing +copy behavior; no independent reviewer approval is claimed. There is no wire or +durable format change, and existing corrupted state is not repaired. diff --git a/miniraft/state.h b/miniraft/state.h index c69f432..052aa12 100644 --- a/miniraft/state.h +++ b/miniraft/state.h @@ -65,14 +65,21 @@ inline void ValidateVoterBinding(const IState& state, const std::vector CloneEntry(const TMessageHolder& entry) { + if (!entry || entry->Type != uint32_t(EMessageType::LOG_ENTRY) || entry->Len < sizeof(TLogEntry)) + throw std::invalid_argument("Invalid in-memory log entry"); + auto copy = NewHoldedMessage(entry->Len); + memcpy(copy.Mes, entry.Mes, entry->Len); + return copy; + } std::vector> Log; std::shared_ptr Snapshot; std::vector BoundVoters; TState() = default; TState(uint64_t currentTerm, uint32_t votedFor, const std::vector>& log) - : Log(log) { + for (const auto& entry : log) Log.push_back(CloneEntry(entry)); CurrentTerm = currentTerm; VotedFor = votedFor; if (!log.empty()) { @@ -80,6 +87,17 @@ struct TState: IState { LastLogTerm = log.back()->Term; } } + TState(const TState& other) : IState(other), Snapshot(other.Snapshot), BoundVoters(other.BoundVoters) { + for (const auto& entry : other.Log) Log.push_back(CloneEntry(entry)); + } + TState& operator=(const TState& other) { + if (this != &other) { + TState copy(other); + static_cast(*this) = copy; + Log.swap(copy.Log); Snapshot.swap(copy.Snapshot); BoundVoters.swap(copy.BoundVoters); + } + return *this; + } void RemoveLast() override { if (Log.empty()) return; @@ -89,14 +107,14 @@ struct TState: IState { } void Append(TMessageHolder entry) override { - Log.emplace_back(std::move(entry)); + Log.emplace_back(CloneEntry(entry)); LastLogIndex = SnapshotIndex + Log.size(); LastLogTerm = Log.back()->Term; } TMessageHolder Get(int64_t index) const override { if (index < 1 || uint64_t(index) <= SnapshotIndex || uint64_t(index) > LastLogIndex) return {}; - return Log[index - SnapshotIndex - 1]; + return CloneEntry(Log[index - SnapshotIndex - 1]); } bool SupportsSnapshots() const override { return true; } diff --git a/test/test_correctness.cpp b/test/test_correctness.cpp index c183b0c..a80e7ae 100644 --- a/test/test_correctness.cpp +++ b/test/test_correctness.cpp @@ -208,6 +208,21 @@ void test_nonleader_proposal_rejection_preserves_store_and_input(void**) { assert_int_equal(entry->Term, 77); } } +void test_mutating_proposal_after_append_cannot_change_committed_command(void**) { + TCluster cluster(3); cluster.Elect(1); cluster.Replicate(1); + TWriteClient leader(cluster.Nodes[0]), follower(cluster.Nodes[1]), other(cluster.Nodes[2]); + auto entry = LogValue("valid"); + auto index = cluster.Nodes[0]->Append(entry); + memcpy(entry->Data, "rogue", 5); + cluster.Replicate(1, true, true); + leader.Poll(); follower.Poll(); other.Poll(); + for (auto* client : {&leader, &follower, &other}) { + assert_int_equal(client->Rsm->Applied.size(), 1); + assert_string_equal(client->Rsm->Applied[0].c_str(), "valid"); + } + assert_true(cluster.Nodes[0]->GetVolatileState()->CommitIndex >= index); + cluster.AssertCommittedPrefix(); +} void test_follower_proposal_cannot_diverge_same_term_committed_command(void**) { TCluster cluster(3); cluster.Elect(1); cluster.Replicate(1); @@ -877,6 +892,7 @@ void test_cluster_restart_preserves_applied_history_and_rebuilds_volatile_applic int main() { const CMUnitTest tests[] = { cmocka_unit_test(test_nonleader_proposal_rejection_preserves_store_and_input), + cmocka_unit_test(test_mutating_proposal_after_append_cannot_change_committed_command), cmocka_unit_test(test_follower_proposal_cannot_diverge_same_term_committed_command), cmocka_unit_test(test_recovered_application_progress_is_not_rewound), cmocka_unit_test(test_application_progress_never_decreases_on_lower_commit_input), diff --git a/test/test_message_safety.cpp b/test/test_message_safety.cpp index fd58c1b..44b446b 100644 --- a/test/test_message_safety.cpp +++ b/test/test_message_safety.cpp @@ -39,6 +39,34 @@ struct TEndpoint : INode { } void Drain() override {} }; +void test_memory_log_does_not_alias_proposed_or_read_entries(void**) { + auto entry = NewHoldedMessage(sizeof(TLogEntry) + 1); + entry->Data[0] = 'a'; + TState state; + state.Append(entry); + entry->Term = 9; entry->Data[0] = 'b'; + auto read = state.Get(1); + assert_int_equal(read->Term, 1); assert_int_equal(read->Data[0], 'a'); + read->Term = 7; read->Data[0] = 'c'; + assert_int_equal(state.Get(1)->Term, 1); assert_int_equal(state.Get(1)->Data[0], 'a'); + assert_int_equal(state.LastLogTerm, 1); +} +void test_memory_log_constructor_owns_initial_entries(void**) { + auto entry = NewHoldedMessage(sizeof(TLogEntry) + 1); + entry->Data[0] = 'x'; + TState state(1, 0, {entry}); + entry->Data[0] = 'y'; entry->Term = 2; + assert_int_equal(state.Get(1)->Data[0], 'x'); assert_int_equal(state.LogTerm(1), 1); +} +void test_memory_state_copies_do_not_share_mutable_log_entries(void**) { + TState source; source.Append(NewHoldedMessage()); + TState copied(source), assigned; assigned = source; + source.Log[0]->Term = 4; + assert_int_equal(copied.LogTerm(1), 1); + assert_int_equal(assigned.LogTerm(1), 1); + copied.Log[0]->Term = 5; + assert_int_equal(assigned.LogTerm(1), 1); +} void test_raft_batches_respect_budget_and_continue_replication(void**) { auto state = std::make_shared(); state->CurrentTerm = 2; @@ -86,6 +114,9 @@ void test_invalid_proposal_does_not_mutate_persistent_state(void**) { } int main() { const CMUnitTest tests[] = { + cmocka_unit_test(test_memory_log_does_not_alias_proposed_or_read_entries), + cmocka_unit_test(test_memory_log_constructor_owns_initial_entries), + cmocka_unit_test(test_memory_state_copies_do_not_share_mutable_log_entries), cmocka_unit_test(test_typed_allocations_reject_undersized_storage), cmocka_unit_test(test_short_wrong_type_and_empty_casts_are_rejected), cmocka_unit_test(test_raft_batches_respect_budget_and_continue_replication), From 6d9c7584afcdd5221be6105a31cc23dc1246011d Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Fri, 13 Nov 2026 09:52:01 -0500 Subject: [PATCH 22/66] Avoid copying log payloads for term checks (#22) --- README.md | 2 ++ docs/term-lookups.md | 26 ++++++++++++++++++++++++++ miniraft/durable.cpp | 5 +++++ miniraft/durable.h | 1 + miniraft/state.h | 12 +++++++++++- test/test_compaction.cpp | 21 +++++++++++++++++++++ test/test_message_safety.cpp | 17 +++++++++++++++++ 7 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 docs/term-lookups.md diff --git a/README.md b/README.md index 2fa418b..1e1273f 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,8 @@ Append calls to leaders, preventing nonleaders from creating conflicting command at the same log index and term. The [memory log ownership follow-up](docs/memory-log-ownership.md) prevents caller-owned entry buffers from changing entries already stored in TState. +The [term lookup follow-up](docs/term-lookups.md) avoids copying full payloads +when Raft needs only a retained entry's term. [![Number of GitHub stars](https://img.shields.io/github/stars/resetius/miniraft-cpp)](https://github.com/resetius/miniraft-cpp/stargazers) ![GitHub commit activity](https://img.shields.io/github/commit-activity/m/resetius/miniraft-cpp) diff --git a/docs/term-lookups.md b/docs/term-lookups.md new file mode 100644 index 0000000..281382a --- /dev/null +++ b/docs/term-lookups.md @@ -0,0 +1,26 @@ +# Term-only storage lookup: v0.1.0-alpha.22 + +The in-memory ownership fix made Get return a defensive copy. LogTerm used Get, +so checking the term of a large entry also copied its entire payload. The durable +backend already copied full entries, so it had the same unnecessary cost. Raft +checks terms during log matching, elections, commitment and compaction. + +IState now has a RetainedLogTerm hook. The in-memory and durable backends read +the term from their retained entry without copying its payload. LogTerm still +checks bounds and returns the snapshot boundary term as before. Full Get calls +continue to return independent copies. The default hook uses Get, preserving +source compatibility for external IState backends that do not override it. +An IState vtable changed: rebuild the library and dependent binaries together. +Backends subclassing TState or TDurableState and overriding Get with different +log semantics must also override RetainedLogTerm consistently. + +Deterministic tests count full-entry reads while repeatedly checking 1 MiB log +entries, verify no Get call is needed in either optimized backend, and verify +mutating a full Get result cannot alter stored data. The durable test also checks +snapshot-boundary and out-of-range behavior. The memory test failed before the +change: 20 term checks invoked 20 full Get calls. No throughput benchmark is +claimed. Legacy and external backends can retain the fallback cost. + +Self-review checked retained-suffix offsets, snapshot boundaries and fallback +behavior. There is no wire or durable-format migration. No independent reviewer +approval is claimed. Voting membership changes and joint consensus remain open. diff --git a/miniraft/durable.cpp b/miniraft/durable.cpp index 7e7f779..144f3b3 100644 --- a/miniraft/durable.cpp +++ b/miniraft/durable.cpp @@ -294,6 +294,11 @@ TMessageHolder TDurableState::Get(int64_t index) const return Clone(Impl->Log[index - SnapshotIndex - 1]); } +uint64_t TDurableState::RetainedLogTerm(int64_t index) const { + if (index < 1 || uint64_t(index) <= SnapshotIndex || uint64_t(index) > LastLogIndex) return 0; + return Impl->Log[index - SnapshotIndex - 1]->Term; +} + std::shared_ptr TDurableState::GetSnapshot() const { return Impl->Snapshot; } void TDurableState::InstallSnapshot(const TSnapshot& snapshot) { diff --git a/miniraft/durable.h b/miniraft/durable.h index 4e10a9d..a97d03e 100644 --- a/miniraft/durable.h +++ b/miniraft/durable.h @@ -19,6 +19,7 @@ class TDurableState : public IState { void RemoveLast() override; void Append(TMessageHolder entry) override; TMessageHolder Get(int64_t index) const override; + uint64_t RetainedLogTerm(int64_t index) const override; void Commit() override; bool SupportsSnapshots() const override { return true; } std::shared_ptr GetSnapshot() const override; diff --git a/miniraft/state.h b/miniraft/state.h index 052aa12..56869f9 100644 --- a/miniraft/state.h +++ b/miniraft/state.h @@ -21,6 +21,12 @@ struct IState: TStateFields { // Absolute indices; only the suffix after SnapshotIndex has log entries. // The boundary term remains available through LogTerm(SnapshotIndex). virtual TMessageHolder Get(int64_t index) const = 0; + // Backends may read a retained entry's term without copying its payload. + // The default keeps existing external backends source compatible. + virtual uint64_t RetainedLogTerm(int64_t index) const { + auto entry = Get(index); + return entry ? entry->Term : 0; + } virtual void Commit() = 0; virtual bool SupportsSnapshots() const { return false; } virtual std::shared_ptr GetSnapshot() const { return {}; } @@ -38,7 +44,7 @@ struct IState: TStateFields { if (index < 1 || uint64_t(index) < SnapshotIndex || index > LastLogIndex) { return 0; } else { - return Get(index)->Term; + return RetainedLogTerm(index); } } }; @@ -116,6 +122,10 @@ struct TState: IState { if (index < 1 || uint64_t(index) <= SnapshotIndex || uint64_t(index) > LastLogIndex) return {}; return CloneEntry(Log[index - SnapshotIndex - 1]); } + uint64_t RetainedLogTerm(int64_t index) const override { + if (index < 1 || uint64_t(index) <= SnapshotIndex || uint64_t(index) > LastLogIndex) return 0; + return Log[index - SnapshotIndex - 1]->Term; + } bool SupportsSnapshots() const override { return true; } bool SupportsVoterBinding() const override { return true; } diff --git a/test/test_compaction.cpp b/test/test_compaction.cpp index ef4766f..8c157a7 100644 --- a/test/test_compaction.cpp +++ b/test/test_compaction.cpp @@ -413,6 +413,26 @@ struct TFiles { } ~TFiles() { std::error_code ignored; std::filesystem::remove_all(Directory, ignored); } }; +struct TCountingDurable : TDurableState { + using TDurableState::TDurableState; + mutable unsigned Reads = 0; + TMessageHolder Get(int64_t index) const override { + ++Reads; return TDurableState::Get(index); + } +}; +void test_durable_term_lookup_avoids_payload_copies_across_compaction(void**) { + TFiles files; TCountingDurable state(files.Name, 1); + auto entry = NewHoldedMessage(sizeof(TLogEntry) + 1024 * 1024); + entry->Data[0] = 'x'; state.Append(entry); state.Append(entry); + state.InstallSnapshot({1, 1, {1}, "image"}); + for (int i = 0; i < 20; ++i) { + assert_int_equal(state.LogTerm(1), 1); assert_int_equal(state.LogTerm(2), 1); + assert_int_equal(state.LogTerm(3), 0); + } + assert_int_equal(state.Reads, 0); + auto read = state.Get(2); read->Data[0] = 'y'; read->Term = 2; + assert_int_equal(state.LogTerm(2), 1); assert_int_equal(state.Get(2)->Data[0], 'x'); +} void test_durable_compacted_boundary_survives_restart(void**) { TFiles files; { TDurableState state(files.Name, 1); ExerciseBoundary(state); } @@ -499,6 +519,7 @@ int main() { cmocka_unit_test(test_snapshot_boundary_index_exhaustion_does_not_append_wrapped_noop), cmocka_unit_test(test_memory_snapshot_boundary_retains_matching_suffix_and_replaces_conflict), #ifdef __linux__ + cmocka_unit_test(test_durable_term_lookup_avoids_payload_copies_across_compaction), cmocka_unit_test(test_durable_compacted_boundary_survives_restart), cmocka_unit_test(test_snapshot_and_suffix_are_atomic_at_every_publication_stage), cmocka_unit_test(test_compaction_reclaims_durable_prefix_bytes), diff --git a/test/test_message_safety.cpp b/test/test_message_safety.cpp index 44b446b..f66d4c5 100644 --- a/test/test_message_safety.cpp +++ b/test/test_message_safety.cpp @@ -39,6 +39,22 @@ struct TEndpoint : INode { } void Drain() override {} }; +struct TCountingState : TState { + mutable unsigned Reads = 0; + TMessageHolder Get(int64_t index) const override { + ++Reads; return TState::Get(index); + } +}; +void test_memory_term_lookup_avoids_copying_full_entry(void**) { + TCountingState state; + auto entry = NewHoldedMessage(sizeof(TLogEntry) + 1024 * 1024); + entry->Term = 1; entry->Data[0] = 'x'; state.Append(entry); + for (int i = 0; i < 20; ++i) assert_int_equal(state.LogTerm(1), 1); + assert_int_equal(state.Reads, 0); + auto copy = state.Get(1); copy->Term = 9; copy->Data[0] = 'y'; + assert_int_equal(state.LogTerm(1), 1); + assert_int_equal(state.Get(1)->Data[0], 'x'); +} void test_memory_log_does_not_alias_proposed_or_read_entries(void**) { auto entry = NewHoldedMessage(sizeof(TLogEntry) + 1); entry->Data[0] = 'a'; @@ -114,6 +130,7 @@ void test_invalid_proposal_does_not_mutate_persistent_state(void**) { } int main() { const CMUnitTest tests[] = { + cmocka_unit_test(test_memory_term_lookup_avoids_copying_full_entry), cmocka_unit_test(test_memory_log_does_not_alias_proposed_or_read_entries), cmocka_unit_test(test_memory_log_constructor_owns_initial_entries), cmocka_unit_test(test_memory_state_copies_do_not_share_mutable_log_entries), From afde6fa08b35cb72416f43980e431f0d25a0866f Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Fri, 13 Nov 2026 09:59:51 -0500 Subject: [PATCH 23/66] Validate message buffer and batch extents before access (#23) --- README.md | 2 ++ docs/message-extents.md | 30 ++++++++++++++++++++++++++++++ miniraft/durable.cpp | 2 +- miniraft/messages.cpp | 3 ++- miniraft/messages.h | 25 +++++++++++++++++++------ miniraft/persist.cpp | 1 + miniraft/raft.cpp | 10 ++++++++-- miniraft/state.h | 2 +- test/test_forwarding.cpp | 22 ++++++++++++++++++++++ test/test_message_safety.cpp | 30 ++++++++++++++++++++++++++++++ 10 files changed, 116 insertions(+), 11 deletions(-) create mode 100644 docs/message-extents.md diff --git a/README.md b/README.md index 1e1273f..16b8ca6 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,8 @@ The [memory log ownership follow-up](docs/memory-log-ownership.md) prevents caller-owned entry buffers from changing entries already stored in TState. The [term lookup follow-up](docs/term-lookups.md) avoids copying full payloads when Raft needs only a retained entry's term. +The [message buffer follow-up](docs/message-extents.md) checks allocated buffer +and batch extents before accepting locally constructed frames. [![Number of GitHub stars](https://img.shields.io/github/stars/resetius/miniraft-cpp)](https://github.com/resetius/miniraft-cpp/stargazers) ![GitHub commit activity](https://img.shields.io/github/commit-activity/m/resetius/miniraft-cpp) diff --git a/docs/message-extents.md b/docs/message-extents.md new file mode 100644 index 0000000..0d442f3 --- /dev/null +++ b/docs/message-extents.md @@ -0,0 +1,30 @@ +# Message buffer extents: v0.1.0-alpha.23 + +Locally constructed message holders exposed a mutable frame length but did not +remember their allocated buffer size. A caller could enlarge an entry's declared +length while leaving it within the protocol maximum; proposal validation then +accepted it and the in-memory log copied beyond the allocation. A deterministic +ASan regression reproduced a heap-buffer-overflow. + +Holders now carry the buffer extent and payload-array capacity through typed +conversions. Frame and batch validation reject lengths or entry counts beyond +those extents. Proposal and command request/reply admission validate the holder +before accessing or copying the data. The storage clone paths check the extent, +and the legacy disk write path fails closed for an invalid buffer. A malformed +forwarded reply is rejected before its client route is consumed. + +NewHoldedMessage and InitPayload populate extents automatically. Callers that +construct holders directly around an external allocation must supply its actual +byte capacity and, for batches, the actual payload-array capacity to the holder +constructor; the default zero capacity fails validation. Holder layout changed, +so rebuild the library and all dependent binaries together. Wire formats and +durable formats are unchanged. The extents are trusted metadata owned by local +C++ code, not a sandbox against callers that forge pointers or mutate capacities. +Application code that bypasses Raft admission remains responsible for its own +validation. + +Deterministic tests cover the original oversized proposal, forged batch count, +oversized child entry, typed conversion and a malformed forwarded reply followed +by a successful valid reply. Self-review checked framing, admission, clones and +forwarded-route ordering. No independent reviewer approval is claimed. Voting +membership changes and joint consensus remain open. diff --git a/miniraft/durable.cpp b/miniraft/durable.cpp index 144f3b3..54c8c5b 100644 --- a/miniraft/durable.cpp +++ b/miniraft/durable.cpp @@ -41,7 +41,7 @@ uint32_t Checksum(const TBytes& bytes, size_t start, const std::array Clone(TMessageHolder entry) { - if (!entry.Mes || entry->Len < sizeof(TLogEntry) || entry->Type != uint32_t(EMessageType::LOG_ENTRY)) + if (!entry.HasValidExtent() || entry->Len < sizeof(TLogEntry) || entry->Type != uint32_t(EMessageType::LOG_ENTRY)) throw std::runtime_error("Invalid durable log entry"); auto copy = NewHoldedMessage(entry->Len); memcpy(copy.Mes, entry.Mes, entry->Len); diff --git a/miniraft/messages.cpp b/miniraft/messages.cpp index 7d0e662..93a421e 100644 --- a/miniraft/messages.cpp +++ b/miniraft/messages.cpp @@ -33,6 +33,7 @@ void ValidateMessageTree(const TMessageHolder& message, const TMessage uint64_t remaining = limits.MaxBatchBytes; auto validate = [&](const auto& frame, bool child) { if (!frame) throw TMessageError("Missing message"); + if (!frame.HasValidExtent()) throw TMessageError("Message length exceeds its buffer"); ValidateMessageHeader(frame->Type, frame->Len, limits); if (frame->Len > remaining) throw TMessageError("Message batch exceeds byte limit"); remaining -= frame->Len; @@ -45,7 +46,7 @@ void ValidateMessageTree(const TMessageHolder& message, const TMessage if (message->Type != uint32_t(EMessageType::APPEND_ENTRIES_REQUEST)) return; auto copy = message; const auto count = copy.Cast()->Nentries; - if (count > limits.MaxEntries || count != message.PayloadSize || (count && !message.Payload)) + if (count > limits.MaxEntries || count != message.PayloadSize || count > message.PayloadCapacity || (count && !message.Payload)) throw TMessageError("Invalid AppendEntries entry count"); for (uint32_t i = 0; i < count; ++i) validate(message.Payload[i], true); } diff --git a/miniraft/messages.h b/miniraft/messages.h index ee49738..f6fc134 100644 --- a/miniraft/messages.h +++ b/miniraft/messages.h @@ -156,9 +156,11 @@ requires std::derived_from struct TMessageHolder { T* Mes; std::shared_ptr RawData; + size_t Capacity = 0; // Allocated bytes accessible starting at Mes. uint32_t PayloadSize = 0; std::shared_ptr[]> Payload; + size_t PayloadCapacity = 0; TMessageHolder() : Mes(nullptr) @@ -169,11 +171,14 @@ struct TMessageHolder { TMessageHolder(U* u, const std::shared_ptr& rawData, uint32_t payloadSize = 0, - const std::shared_ptr[]>& payload = {}) + const std::shared_ptr[]>& payload = {}, size_t capacity = 0, + size_t payloadCapacity = 0) : Mes(u) , RawData(rawData) + , Capacity(capacity) , PayloadSize(payloadSize) , Payload(payload) + , PayloadCapacity(payloadCapacity) { } template @@ -181,13 +186,16 @@ struct TMessageHolder { TMessageHolder(const TMessageHolder& other) : Mes(other.Mes) , RawData(other.RawData) + , Capacity(other.Capacity) , PayloadSize(other.PayloadSize) , Payload(other.Payload) + , PayloadCapacity(other.PayloadCapacity) { } void InitPayload(uint32_t size) { PayloadSize = size; Payload = std::shared_ptr[]>(new TMessageHolder[size]); + PayloadCapacity = size; } T* operator->() { @@ -202,17 +210,22 @@ struct TMessageHolder { return !!Mes; } + bool HasValidExtent() const { + return Mes && RawData && Capacity >= sizeof(TMessage) && + Mes->Len >= sizeof(TMessage) && Mes->Len <= Capacity; + } + bool IsEx() const { - return Mes && Mes->Len >= sizeof(TMessageEx) && (2 <= Mes->Type && Mes->Type <= 7); + return HasValidExtent() && Mes->Len >= sizeof(TMessageEx) && (2 <= Mes->Type && Mes->Type <= 7); } template requires std::derived_from TMessageHolder Cast() { - if (!Mes || Mes->Len < sizeof(U) || + if (!HasValidExtent() || Mes->Len < sizeof(U) || (U::MessageType != EMessageType::NONE && Mes->Type != uint32_t(U::MessageType))) throw std::bad_cast(); - return TMessageHolder(static_cast(Mes), RawData, PayloadSize, Payload); + return TMessageHolder(static_cast(Mes), RawData, PayloadSize, Payload, Capacity, PayloadCapacity); } template @@ -233,7 +246,7 @@ struct TMessageHolder { } }; - return Mes && Mes->Len >= sizeof(U) && Mes->Type == static_cast(U::MessageType) + return HasValidExtent() && Mes->Len >= sizeof(U) && Mes->Type == static_cast(U::MessageType) ? Maybe { *this } : Maybe { }; } @@ -258,7 +271,7 @@ requires std::derived_from TMessageHolder NewHoldedMessage(uint32_t type, size_t len) { T* mes = NewMessage(type, len); - return TMessageHolder(mes, std::shared_ptr(reinterpret_cast(mes))); + return TMessageHolder(mes, std::shared_ptr(reinterpret_cast(mes)), 0, {}, len); } template diff --git a/miniraft/persist.cpp b/miniraft/persist.cpp index dce7979..4c44cbd 100644 --- a/miniraft/persist.cpp +++ b/miniraft/persist.cpp @@ -114,6 +114,7 @@ TMessageHolder TDiskState::Read() const void TDiskState::Write(TMessageHolder message) { + if (!message.HasValidExtent()) Fail("Invalid Raft log entry buffer"); if (!Entries.write(reinterpret_cast(message.Mes), message->Len)) { Fail("Cannot write Raft log entry"); } diff --git a/miniraft/raft.cpp b/miniraft/raft.cpp index bb72989..1a5ac10 100644 --- a/miniraft/raft.cpp +++ b/miniraft/raft.cpp @@ -686,9 +686,9 @@ uint64_t TRaft::Append(TMessageHolder entry) { if (!IsVoter(Id)) throw std::logic_error("A learner cannot propose log entries"); if (StateName != EState::LEADER) throw std::logic_error("Only a leader can propose log entries"); if (State->LastLogIndex >= uint64_t(INT64_MAX)) throw std::overflow_error("Raft log index exhausted"); - if (!entry || entry->Type != uint32_t(EMessageType::LOG_ENTRY)) + if (!entry.HasValidExtent() || entry->Type != uint32_t(EMessageType::LOG_ENTRY)) throw TMessageError("Invalid proposed log entry"); - ValidateMessageHeader(entry->Type, entry->Len); + ValidateMessageTree(entry); entry->Term = State->CurrentTerm; State->Append(std::move(entry)); return State->LastLogIndex; @@ -1017,6 +1017,9 @@ void TRequestProcessor::OnWriteRequest(TMessageHolder command, } void TRequestProcessor::OnCommandRequest(TMessageHolder command, const std::shared_ptr& replyTo) { + if (!command.HasValidExtent() || command->Type != uint32_t(EMessageType::COMMAND_REQUEST)) + throw TMessageError("Invalid command request"); + ValidateMessageTree(command); Rsm->Validate(command); PruneRequests(); auto flags = command->Flags; @@ -1030,6 +1033,9 @@ void TRequestProcessor::OnCommandRequest(TMessageHolder command void TRequestProcessor::OnCommandResponse(TMessageHolder command, const std::shared_ptr& peer) { + if (!command.HasValidExtent() || command->Type != uint32_t(EMessageType::COMMAND_RESPONSE)) + throw TMessageError("Invalid command response"); + ValidateMessageTree(command); PruneRequests(); auto it = Forwarded.find(command->Cookie); if (it == Forwarded.end() || !peer || it->second.Peer != peer) return; diff --git a/miniraft/state.h b/miniraft/state.h index 56869f9..85f246a 100644 --- a/miniraft/state.h +++ b/miniraft/state.h @@ -72,7 +72,7 @@ inline void ValidateVoterBinding(const IState& state, const std::vector CloneEntry(const TMessageHolder& entry) { - if (!entry || entry->Type != uint32_t(EMessageType::LOG_ENTRY) || entry->Len < sizeof(TLogEntry)) + if (!entry.HasValidExtent() || entry->Type != uint32_t(EMessageType::LOG_ENTRY) || entry->Len < sizeof(TLogEntry)) throw std::invalid_argument("Invalid in-memory log entry"); auto copy = NewHoldedMessage(entry->Len); memcpy(copy.Mes, entry.Mes, entry->Len); diff --git a/test/test_forwarding.cpp b/test/test_forwarding.cpp index 4e8ab9b..c5d4193 100644 --- a/test/test_forwarding.cpp +++ b/test/test_forwarding.cpp @@ -64,6 +64,27 @@ struct TFollower { } }; +void test_malformed_forwarded_reply_does_not_consume_route(void**) { + TFollower follower; TClient client; + follower.Request(client, 42); + auto response = follower.Response(0); + response->Len += 8; + bool rejected = false; + try { follower.Processor.OnCommandResponse(response, follower.Leader); } + catch (const TMessageError&) { rejected = true; } + assert_true(rejected); assert_true(client.Replies.empty()); + auto wrong = follower.Response(0); + wrong->Type = uint32_t(EMessageType::LOG_ENTRY); + rejected = false; + try { follower.Processor.OnCommandResponse(wrong, follower.Leader); } + catch (const TMessageError&) { rejected = true; } + assert_true(rejected); assert_true(client.Replies.empty()); + follower.Reply(0); + assert_int_equal(client.Replies.size(), 1); + assert_int_equal(client.Replies[0].Cookie, 42); + assert_string_equal(client.Replies[0].Data.c_str(), "answer"); +} + void test_nonzero_client_cookies_are_forwarded(void**) { for (uint32_t cookie : {42u, UINT32_MAX}) { TFollower follower; @@ -225,6 +246,7 @@ void test_peer_disconnect_cancels_once_and_does_not_reuse_cookies(void**) { int main() { const CMUnitTest tests[] = { + cmocka_unit_test(test_malformed_forwarded_reply_does_not_consume_route), cmocka_unit_test(test_nonzero_client_cookies_are_forwarded), cmocka_unit_test(test_zero_client_cookie_is_restored), cmocka_unit_test(test_forwarding_copies_request_and_marks_one_hop), diff --git a/test/test_message_safety.cpp b/test/test_message_safety.cpp index f66d4c5..9b0d05b 100644 --- a/test/test_message_safety.cpp +++ b/test/test_message_safety.cpp @@ -127,6 +127,34 @@ void test_invalid_proposal_does_not_mutate_persistent_state(void**) { assert_int_equal(entry->Term, 99); } } +void test_declared_length_cannot_exceed_actual_proposal_buffer(void**) { + auto state = std::make_shared(); + TRaft raft(state, 1, {}); raft.Become(EState::LEADER); + auto entry = NewHoldedMessage(); + entry->Len = sizeof(TLogEntry) + 8; + bool rejected = false; + try { raft.Append(entry); } catch (const TMessageError&) { rejected = true; } + assert_true(rejected); + assert_int_equal(state->LastLogIndex, 0); +} +void test_invalid_holder_extents_are_rejected_before_batch_access(void**) { + auto shortEntry = NewHoldedMessage(); + shortEntry->Len += 8; + assert_false(shortEntry.Maybe()); + bool rejected = false; + try { ValidateMessageTree(shortEntry); } catch (const TMessageError&) { rejected = true; } + assert_true(rejected); + auto append = NewHoldedMessage(TAppendEntriesRequest{.Nentries = 2}); + append.InitPayload(1); append.Payload[0] = NewHoldedMessage(); + append.PayloadSize = 2; + rejected = false; + try { ValidateMessageTree(append); } catch (const TMessageError&) { rejected = true; } + assert_true(rejected); + append.PayloadSize = 1; append->Nentries = 1; append.Payload[0] = shortEntry; + rejected = false; + try { ValidateMessageTree(append); } catch (const TMessageError&) { rejected = true; } + assert_true(rejected); +} } int main() { const CMUnitTest tests[] = { @@ -138,6 +166,8 @@ int main() { cmocka_unit_test(test_short_wrong_type_and_empty_casts_are_rejected), cmocka_unit_test(test_raft_batches_respect_budget_and_continue_replication), cmocka_unit_test(test_invalid_proposal_does_not_mutate_persistent_state), + cmocka_unit_test(test_declared_length_cannot_exceed_actual_proposal_buffer), + cmocka_unit_test(test_invalid_holder_extents_are_rejected_before_batch_access), }; return cmocka_run_group_tests(tests, nullptr, nullptr); } From 452377c58285444e8967bf1ca3377eee9f055dc4 Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Fri, 13 Nov 2026 10:11:08 -0500 Subject: [PATCH 24/66] Add joint quorum arithmetic foundation (#24) --- README.md | 2 + docs/joint-quorum.md | 35 +++++++++++++++ miniraft/quorum.h | 48 ++++++++++++++++++++ miniraft/raft.cpp | 37 ++++++++-------- miniraft/raft.h | 3 ++ test/CMakeLists.txt | 1 + test/test_quorum.cpp | 101 +++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 207 insertions(+), 20 deletions(-) create mode 100644 docs/joint-quorum.md create mode 100644 miniraft/quorum.h create mode 100644 test/test_quorum.cpp diff --git a/README.md b/README.md index 16b8ca6..2780ba2 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,8 @@ The [memory log ownership follow-up](docs/memory-log-ownership.md) prevents caller-owned entry buffers from changing entries already stored in TState. The [term lookup follow-up](docs/term-lookups.md) avoids copying full payloads when Raft needs only a retained entry's term. +The [joint quorum foundation](docs/joint-quorum.md) adds and tests dual-majority +arithmetic for future membership changes; live voter reconfiguration remains disabled. The [message buffer follow-up](docs/message-extents.md) checks allocated buffer and batch extents before accepting locally constructed frames. diff --git a/docs/joint-quorum.md b/docs/joint-quorum.md new file mode 100644 index 0000000..3bf88bc --- /dev/null +++ b/docs/joint-quorum.md @@ -0,0 +1,35 @@ +# Joint quorum arithmetic: v0.1.0-alpha.24 + +This step prepares the election, commit and strong-read quorum calculations for +joint consensus. A joint quorum requires a strict majority of both the old and +new voter sets. Its replicated-index or read-ack watermark is the lower of the +two majority watermarks. Shared voters count in each set; outsiders and learners +do not contribute. Even-sized groups require more than half. + +TQuorum validates positive, sorted, unique, nonempty voter sets and computes +stable or joint election results and watermarks. Its value callback must return +the same value for a voter throughout a call. TVolatileState can apply this +arithmetic to commit advancement, retaining the current-term commit restriction +and monotonic commit index. Existing fixed-membership election, commit and strong +read paths now use the same stable-quorum arithmetic; they do not enter joint mode. + +Tests exhaust every pair of nonempty old/new subsets of five IDs and every vote +subset, including disjoint and overlapping sets, even-sized groups, absent self +and outsider votes. They independently enumerate thresholds for joint commit/read +watermarks and check current-term gating. Separate stable-set tests compare the +calculation with strict-majority order statistics for sizes 1–8. Existing Raft, +partition, learner, snapshot and read tests exercise the fixed-membership paths. + +This is arithmetic foundation, not a reconfiguration API. It does not replicate +configuration entries, change roles, elect/remove leaders, bind joint membership +durably or encode configuration history in snapshots. A safe next step must define +configuration entry ordering and recovery, activation and rollback of uncommitted +configurations, promotion readiness, old/new quorum use at every protocol phase, +and snapshot installation across transitions. Until those pieces are implemented +and tested, stored voter bindings remain immutable and promotion stays disabled. + +The change adds a public type and changes TRaft's in-memory layout; rebuild the +library and dependents together. Wire and durable formats are unchanged. +Self-review checked overlap, strict majorities, current-term commitment, stable +path equivalence and learner exclusion. No independent reviewer approval is +claimed. There is no live membership change to exercise or benchmark yet. diff --git a/miniraft/quorum.h b/miniraft/quorum.h new file mode 100644 index 0000000..b86cc6f --- /dev/null +++ b/miniraft/quorum.h @@ -0,0 +1,48 @@ +#pragma once + +#include "state.h" +#include +#include +#include +#include +#include + +// Quorum arithmetic only. This does not activate or persist a membership change. +class TQuorum { +public: + explicit TQuorum(std::vector voters) : Old(std::move(voters)) { + ValidateFixedVoters(Old); + } + TQuorum(std::vector oldVoters, std::vector newVoters) + : Old(std::move(oldVoters)), New(std::move(newVoters)) { + ValidateFixedVoters(Old); + ValidateFixedVoters(New); + } + + // Highest threshold acknowledged by a strict majority of each voter set. + // Missing voters should return zero. Overlapping voters contribute once to + // each set; members outside both sets cannot affect the result. + // The callback must return a stable value for an ID during the call. + template uint64_t Value(F value, std::vector* scratch = nullptr) const { + std::vector local; + auto& values = scratch ? *scratch : local; + const auto majority = [&](const auto& voters) { + values.clear(); + values.reserve(voters.size()); + for (auto id : voters) values.push_back(value(id)); + auto middle = values.begin() + (values.size() - 1) / 2; + std::nth_element(values.begin(), middle, values.end()); + return *middle; + }; + const auto old = majority(Old); + return New.empty() ? old : std::min(old, majority(New)); + } + + bool HasQuorum(const std::unordered_set& votes, uint32_t self = 0) const { + return Value([&](uint32_t id) -> uint64_t { + return (self != 0 && id == self) || votes.contains(id); + }) != 0; + } +private: + std::vector Old, New; +}; diff --git a/miniraft/raft.cpp b/miniraft/raft.cpp index 1a5ac10..13cca44 100644 --- a/miniraft/raft.cpp +++ b/miniraft/raft.cpp @@ -100,14 +100,15 @@ TVolatileState& TVolatileState::CommitAdvance(int nservers, const IState& state) } TVolatileState& TVolatileState::CommitAdvance(const std::vector& voters, uint32_t self, const IState& state) { - ValidateFixedVoters(voters); - Indices.clear(); - for (auto id : voters) { + return CommitAdvance(TQuorum(voters), self, state); +} + +TVolatileState& TVolatileState::CommitAdvance(const TQuorum& quorum, uint32_t self, const IState& state) { + const auto quorumIndex = quorum.Value([&](uint32_t id) { auto match = MatchIndex.find(id); - Indices.push_back(id == self ? state.LastLogIndex : match == MatchIndex.end() ? 0 : match->second); - } - std::sort(Indices.begin(), Indices.end()); - const auto index = std::max(CommitIndex, Indices[(voters.size() - 1) / 2]); + return id == self ? state.LastLogIndex : match == MatchIndex.end() ? uint64_t(0) : match->second; + }, &Indices); + const auto index = std::max(CommitIndex, quorumIndex); if (state.LogTerm(index) == state.CurrentTerm) CommitIndex = index; return *this; } @@ -190,6 +191,7 @@ TRaft::TRaft(std::shared_ptr state, int node, const TNodeDict& nodes, st if (!std::binary_search(Learners.begin(), Learners.end(), id)) Voters.push_back(id); std::sort(Voters.begin(), Voters.end()); ValidateFixedVoters(Voters); + VotingQuorum.emplace(Voters); if (!IsVoter(Id) && State->VotedFor) throw std::invalid_argument("A learner store must not contain a vote"); Nservers = Voters.size(); MinVotes = Nservers / 2 + 1; @@ -337,7 +339,7 @@ void TRaft::OnAppendEntries(TMessageHolder message) { .SetRpcDue(nodeId, ITimeSource::Time{}) .SetBatchSize(nodeId, 1024) .SetBackOff(nodeId, 1) - .CommitAdvance(Voters, Id, *State); + .CommitAdvance(*VotingQuorum, Id, *State); } else if (message->MatchIndex > 0) { // A compacted-prefix hint is not replication evidence. Probe its boundary // with AppendEntries before advancing MatchIndex or confirming reads. @@ -576,7 +578,7 @@ void TRaft::LeaderTimeout(ITimeSource::Time now) { } if (Nservers == 1) { - VolatileState->CommitAdvance(Voters, Id, *State); + VolatileState->CommitAdvance(*VotingQuorum, Id, *State); } } @@ -595,15 +597,11 @@ uint64_t TRaft::ApproveRead() { uint64_t TRaft::CommitSeqno() const { if (StateName != EState::LEADER || VolatileState->CommitIndex == 0 || State->LogTerm(VolatileState->CommitIndex) != State->CurrentTerm) return 0; - std::vector acknowledgements{Seqno - 1}; // leader's own confirmation - acknowledgements.reserve(Nservers); - for (auto id : Voters) { - if (id == Id) continue; + return VotingQuorum->Value([&](uint32_t id) -> uint64_t { + if (id == Id) return Seqno - 1; // leader's own confirmation auto it = VolatileState->ReadAckSeqno.find(id); - acknowledgements.push_back(it == VolatileState->ReadAckSeqno.end() ? 0 : it->second); - } - std::sort(acknowledgements.begin(), acknowledgements.end()); - return acknowledgements[(Nservers - 1) / 2]; + return it == VolatileState->ReadAckSeqno.end() ? 0 : it->second; + }); } uint64_t TRaft::NextSeqno() { @@ -632,8 +630,7 @@ void TRaft::ProcessTimeout(ITimeSource::Time now) { } if (StateName == EState::CANDIDATE) { - int nvotes = VolatileState->Votes.size()+1; - if (nvotes >= MinVotes) { + if (VotingQuorum->HasQuorum(VolatileState->Votes, Id)) { if (State->LastLogIndex >= uint64_t(INT64_MAX)) throw std::overflow_error("Raft log index exhausted"); auto value = State->LastLogIndex+1; decltype(VolatileState->NextIndex) nextIndex; @@ -770,7 +767,7 @@ void TRaft::OnInstallSnapshot(TMessageHolder response) if (response->NextOffset != rpc.Bytes->size()) return; const auto index = std::max(VolatileState->MatchIndex[response->Src], response->SnapshotIndex); (*VolatileState).SetMatchIndex(response->Src, index).SetNextIndex(response->Src, index + 1) - .SetBatchSize(response->Src, 1024).SetBackOff(response->Src, 1).CommitAdvance(Voters, Id, *State); + .SetBatchSize(response->Src, 1024).SetBackOff(response->Src, 1).CommitAdvance(*VotingQuorum, Id, *State); VolatileState->SnapshotRpcs.erase(found); // Snapshot completion is not a fresh-read acknowledgement. AppendEntries // must still establish the next read barrier and replicate the suffix. diff --git a/miniraft/raft.h b/miniraft/raft.h index 97e89d4..816d164 100644 --- a/miniraft/raft.h +++ b/miniraft/raft.h @@ -12,6 +12,7 @@ #include "messages.h" #include "timesource.h" #include "state.h" +#include "quorum.h" struct INode { virtual ~INode() = default; @@ -85,6 +86,7 @@ struct TVolatileState { // Compatibility helper for all-voter fixtures; mixed roles require IDs below. TVolatileState& CommitAdvance(int nservers, const IState& state); TVolatileState& CommitAdvance(const std::vector& voters, uint32_t self, const IState& state); + TVolatileState& CommitAdvance(const TQuorum& quorum, uint32_t self, const IState& state); TVolatileState& SetCommitIndex(uint64_t index); TVolatileState& SetElectionDue(ITimeSource::Time); TVolatileState& SetNextIndex(uint32_t id, uint64_t nextIndex); @@ -195,6 +197,7 @@ class TRaft { uint32_t Id; TNodeDict Nodes; std::vector Voters, Learners; + std::optional VotingQuorum; int MinVotes; int Npeers; int Nservers; diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index d94c824..7405616 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -13,6 +13,7 @@ endmacro() if (CMOCKA_FOUND) ut(test_raft test_raft.cpp miniraft) +ut(test_quorum test_quorum.cpp miniraft) ut(test_correctness test_correctness.cpp miniraft) ut(test_forwarding test_forwarding.cpp miniraft) ut(test_message_safety test_message_safety.cpp miniraft) diff --git a/test/test_quorum.cpp b/test/test_quorum.cpp new file mode 100644 index 0000000..52ffe97 --- /dev/null +++ b/test/test_quorum.cpp @@ -0,0 +1,101 @@ +#include +#include +#include +#include +#include +#include +extern "C" { +#include +} +namespace { +std::vector Members(unsigned mask) { + std::vector out; + for (uint32_t id = 1; id <= 5; ++id) if (mask & (1u << (id - 1))) out.push_back(id); + return out; +} +bool Majority(unsigned members, unsigned votes) { + return std::popcount(members & votes) > std::popcount(members) / 2; +} +void test_stable_quorum_matches_strict_majority_order_statistic(void**) { + for (uint32_t count = 1; count <= 8; ++count) { + std::vector voters; + for (uint32_t id = 1; id <= count; ++id) voters.push_back(id); + TQuorum quorum(voters); + for (uint64_t seed = 0; seed < 64; ++seed) { + auto value = [&](uint32_t id) { return (seed * 7 + id * id * 3) % 17; }; + std::vector expected; + for (auto id : voters) expected.push_back(value(id)); + std::sort(expected.begin(), expected.end()); + assert_int_equal(quorum.Value(value), expected[(count - 1) / 2]); + } + } +} +void test_all_small_joint_election_quorums(void**) { + for (unsigned old = 1; old < 32; ++old) for (unsigned next = 1; next < 32; ++next) { + TQuorum quorum(Members(old), Members(next)); + for (unsigned mask = 0; mask < 32; ++mask) { + std::unordered_set votes{99}; + for (auto id : Members(mask)) votes.insert(id); + assert_int_equal(quorum.HasQuorum(votes), Majority(old, mask) && Majority(next, mask)); + for (uint32_t self = 1; self <= 6; ++self) { + const auto withSelf = self <= 5 ? mask | (1u << (self - 1)) : mask; + assert_int_equal(quorum.HasQuorum(votes, self), Majority(old, withSelf) && Majority(next, withSelf)); + } + } + } +} +void test_joint_commit_and_read_watermarks_need_both_majorities(void**) { + for (unsigned old = 1; old < 32; ++old) for (unsigned next = 1; next < 32; ++next) { + TQuorum quorum(Members(old), Members(next)); + for (unsigned pattern = 0; pattern < 32; ++pattern) { + const auto value = [&](uint32_t id) -> uint64_t { + return pattern & (1u << (id - 1)) ? 11 - id : id % 3; + }; + uint64_t expected = 0; + for (uint64_t threshold = 1; threshold <= 10; ++threshold) { + unsigned matched = 0; + for (uint32_t id = 1; id <= 5; ++id) if (value(id) >= threshold) matched |= 1u << (id - 1); + if (Majority(old, matched) && Majority(next, matched)) expected = threshold; + } + assert_int_equal(quorum.Value(value), expected); + } + } +} +void test_invalid_configurations_are_rejected(void**) { + for (auto bad : {std::vector{}, std::vector{0}, + std::vector{1, 1}, std::vector{2, 1}}) { + bool rejected = false; + try { TQuorum invalid(bad); } catch (const std::invalid_argument&) { rejected = true; } + assert_true(rejected); rejected = false; + try { TQuorum invalid({1}, bad); } catch (const std::invalid_argument&) { rejected = true; } + assert_true(rejected); + } +} +void test_joint_commit_respects_both_majorities_and_current_term(void**) { + TState state; state.CurrentTerm = 2; + auto old = NewHoldedMessage(); old->Term = 1; state.Append(old); + auto current = NewHoldedMessage(); current->Term = 2; state.Append(current); + TQuorum quorum({1, 2, 3}, {3, 4, 5}); + TVolatileState progress; + progress.MatchIndex[2] = 2; progress.MatchIndex[99] = 2; + progress.CommitAdvance(quorum, 1, state); assert_int_equal(progress.CommitIndex, 0); + progress.MatchIndex[3] = 1; progress.MatchIndex[4] = 1; + progress.CommitAdvance(quorum, 1, state); assert_int_equal(progress.CommitIndex, 0); + progress.MatchIndex[3] = 2; + progress.CommitAdvance(quorum, 1, state); assert_int_equal(progress.CommitIndex, 0); + progress.MatchIndex[4] = 2; + progress.CommitAdvance(quorum, 1, state); assert_int_equal(progress.CommitIndex, 2); + progress.MatchIndex.clear(); + progress.CommitAdvance(quorum, 1, state); assert_int_equal(progress.CommitIndex, 2); +} +} +int main() { + const CMUnitTest tests[] = { + cmocka_unit_test(test_stable_quorum_matches_strict_majority_order_statistic), + cmocka_unit_test(test_all_small_joint_election_quorums), + cmocka_unit_test(test_joint_commit_and_read_watermarks_need_both_majorities), + cmocka_unit_test(test_invalid_configurations_are_rejected), + cmocka_unit_test(test_joint_commit_respects_both_majorities_and_current_term), + }; + return cmocka_run_group_tests(tests, nullptr, nullptr); +} From 5ae2203272e1e2dfc22129c6f0e877009d4d8ff5 Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Fri, 13 Nov 2026 14:47:59 -0500 Subject: [PATCH 25/66] Add joint snapshot metadata with fail-closed runtime guards (#25) --- README.md | 3 ++ docs/joint-snapshots.md | 46 +++++++++++++++++++ miniraft/durable.cpp | 2 + miniraft/raft.cpp | 35 ++++++++++++-- miniraft/snapshot.cpp | 48 ++++++++++++++++---- miniraft/snapshot.h | 5 ++ miniraft/state.h | 2 + test/test_compaction.cpp | 36 +++++++++++++++ test/test_snapshots.cpp | 98 ++++++++++++++++++++++++++++++++++++++++ 9 files changed, 263 insertions(+), 12 deletions(-) create mode 100644 docs/joint-snapshots.md diff --git a/README.md b/README.md index 2780ba2..982bdb0 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,9 @@ The [term lookup follow-up](docs/term-lookups.md) avoids copying full payloads when Raft needs only a retained entry's term. The [joint quorum foundation](docs/joint-quorum.md) adds and tests dual-majority arithmetic for future membership changes; live voter reconfiguration remains disabled. +The [joint snapshot metadata foundation](docs/joint-snapshots.md) adds a versioned +representation for a joint configuration at a snapshot boundary; runtime adoption +and membership changes remain disabled. The [message buffer follow-up](docs/message-extents.md) checks allocated buffer and batch extents before accepting locally constructed frames. diff --git a/docs/joint-snapshots.md b/docs/joint-snapshots.md new file mode 100644 index 0000000..a7b6211 --- /dev/null +++ b/docs/joint-snapshots.md @@ -0,0 +1,46 @@ +# Joint snapshot metadata foundation: v0.1.0-alpha.25 + +This step adds an experimental snapshot representation for a joint configuration +effective at the included snapshot boundary. MRFTS002 records the old and new +sorted voter sets plus the configuration entry's index and term. Its index must +be positive and no greater than the snapshot index; its term must be positive and +no greater than the snapshot term. Both voter sets are nonempty, positive, sorted, +unique and individually bounded. The codec validates structure, not whether the +configuration was committed or belongs to an authenticated cluster. + +The v2 byte layout is magic[8], snapshot index[8], snapshot term[8], old voter +count[4], application data length[8], configuration index[8], configuration +term[8], new voter count[4], old voter IDs[4 each], new voter IDs[4 each], opaque +application data, and CRC32[4] over all preceding bytes. Lengths and counts are +bounded before allocation. Fixed snapshots continue to encode as MRFTS001 with +exactly the previous bytes; a golden v1 image tests compatibility. The standalone +snapshot store can export and load v2 images for testing or offline tooling. + +This is a format foundation, not live joint consensus or complete configuration +history. Current Raft nodes reject joint snapshot restore, installation and +durable recovery; compaction still creates only fixed-voter snapshots. A covered +snapshot is acknowledged only after its prefix identifies the supported v1 +format, including when the prefix arrives in several chunks. Unsupported joint +images are rejected without replacing application data or publishing a joint +snapshot. A valid higher-term RPC header can still update durable hard state +before its unsupported snapshot contents are recognized. +These rules avoid falsely acknowledging a joint image merely because its index +is already committed locally. + +Older binaries do not understand MRFTS002 and may acknowledge an already-covered +image without checking its version. Do not send or activate joint snapshots in a +mixed-version cluster. Future live use requires an upgrade of every voter, a +defined compatibility protocol, replicated configuration entries, durable and +snapshot recovery of configuration history, transition rollback rules and dual +quorum handling throughout the protocol. This experimental v2 format may change +before those semantics are implemented. A standalone export is not proof that a +configuration is committed and is not an authoritative Raft checkpoint. + +Tests cover a golden v1 image, joint round-trip, every-byte corruption and +truncation, semantic corruption with valid CRC, bounds and ordering, standalone +publication, rejected memory/durable installation and durable recovery without +rewriting the image, and network rejection both above and below the commit boundary. +They also cover a supported v1 prefix split across chunks. The snapshot struct's +in-memory layout changes; rebuild the library and dependents together. Self-review +checked version dispatch, bounds, immutable voter binding, covered acknowledgments +and fail-closed recovery. No independent reviewer approval is claimed. diff --git a/miniraft/durable.cpp b/miniraft/durable.cpp index 54c8c5b..8e3e20e 100644 --- a/miniraft/durable.cpp +++ b/miniraft/durable.cpp @@ -167,6 +167,7 @@ TDurableState::TDurableState(const std::string& name, uint32_t id, THook hook) if (!length && compacted) throw std::runtime_error("Missing embedded snapshot"); if (length) { auto snapshot = DecodeSnapshot(std::string_view(reinterpret_cast(bytes.data() + position), size_t(length))); + if (snapshot.IsJoint()) throw std::runtime_error("Joint snapshot recovery is unsupported"); if (snapshot.Term > CurrentTerm) throw std::runtime_error("Future durable snapshot term"); if (configured && snapshot.Voters != Impl->Voters) throw std::runtime_error("Durable snapshot/voter mismatch"); position += size_t(length); @@ -302,6 +303,7 @@ uint64_t TDurableState::RetainedLogTerm(int64_t index) const { std::shared_ptr TDurableState::GetSnapshot() const { return Impl->Snapshot; } void TDurableState::InstallSnapshot(const TSnapshot& snapshot) { + if (snapshot.IsJoint()) throw std::invalid_argument("Joint snapshot installation is unsupported"); EncodeSnapshot(snapshot); if (!Impl->Voters.empty() && snapshot.Voters != Impl->Voters) throw std::invalid_argument("Snapshot would change bound voters"); diff --git a/miniraft/raft.cpp b/miniraft/raft.cpp index 13cca44..b390a27 100644 --- a/miniraft/raft.cpp +++ b/miniraft/raft.cpp @@ -198,6 +198,8 @@ TRaft::TRaft(std::shared_ptr state, int node, const TNodeDict& nodes, st if (!Learners.empty() && !State->SupportsVoterBinding()) throw std::invalid_argument("Learners require voter-binding-capable storage"); const auto bound = State->GetBoundVoters(); + if (auto snapshot = State->GetSnapshot(); snapshot && snapshot->IsJoint()) + throw std::invalid_argument("Joint snapshot recovery is unsupported"); const bool fresh = State->CurrentTerm == 1 && !State->VotedFor && !State->LastLogIndex; if (State->SupportsVoterBinding() && (!bound.empty() || fresh || State->GetSnapshot() || !Learners.empty())) State->BindVoters(Voters); @@ -793,8 +795,32 @@ void TRaft::OnInstallSnapshot(ITimeSource::Time now, TMessageHolderSnapshotIndex <= VolatileState->CommitIndex) { const auto term = State->LogTerm(request->SnapshotIndex); const bool covered = request->SnapshotIndex < State->SnapshotIndex || term == request->SnapshotTerm; - if (IncomingSnapshot && IncomingSnapshot->Index <= VolatileState->CommitIndex) IncomingSnapshot.reset(); - reply(covered, covered, covered ? request->TotalBytes : 0); + if (!covered) { reply(false, false, 0); return; } + // A covered image needs no application replacement, but must first be + // identified as a supported fixed-membership format. Accumulate a short + // prefix when a peer splits the magic across multiple valid chunks. + const bool same = IncomingSnapshot && IncomingSnapshot->Term == request->Term && + IncomingSnapshot->Leader == request->LeaderId && IncomingSnapshot->Index == request->SnapshotIndex && + IncomingSnapshot->BoundaryTerm == request->SnapshotTerm && IncomingSnapshot->Total == request->TotalBytes; + if (!same) { + if (request->Offset != 0 || (IncomingSnapshot && IncomingSnapshot->Term == request->Term && + IncomingSnapshot->Index > request->SnapshotIndex)) { reply(false, false, 0); return; } + IncomingSnapshot = TIncomingSnapshot{request->Term, request->SnapshotIndex, request->SnapshotTerm, + request->TotalBytes, request->LeaderId, {}}; + } + auto& prefix = IncomingSnapshot->Bytes; + const auto count = request->Len - sizeof(TInstallSnapshotRequest); + if (request->Offset > prefix.size()) { reply(false, false, 0); return; } + if (request->Offset < prefix.size()) { + if (count > prefix.size() - request->Offset || + std::string_view(prefix).substr(request->Offset, count) != std::string_view(request->Data, count)) { + IncomingSnapshot.reset(); reply(false, false, 0); return; + } + } else prefix.append(request->Data, count); + if (prefix.size() < 8) { reply(true, false, prefix.size()); return; } + const bool supported = std::string_view(prefix).substr(0, 8) == "MRFTS001"; + IncomingSnapshot.reset(); + reply(supported, supported, supported ? request->TotalBytes : 0); return; } auto application = SnapshotApplication.lock(); @@ -817,11 +843,14 @@ void TRaft::OnInstallSnapshot(ITimeSource::Time now, TMessageHolderData, count); + if (bytes.size() >= 8 && std::string_view(bytes).substr(0, 8) != "MRFTS001") { + IncomingSnapshot.reset(); reply(false, false, 0); return; + } if (bytes.size() < request->TotalBytes) { reply(true, false, bytes.size()); return; } TSnapshot snapshot; try { snapshot = DecodeSnapshot(bytes); - if (snapshot.Index != request->SnapshotIndex || snapshot.Term != request->SnapshotTerm || snapshot.Voters != GetVoterIds()) + if (snapshot.IsJoint() || snapshot.Index != request->SnapshotIndex || snapshot.Term != request->SnapshotTerm || snapshot.Voters != GetVoterIds()) throw TMessageError("Snapshot metadata mismatch"); application->ValidateSnapshot(snapshot.Data, snapshot.Index); } catch (const std::exception&) { diff --git a/miniraft/snapshot.cpp b/miniraft/snapshot.cpp index b2d1a10..cf3de33 100644 --- a/miniraft/snapshot.cpp +++ b/miniraft/snapshot.cpp @@ -15,20 +15,33 @@ namespace { using namespace NSnapshotCodec; constexpr std::string_view Magic = "MRFTS001"; +constexpr std::string_view JointMagic = "MRFTS002"; void Validate(const TSnapshot& snapshot) { if (!snapshot.Index || snapshot.Index > uint64_t(INT64_MAX) || !snapshot.Term || snapshot.Voters.empty() || snapshot.Voters.size() > TSnapshot::MaxVoters || - snapshot.Data.size() > TSnapshot::MaxBytes - 40 - 4 * snapshot.Voters.size()) + snapshot.JointVoters.size() > TSnapshot::MaxVoters || + snapshot.Data.size() > TSnapshot::MaxBytes - (snapshot.IsJoint() ? 60 : 40) - + 4 * (snapshot.Voters.size() + snapshot.JointVoters.size())) throw std::runtime_error("Invalid snapshot metadata/size"); - uint32_t previous = 0; - for (auto voter : snapshot.Voters) { - if (voter <= previous) throw std::runtime_error("Snapshot voters must be positive, sorted and unique"); - previous = voter; + const auto validateVoters = [](const auto& voters) { + uint32_t previous = 0; + for (auto voter : voters) { + if (voter <= previous) throw std::runtime_error("Snapshot voters must be positive, sorted and unique"); + previous = voter; + } + }; + validateVoters(snapshot.Voters); + if (snapshot.IsJoint()) { + if (snapshot.JointVoters.empty() || !snapshot.ConfigurationIndex || + snapshot.ConfigurationIndex > snapshot.Index || !snapshot.ConfigurationTerm || + snapshot.ConfigurationTerm > snapshot.Term) + throw std::runtime_error("Invalid joint snapshot configuration boundary"); + validateVoters(snapshot.JointVoters); } } void CheckLocalLog(const TSnapshot& snapshot, const TRaft& raft) { Validate(snapshot); - if (snapshot.Voters != raft.GetVoterIds() || snapshot.Index > raft.GetLastIndex() || + if (snapshot.IsJoint() || snapshot.Voters != raft.GetVoterIds() || snapshot.Index > raft.GetLastIndex() || snapshot.Term > raft.GetState()->CurrentTerm || raft.GetState()->LogTerm(snapshot.Index) != snapshot.Term) throw std::runtime_error("Snapshot does not match retained log and fixed configuration"); @@ -54,12 +67,18 @@ void Sync(int fd) { std::string EncodeSnapshot(const TSnapshot& snapshot) { Validate(snapshot); - std::string out(Magic); + std::string out(snapshot.IsJoint() ? JointMagic : Magic); Put(out, snapshot.Index, 8); Put(out, snapshot.Term, 8); Put(out, snapshot.Voters.size(), 4); Put(out, snapshot.Data.size(), 8); + if (snapshot.IsJoint()) { + Put(out, snapshot.ConfigurationIndex, 8); + Put(out, snapshot.ConfigurationTerm, 8); + Put(out, snapshot.JointVoters.size(), 4); + } for (auto voter : snapshot.Voters) Put(out, voter, 4); + for (auto voter : snapshot.JointVoters) Put(out, voter, 4); Append(out, snapshot.Data); Put(out, Checksum(out), 4); return out; @@ -72,15 +91,26 @@ TSnapshot DecodeSnapshot(std::string_view bytes) { if (Checksum(bytes.substr(0, bytes.size() - 4)) != trailer.Take(4)) throw std::runtime_error("Snapshot checksum mismatch"); TReader reader{bytes.substr(0, bytes.size() - 4)}; - if (reader.Read(Magic.size()) != Magic) throw std::runtime_error("Unknown snapshot format"); + const auto magic = reader.Read(Magic.size()); + const bool joint = magic == JointMagic; + if (!joint && magic != Magic) throw std::runtime_error("Unknown snapshot format"); TSnapshot snapshot; snapshot.Index = reader.Take(8); snapshot.Term = reader.Take(8); const auto count = reader.Take(4); const auto length = reader.Take(8); - if (count > TSnapshot::MaxVoters || count > reader.Input.size() / 4) + uint64_t newCount = 0; + if (joint) { + snapshot.ConfigurationIndex = reader.Take(8); + snapshot.ConfigurationTerm = reader.Take(8); + newCount = reader.Take(4); + } + if (count > TSnapshot::MaxVoters || newCount > TSnapshot::MaxVoters || + count + newCount > reader.Input.size() / 4) throw std::runtime_error("Invalid snapshot voter count"); for (uint32_t i = 0; i < count; ++i) snapshot.Voters.push_back(reader.Take(4)); + for (uint32_t i = 0; i < newCount; ++i) snapshot.JointVoters.push_back(reader.Take(4)); + if (joint && !snapshot.IsJoint()) throw std::runtime_error("Empty joint snapshot configuration"); if (length > reader.Input.size()) throw std::runtime_error("Invalid snapshot data length"); snapshot.Data = reader.Read(size_t(length)); if (!reader.Input.empty()) throw std::runtime_error("Trailing snapshot data"); diff --git a/miniraft/snapshot.h b/miniraft/snapshot.h index 710fda0..ef819df 100644 --- a/miniraft/snapshot.h +++ b/miniraft/snapshot.h @@ -18,6 +18,11 @@ struct TSnapshot { uint64_t Index = 0, Term = 0; std::vector Voters; std::string Data; + // Optional joint configuration effective at this snapshot boundary. + // Runtime installation is disabled until membership recovery is implemented. + std::vector JointVoters; + uint64_t ConfigurationIndex = 0, ConfigurationTerm = 0; + bool IsJoint() const { return !JointVoters.empty() || ConfigurationIndex || ConfigurationTerm; } bool operator==(const TSnapshot&) const = default; }; std::string EncodeSnapshot(const TSnapshot& snapshot); diff --git a/miniraft/state.h b/miniraft/state.h index 85f246a..9b13f2e 100644 --- a/miniraft/state.h +++ b/miniraft/state.h @@ -64,6 +64,7 @@ inline void ValidateVoterBinding(const IState& state, const std::vectorIsJoint()) throw std::invalid_argument("Joint snapshot recovery is unsupported"); if (snapshot->Voters != voters) throw std::invalid_argument("Snapshot voter configuration mismatch"); } else if (state.CurrentTerm != 1 || state.VotedFor || state.LastLogIndex) { throw std::invalid_argument("Unbound nonfresh store needs a verified snapshot before enabling learners"); @@ -136,6 +137,7 @@ struct TState: IState { } std::shared_ptr GetSnapshot() const override { return Snapshot; } void InstallSnapshot(const TSnapshot& snapshot) override { + if (snapshot.IsJoint()) throw std::invalid_argument("Joint snapshot installation is unsupported"); EncodeSnapshot(snapshot); // Validate before changing any state. if (!BoundVoters.empty() && snapshot.Voters != BoundVoters) throw std::invalid_argument("Snapshot would change bound voters"); diff --git a/test/test_compaction.cpp b/test/test_compaction.cpp index 8c157a7..ddef6a7 100644 --- a/test/test_compaction.cpp +++ b/test/test_compaction.cpp @@ -314,6 +314,41 @@ TMessageHolder Chunk(const TSnapshot& snapshot, std::st message->Offset = offset; message->TotalBytes = bytes.size(); message->Done = offset + count == bytes.size(); message->LeaderId = 1; memcpy(message->Data, bytes.data() + offset, count); return message; } +void test_joint_snapshot_is_rejected_even_if_boundary_is_already_covered(void**) { + TKv source; source.Write(source.Prepare(Command("a", "old")), 5); + TSnapshot fixed{5, 1, {1, 2}, source.SaveSnapshot()}; + std::vector> replies; + auto peer = std::make_shared([&](auto message) { replies.push_back(message.template Cast()); }); + auto state = std::make_shared(); + auto raft = std::make_shared(state, 2, TNodeDict{{1, peer}}); + auto app = std::make_shared(); TRequestProcessor processor(raft, app, {}); + auto fixedBytes = EncodeSnapshot(fixed); + raft->Process({}, Chunk(fixed, fixedBytes, 0, fixedBytes.size())); + assert_true(replies.back()->Installed); assert_int_equal(app->Loads, 1); + auto joint = fixed; joint.JointVoters = {2, 3}; joint.ConfigurationIndex = 4; joint.ConfigurationTerm = 1; + const auto bytes = EncodeSnapshot(joint); + const auto before = static_cast(*state); + raft->Process({}, Chunk(joint, bytes, 0, bytes.size())); + assert_false(replies.back()->Success); assert_false(replies.back()->Installed); + assert_true(static_cast(*state) == before); assert_int_equal(app->Loads, 1); + // A supported covered snapshot can also identify its magic across chunks. + raft->Process({}, Chunk(fixed, fixedBytes, 0, 4)); + assert_true(replies.back()->Success); assert_false(replies.back()->Installed); + assert_int_equal(replies.back()->NextOffset, 4); + raft->Process({}, Chunk(fixed, fixedBytes, 4, 4)); + assert_true(replies.back()->Success); assert_true(replies.back()->Installed); + assert_int_equal(replies.back()->NextOffset, fixedBytes.size()); + raft->Process({}, Chunk(joint, bytes, 0, 4)); + assert_true(replies.back()->Success); assert_false(replies.back()->Installed); + raft->Process({}, Chunk(joint, bytes, 4, 4)); + assert_false(replies.back()->Success); assert_false(replies.back()->Installed); + joint.Index = 6; + const auto newer = EncodeSnapshot(joint); + raft->Process({}, Chunk(joint, newer, 0, newer.size())); + assert_false(replies.back()->Success); assert_false(replies.back()->Installed); + assert_true(static_cast(*state) == before); assert_int_equal(app->Loads, 1); + assert_true(Read(*app, "a") == "old"); +} void test_corrupt_snapshot_and_invalid_headers_cannot_mutate_application(void**) { TKv source; source.Write(source.Prepare(Command("a", "new")), 5); TSnapshot snapshot{5, 1, {1, 2}, source.SaveSnapshot()}; @@ -515,6 +550,7 @@ int main() { cmocka_unit_test(test_partial_and_installed_snapshot_acks_do_not_confirm_strong_reads), cmocka_unit_test(test_compacted_follower_hint_requires_boundary_probe_before_match), cmocka_unit_test(test_corrupt_snapshot_and_invalid_headers_cannot_mutate_application), + cmocka_unit_test(test_joint_snapshot_is_rejected_even_if_boundary_is_already_covered), cmocka_unit_test(test_remote_snapshot_retains_suffix_without_applying_it_before_commit), cmocka_unit_test(test_snapshot_boundary_index_exhaustion_does_not_append_wrapped_noop), cmocka_unit_test(test_memory_snapshot_boundary_retains_matching_suffix_and_replaces_conflict), diff --git a/test/test_snapshots.cpp b/test/test_snapshots.cpp index e875cac..a60ba52 100644 --- a/test/test_snapshots.cpp +++ b/test/test_snapshots.cpp @@ -46,6 +46,12 @@ void Apply(TKv& kv, const std::string& key, const std::string& value, uint64_t i kv.Write(kv.Prepare(Command(key, value)), index); } TSnapshot Sample() { return {2, 1, {1, 2, 3}, std::string("a\0b", 3)}; } +TSnapshot JointSample() { + auto snapshot = Sample(); + snapshot.JointVoters = {3, 4, 5}; + snapshot.ConfigurationIndex = 1; snapshot.ConfigurationTerm = 1; + return snapshot; +} void FixChecksum(std::string& bytes) { bytes.resize(bytes.size() - 4); NSnapshotCodec::Put(bytes, NSnapshotCodec::Checksum(bytes), 4); @@ -74,6 +80,65 @@ void test_snapshot_codec_roundtrip_and_all_byte_corruption(void**) { Reject([&] { DecodeSnapshot(std::string_view(encoded).substr(0, i)); }); } } +void test_joint_snapshot_codec_roundtrip_and_corruption(void**) { + auto fixed = EncodeSnapshot(Sample()); + // Golden v1 image, including the original CRC and embedded NUL byte. + const std::string hex = "4d5246545330303102000000000000000100000000000000030000000300000000000000010000000200000003000000610062e0c541c6"; + std::string golden; + for (size_t i = 0; i < hex.size(); i += 2) golden.push_back(char(std::stoul(hex.substr(i, 2), nullptr, 16))); + assert_true(fixed == golden); + assert_true(DecodeSnapshot(golden) == Sample()); + assert_false(DecodeSnapshot(fixed).IsJoint()); + const auto original = JointSample(); + const auto encoded = EncodeSnapshot(original); + assert_true(encoded.substr(0, 8) == "MRFTS002"); + assert_true(DecodeSnapshot(encoded) == original); + for (size_t i = 0; i < encoded.size(); ++i) { + auto damaged = encoded; damaged[i] ^= 1; + Reject([&] { DecodeSnapshot(damaged); }); + Reject([&] { DecodeSnapshot(std::string_view(encoded).substr(0, i)); }); + } +} +void test_joint_snapshot_metadata_rejects_invalid_boundaries_and_voters(void**) { + for (int mode = 0; mode < 9; ++mode) { + auto bad = JointSample(); + if (mode == 0) bad.JointVoters.clear(); + if (mode == 1) bad.JointVoters = {0, 1}; + if (mode == 2) bad.JointVoters = {1, 1}; + if (mode == 3) bad.JointVoters = {2, 1}; + if (mode == 4) bad.JointVoters.resize(TSnapshot::MaxVoters + 1, 1); + if (mode == 5) bad.ConfigurationIndex = 0; + if (mode == 6) bad.ConfigurationIndex = bad.Index + 1; + if (mode == 7) bad.ConfigurationTerm = 0; + if (mode == 8) bad.ConfigurationTerm = bad.Term + 1; + Reject([&] { EncodeSnapshot(bad); }); + } + for (size_t offset : {size_t(36), size_t(44), size_t(52)}) { + auto damaged = EncodeSnapshot(JointSample()); + const auto width = offset == 52 ? 4 : 8; + for (size_t i = offset; i < offset + width; ++i) damaged[i] = char(0xff); + FixChecksum(damaged); Reject([&] { DecodeSnapshot(damaged); }); + } + auto empty = EncodeSnapshot(JointSample()); + for (size_t i = 36; i < 56; ++i) empty[i] = 0; + FixChecksum(empty); Reject([&] { DecodeSnapshot(empty); }); +} +void test_joint_snapshot_cannot_be_installed_or_restored_by_fixed_runtime(void**) { + auto snapshot = JointSample(); + auto state = std::make_shared(); + state->Append(NewHoldedMessage()); state->Append(NewHoldedMessage()); + const auto before = static_cast(*state); + Reject([&] { state->InstallSnapshot(snapshot); }); + assert_true(static_cast(*state) == before); + TRaft raft(state, 1, {{2, std::make_shared()}, {3, std::make_shared()}}); + TKv app; + Reject([&] { RestoreSnapshot(snapshot, raft, app); }); + assert_int_equal(app.LastAppliedIndex, 0); + auto recovered = std::make_shared(); + recovered->Snapshot = std::make_shared(snapshot); + Reject([&] { TRaft unsupported(recovered, 1, {{2, std::make_shared()}, {3, std::make_shared()}}); }); + assert_true(recovered->GetBoundVoters().empty()); +} void test_snapshot_metadata_and_declared_lengths_are_bounded(void**) { auto snapshot = Sample(); for (uint64_t index : {uint64_t(0), UINT64_MAX}) { @@ -175,6 +240,35 @@ void test_restore_rejects_mismatched_log_configuration_and_used_application(void assert_true(Read(restored, "a") == "one"); } #ifdef __linux__ +void test_joint_snapshot_export_roundtrips_but_durable_state_rejects_it(void**) { + TFiles files; + { TSnapshotStore exportStore(files.Name, 1); exportStore.Save(JointSample()); + assert_true(exportStore.Load().value() == JointSample()); } + const auto path = files.Name + ".durable.1"; + { + auto state = std::make_shared(files.Name, 1); + TRaft raft(state, 1, {{2, std::make_shared()}, {3, std::make_shared()}}); + const auto before = static_cast(*state); + std::ifstream input(path, std::ios::binary); std::string bytes(std::istreambuf_iterator(input), {}); input.close(); + Reject([&] { state->InstallSnapshot(JointSample()); }); + assert_true(static_cast(*state) == before); + std::ifstream after(path, std::ios::binary); + assert_true(std::string(std::istreambuf_iterator(after), {}) == bytes); + } + // Synthesize a well-checksummed v3 store containing the unsupported image. + std::ifstream input(path, std::ios::binary); std::string bytes(std::istreambuf_iterator(input), {}); input.close(); + assert_true(bytes.substr(0, 8) == "MRFTD003"); + const auto image = EncodeSnapshot(JointSample()); + for (int i = 0; i < 8; ++i) bytes[56 + i] = char(uint64_t(image.size()) >> (8 * i)); + bytes.insert(64, image); + for (int i = 0; i < 8; ++i) bytes[8 + i] = char(uint64_t(bytes.size() - 20) >> (8 * i)); + const auto crc = NSnapshotCodec::Checksum(bytes.substr(0, 8) + bytes.substr(20)); + for (int i = 0; i < 4; ++i) bytes[16 + i] = char(crc >> (8 * i)); + { std::ofstream output(path, std::ios::binary | std::ios::trunc); output.write(bytes.data(), bytes.size()); } + Reject([&] { TDurableState state(files.Name, 1); }); + std::ifstream after(path, std::ios::binary); + assert_true(std::string(std::istreambuf_iterator(after), {}) == bytes); +} void test_snapshot_store_rejects_oversized_files_and_temp_symlinks(void**) { TFiles files; TSnapshotStore store(files.Name, 1); @@ -287,12 +381,16 @@ void test_durable_kv_restart_restores_checkpoint_and_replays_only_tail(void**) { int main() { const CMUnitTest tests[] = { cmocka_unit_test(test_snapshot_codec_roundtrip_and_all_byte_corruption), + cmocka_unit_test(test_joint_snapshot_codec_roundtrip_and_corruption), + cmocka_unit_test(test_joint_snapshot_metadata_rejects_invalid_boundaries_and_voters), + cmocka_unit_test(test_joint_snapshot_cannot_be_installed_or_restored_by_fixed_runtime), cmocka_unit_test(test_snapshot_metadata_and_declared_lengths_are_bounded), cmocka_unit_test(test_kv_snapshot_is_canonical_and_binary_safe), cmocka_unit_test(test_kv_invalid_snapshot_preserves_existing_state_and_progress), cmocka_unit_test(test_capture_requires_committed_applied_progress), cmocka_unit_test(test_restore_rejects_mismatched_log_configuration_and_used_application), #ifdef __linux__ + cmocka_unit_test(test_joint_snapshot_export_roundtrips_but_durable_state_rejects_it), cmocka_unit_test(test_snapshot_store_rejects_oversized_files_and_temp_symlinks), cmocka_unit_test(test_snapshot_store_roundtrip_lock_and_corrupt_recovery), cmocka_unit_test(test_snapshot_crash_at_each_publication_boundary), From db5794b50e1b8378eaf57605d523453142d73fc7 Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Fri, 13 Nov 2026 14:54:24 -0500 Subject: [PATCH 26/66] Add fail-closed configuration entry foundation (#26) --- README.md | 2 + docs/configuration-entries.md | 50 +++++++++ miniraft/CMakeLists.txt | 1 + miniraft/configuration.cpp | 68 ++++++++++++ miniraft/configuration.h | 17 +++ miniraft/durable.cpp | 5 + miniraft/durable.h | 1 + miniraft/messages.h | 4 +- miniraft/raft.cpp | 12 +++ miniraft/state.h | 11 ++ test/CMakeLists.txt | 1 + test/test_configuration.cpp | 195 ++++++++++++++++++++++++++++++++++ 12 files changed, 366 insertions(+), 1 deletion(-) create mode 100644 docs/configuration-entries.md create mode 100644 miniraft/configuration.cpp create mode 100644 miniraft/configuration.h create mode 100644 test/test_configuration.cpp diff --git a/README.md b/README.md index 982bdb0..b8e7be9 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,8 @@ arithmetic for future membership changes; live voter reconfiguration remains dis The [joint snapshot metadata foundation](docs/joint-snapshots.md) adds a versioned representation for a joint configuration at a snapshot boundary; runtime adoption and membership changes remain disabled. +The [configuration entry foundation](docs/configuration-entries.md) defines an +experimental entry codec and rejects that entry kind in the fixed-membership runtime. The [message buffer follow-up](docs/message-extents.md) checks allocated buffer and batch extents before accepting locally constructed frames. diff --git a/docs/configuration-entries.md b/docs/configuration-entries.md new file mode 100644 index 0000000..7d1d3ec --- /dev/null +++ b/docs/configuration-entries.md @@ -0,0 +1,50 @@ +# Configuration entry foundation: v0.1.0-alpha.26 + +This milestone defines an experimental representation for a joint or stable +configuration log entry. It does not enable live membership changes. Each record +contains its phase, entry term, previous configuration entry index and term, and +the old and new sorted voter sets. The stable record carries both sets as context; +future transition logic must decide how to validate and apply it. + +The MRFTC001 payload is magic[8], phase[4] (joint=1, stable=2), entry term[8], +previous configuration index[8], previous configuration term[8], old count[4], +new count[4], old voter IDs[4 each], new voter IDs[4 each], and CRC32[4] over +the preceding payload. The entry's outer term must equal the encoded term, and +its flags must equal EConfiguration, the high bit of the 64-bit flags field. +Counts, lengths, ordering and bounds are checked before allocation. A joint +entry may refer to an implicit bootstrap configuration with zero previous index +and term; a stable entry requires a nonzero predecessor. The codec checks shape, +not log position, predecessor identity, commitment, role eligibility or cluster +authentication. CRC is an accidental-corruption check, not authentication. + +Current Raft rejects entries carrying the reserved bit in local proposals and +incoming AppendEntries, before changing term or log. Startup scans the retained +suffix and rejects such entries before voter binding or serving traffic; the +application path also refuses them. Joint snapshots remain rejected as described +in [joint snapshot metadata](joint-snapshots.md). The codec and storage backends +can represent records for deterministic testing and offline tools, but they are +not an active replication or recovery protocol. + +Compatibility matters: previous versions allowed custom entry flags. Before +upgrading, audit applications and retained logs for use of the high bit. A node +with that bit in its retained suffix now refuses startup; this release does not +rewrite or migrate it. A fixed snapshot may already contain effects of older +entries, which this suffix scan cannot detect. Older binaries may apply an unknown +configuration entry as application data, so no future version should replicate +these entries until every voter runs a compatible implementation. Do not delete +or edit committed state to bypass rejection. + +The optimized memory and durable backends scan flags without copying payloads. +External and legacy backends use the default Get-based fallback, so startup can +read the full retained log; the interface's vtable changed and dependent binaries +must be rebuilt. Backends that override Get with different semantics must make +RetainedLogFlags consistent. No existing durable file or wire frame layout changed. + +Tests cover both phases, bootstrap and maximum-size records, every-byte corruption +and truncation, semantic corruption with valid CRC, malformed frames, rejected RPCs +and proposals, application isolation, compacted suffixes and restart rejection +without rewriting durable or legacy files. Self-review checked fail-closed ordering, +the flag scan, predecessor shape and compatibility limits. No independent reviewer +approval is claimed. Dynamic promotion/removal, configuration activation and +rollback, joint snapshot recovery, discovery and full configuration history remain +open; this experimental encoding may change when those semantics are implemented. diff --git a/miniraft/CMakeLists.txt b/miniraft/CMakeLists.txt index e2a1ab5..58a1a6f 100644 --- a/miniraft/CMakeLists.txt +++ b/miniraft/CMakeLists.txt @@ -4,6 +4,7 @@ add_library(miniraft persist.cpp durable.cpp snapshot.cpp + configuration.cpp ) target_include_directories(miniraft PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/..) diff --git a/miniraft/configuration.cpp b/miniraft/configuration.cpp new file mode 100644 index 0000000..3477085 --- /dev/null +++ b/miniraft/configuration.cpp @@ -0,0 +1,68 @@ +#include "configuration.h" +#include "snapshot_codec.h" +#include "state.h" +#include +#include +#include +#include + +namespace { +using namespace NSnapshotCodec; +constexpr std::string_view Magic = "MRFTC001"; +constexpr size_t MaxPayload = 48 + 8 * TSnapshot::MaxVoters; +void Validate(const TConfigurationRecord& record) { + if ((record.Phase != EConfigurationPhase::Joint && record.Phase != EConfigurationPhase::Stable) || + !record.Term || record.PreviousIndex > uint64_t(INT64_MAX) || + ((record.PreviousIndex == 0) != (record.PreviousTerm == 0)) || + record.PreviousTerm > record.Term || + (record.Phase == EConfigurationPhase::Stable && !record.PreviousIndex)) + throw std::invalid_argument("Invalid configuration entry metadata"); + ValidateFixedVoters(record.OldVoters); + ValidateFixedVoters(record.NewVoters); +} +} + +TMessageHolder EncodeConfigurationEntry(const TConfigurationRecord& record) { + Validate(record); + std::string bytes(Magic); + Put(bytes, uint32_t(record.Phase), 4); + Put(bytes, record.Term, 8); + Put(bytes, record.PreviousIndex, 8); + Put(bytes, record.PreviousTerm, 8); + Put(bytes, record.OldVoters.size(), 4); + Put(bytes, record.NewVoters.size(), 4); + for (auto id : record.OldVoters) Put(bytes, id, 4); + for (auto id : record.NewVoters) Put(bytes, id, 4); + Put(bytes, Checksum(bytes), 4); + auto entry = NewHoldedMessage(sizeof(TLogEntry) + bytes.size()); + entry->Term = record.Term; entry->Flags = TLogEntry::EConfiguration; + memcpy(entry->Data, bytes.data(), bytes.size()); + return entry; +} + +TConfigurationRecord DecodeConfigurationEntry(TMessageHolder entry) { + if (!entry.HasValidExtent() || entry->Type != uint32_t(EMessageType::LOG_ENTRY) || + entry->Len < sizeof(TLogEntry) || entry->Flags != TLogEntry::EConfiguration || entry.PayloadSize) + throw std::invalid_argument("Invalid configuration entry frame"); + const auto length = entry->Len - sizeof(TLogEntry); + if (length < 56 || length > MaxPayload) throw std::invalid_argument("Invalid configuration entry size"); + const std::string_view bytes(entry->Data, length); + TReader trailer{bytes.substr(length - 4)}; + if (Checksum(bytes.substr(0, length - 4)) != trailer.Take(4)) + throw std::invalid_argument("Configuration entry checksum mismatch"); + TReader reader{bytes.substr(0, length - 4)}; + if (reader.Read(Magic.size()) != Magic) throw std::invalid_argument("Unknown configuration entry format"); + TConfigurationRecord record; + record.Phase = EConfigurationPhase(reader.Take(4)); + record.Term = reader.Take(8); record.PreviousIndex = reader.Take(8); record.PreviousTerm = reader.Take(8); + const auto oldCount = reader.Take(4), newCount = reader.Take(4); + if (oldCount > TSnapshot::MaxVoters || newCount > TSnapshot::MaxVoters || + oldCount + newCount > reader.Input.size() / 4) + throw std::invalid_argument("Invalid configuration voter counts"); + for (uint64_t i = 0; i < oldCount; ++i) record.OldVoters.push_back(reader.Take(4)); + for (uint64_t i = 0; i < newCount; ++i) record.NewVoters.push_back(reader.Take(4)); + if (!reader.Input.empty() || entry->Term != record.Term) + throw std::invalid_argument("Configuration entry term or length mismatch"); + Validate(record); + return record; +} diff --git a/miniraft/configuration.h b/miniraft/configuration.h new file mode 100644 index 0000000..482b492 --- /dev/null +++ b/miniraft/configuration.h @@ -0,0 +1,17 @@ +#pragma once + +#include "messages.h" +#include +#include + +// Experimental record encoding only. Current Raft rejects these entries. +enum class EConfigurationPhase : uint32_t { Joint = 1, Stable = 2 }; +struct TConfigurationRecord { + EConfigurationPhase Phase = EConfigurationPhase::Joint; + uint64_t Term = 0, PreviousIndex = 0, PreviousTerm = 0; + std::vector OldVoters, NewVoters; + bool operator==(const TConfigurationRecord&) const = default; +}; + +TMessageHolder EncodeConfigurationEntry(const TConfigurationRecord& record); +TConfigurationRecord DecodeConfigurationEntry(TMessageHolder entry); diff --git a/miniraft/durable.cpp b/miniraft/durable.cpp index 8e3e20e..603fae2 100644 --- a/miniraft/durable.cpp +++ b/miniraft/durable.cpp @@ -299,6 +299,11 @@ uint64_t TDurableState::RetainedLogTerm(int64_t index) const { if (index < 1 || uint64_t(index) <= SnapshotIndex || uint64_t(index) > LastLogIndex) return 0; return Impl->Log[index - SnapshotIndex - 1]->Term; } +uint64_t TDurableState::RetainedLogFlags(int64_t index) const { + if (index < 1 || uint64_t(index) <= SnapshotIndex || uint64_t(index) > LastLogIndex) + throw std::out_of_range("Missing retained log entry"); + return Impl->Log[index - SnapshotIndex - 1]->Flags; +} std::shared_ptr TDurableState::GetSnapshot() const { return Impl->Snapshot; } diff --git a/miniraft/durable.h b/miniraft/durable.h index a97d03e..8c4ea24 100644 --- a/miniraft/durable.h +++ b/miniraft/durable.h @@ -20,6 +20,7 @@ class TDurableState : public IState { void Append(TMessageHolder entry) override; TMessageHolder Get(int64_t index) const override; uint64_t RetainedLogTerm(int64_t index) const override; + uint64_t RetainedLogFlags(int64_t index) const override; void Commit() override; bool SupportsSnapshots() const override { return true; } std::shared_ptr GetSnapshot() const override; diff --git a/miniraft/messages.h b/miniraft/messages.h index f6fc134..8bbd6de 100644 --- a/miniraft/messages.h +++ b/miniraft/messages.h @@ -44,9 +44,11 @@ static_assert(sizeof(TMessage) == 8); struct TLogEntry: public TMessage { static constexpr EMessageType MessageType = EMessageType::LOG_ENTRY; - enum EFlags { + enum EFlags : uint64_t { ENone = 0, EStub = 1, + // Reserved for configuration entries; live use remains disabled. + EConfiguration = uint64_t(1) << 63, }; uint64_t Term = 1; uint64_t Flags = 0; diff --git a/miniraft/raft.cpp b/miniraft/raft.cpp index b390a27..61202c2 100644 --- a/miniraft/raft.cpp +++ b/miniraft/raft.cpp @@ -200,6 +200,13 @@ TRaft::TRaft(std::shared_ptr state, int node, const TNodeDict& nodes, st const auto bound = State->GetBoundVoters(); if (auto snapshot = State->GetSnapshot(); snapshot && snapshot->IsJoint()) throw std::invalid_argument("Joint snapshot recovery is unsupported"); + if (State->SnapshotIndex > State->LastLogIndex || State->LastLogIndex > uint64_t(INT64_MAX)) + throw std::invalid_argument("Invalid retained log bounds"); + // Reject before binding voters or serving traffic. Current Raft cannot + // interpret configuration entries, including uncommitted suffix entries. + for (uint64_t index = State->SnapshotIndex + 1; index <= State->LastLogIndex; ++index) + if (State->RetainedLogFlags(index) & TLogEntry::EConfiguration) + throw std::invalid_argument("Configuration log recovery is unsupported"); const bool fresh = State->CurrentTerm == 1 && !State->VotedFor && !State->LastLogIndex; if (State->SupportsVoterBinding() && (!bound.empty() || fresh || State->GetSnapshot() || !Learners.empty())) State->BindVoters(Voters); @@ -539,6 +546,7 @@ bool TRaft::ValidateRpc(TMessageHolder message) const { auto previousTerm = append->PrevLogTerm; for (uint32_t i = 0; i < message.PayloadSize; ++i) { auto entry = message.Payload[i].Cast(); + if (entry->Flags & TLogEntry::EConfiguration) return false; if (!entry->Term || entry->Term < previousTerm || entry->Term > header->Term) return false; previousTerm = entry->Term; const auto index = append->PrevLogIndex + i + 1; @@ -688,6 +696,8 @@ uint64_t TRaft::Append(TMessageHolder entry) { if (!entry.HasValidExtent() || entry->Type != uint32_t(EMessageType::LOG_ENTRY)) throw TMessageError("Invalid proposed log entry"); ValidateMessageTree(entry); + if (entry->Flags & TLogEntry::EConfiguration) + throw TMessageError("Configuration proposals are unsupported"); entry->Term = State->CurrentTerm; State->Append(std::move(entry)); return State->LastLogIndex; @@ -1090,6 +1100,8 @@ void TRequestProcessor::ProcessCommitted() { auto& state = Raft->GetState(); for (auto i = Rsm->LastAppliedIndex+1; i <= commitIndex; i++) { auto entry = state->Get(i); + if (entry->Flags & TLogEntry::EConfiguration) + throw std::logic_error("Configuration application is unsupported"); if (entry->Flags == TLogEntry::EStub) { continue; } diff --git a/miniraft/state.h b/miniraft/state.h index 9b13f2e..f8ac489 100644 --- a/miniraft/state.h +++ b/miniraft/state.h @@ -27,6 +27,12 @@ struct IState: TStateFields { auto entry = Get(index); return entry ? entry->Term : 0; } + // Used during startup to reject unsupported retained entry kinds. + virtual uint64_t RetainedLogFlags(int64_t index) const { + auto entry = Get(index); + if (!entry) throw std::runtime_error("Missing retained log entry"); + return entry->Flags; + } virtual void Commit() = 0; virtual bool SupportsSnapshots() const { return false; } virtual std::shared_ptr GetSnapshot() const { return {}; } @@ -127,6 +133,11 @@ struct TState: IState { if (index < 1 || uint64_t(index) <= SnapshotIndex || uint64_t(index) > LastLogIndex) return 0; return Log[index - SnapshotIndex - 1]->Term; } + uint64_t RetainedLogFlags(int64_t index) const override { + if (index < 1 || uint64_t(index) <= SnapshotIndex || uint64_t(index) > LastLogIndex) + throw std::out_of_range("Missing retained log entry"); + return Log[index - SnapshotIndex - 1]->Flags; + } bool SupportsSnapshots() const override { return true; } bool SupportsVoterBinding() const override { return true; } diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 7405616..7ec08da 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -14,6 +14,7 @@ endmacro() if (CMOCKA_FOUND) ut(test_raft test_raft.cpp miniraft) ut(test_quorum test_quorum.cpp miniraft) +ut(test_configuration test_configuration.cpp miniraft) ut(test_correctness test_correctness.cpp miniraft) ut(test_forwarding test_forwarding.cpp miniraft) ut(test_message_safety test_message_safety.cpp miniraft) diff --git a/test/test_configuration.cpp b/test/test_configuration.cpp new file mode 100644 index 0000000..6e4943d --- /dev/null +++ b/test/test_configuration.cpp @@ -0,0 +1,195 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +extern "C" { +#include +} +namespace { +template void Reject(F f) { + bool rejected = false; + try { f(); } catch (const std::exception&) { rejected = true; } + assert_true(rejected); +} +TConfigurationRecord Joint() { + return {EConfigurationPhase::Joint, 3, 2, 2, {1, 2, 3}, {3, 4, 5}}; +} +TMessageHolder Copy(TMessageHolder entry) { + auto copy = NewHoldedMessage(entry->Len); + memcpy(copy.Mes, entry.Mes, entry->Len); return copy; +} +void FixChecksum(TMessageHolder entry) { + const auto length = entry->Len - sizeof(TLogEntry); + const auto crc = NSnapshotCodec::Checksum(std::string_view(entry->Data, length - 4)); + for (int i = 0; i < 4; ++i) entry->Data[length - 4 + i] = char(crc >> (8 * i)); +} +struct TEndpoint : INode { + unsigned Sent = 0; + void Send(TMessageHolder) override { ++Sent; } + void Drain() override {} +}; +void test_configuration_roundtrips_both_phases_and_bounded_maximum(void**) { + auto record = Joint(); + for (auto phase : {EConfigurationPhase::Joint, EConfigurationPhase::Stable}) { + record.Phase = phase; + auto entry = EncodeConfigurationEntry(record); + assert_int_equal(entry->Flags, TLogEntry::EConfiguration); + assert_true(DecodeConfigurationEntry(entry) == record); + } + record = {EConfigurationPhase::Joint, 1, 0, 0, {1}, {1, 2}}; + assert_true(DecodeConfigurationEntry(EncodeConfigurationEntry(record)) == record); + record.OldVoters.clear(); record.NewVoters.clear(); + for (uint32_t id = 1; id <= TSnapshot::MaxVoters; ++id) { + record.OldVoters.push_back(id); record.NewVoters.push_back(id + TSnapshot::MaxVoters); + } + auto max = EncodeConfigurationEntry(record); + assert_int_equal(max->Len, sizeof(TLogEntry) + 48 + 8 * TSnapshot::MaxVoters); + assert_true(DecodeConfigurationEntry(max) == record); +} +void test_configuration_rejects_invalid_fields_and_valid_crc_corruption(void**) { + for (int mode = 0; mode < 12; ++mode) { + auto record = Joint(); + if (mode == 0) record.Phase = EConfigurationPhase(0); + if (mode == 1) record.Term = 0; + if (mode == 2) record.PreviousIndex = UINT64_MAX; + if (mode == 3) record.PreviousIndex = 0; + if (mode == 4) record.PreviousTerm = 0; + if (mode == 5) record.PreviousTerm = 4; + if (mode == 6) record.OldVoters.clear(); + if (mode == 7) record.NewVoters = {0, 1}; + if (mode == 8) record.NewVoters = {1, 1}; + if (mode == 9) record.NewVoters = {3, 2}; + if (mode == 10) { record.Phase = EConfigurationPhase::Stable; record.PreviousIndex = record.PreviousTerm = 0; } + if (mode == 11) record.OldVoters.resize(TSnapshot::MaxVoters + 1, 1); + Reject([&] { EncodeConfigurationEntry(record); }); + } + auto original = EncodeConfigurationEntry(Joint()); + const auto payload = original->Len - sizeof(TLogEntry); + for (size_t i = 0; i < payload; ++i) { + auto bad = Copy(original); bad->Data[i] ^= 1; + Reject([&] { DecodeConfigurationEntry(bad); }); + bad = Copy(original); bad->Len = sizeof(TLogEntry) + i; + Reject([&] { DecodeConfigurationEntry(bad); }); + } + for (size_t offset : {size_t(8), size_t(12), size_t(20), size_t(28), size_t(36), size_t(40), size_t(44), size_t(56)}) { + auto bad = Copy(original); + const auto width = offset == 12 || offset == 20 || offset == 28 ? 8 : 4; + for (size_t i = offset; i < offset + width; ++i) bad->Data[i] = offset >= 44 ? 0 : char(0xff); + FixChecksum(bad); Reject([&] { DecodeConfigurationEntry(bad); }); + } + auto bad = Copy(original); bad->Term++; Reject([&] { DecodeConfigurationEntry(bad); }); + bad = Copy(original); bad->Flags |= TLogEntry::EStub; Reject([&] { DecodeConfigurationEntry(bad); }); + bad = Copy(original); bad->Len += 1; Reject([&] { DecodeConfigurationEntry(bad); }); + bad = Copy(original); bad.InitPayload(1); Reject([&] { DecodeConfigurationEntry(bad); }); +} +void test_configuration_rpc_and_proposal_are_rejected_before_mutation(void**) { + auto peer = std::make_shared(); auto state = std::make_shared(); + TRaft follower(state, 2, {{1, peer}}); + auto record = Joint(); record.Term = 2; record.PreviousIndex = record.PreviousTerm = 0; + auto config = EncodeConfigurationEntry(record); + auto request = NewHoldedMessage(TMessageEx{.Src = 1, .Dst = 2, .Term = 2}, + TAppendEntriesRequest{.LeaderId = 1, .Nentries = 1}); + request.InitPayload(1); request.Payload[0] = config; + const auto before = static_cast(*state); + follower.Process({}, request); + assert_true(static_cast(*state) == before); assert_int_equal(peer->Sent, 0); + config->Flags |= TLogEntry::EStub; follower.Process({}, request); + assert_true(static_cast(*state) == before); assert_int_equal(peer->Sent, 0); + auto normal = NewHoldedMessage(); normal->Term = 2; request.Payload[0] = normal; + follower.Process({}, request); assert_int_equal(state->CurrentTerm, 2); assert_int_equal(state->LastLogIndex, 1); + assert_int_equal(peer->Sent, 1); + auto leaderState = std::make_shared(); TRaft leader(leaderState, 1, {}); + leader.ProcessTimeout({}); assert_int_equal(leader.CurrentStateName(), EState::LEADER); + const auto leaderBefore = static_cast(*leaderState); + auto proposed = EncodeConfigurationEntry(record); + Reject([&] { leader.Append(proposed); }); + assert_true(static_cast(*leaderState) == leaderBefore); assert_int_equal(proposed->Term, 2); +} +void test_unsupported_configuration_never_reaches_application_or_binds_voters(void**) { + auto state = std::make_shared(); auto record = Joint(); record.Term = 1; + record.PreviousIndex = record.PreviousTerm = 0; + state->Append(EncodeConfigurationEntry(record)); + Reject([&] { TRaft invalid(state, 1, {}); }); + assert_true(state->GetBoundVoters().empty()); + auto live = std::make_shared(); auto raft = std::make_shared(live, 1, TNodeDict{}); + auto app = std::make_shared(); TRequestProcessor processor(raft, app, {}); + live->Append(EncodeConfigurationEntry(record)); + const_cast(raft->GetVolatileState())->CommitIndex = 1; + Reject([&] { processor.ProcessCommitted(); }); + assert_int_equal(app->LastAppliedIndex, 0); + auto compacted = std::make_shared(); + { TRaft bootstrap(compacted, 1, {}); } + compacted->Append(NewHoldedMessage()); + compacted->InstallSnapshot({1, 1, {1}, "fixed image"}); + compacted->Append(EncodeConfigurationEntry(record)); + const auto fields = static_cast(*compacted); + Reject([&] { TRaft invalid(compacted, 1, {}); }); + assert_true(static_cast(*compacted) == fields); + assert_true(compacted->GetSnapshot()->Data == "fixed image"); +} +#ifdef __linux__ +void test_durable_restart_rejects_reserved_entry_without_rewriting(void**) { + auto dir = std::filesystem::temp_directory_path() / ("miniraft-config-" + + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count())); + assert_true(std::filesystem::create_directory(dir)); + const auto name = (dir / "raft").string(); const auto path = name + ".durable.1"; + auto record = Joint(); record.Term = 1; record.PreviousIndex = record.PreviousTerm = 0; + { auto state = std::make_shared(name, 1); + { TRaft bootstrap(state, 1, {}); } + state->Append(EncodeConfigurationEntry(record)); } + std::ifstream input(path, std::ios::binary); std::string before(std::istreambuf_iterator(input), {}); input.close(); + { auto state = std::make_shared(name, 1); + const auto fields = static_cast(*state); + Reject([&] { TRaft invalid(state, 1, {}); }); + assert_true(static_cast(*state) == fields); } + std::ifstream after(path, std::ios::binary); + assert_true(std::string(std::istreambuf_iterator(after), {}) == before); + std::error_code ignored; std::filesystem::remove_all(dir, ignored); +} +void test_legacy_restart_uses_safe_flag_lookup_and_preserves_files(void**) { + auto dir = std::filesystem::temp_directory_path() / ("miniraft-legacy-config-" + + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count())); + assert_true(std::filesystem::create_directory(dir)); + const auto name = (dir / "raft").string(); + auto record = Joint(); record.Term = 1; record.PreviousIndex = record.PreviousTerm = 0; + { TDiskState state(name, 2); state.Append(EncodeConfigurationEntry(record)); } + std::vector before; + for (auto suffix : {".entries.2", ".index.2", ".state.2"}) { + std::ifstream input(name + suffix, std::ios::binary); + before.emplace_back(std::istreambuf_iterator(input), std::istreambuf_iterator()); + } + { auto state = std::make_shared(name, 2); + const auto fields = static_cast(*state); + Reject([&] { TRaft invalid(state, 2, {}); }); + assert_true(static_cast(*state) == fields); } + size_t i = 0; + for (auto suffix : {".entries.2", ".index.2", ".state.2"}) { + std::ifstream input(name + suffix, std::ios::binary); + assert_true(std::string(std::istreambuf_iterator(input), {}) == before[i++]); + } + std::error_code ignored; std::filesystem::remove_all(dir, ignored); +} +#endif +} +int main() { + const CMUnitTest tests[] = { + cmocka_unit_test(test_configuration_roundtrips_both_phases_and_bounded_maximum), + cmocka_unit_test(test_configuration_rejects_invalid_fields_and_valid_crc_corruption), + cmocka_unit_test(test_configuration_rpc_and_proposal_are_rejected_before_mutation), + cmocka_unit_test(test_unsupported_configuration_never_reaches_application_or_binds_voters), +#ifdef __linux__ + cmocka_unit_test(test_durable_restart_rejects_reserved_entry_without_rewriting), + cmocka_unit_test(test_legacy_restart_uses_safe_flag_lookup_and_preserves_files), +#endif + }; + return cmocka_run_group_tests(tests, nullptr, nullptr); +} From 99efd7d49626d981d3d7c7bc9620e652f03afb6d Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Fri, 13 Nov 2026 15:00:23 -0500 Subject: [PATCH 27/66] Validate offline configuration history and snapshot identities (#27) --- README.md | 2 + docs/configuration-history.md | 48 +++++++++++++++ docs/joint-snapshots.md | 3 + miniraft/configuration.cpp | 57 ++++++++++++++++++ miniraft/configuration.h | 31 ++++++++++ miniraft/snapshot.cpp | 5 +- miniraft/snapshot.h | 2 + test/test_configuration.cpp | 109 ++++++++++++++++++++++++++++++++++ test/test_snapshots.cpp | 7 ++- 9 files changed, 262 insertions(+), 2 deletions(-) create mode 100644 docs/configuration-history.md diff --git a/README.md b/README.md index b8e7be9..5e2c81a 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,8 @@ representation for a joint configuration at a snapshot boundary; runtime adoptio and membership changes remain disabled. The [configuration entry foundation](docs/configuration-entries.md) defines an experimental entry codec and rejects that entry kind in the fixed-membership runtime. +The [configuration history validator](docs/configuration-history.md) checks an +offline retained-log sequence against a trusted baseline or joint snapshot. The [message buffer follow-up](docs/message-extents.md) checks allocated buffer and batch extents before accepting locally constructed frames. diff --git a/docs/configuration-history.md b/docs/configuration-history.md new file mode 100644 index 0000000..1681a85 --- /dev/null +++ b/docs/configuration-history.md @@ -0,0 +1,48 @@ +# Offline configuration history validation: v0.1.0-alpha.27 + +TConfigurationHistory checks a sequence of retained log entries against a trusted +configuration baseline. It joins the experimental configuration-entry codec with +the joint-snapshot metadata and checks that joint and stable records have the +expected predecessor index/term and voter sets. A stable configuration can advance +to a joint one; that joint configuration can advance to a stable record naming the +same old and new sets. A successful stable transition selects the new voters. + +The caller visits every retained entry in order: Observe handles ordinary entries, +and Advance handles configuration entries. Both return a new history value and +leave the original unchanged on success or failure. They reject index gaps, +nonpositive or decreasing terms, malformed records, wrong predecessors, wrong voter +sets and invalid phase transitions. A caller can retain a previous value and scan +a replacement branch from that point. This models validation of rollback; it does +not truncate a live log or decide when a configuration becomes active. + +FromJointSnapshot validates the snapshot metadata and starts at its included +index and term, using its joint configuration identity. Stable accepts an explicit +configuration identity and scanned-prefix boundary. The caller must establish that +baseline independently: a v1 fixed snapshot contains voter IDs but does not prove +the latest configuration entry's identity. Passing a zero identity asserts that no +earlier configuration entry exists. Decode and authenticate any persisted input as +appropriate before supplying it. The validator does not prove commitment, cluster +identity, promotion readiness, log completeness outside the supplied sequence or +the correctness of the caller's trusted baseline. + +Joint snapshot metadata now rejects a configuration term different from the +snapshot term when both indices are equal: one log index cannot have two terms. +Earlier alpha.25 tooling could encode such structurally inconsistent v2 images; +they are now rejected and are not rewritten automatically. The live runtime never +accepted joint snapshots. Fixed v1 snapshot encoding and existing durable formats +are unchanged. + +Tests cover joint/stable chains, alternate branches, joint snapshot boundaries, +invalid baseline identities, skipped entries, stale terms, wrong sets and +predecessors, malformed entries, index exhaustion and CRC-valid snapshot metadata +corruption. A durable test scans configuration entries after a fixed snapshot and +restart, confirms the fixture's known bootstrap baseline, and verifies that Raft +still rejects the unsupported entries without rewriting storage. Full tests also +cover existing snapshot installation and learner catch-up. + +This is offline validation, not live reconfiguration. Joint entry activation and +rollback, durable configuration recovery, snapshot installation across membership +transitions, promotion gates and transport discovery remain open. Self-review +checked contiguous scanning, term and predecessor relationships, immutable branch +results, snapshot boundary consistency and the unknown-v1-identity case. No +independent reviewer approval is claimed. diff --git a/docs/joint-snapshots.md b/docs/joint-snapshots.md index a7b6211..3afe7f9 100644 --- a/docs/joint-snapshots.md +++ b/docs/joint-snapshots.md @@ -1,5 +1,8 @@ # Joint snapshot metadata foundation: v0.1.0-alpha.25 +The [alpha.27 follow-up](configuration-history.md) requires the configuration +term to equal the snapshot term when both refer to the same index. + This step adds an experimental snapshot representation for a joint configuration effective at the included snapshot boundary. MRFTS002 records the old and new sorted voter sets plus the configuration entry's index and term. Its index must diff --git a/miniraft/configuration.cpp b/miniraft/configuration.cpp index 3477085..f0c4ac8 100644 --- a/miniraft/configuration.cpp +++ b/miniraft/configuration.cpp @@ -5,6 +5,7 @@ #include #include #include +#include namespace { using namespace NSnapshotCodec; @@ -66,3 +67,59 @@ TConfigurationRecord DecodeConfigurationEntry(TMessageHolder entry) { Validate(record); return record; } + +TConfigurationHistory TConfigurationHistory::Stable(std::vector voters, + uint64_t configurationIndex, uint64_t configurationTerm, uint64_t throughIndex, uint64_t throughTerm) { + ValidateFixedVoters(voters); + if (throughIndex > uint64_t(INT64_MAX) || configurationIndex > throughIndex || + ((configurationIndex == 0) != (configurationTerm == 0)) || + ((throughIndex == 0) != (throughTerm == 0)) || configurationTerm > throughTerm || + (configurationIndex && configurationIndex == throughIndex && configurationTerm != throughTerm)) + throw std::invalid_argument("Invalid configuration history baseline"); + TConfigurationHistory history; + history.Old = std::move(voters); history.ConfigIndex = configurationIndex; + history.ConfigTerm = configurationTerm; history.Through = throughIndex; history.ThroughLogTerm = throughTerm; + return history; +} + +TConfigurationHistory TConfigurationHistory::FromJointSnapshot(const TSnapshot& snapshot) { + ValidateSnapshot(snapshot); + if (!snapshot.IsJoint()) throw std::invalid_argument("Snapshot does not identify a joint configuration"); + TConfigurationHistory history; + history.Old = snapshot.Voters; history.New = snapshot.JointVoters; + history.ConfigIndex = snapshot.ConfigurationIndex; history.ConfigTerm = snapshot.ConfigurationTerm; + history.Through = snapshot.Index; history.ThroughLogTerm = snapshot.Term; + return history; +} + +TConfigurationHistory TConfigurationHistory::Advance(uint64_t index, TMessageHolder entry) const { + const auto record = DecodeConfigurationEntry(std::move(entry)); + if (Through >= uint64_t(INT64_MAX) || index != Through + 1 || record.Term < ThroughLogTerm || + record.PreviousIndex != ConfigIndex || record.PreviousTerm != ConfigTerm) + throw std::invalid_argument("Configuration entry does not extend the trusted history"); + TConfigurationHistory next = *this; + if (!IsJoint()) { + if (record.Phase != EConfigurationPhase::Joint || record.OldVoters != Old) + throw std::invalid_argument("Expected a joint configuration from the stable voter set"); + next.New = record.NewVoters; + } else { + if (record.Phase != EConfigurationPhase::Stable || record.OldVoters != Old || record.NewVoters != New) + throw std::invalid_argument("Expected the stable successor of the joint configuration"); + next.Old = New; next.New.clear(); + } + next.ConfigIndex = next.Through = index; + next.ConfigTerm = next.ThroughLogTerm = record.Term; + return next; +} + +TConfigurationHistory TConfigurationHistory::Observe(uint64_t index, TMessageHolder entry) const { + if (!entry.HasValidExtent() || entry->Type != uint32_t(EMessageType::LOG_ENTRY) || + entry->Len < sizeof(TLogEntry) || entry.PayloadSize) + throw std::invalid_argument("Invalid ordinary log entry"); + if ((entry->Flags & TLogEntry::EConfiguration) || !entry->Term || + Through >= uint64_t(INT64_MAX) || index != Through + 1 || entry->Term < ThroughLogTerm) + throw std::invalid_argument("Ordinary entry does not extend the trusted history"); + TConfigurationHistory next = *this; + next.Through = index; next.ThroughLogTerm = entry->Term; + return next; +} diff --git a/miniraft/configuration.h b/miniraft/configuration.h index 482b492..a89feea 100644 --- a/miniraft/configuration.h +++ b/miniraft/configuration.h @@ -1,6 +1,7 @@ #pragma once #include "messages.h" +#include "snapshot.h" #include #include @@ -15,3 +16,33 @@ struct TConfigurationRecord { TMessageHolder EncodeConfigurationEntry(const TConfigurationRecord& record); TConfigurationRecord DecodeConfigurationEntry(TMessageHolder entry); + +// Offline transition validation only; does not establish commitment, activate +// roles, or make a fixed-membership Raft node accept configuration entries. +class TConfigurationHistory { +public: + TConfigurationHistory(const TConfigurationHistory&) = default; + TConfigurationHistory& operator=(const TConfigurationHistory&) = default; + // The caller must supply a trusted identity and scanned-prefix boundary. + // A zero configuration identity asserts that no configuration entry exists + // in that prefix; a v1 snapshot alone does not prove this assertion. + static TConfigurationHistory Stable(std::vector voters, + uint64_t configurationIndex, uint64_t configurationTerm, + uint64_t throughIndex, uint64_t throughTerm); + static TConfigurationHistory FromJointSnapshot(const TSnapshot& snapshot); + // Visit every retained entry in index order; neither method changes *this. + TConfigurationHistory Observe(uint64_t index, TMessageHolder entry) const; + TConfigurationHistory Advance(uint64_t index, TMessageHolder entry) const; + + bool IsJoint() const { return !New.empty(); } + const std::vector& OldVoters() const { return Old; } + const std::vector& NewVoters() const { return New; } + uint64_t ConfigurationIndex() const { return ConfigIndex; } + uint64_t ConfigurationTerm() const { return ConfigTerm; } + uint64_t ThroughIndex() const { return Through; } + uint64_t ThroughTerm() const { return ThroughLogTerm; } +private: + TConfigurationHistory() = default; + std::vector Old, New; + uint64_t ConfigIndex = 0, ConfigTerm = 0, Through = 0, ThroughLogTerm = 0; +}; diff --git a/miniraft/snapshot.cpp b/miniraft/snapshot.cpp index cf3de33..a0ba243 100644 --- a/miniraft/snapshot.cpp +++ b/miniraft/snapshot.cpp @@ -34,7 +34,8 @@ void Validate(const TSnapshot& snapshot) { if (snapshot.IsJoint()) { if (snapshot.JointVoters.empty() || !snapshot.ConfigurationIndex || snapshot.ConfigurationIndex > snapshot.Index || !snapshot.ConfigurationTerm || - snapshot.ConfigurationTerm > snapshot.Term) + snapshot.ConfigurationTerm > snapshot.Term || + (snapshot.ConfigurationIndex == snapshot.Index && snapshot.ConfigurationTerm != snapshot.Term)) throw std::runtime_error("Invalid joint snapshot configuration boundary"); validateVoters(snapshot.JointVoters); } @@ -65,6 +66,8 @@ void Sync(int fd) { #endif } +void ValidateSnapshot(const TSnapshot& snapshot) { Validate(snapshot); } + std::string EncodeSnapshot(const TSnapshot& snapshot) { Validate(snapshot); std::string out(snapshot.IsJoint() ? JointMagic : Magic); diff --git a/miniraft/snapshot.h b/miniraft/snapshot.h index ef819df..6ae01e2 100644 --- a/miniraft/snapshot.h +++ b/miniraft/snapshot.h @@ -27,6 +27,8 @@ struct TSnapshot { }; std::string EncodeSnapshot(const TSnapshot& snapshot); TSnapshot DecodeSnapshot(std::string_view bytes); +// Validate metadata and size without copying the application image. +void ValidateSnapshot(const TSnapshot& snapshot); TSnapshot CaptureSnapshot(const TRaft& raft, const IRsm& rsm); // Startup only, with a fresh application and a matching local boundary/log. void RestoreSnapshot(const TSnapshot& snapshot, const TRaft& raft, IRsm& rsm); diff --git a/test/test_configuration.cpp b/test/test_configuration.cpp index 6e4943d..0882310 100644 --- a/test/test_configuration.cpp +++ b/test/test_configuration.cpp @@ -37,6 +37,77 @@ struct TEndpoint : INode { void Send(TMessageHolder) override { ++Sent; } void Drain() override {} }; +TMessageHolder Ordinary(uint64_t term) { + auto entry = NewHoldedMessage(); entry->Term = term; return entry; +} +void test_history_validates_joint_stable_chain_and_independent_rollback(void**) { + auto initial = TConfigurationHistory::Stable({1, 2, 3}, 0, 0, 0, 0); + auto prefix = initial.Observe(1, Ordinary(1)); + auto joint = Joint(); joint.Term = 2; joint.PreviousIndex = joint.PreviousTerm = 0; + auto pending = prefix.Advance(2, EncodeConfigurationEntry(joint)); + assert_true(pending.IsJoint()); + assert_true(pending.OldVoters() == joint.OldVoters); assert_true(pending.NewVoters() == joint.NewVoters); + assert_int_equal(pending.ConfigurationIndex(), 2); assert_int_equal(pending.ConfigurationTerm(), 2); + auto scanned = pending.Observe(3, Ordinary(2)); + auto final = joint; final.Phase = EConfigurationPhase::Stable; final.Term = 3; + final.PreviousIndex = 2; final.PreviousTerm = 2; + auto stable = scanned.Advance(4, EncodeConfigurationEntry(final)); + assert_false(stable.IsJoint()); assert_true(stable.OldVoters() == joint.NewVoters); + assert_int_equal(stable.ConfigurationIndex(), 4); assert_int_equal(stable.ThroughTerm(), 3); + // A replacement branch starts from the unchanged prior prefix. + auto replacement = joint; replacement.Term = 3; replacement.NewVoters = {3, 5, 6}; + auto alternate = prefix.Advance(2, EncodeConfigurationEntry(replacement)); + assert_true(alternate.NewVoters() == replacement.NewVoters); + assert_true(pending.NewVoters() == joint.NewVoters); + assert_false(prefix.IsJoint()); assert_int_equal(prefix.ThroughIndex(), 1); +} +void test_history_from_joint_snapshot_requires_a_consistent_identity(void**) { + TSnapshot snapshot{8, 4, {1, 2, 3}, "opaque"}; + snapshot.JointVoters = {3, 4, 5}; snapshot.ConfigurationIndex = 5; snapshot.ConfigurationTerm = 3; + auto history = TConfigurationHistory::FromJointSnapshot(snapshot); + assert_true(history.IsJoint()); assert_int_equal(history.ConfigurationIndex(), 5); + assert_int_equal(history.ConfigurationTerm(), 3); assert_int_equal(history.ThroughIndex(), 8); + assert_int_equal(history.ThroughTerm(), 4); + auto final = Joint(); final.Phase = EConfigurationPhase::Stable; final.Term = 4; + final.PreviousIndex = 5; final.PreviousTerm = 3; + auto stable = history.Advance(9, EncodeConfigurationEntry(final)); + assert_false(stable.IsJoint()); assert_true(stable.OldVoters() == final.NewVoters); + auto fixed = snapshot; fixed.JointVoters.clear(); fixed.ConfigurationIndex = fixed.ConfigurationTerm = 0; + Reject([&] { TConfigurationHistory::FromJointSnapshot(fixed); }); + auto inconsistent = snapshot; inconsistent.ConfigurationIndex = inconsistent.Index; + Reject([&] { TConfigurationHistory::FromJointSnapshot(inconsistent); }); + Reject([&] { TConfigurationHistory::Stable({1}, 4, 2, 4, 3); }); + Reject([&] { TConfigurationHistory::Stable({1}, 5, 1, 4, 3); }); + Reject([&] { TConfigurationHistory::Stable({1}, 0, 0, 1, 0); }); +} +void test_history_rejects_skips_wrong_predecessors_and_term_regression(void**) { + auto base = TConfigurationHistory::Stable({1, 2, 3}, 0, 0, 0, 0); + auto joint = Joint(); joint.Term = 2; joint.PreviousIndex = joint.PreviousTerm = 0; + auto entry = EncodeConfigurationEntry(joint); + Reject([&] { base.Advance(2, entry); }); + Reject([&] { base.Observe(2, Ordinary(1)); }); + Reject([&] { base.Observe(1, entry); }); + Reject([&] { base.Advance(1, Ordinary(1)); }); + auto wrong = joint; wrong.OldVoters = {1, 2, 4}; + Reject([&] { base.Advance(1, EncodeConfigurationEntry(wrong)); }); + auto prefix = base.Observe(1, Ordinary(3)); + Reject([&] { prefix.Advance(2, entry); }); + Reject([&] { prefix.Observe(2, Ordinary(2)); }); + auto valid = joint; valid.Term = 3; + auto pending = prefix.Advance(2, EncodeConfigurationEntry(valid)); + wrong = valid; wrong.Phase = EConfigurationPhase::Stable; wrong.PreviousIndex = 2; wrong.PreviousTerm = 3; + wrong.NewVoters = {3, 4, 6}; + Reject([&] { pending.Advance(3, EncodeConfigurationEntry(wrong)); }); + wrong = valid; wrong.PreviousIndex = 2; wrong.PreviousTerm = 3; + Reject([&] { pending.Advance(3, EncodeConfigurationEntry(wrong)); }); + auto corrupt = EncodeConfigurationEntry(valid); corrupt->Data[0] ^= 1; + Reject([&] { prefix.Advance(2, corrupt); }); + auto exhausted = TConfigurationHistory::Stable({1}, 0, 0, uint64_t(INT64_MAX), 1); + Reject([&] { exhausted.Observe(UINT64_MAX, Ordinary(1)); }); + Reject([&] { exhausted.Advance(UINT64_MAX, entry); }); + assert_false(prefix.IsJoint()); assert_int_equal(prefix.ThroughIndex(), 1); + assert_true(pending.IsJoint()); assert_int_equal(pending.ThroughIndex(), 2); +} void test_configuration_roundtrips_both_phases_and_bounded_maximum(void**) { auto record = Joint(); for (auto phase : {EConfigurationPhase::Joint, EConfigurationPhase::Stable}) { @@ -178,10 +249,47 @@ void test_legacy_restart_uses_safe_flag_lookup_and_preserves_files(void**) { } std::error_code ignored; std::filesystem::remove_all(dir, ignored); } +void test_offline_history_scan_after_durable_snapshot_and_restart(void**) { + auto dir = std::filesystem::temp_directory_path() / ("miniraft-history-" + + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count())); + assert_true(std::filesystem::create_directory(dir)); + const auto name = (dir / "raft").string(); const auto path = name + ".durable.1"; + TConfigurationRecord joint{EConfigurationPhase::Joint, 2, 0, 0, {1}, {1, 2}}; + auto final = joint; final.Phase = EConfigurationPhase::Stable; final.Term = 3; + final.PreviousIndex = 2; final.PreviousTerm = 2; + { + auto state = std::make_shared(name, 1); + { TRaft bootstrap(state, 1, {}); } + state->Append(Ordinary(1)); state->InstallSnapshot({1, 1, {1}, "fixed image"}); + state->CurrentTerm = 3; state->Commit(); + state->Append(EncodeConfigurationEntry(joint)); state->Append(Ordinary(2)); + state->Append(EncodeConfigurationEntry(final)); + } + std::ifstream input(path, std::ios::binary); std::string before(std::istreambuf_iterator(input), {}); input.close(); + { + auto state = std::make_shared(name, 1); + // This fixture explicitly knows that index 1 precedes any config entry. + auto history = TConfigurationHistory::Stable({1}, 0, 0, state->SnapshotIndex, state->SnapshotTerm); + history = history.Advance(2, state->Get(2)); + assert_true(history.IsJoint()); + history = history.Observe(3, state->Get(3)); + history = history.Advance(4, state->Get(4)); + assert_false(history.IsJoint()); assert_true(history.OldVoters() == final.NewVoters); + assert_int_equal(history.ConfigurationIndex(), 4); + Reject([&] { TRaft disabled(state, 1, {}); }); + assert_true(state->GetSnapshot()->Data == "fixed image"); + } + std::ifstream after(path, std::ios::binary); + assert_true(std::string(std::istreambuf_iterator(after), {}) == before); + std::error_code ignored; std::filesystem::remove_all(dir, ignored); +} #endif } int main() { const CMUnitTest tests[] = { + cmocka_unit_test(test_history_validates_joint_stable_chain_and_independent_rollback), + cmocka_unit_test(test_history_from_joint_snapshot_requires_a_consistent_identity), + cmocka_unit_test(test_history_rejects_skips_wrong_predecessors_and_term_regression), cmocka_unit_test(test_configuration_roundtrips_both_phases_and_bounded_maximum), cmocka_unit_test(test_configuration_rejects_invalid_fields_and_valid_crc_corruption), cmocka_unit_test(test_configuration_rpc_and_proposal_are_rejected_before_mutation), @@ -189,6 +297,7 @@ int main() { #ifdef __linux__ cmocka_unit_test(test_durable_restart_rejects_reserved_entry_without_rewriting), cmocka_unit_test(test_legacy_restart_uses_safe_flag_lookup_and_preserves_files), + cmocka_unit_test(test_offline_history_scan_after_durable_snapshot_and_restart), #endif }; return cmocka_run_group_tests(tests, nullptr, nullptr); diff --git a/test/test_snapshots.cpp b/test/test_snapshots.cpp index a60ba52..7a80623 100644 --- a/test/test_snapshots.cpp +++ b/test/test_snapshots.cpp @@ -100,7 +100,7 @@ void test_joint_snapshot_codec_roundtrip_and_corruption(void**) { } } void test_joint_snapshot_metadata_rejects_invalid_boundaries_and_voters(void**) { - for (int mode = 0; mode < 9; ++mode) { + for (int mode = 0; mode < 10; ++mode) { auto bad = JointSample(); if (mode == 0) bad.JointVoters.clear(); if (mode == 1) bad.JointVoters = {0, 1}; @@ -111,6 +111,7 @@ void test_joint_snapshot_metadata_rejects_invalid_boundaries_and_voters(void**) if (mode == 6) bad.ConfigurationIndex = bad.Index + 1; if (mode == 7) bad.ConfigurationTerm = 0; if (mode == 8) bad.ConfigurationTerm = bad.Term + 1; + if (mode == 9) { bad.Term = 2; bad.ConfigurationIndex = bad.Index; bad.ConfigurationTerm = 1; } Reject([&] { EncodeSnapshot(bad); }); } for (size_t offset : {size_t(36), size_t(44), size_t(52)}) { @@ -122,6 +123,10 @@ void test_joint_snapshot_metadata_rejects_invalid_boundaries_and_voters(void**) auto empty = EncodeSnapshot(JointSample()); for (size_t i = 36; i < 56; ++i) empty[i] = 0; FixChecksum(empty); Reject([&] { DecodeSnapshot(empty); }); + auto mismatch = EncodeSnapshot(JointSample()); + mismatch[16] = 2; // snapshot term + mismatch[36] = 2; // configuration index equals the snapshot index + FixChecksum(mismatch); Reject([&] { DecodeSnapshot(mismatch); }); } void test_joint_snapshot_cannot_be_installed_or_restored_by_fixed_runtime(void**) { auto snapshot = JointSample(); From f510e9922b40e0e109143dd78083050c0c62d26c Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Fri, 13 Nov 2026 15:06:22 -0500 Subject: [PATCH 28/66] Validate configuration history across retained logs (#28) --- README.md | 2 + docs/configuration-scan.md | 30 +++++++++++++ miniraft/configuration.cpp | 66 ++++++++++++++++++++++++++-- miniraft/configuration.h | 17 ++++++++ test/test_configuration.cpp | 85 +++++++++++++++++++++++++++++++++++++ 5 files changed, 197 insertions(+), 3 deletions(-) create mode 100644 docs/configuration-scan.md diff --git a/README.md b/README.md index 5e2c81a..477651f 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,8 @@ The [configuration entry foundation](docs/configuration-entries.md) defines an experimental entry codec and rejects that entry kind in the fixed-membership runtime. The [configuration history validator](docs/configuration-history.md) checks an offline retained-log sequence against a trusted baseline or joint snapshot. +The [configuration log scanner](docs/configuration-scan.md) validates a retained +suffix at a caller-supplied commit boundary and at its end. The [message buffer follow-up](docs/message-extents.md) checks allocated buffer and batch extents before accepting locally constructed frames. diff --git a/docs/configuration-scan.md b/docs/configuration-scan.md new file mode 100644 index 0000000..b707411 --- /dev/null +++ b/docs/configuration-scan.md @@ -0,0 +1,30 @@ +# Offline configuration log scan + +`ScanConfigurationHistory` validates a retained suffix starting at a trusted +configuration history at the snapshot boundary. It returns the history at the +caller-supplied commit index and at the end of the suffix. The two results make +it possible to distinguish the configuration in that committed prefix from a +later configuration record, including after a suffix is replaced. A malformed +entry anywhere in the suffix fails the entire scan; the scanner does not decide +whether a damaged tail may be truncated. + +The caller must independently establish the baseline and commit index: the scan +does not prove that any entry is committed, recover a commit index, authenticate +the snapshot, or establish cluster identity. A fixed v1 snapshot does not identify +its latest configuration entry, so its configuration identity must come from +other trusted information. A joint snapshot contains that identity and the scan +checks that the supplied baseline matches it. The scan checks snapshot and log +boundaries, bound fixed voters, terms, configuration predecessors, and transitions. +It requires a consistent, externally serialized view of the state; it does not +mutate storage, apply entries, activate voters, or change Raft roles. + +For ordinary entries, the scan trusts term and flag metadata from the storage +backend, which must validate its own records. Memory and durable backends avoid +copying ordinary payloads; external backends using default metadata accessors may +read full entries. Configuration records are read and validated in full. + +Joint snapshots and configuration entries remain rejected by the live runtime and +its storage installation paths. Tests use a synthetic joint snapshot for offline +validation and a real fixed snapshot across durable restart. Live membership +activation, commit rules, snapshot installation across transitions, and recovery +of a proven configuration remain future work. diff --git a/miniraft/configuration.cpp b/miniraft/configuration.cpp index f0c4ac8..171e2b0 100644 --- a/miniraft/configuration.cpp +++ b/miniraft/configuration.cpp @@ -116,10 +116,70 @@ TConfigurationHistory TConfigurationHistory::Observe(uint64_t index, TMessageHol if (!entry.HasValidExtent() || entry->Type != uint32_t(EMessageType::LOG_ENTRY) || entry->Len < sizeof(TLogEntry) || entry.PayloadSize) throw std::invalid_argument("Invalid ordinary log entry"); - if ((entry->Flags & TLogEntry::EConfiguration) || !entry->Term || - Through >= uint64_t(INT64_MAX) || index != Through + 1 || entry->Term < ThroughLogTerm) + return ObserveMetadata(index, entry->Term, entry->Flags); +} + +TConfigurationHistory TConfigurationHistory::ObserveMetadata(uint64_t index, uint64_t term, uint64_t flags) const { + if ((flags & TLogEntry::EConfiguration) || !term || + Through >= uint64_t(INT64_MAX) || index != Through + 1 || term < ThroughLogTerm) throw std::invalid_argument("Ordinary entry does not extend the trusted history"); TConfigurationHistory next = *this; - next.Through = index; next.ThroughLogTerm = entry->Term; + next.Through = index; next.ThroughLogTerm = term; return next; } + +TConfigurationScan ScanConfigurationHistory(const IState& state, + TConfigurationHistory baseline, uint64_t commitIndex) { + if (!state.CurrentTerm || state.LastLogIndex > uint64_t(INT64_MAX) || + state.SnapshotIndex > state.LastLogIndex || state.SnapshotTerm > state.CurrentTerm || + state.LastLogTerm > state.CurrentTerm || + baseline.ThroughIndex() != state.SnapshotIndex || baseline.ThroughTerm() != state.SnapshotTerm || + commitIndex < state.SnapshotIndex || commitIndex > state.LastLogIndex) + throw std::invalid_argument("Invalid configuration scan boundaries"); + + const auto snapshot = state.GetSnapshot(); + if (state.SnapshotIndex) { + if (!snapshot) throw std::invalid_argument("Missing configuration scan snapshot"); + ValidateSnapshot(*snapshot); + if (snapshot->Index != state.SnapshotIndex || snapshot->Term != state.SnapshotTerm) + throw std::invalid_argument("Configuration scan snapshot boundary mismatch"); + if (snapshot->IsJoint()) { + const auto expected = TConfigurationHistory::FromJointSnapshot(*snapshot); + if (!baseline.IsJoint() || baseline.OldVoters() != expected.OldVoters() || + baseline.NewVoters() != expected.NewVoters() || + baseline.ConfigurationIndex() != expected.ConfigurationIndex() || + baseline.ConfigurationTerm() != expected.ConfigurationTerm()) + throw std::invalid_argument("Joint snapshot baseline mismatch"); + } else if (baseline.IsJoint() || baseline.OldVoters() != snapshot->Voters) { + throw std::invalid_argument("Fixed snapshot baseline mismatch"); + } + } else if (snapshot || state.SnapshotTerm || baseline.IsJoint()) { + throw std::invalid_argument("Invalid empty snapshot boundary"); + } + if (!baseline.IsJoint()) { + const auto bound = state.GetBoundVoters(); + if (!bound.empty() && bound != baseline.OldVoters()) + throw std::invalid_argument("Configuration baseline differs from bound voters"); + } + + auto latest = baseline; + auto committed = baseline; + for (uint64_t index = state.SnapshotIndex + 1; index <= state.LastLogIndex; ++index) { + const auto term = state.RetainedLogTerm(index); + const auto flags = state.RetainedLogFlags(index); + if (!term || term > state.CurrentTerm) throw std::invalid_argument("Invalid retained log term"); + if (flags & TLogEntry::EConfiguration) { + auto entry = state.Get(index); + if (!entry.HasValidExtent() || entry->Len < sizeof(TLogEntry) || + entry->Term != term || entry->Flags != flags) + throw std::invalid_argument("Inconsistent configuration entry metadata"); + latest = latest.Advance(index, entry); + } else { + latest = latest.ObserveMetadata(index, term, flags); + } + if (index == commitIndex) committed = latest; + } + if (latest.ThroughTerm() != state.LastLogTerm) + throw std::invalid_argument("Configuration scan last term mismatch"); + return {committed, latest}; +} diff --git a/miniraft/configuration.h b/miniraft/configuration.h index a89feea..9834af9 100644 --- a/miniraft/configuration.h +++ b/miniraft/configuration.h @@ -32,6 +32,8 @@ class TConfigurationHistory { static TConfigurationHistory FromJointSnapshot(const TSnapshot& snapshot); // Visit every retained entry in index order; neither method changes *this. TConfigurationHistory Observe(uint64_t index, TMessageHolder entry) const; + // Metadata must come from a trusted, already validated storage backend. + TConfigurationHistory ObserveMetadata(uint64_t index, uint64_t term, uint64_t flags) const; TConfigurationHistory Advance(uint64_t index, TMessageHolder entry) const; bool IsJoint() const { return !New.empty(); } @@ -46,3 +48,18 @@ class TConfigurationHistory { std::vector Old, New; uint64_t ConfigIndex = 0, ConfigTerm = 0, Through = 0, ThroughLogTerm = 0; }; + +struct IState; +struct TConfigurationScan { + TConfigurationHistory Committed; + TConfigurationHistory Latest; + bool HasUncommittedConfiguration() const { + return Committed.ConfigurationIndex() != Latest.ConfigurationIndex() || + Committed.ConfigurationTerm() != Latest.ConfigurationTerm(); + } +}; + +// Offline, read-only validation. The caller supplies and must substantiate both +// the baseline and the commit boundary; neither is proven by this scan. +TConfigurationScan ScanConfigurationHistory(const IState& state, + TConfigurationHistory baseline, uint64_t commitIndex); diff --git a/test/test_configuration.cpp b/test/test_configuration.cpp index 0882310..6572366 100644 --- a/test/test_configuration.cpp +++ b/test/test_configuration.cpp @@ -108,6 +108,84 @@ void test_history_rejects_skips_wrong_predecessors_and_term_regression(void**) { assert_false(prefix.IsJoint()); assert_int_equal(prefix.ThroughIndex(), 1); assert_true(pending.IsJoint()); assert_int_equal(pending.ThroughIndex(), 2); } +struct TCountingState : TState { + mutable unsigned Gets = 0; + TMessageHolder Get(int64_t index) const override { ++Gets; return TState::Get(index); } +}; +struct TMismatchedMetadataState : TState { + uint64_t RetainedLogTerm(int64_t index) const override { return TState::RetainedLogTerm(index) + 1; } +}; +void test_scan_separates_commit_and_replaced_suffix(void**) { + TCountingState state; state.CurrentTerm = 3; + auto large = NewHoldedMessage(sizeof(TLogEntry) + 1024 * 1024); large->Term = 1; + TConfigurationRecord joint{EConfigurationPhase::Joint, 2, 0, 0, {1}, {1, 2}}; + auto final = joint; final.Phase = EConfigurationPhase::Stable; final.Term = 3; + final.PreviousIndex = 2; final.PreviousTerm = 2; + state.Append(large); state.Append(EncodeConfigurationEntry(joint)); + state.Append(Ordinary(2)); state.Append(EncodeConfigurationEntry(final)); + auto base = TConfigurationHistory::Stable({1}, 0, 0, 0, 0); + auto scan = ScanConfigurationHistory(state, base, 2); + assert_true(scan.Committed.IsJoint()); assert_int_equal(scan.Committed.ThroughIndex(), 2); + assert_false(scan.Latest.IsJoint()); assert_true(scan.Latest.OldVoters() == final.NewVoters); + assert_true(scan.HasUncommittedConfiguration()); assert_int_equal(state.Gets, 2); + scan = ScanConfigurationHistory(state, base, 4); + assert_false(scan.HasUncommittedConfiguration()); assert_int_equal(scan.Committed.ThroughIndex(), 4); + state.RemoveLast(); + scan = ScanConfigurationHistory(state, base, 2); + assert_true(scan.Latest.IsJoint()); assert_false(scan.HasUncommittedConfiguration()); + state.Append(EncodeConfigurationEntry(final)); + scan = ScanConfigurationHistory(state, base, 1); + assert_false(scan.Committed.IsJoint()); assert_true(scan.HasUncommittedConfiguration()); + Reject([&] { base.ObserveMetadata(1, 1, TLogEntry::EConfiguration); }); + Reject([&] { base.ObserveMetadata(2, 1, 0); }); +} +void test_scan_from_joint_snapshot_and_rejects_mismatched_baselines(void**) { + TSnapshot snapshot{2, 2, {1, 2}, "image"}; + snapshot.JointVoters = {2, 3}; snapshot.ConfigurationIndex = 2; snapshot.ConfigurationTerm = 2; + // Synthetic offline state: runtime storage deliberately rejects joint snapshots. + TState state; state.Snapshot = std::make_shared(snapshot); + state.SnapshotIndex = state.LastLogIndex = 2; state.SnapshotTerm = state.LastLogTerm = 2; + state.CurrentTerm = 3; state.Append(Ordinary(2)); + TConfigurationRecord final{EConfigurationPhase::Stable, 3, 2, 2, {1, 2}, {2, 3}}; + state.Append(EncodeConfigurationEntry(final)); + auto base = TConfigurationHistory::FromJointSnapshot(snapshot); + auto scan = ScanConfigurationHistory(state, base, 2); + assert_true(scan.Committed.IsJoint()); assert_false(scan.Latest.IsJoint()); + assert_true(scan.Latest.OldVoters() == final.NewVoters); assert_true(scan.HasUncommittedConfiguration()); + assert_false(ScanConfigurationHistory(state, base, 4).HasUncommittedConfiguration()); + Reject([&] { ScanConfigurationHistory(state, base, 1); }); + Reject([&] { ScanConfigurationHistory(state, base, 5); }); + Reject([&] { ScanConfigurationHistory(state, TConfigurationHistory::Stable({1, 2}, 0, 0, 2, 2), 2); }); + auto wrong = snapshot; wrong.JointVoters = {2, 4}; + Reject([&] { ScanConfigurationHistory(state, TConfigurationHistory::FromJointSnapshot(wrong), 2); }); + state.LastLogTerm = 2; + Reject([&] { ScanConfigurationHistory(state, base, 2); }); +} +void test_scan_fails_closed_on_invalid_suffix(void**) { + TState state; state.CurrentTerm = 3; + auto base = TConfigurationHistory::Stable({1}, 0, 0, 0, 0); + state.Append(Ordinary(2)); + TConfigurationRecord wrong{EConfigurationPhase::Joint, 3, 1, 2, {1}, {1, 2}}; + state.Append(EncodeConfigurationEntry(wrong)); + const auto fields = static_cast(state); + Reject([&] { ScanConfigurationHistory(state, base, 1); }); + assert_true(static_cast(state) == fields); assert_int_equal(base.ThroughIndex(), 0); + state.RemoveLast(); state.Append(Ordinary(1)); + Reject([&] { ScanConfigurationHistory(state, base, 1); }); + state.RemoveLast(); state.Append(Ordinary(3)); + state.BoundVoters = {2}; + Reject([&] { ScanConfigurationHistory(state, base, 1); }); + state.BoundVoters.clear(); state.CurrentTerm = 2; + Reject([&] { ScanConfigurationHistory(state, base, 1); }); + TState corrupt; corrupt.CurrentTerm = 3; + TConfigurationRecord joint{EConfigurationPhase::Joint, 2, 0, 0, {1}, {1, 2}}; + auto damaged = EncodeConfigurationEntry(joint); damaged->Data[0] ^= 1; + corrupt.Append(damaged); + Reject([&] { ScanConfigurationHistory(corrupt, base, 0); }); + TMismatchedMetadataState mismatch; mismatch.CurrentTerm = 3; + mismatch.Append(EncodeConfigurationEntry(joint)); + Reject([&] { ScanConfigurationHistory(mismatch, base, 0); }); +} void test_configuration_roundtrips_both_phases_and_bounded_maximum(void**) { auto record = Joint(); for (auto phase : {EConfigurationPhase::Joint, EConfigurationPhase::Stable}) { @@ -270,6 +348,10 @@ void test_offline_history_scan_after_durable_snapshot_and_restart(void**) { auto state = std::make_shared(name, 1); // This fixture explicitly knows that index 1 precedes any config entry. auto history = TConfigurationHistory::Stable({1}, 0, 0, state->SnapshotIndex, state->SnapshotTerm); + const auto scan = ScanConfigurationHistory(*state, history, 2); + assert_true(scan.Committed.IsJoint()); assert_false(scan.Latest.IsJoint()); + assert_true(scan.Latest.OldVoters() == final.NewVoters); + assert_true(scan.HasUncommittedConfiguration()); history = history.Advance(2, state->Get(2)); assert_true(history.IsJoint()); history = history.Observe(3, state->Get(3)); @@ -290,6 +372,9 @@ int main() { cmocka_unit_test(test_history_validates_joint_stable_chain_and_independent_rollback), cmocka_unit_test(test_history_from_joint_snapshot_requires_a_consistent_identity), cmocka_unit_test(test_history_rejects_skips_wrong_predecessors_and_term_regression), + cmocka_unit_test(test_scan_separates_commit_and_replaced_suffix), + cmocka_unit_test(test_scan_from_joint_snapshot_and_rejects_mismatched_baselines), + cmocka_unit_test(test_scan_fails_closed_on_invalid_suffix), cmocka_unit_test(test_configuration_roundtrips_both_phases_and_bounded_maximum), cmocka_unit_test(test_configuration_rejects_invalid_fields_and_valid_crc_corruption), cmocka_unit_test(test_configuration_rpc_and_proposal_are_rejected_before_mutation), From b20fba67a5a18fd88491e8695a55a6e098259d14 Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Fri, 13 Nov 2026 19:47:45 -0500 Subject: [PATCH 29/66] Reject snapshot terms contradicting committed history (#29) --- README.md | 2 + docs/snapshot-committed-terms.md | 23 ++++++++++ miniraft/raft.cpp | 6 +++ test/test_compaction.cpp | 75 ++++++++++++++++++++++++++++++++ 4 files changed, 106 insertions(+) create mode 100644 docs/snapshot-committed-terms.md diff --git a/README.md b/README.md index 477651f..9dcb0df 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,8 @@ The [configuration history validator](docs/configuration-history.md) checks an offline retained-log sequence against a trusted baseline or joint snapshot. The [configuration log scanner](docs/configuration-scan.md) validates a retained suffix at a caller-supplied commit boundary and at its end. +The [snapshot term validation fix](docs/snapshot-committed-terms.md) rejects +snapshot boundaries that contradict the follower's committed prefix. The [message buffer follow-up](docs/message-extents.md) checks allocated buffer and batch extents before accepting locally constructed frames. diff --git a/docs/snapshot-committed-terms.md b/docs/snapshot-committed-terms.md new file mode 100644 index 0000000..7e82ae9 --- /dev/null +++ b/docs/snapshot-committed-terms.md @@ -0,0 +1,23 @@ +# Snapshot boundary terms and committed prefixes + +An InstallSnapshot request could previously advertise a snapshot at an index +after the follower's committed prefix but with an older boundary term. Such a +history cannot be a Raft log: entry terms do not decrease as indices increase. +The follower could accept that request, update its current term, replace its +snapshot and application state, and acknowledge the impossible boundary. + +Incoming snapshot requests are now checked before term or role changes. A +snapshot after the committed prefix must have a boundary term at least as large +as the term at the follower's commit index. If the requested boundary itself is +committed and still known locally, its term must match. A boundary older than the +local snapshot cannot be checked this way because its individual term has been +compacted. Invalid requests are dropped, consistent with other malformed RPCs. +Each chunk is checked again, including when commitment advances mid-transfer. + +Deterministic tests reproduce the prior mutation with both retained and compacted +committed prefixes, check higher-term invalid requests leave hard state and the +application unchanged, exercise a commit advancing during transfer, and confirm a +valid later snapshot still installs. This check detects these specific committed +term contradictions; it does not authenticate peers or prove snapshot contents +match the committed application state. Membership changes and joint snapshot +installation remain disabled. No wire or storage format changes. diff --git a/miniraft/raft.cpp b/miniraft/raft.cpp index 61202c2..684e863 100644 --- a/miniraft/raft.cpp +++ b/miniraft/raft.cpp @@ -524,8 +524,14 @@ bool TRaft::ValidateRpc(TMessageHolder message) const { case EMessageType::INSTALL_SNAPSHOT_REQUEST: { auto request = message.Cast(); const auto length = request->Len - sizeof(TInstallSnapshotRequest); + // A later boundary cannot have an older term than the committed prefix. + // For an already committed, retained boundary, the terms must match. return request->LeaderId == header->Src && request->SnapshotIndex > 0 && request->SnapshotIndex <= maxIndex && request->SnapshotTerm > 0 && request->SnapshotTerm <= header->Term && request->Done <= 1 && + (request->SnapshotIndex <= VolatileState->CommitIndex || + request->SnapshotTerm >= State->LogTerm(VolatileState->CommitIndex)) && + (request->SnapshotIndex > VolatileState->CommitIndex || request->SnapshotIndex < State->SnapshotIndex || + request->SnapshotTerm == State->LogTerm(request->SnapshotIndex)) && request->TotalBytes >= 44 && request->TotalBytes <= TSnapshot::MaxBytes && request->Offset <= request->TotalBytes && length > 0 && length <= TInstallSnapshotRequest::MaxChunkBytes && length <= request->TotalBytes - request->Offset && diff --git a/test/test_compaction.cpp b/test/test_compaction.cpp index ddef6a7..7e24ef5 100644 --- a/test/test_compaction.cpp +++ b/test/test_compaction.cpp @@ -388,6 +388,79 @@ void test_corrupt_snapshot_and_invalid_headers_cannot_mutate_application(void**) assert_int_equal(app->LastAppliedIndex, 5); assert_int_equal(app->Loads, 1); } +void test_newer_snapshot_cannot_regress_committed_term(void**) { + for (bool compacted : {false, true}) { + std::vector> replies; + auto peer = std::make_shared([&](auto message) { + if (message->Type == uint32_t(EMessageType::INSTALL_SNAPSHOT_RESPONSE)) + replies.push_back(message.template Cast()); + }); + auto state = std::make_shared(); + auto raft = std::make_shared(state, 2, TNodeDict{{1, peer}}); + auto app = std::make_shared(); TRequestProcessor processor(raft, app, {}); + state->CurrentTerm = 2; + for (uint64_t index = 1; index <= 3; ++index) { + auto entry = app->Prepare(Command("key", std::to_string(index))); + entry->Term = index == 1 ? 1 : 2; state->Append(entry); + } + auto heartbeat = NewHoldedMessage(TMessageEx{.Src = 1, .Dst = 2, .Term = 2}, + TAppendEntriesRequest{.PrevLogIndex = 3, .PrevLogTerm = 2, .LeaderCommit = 3, .LeaderId = 1}); + raft->Process({}, heartbeat); processor.ProcessCommitted(); + if (compacted) raft->Compact(); + assert_int_equal(app->LastAppliedIndex, 3); assert_true(Read(*app, "key") == "3"); + TKv source; source.Write(source.Prepare(Command("key", "future")), 4); + TSnapshot snapshot{4, 1, {1, 2}, source.SaveSnapshot()}; + auto bytes = EncodeSnapshot(snapshot); auto request = Chunk(snapshot, bytes, 0, bytes.size()); + request->Term = 9; // An invalid higher-term request must not change hard state. + const auto before = static_cast(*state); + raft->Process({}, request); + assert_true(static_cast(*state) == before); + assert_true(replies.empty()); assert_int_equal(app->LastAppliedIndex, 3); + assert_true(Read(*app, "key") == "3"); assert_int_equal(app->Loads, 0); + auto conflicting = snapshot; conflicting.Index = 3; + const auto conflictingBytes = EncodeSnapshot(conflicting); + auto covered = Chunk(conflicting, conflictingBytes, 0, conflictingBytes.size()); covered->Term = 9; + raft->Process({}, covered); + assert_true(static_cast(*state) == before); assert_true(replies.empty()); + snapshot.Term = 2; bytes = EncodeSnapshot(snapshot); + raft->Process({}, Chunk(snapshot, bytes, 0, bytes.size())); + assert_true(replies.back()->Installed); assert_int_equal(state->SnapshotIndex, 4); + assert_int_equal(app->LastAppliedIndex, 4); assert_true(Read(*app, "key") == "future"); + } +} + +void test_snapshot_is_rechecked_when_commit_advances_mid_transfer(void**) { + std::vector> replies; + auto peer = std::make_shared([&](auto message) { + if (message->Type == uint32_t(EMessageType::INSTALL_SNAPSHOT_RESPONSE)) + replies.push_back(message.template Cast()); + }); + auto state = std::make_shared(); auto raft = std::make_shared(state, 2, TNodeDict{{1, peer}}); + auto app = std::make_shared(); TRequestProcessor processor(raft, app, {}); + state->CurrentTerm = 2; + for (uint64_t index = 1; index <= 3; ++index) { + auto entry = app->Prepare(Command("key", std::to_string(index))); + entry->Term = index == 1 ? 1 : 2; state->Append(entry); + } + auto heartbeat = NewHoldedMessage(TMessageEx{.Src = 1, .Dst = 2, .Term = 3}, + TAppendEntriesRequest{.PrevLogIndex = 3, .PrevLogTerm = 2, .LeaderCommit = 1, .LeaderId = 1}); + raft->Process({}, heartbeat); processor.ProcessCommitted(); + TKv source; source.Write(source.Prepare(Command("key", "future")), 4); + TSnapshot snapshot{4, 1, {1, 2}, source.SaveSnapshot()}; + const auto bytes = EncodeSnapshot(snapshot); + raft->Process({}, Chunk(snapshot, bytes, 0, 8)); + assert_true(replies.back()->Success); assert_false(replies.back()->Installed); + heartbeat.Cast()->LeaderCommit = 3; + raft->Process({}, heartbeat); processor.ProcessCommitted(); + const auto before = static_cast(*state); const auto replyCount = replies.size(); + raft->Process({}, Chunk(snapshot, bytes, 8, bytes.size() - 8)); + assert_int_equal(replies.size(), replyCount); assert_true(static_cast(*state) == before); + assert_int_equal(app->LastAppliedIndex, 3); assert_true(Read(*app, "key") == "3"); + snapshot.Term = 2; const auto valid = EncodeSnapshot(snapshot); + raft->Process({}, Chunk(snapshot, valid, 0, valid.size())); + assert_true(replies.back()->Installed); assert_true(Read(*app, "key") == "future"); +} + void test_remote_snapshot_retains_suffix_without_applying_it_before_commit(void**) { auto state = std::make_shared(); TKv prefix; @@ -550,6 +623,8 @@ int main() { cmocka_unit_test(test_partial_and_installed_snapshot_acks_do_not_confirm_strong_reads), cmocka_unit_test(test_compacted_follower_hint_requires_boundary_probe_before_match), cmocka_unit_test(test_corrupt_snapshot_and_invalid_headers_cannot_mutate_application), + cmocka_unit_test(test_newer_snapshot_cannot_regress_committed_term), + cmocka_unit_test(test_snapshot_is_rechecked_when_commit_advances_mid_transfer), cmocka_unit_test(test_joint_snapshot_is_rejected_even_if_boundary_is_already_covered), cmocka_unit_test(test_remote_snapshot_retains_suffix_without_applying_it_before_commit), cmocka_unit_test(test_snapshot_boundary_index_exhaustion_does_not_append_wrapped_noop), From d202c36b9b98a42cb980d8998812c775ebf06342 Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Fri, 13 Nov 2026 19:53:36 -0500 Subject: [PATCH 30/66] Reject snapshot acknowledgements that skip bytes (#30) --- docs/snapshot-retries.md | 5 +++++ miniraft/raft.cpp | 2 +- test/test_compaction.cpp | 16 ++++++++++++++++ 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/docs/snapshot-retries.md b/docs/snapshot-retries.md index 14b4e98..27b6d72 100644 --- a/docs/snapshot-retries.md +++ b/docs/snapshot-retries.md @@ -1,5 +1,10 @@ # Snapshot retry progress: v0.1.0-alpha.18 +Follow-up: partial acknowledgements must report exactly the end of the +outstanding chunk. Previously a larger offset could skip unsent bytes. An +invalid acknowledgement now leaves the chunk outstanding for retry; a +deterministic regression verifies retry and eventual installation. + The [alpha.19 follow-up](snapshot-pinning.md) also pins an active transfer across local compaction, replacing the image-switching behavior described below. diff --git a/miniraft/raft.cpp b/miniraft/raft.cpp index 684e863..d669b65 100644 --- a/miniraft/raft.cpp +++ b/miniraft/raft.cpp @@ -790,7 +790,7 @@ void TRaft::OnInstallSnapshot(TMessageHolder response) // Snapshot completion is not a fresh-read acknowledgement. AppendEntries // must still establish the next read barrier and replicate the suffix. } else { - if (response->NextOffset < rpc.SentEnd || response->NextOffset >= rpc.Bytes->size()) return; + if (response->NextOffset != rpc.SentEnd || response->NextOffset >= rpc.Bytes->size()) return; rpc.Offset = response->NextOffset; rpc.Seqno = 0; // Next chunk gets a new identity; old replies stay stale. } diff --git a/test/test_compaction.cpp b/test/test_compaction.cpp index 7e24ef5..2ef8d95 100644 --- a/test/test_compaction.cpp +++ b/test/test_compaction.cpp @@ -177,6 +177,21 @@ void test_partial_transfer_restart_and_delayed_ack_are_safe(void**) { assert_int_equal(cluster.Apps[2]->Loads, 1); assert_true(Read(*cluster.Apps[2], "c") == std::string(60000, 'c')); } +void test_snapshot_partial_ack_cannot_skip_unsent_bytes(void**) { + TCluster cluster; cluster.Prepare(); + auto chunk = cluster.SendChunk(); + const auto end = chunk->Offset + chunk->Len - sizeof(TInstallSnapshotRequest); + assert_true(end + 1 < chunk->TotalBytes); + auto invalid = NewHoldedMessage(TMessageEx{.Src = 3, .Dst = 1, .Term = chunk->Term, .Seqno = chunk->Seqno}, + TInstallSnapshotResponse{.SnapshotIndex = chunk->SnapshotIndex, .NextOffset = end + 1, .Success = 1}); + cluster.Deliver(invalid); + auto retry = cluster.SendChunk(); + assert_int_equal(retry->Offset, chunk->Offset); assert_int_equal(retry->Seqno, chunk->Seqno); + cluster.AckChunk(retry); auto next = cluster.SendChunk(); + assert_int_equal(next->Offset, end); + cluster.AckChunk(next); cluster.Pump(1, 80, true); + assert_int_equal(cluster.Apps[2]->Loads, 1); +} void ExerciseDelayedSnapshotAcks(bool readTraffic) { for (auto learners : {std::vector{}, std::vector{3}}) { TCluster cluster(learners); cluster.Prepare(); @@ -616,6 +631,7 @@ int main() { cmocka_unit_test(test_compaction_lagging_follower_installs_snapshot_then_tail), cmocka_unit_test(test_simultaneous_startup_candidates_use_distinct_election_deadlines), cmocka_unit_test(test_partial_transfer_restart_and_delayed_ack_are_safe), + cmocka_unit_test(test_snapshot_partial_ack_cannot_skip_unsent_bytes), cmocka_unit_test(test_snapshot_timeout_retries_do_not_starve_delayed_acks), cmocka_unit_test(test_snapshot_read_traffic_does_not_starve_delayed_acks), cmocka_unit_test(test_compaction_during_each_chunk_cannot_restart_snapshot_catchup), From ff79596ffddf888c4a82d10cd0e6d60c2d62eb94 Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Fri, 13 Nov 2026 20:01:40 -0500 Subject: [PATCH 31/66] Reject snapshot storage boundary term regression (#31) --- docs/snapshot-committed-terms.md | 5 +++++ miniraft/durable.cpp | 2 +- miniraft/state.h | 2 +- test/test_compaction.cpp | 34 ++++++++++++++++++++++++++++++++ 4 files changed, 41 insertions(+), 2 deletions(-) diff --git a/docs/snapshot-committed-terms.md b/docs/snapshot-committed-terms.md index 7e82ae9..43106af 100644 --- a/docs/snapshot-committed-terms.md +++ b/docs/snapshot-committed-terms.md @@ -21,3 +21,8 @@ valid later snapshot still installs. This check detects these specific committed term contradictions; it does not authenticate peers or prove snapshot contents match the committed application state. Membership changes and joint snapshot installation remain disabled. No wire or storage format changes. + +The in-memory and durable storage APIs also reject a replacement snapshot whose +term is below the existing compacted boundary term. This protects direct storage +callers even when they bypass RPC validation. Rejection leaves the selected +snapshot, retained suffix and durable image unchanged. diff --git a/miniraft/durable.cpp b/miniraft/durable.cpp index 603fae2..5eab3df 100644 --- a/miniraft/durable.cpp +++ b/miniraft/durable.cpp @@ -312,7 +312,7 @@ void TDurableState::InstallSnapshot(const TSnapshot& snapshot) { EncodeSnapshot(snapshot); if (!Impl->Voters.empty() && snapshot.Voters != Impl->Voters) throw std::invalid_argument("Snapshot would change bound voters"); - if (snapshot.Index <= SnapshotIndex || snapshot.Term > CurrentTerm) + if (snapshot.Index <= SnapshotIndex || snapshot.Term < SnapshotTerm || snapshot.Term > CurrentTerm) throw std::invalid_argument("Obsolete or future durable snapshot"); auto image = std::make_shared(snapshot); std::vector> suffix; diff --git a/miniraft/state.h b/miniraft/state.h index f8ac489..b99c962 100644 --- a/miniraft/state.h +++ b/miniraft/state.h @@ -152,7 +152,7 @@ struct TState: IState { EncodeSnapshot(snapshot); // Validate before changing any state. if (!BoundVoters.empty() && snapshot.Voters != BoundVoters) throw std::invalid_argument("Snapshot would change bound voters"); - if (snapshot.Index <= SnapshotIndex || snapshot.Term > CurrentTerm) + if (snapshot.Index <= SnapshotIndex || snapshot.Term < SnapshotTerm || snapshot.Term > CurrentTerm) throw std::invalid_argument("Obsolete or future snapshot"); auto image = std::make_shared(snapshot); std::vector> suffix; diff --git a/test/test_compaction.cpp b/test/test_compaction.cpp index 2ef8d95..a824dac 100644 --- a/test/test_compaction.cpp +++ b/test/test_compaction.cpp @@ -476,6 +476,38 @@ void test_snapshot_is_rechecked_when_commit_advances_mid_transfer(void**) { assert_true(replies.back()->Installed); assert_true(Read(*app, "key") == "future"); } +void CheckSnapshotStorageTerm(IState& state, std::function image = [] { return std::string{}; }) { + state.CurrentTerm = 2; state.Commit(); + state.InstallSnapshot({2, 2, {1}, "committed"}); + auto entry = NewHoldedMessage(); entry->Term = 2; state.Append(entry); + const auto before = static_cast(state); + const auto beforeImage = image(); + Reject([&] { state.InstallSnapshot({4, 1, {1}, "regressed"}); }); + assert_true(static_cast(state) == before); + assert_true(image() == beforeImage); + assert_true(state.GetSnapshot()->Data == "committed"); assert_int_equal(state.LogTerm(3), 2); + state.InstallSnapshot({4, 2, {1}, "valid"}); + assert_int_equal(state.SnapshotIndex, 4); assert_true(state.GetSnapshot()->Data == "valid"); +} +void test_memory_snapshot_storage_rejects_term_regression(void**) { + TState state; CheckSnapshotStorageTerm(state); +} +#ifdef __linux__ +void test_durable_snapshot_storage_rejects_term_regression(void**) { + auto dir = std::filesystem::temp_directory_path() / ("miniraft-term-boundary-" + + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count())); + assert_true(std::filesystem::create_directory(dir)); + auto name = (dir / "raft").string(); + { TDurableState state(name, 1); CheckSnapshotStorageTerm(state, [&] { + std::ifstream input(name + ".durable.1", std::ios::binary); + return std::string(std::istreambuf_iterator(input), {}); + }); } + { TDurableState state(name, 1); assert_int_equal(state.SnapshotIndex, 4); + assert_int_equal(state.SnapshotTerm, 2); assert_true(state.GetSnapshot()->Data == "valid"); } + std::error_code ignored; std::filesystem::remove_all(dir, ignored); +} +#endif + void test_remote_snapshot_retains_suffix_without_applying_it_before_commit(void**) { auto state = std::make_shared(); TKv prefix; @@ -641,11 +673,13 @@ int main() { cmocka_unit_test(test_corrupt_snapshot_and_invalid_headers_cannot_mutate_application), cmocka_unit_test(test_newer_snapshot_cannot_regress_committed_term), cmocka_unit_test(test_snapshot_is_rechecked_when_commit_advances_mid_transfer), + cmocka_unit_test(test_memory_snapshot_storage_rejects_term_regression), cmocka_unit_test(test_joint_snapshot_is_rejected_even_if_boundary_is_already_covered), cmocka_unit_test(test_remote_snapshot_retains_suffix_without_applying_it_before_commit), cmocka_unit_test(test_snapshot_boundary_index_exhaustion_does_not_append_wrapped_noop), cmocka_unit_test(test_memory_snapshot_boundary_retains_matching_suffix_and_replaces_conflict), #ifdef __linux__ + cmocka_unit_test(test_durable_snapshot_storage_rejects_term_regression), cmocka_unit_test(test_durable_term_lookup_avoids_payload_copies_across_compaction), cmocka_unit_test(test_durable_compacted_boundary_survives_restart), cmocka_unit_test(test_snapshot_and_suffix_are_atomic_at_every_publication_stage), From 005b14a1fb55977e19b492bf03444e400ef8f21c Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Fri, 13 Nov 2026 20:06:36 -0500 Subject: [PATCH 32/66] Reject storage appends beyond maximum log index (#32) --- docs/compaction.md | 5 ++++- miniraft/durable.cpp | 1 + miniraft/state.h | 1 + test/test_compaction.cpp | 29 +++++++++++++++++++++++++++++ 4 files changed, 35 insertions(+), 1 deletion(-) diff --git a/docs/compaction.md b/docs/compaction.md index 8ee2731..b5d40ce 100644 --- a/docs/compaction.md +++ b/docs/compaction.md @@ -39,7 +39,10 @@ entries after SnapshotIndex, while LogTerm at the boundary returns SnapshotTerm. LastLogIndex and all RPC/application indices remain absolute. Appends, truncation, votes, current-term commitment and strong reads use these absolute boundaries. Truncation cannot remove a snapshot. Election no-op creation also checks signed -index exhaustion, now reachable without allocating an enormous prefix. +index exhaustion, now reachable without allocating an enormous prefix. Memory +and durable storage reject a direct append once the last index is INT64_MAX, +before mutating the log or publishing an image. Appending the entry at INT64_MAX +itself remains valid when the previous index is INT64_MAX - 1. An installation retains the suffix only when the local entry at the included index has the same term. Otherwise its uncommitted log is discarded. A follower diff --git a/miniraft/durable.cpp b/miniraft/durable.cpp index 5eab3df..b3ea345 100644 --- a/miniraft/durable.cpp +++ b/miniraft/durable.cpp @@ -274,6 +274,7 @@ void TDurableState::Commit() void TDurableState::Append(TMessageHolder entry) { + if (LastLogIndex >= uint64_t(INT64_MAX)) throw std::overflow_error("Raft log index exhausted"); Impl->Log.push_back(Clone(entry)); LastLogIndex = SnapshotIndex + Impl->Log.size(); LastLogTerm = Impl->Log.back()->Term; diff --git a/miniraft/state.h b/miniraft/state.h index b99c962..afebc93 100644 --- a/miniraft/state.h +++ b/miniraft/state.h @@ -120,6 +120,7 @@ struct TState: IState { } void Append(TMessageHolder entry) override { + if (LastLogIndex >= uint64_t(INT64_MAX)) throw std::overflow_error("Raft log index exhausted"); Log.emplace_back(CloneEntry(entry)); LastLogIndex = SnapshotIndex + Log.size(); LastLogTerm = Log.back()->Term; diff --git a/test/test_compaction.cpp b/test/test_compaction.cpp index a824dac..7a3f34f 100644 --- a/test/test_compaction.cpp +++ b/test/test_compaction.cpp @@ -539,6 +539,33 @@ void test_snapshot_boundary_index_exhaustion_does_not_append_wrapped_noop(void** assert_true(state->LastLogIndex == uint64_t(INT64_MAX)); assert_int_equal(state->LastLogTerm, 1); } +void CheckStorageIndexExhaustion(IState& state, std::function image = [] { return std::string{}; }) { + state.InstallSnapshot({uint64_t(INT64_MAX) - 1, 1, {1}, "boundary"}); + auto entry = NewHoldedMessage(); entry->Term = 1; state.Append(entry); + assert_true(state.LastLogIndex == uint64_t(INT64_MAX)); + const auto before = static_cast(state); const auto beforeImage = image(); + Reject([&] { state.Append(entry); }); + assert_true(static_cast(state) == before); assert_true(image() == beforeImage); + assert_int_equal(state.LogTerm(INT64_MAX), 1); assert_true(state.GetSnapshot()->Data == "boundary"); +} +void test_memory_storage_rejects_index_exhaustion(void**) { + TState state; CheckStorageIndexExhaustion(state); +} +#ifdef __linux__ +void test_durable_storage_rejects_index_exhaustion(void**) { + auto dir = std::filesystem::temp_directory_path() / ("miniraft-index-limit-" + + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count())); + assert_true(std::filesystem::create_directory(dir)); + auto name = (dir / "raft").string(); + { TDurableState state(name, 1); CheckStorageIndexExhaustion(state, [&] { + std::ifstream input(name + ".durable.1", std::ios::binary); + return std::string(std::istreambuf_iterator(input), {}); + }); } + { TDurableState state(name, 1); assert_true(state.LastLogIndex == uint64_t(INT64_MAX)); + assert_int_equal(state.LogTerm(INT64_MAX), 1); } + std::error_code ignored; std::filesystem::remove_all(dir, ignored); +} +#endif void ExerciseBoundary(IState& state) { state.CurrentTerm = 3; state.VotedFor = 2; state.Commit(); @@ -677,9 +704,11 @@ int main() { cmocka_unit_test(test_joint_snapshot_is_rejected_even_if_boundary_is_already_covered), cmocka_unit_test(test_remote_snapshot_retains_suffix_without_applying_it_before_commit), cmocka_unit_test(test_snapshot_boundary_index_exhaustion_does_not_append_wrapped_noop), + cmocka_unit_test(test_memory_storage_rejects_index_exhaustion), cmocka_unit_test(test_memory_snapshot_boundary_retains_matching_suffix_and_replaces_conflict), #ifdef __linux__ cmocka_unit_test(test_durable_snapshot_storage_rejects_term_regression), + cmocka_unit_test(test_durable_storage_rejects_index_exhaustion), cmocka_unit_test(test_durable_term_lookup_avoids_payload_copies_across_compaction), cmocka_unit_test(test_durable_compacted_boundary_survives_restart), cmocka_unit_test(test_snapshot_and_suffix_are_atomic_at_every_publication_stage), From 326033e2ef557dbb0b7fa5fba0dafd3a05fdfe1e Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Fri, 13 Nov 2026 20:19:14 -0500 Subject: [PATCH 33/66] Reject covered snapshots with impossible terms (#33) --- docs/snapshot-committed-terms.md | 5 +++-- miniraft/raft.cpp | 4 +++- test/test_compaction.cpp | 22 ++++++++++++++++++++++ 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/docs/snapshot-committed-terms.md b/docs/snapshot-committed-terms.md index 43106af..c196990 100644 --- a/docs/snapshot-committed-terms.md +++ b/docs/snapshot-committed-terms.md @@ -10,8 +10,9 @@ Incoming snapshot requests are now checked before term or role changes. A snapshot after the committed prefix must have a boundary term at least as large as the term at the follower's commit index. If the requested boundary itself is committed and still known locally, its term must match. A boundary older than the -local snapshot cannot be checked this way because its individual term has been -compacted. Invalid requests are dropped, consistent with other malformed RPCs. +local snapshot cannot be checked for equality because its individual term has been +compacted, but its term must not exceed the local snapshot term. Invalid requests +are dropped, consistent with other malformed RPCs. Each chunk is checked again, including when commitment advances mid-transfer. Deterministic tests reproduce the prior mutation with both retained and compacted diff --git a/miniraft/raft.cpp b/miniraft/raft.cpp index d669b65..1ed62e0 100644 --- a/miniraft/raft.cpp +++ b/miniraft/raft.cpp @@ -525,13 +525,15 @@ bool TRaft::ValidateRpc(TMessageHolder message) const { auto request = message.Cast(); const auto length = request->Len - sizeof(TInstallSnapshotRequest); // A later boundary cannot have an older term than the committed prefix. - // For an already committed, retained boundary, the terms must match. + // For an already committed, retained boundary, the terms must match; + // an older compacted boundary cannot have a newer term than the snapshot. return request->LeaderId == header->Src && request->SnapshotIndex > 0 && request->SnapshotIndex <= maxIndex && request->SnapshotTerm > 0 && request->SnapshotTerm <= header->Term && request->Done <= 1 && (request->SnapshotIndex <= VolatileState->CommitIndex || request->SnapshotTerm >= State->LogTerm(VolatileState->CommitIndex)) && (request->SnapshotIndex > VolatileState->CommitIndex || request->SnapshotIndex < State->SnapshotIndex || request->SnapshotTerm == State->LogTerm(request->SnapshotIndex)) && + (request->SnapshotIndex >= State->SnapshotIndex || request->SnapshotTerm <= State->SnapshotTerm) && request->TotalBytes >= 44 && request->TotalBytes <= TSnapshot::MaxBytes && request->Offset <= request->TotalBytes && length > 0 && length <= TInstallSnapshotRequest::MaxChunkBytes && length <= request->TotalBytes - request->Offset && diff --git a/test/test_compaction.cpp b/test/test_compaction.cpp index 7a3f34f..7562416 100644 --- a/test/test_compaction.cpp +++ b/test/test_compaction.cpp @@ -403,6 +403,27 @@ void test_corrupt_snapshot_and_invalid_headers_cannot_mutate_application(void**) assert_int_equal(app->LastAppliedIndex, 5); assert_int_equal(app->Loads, 1); } +void test_older_snapshot_cannot_exceed_compacted_boundary_term(void**) { + std::vector> replies; + auto peer = std::make_shared([&](auto message) { replies.push_back(message.template Cast()); }); + auto state = std::make_shared(); state->CurrentTerm = 2; + TKv image; image.Write(image.Prepare(Command("key", "committed")), 5); + state->InstallSnapshot({5, 2, {1, 2}, image.SaveSnapshot()}); + auto raft = std::make_shared(state, 2, TNodeDict{{1, peer}}); + auto app = std::make_shared(); TRequestProcessor processor(raft, app, {}); + const auto before = static_cast(*state); + TSnapshot older{3, 3, {1, 2}, image.SaveSnapshot()}; + auto bytes = EncodeSnapshot(older); auto invalid = Chunk(older, bytes, 0, bytes.size()); invalid->Term = 9; + raft->Process({}, invalid); + assert_true(replies.empty()); assert_true(static_cast(*state) == before); + assert_int_equal(app->Loads, 1); assert_true(Read(*app, "key") == "committed"); + for (uint64_t term : {1, 2}) { + older.Term = term; bytes = EncodeSnapshot(older); + raft->Process({}, Chunk(older, bytes, 0, bytes.size())); + assert_true(replies.back()->Success); assert_true(replies.back()->Installed); + assert_int_equal(app->Loads, 1); assert_int_equal(state->SnapshotIndex, 5); + } +} void test_newer_snapshot_cannot_regress_committed_term(void**) { for (bool compacted : {false, true}) { std::vector> replies; @@ -699,6 +720,7 @@ int main() { cmocka_unit_test(test_compacted_follower_hint_requires_boundary_probe_before_match), cmocka_unit_test(test_corrupt_snapshot_and_invalid_headers_cannot_mutate_application), cmocka_unit_test(test_newer_snapshot_cannot_regress_committed_term), + cmocka_unit_test(test_older_snapshot_cannot_exceed_compacted_boundary_term), cmocka_unit_test(test_snapshot_is_rechecked_when_commit_advances_mid_transfer), cmocka_unit_test(test_memory_snapshot_storage_rejects_term_regression), cmocka_unit_test(test_joint_snapshot_is_rejected_even_if_boundary_is_already_covered), From 47218e8d98edb4ce8eca15ecb8cc0a90d9341a21 Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Fri, 13 Nov 2026 20:24:39 -0500 Subject: [PATCH 34/66] Add offline stable configuration snapshot metadata (#34) --- README.md | 2 + docs/configuration-scan.md | 2 +- docs/stable-configuration-snapshots.md | 28 +++++++++++++ miniraft/configuration.cpp | 12 +++++- miniraft/configuration.h | 1 + miniraft/durable.cpp | 4 +- miniraft/raft.cpp | 6 +-- miniraft/snapshot.cpp | 23 +++++----- miniraft/snapshot.h | 5 ++- miniraft/state.h | 4 +- test/test_compaction.cpp | 9 ++++ test/test_configuration.cpp | 26 ++++++++++++ test/test_snapshots.cpp | 58 ++++++++++++++++++++++++++ 13 files changed, 159 insertions(+), 21 deletions(-) create mode 100644 docs/stable-configuration-snapshots.md diff --git a/README.md b/README.md index 9dcb0df..5203bc5 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,8 @@ The [configuration history validator](docs/configuration-history.md) checks an offline retained-log sequence against a trusted baseline or joint snapshot. The [configuration log scanner](docs/configuration-scan.md) validates a retained suffix at a caller-supplied commit boundary and at its end. +The [stable configuration snapshot foundation](docs/stable-configuration-snapshots.md) +preserves configuration identity for offline recovery validation. The [snapshot term validation fix](docs/snapshot-committed-terms.md) rejects snapshot boundaries that contradict the follower's committed prefix. The [message buffer follow-up](docs/message-extents.md) checks allocated buffer diff --git a/docs/configuration-scan.md b/docs/configuration-scan.md index b707411..ebe7d1d 100644 --- a/docs/configuration-scan.md +++ b/docs/configuration-scan.md @@ -12,7 +12,7 @@ The caller must independently establish the baseline and commit index: the scan does not prove that any entry is committed, recover a commit index, authenticate the snapshot, or establish cluster identity. A fixed v1 snapshot does not identify its latest configuration entry, so its configuration identity must come from -other trusted information. A joint snapshot contains that identity and the scan +other trusted information. Experimental stable and joint snapshots contain that identity and the scan checks that the supplied baseline matches it. The scan checks snapshot and log boundaries, bound fixed voters, terms, configuration predecessors, and transitions. It requires a consistent, externally serialized view of the state; it does not diff --git a/docs/stable-configuration-snapshots.md b/docs/stable-configuration-snapshots.md new file mode 100644 index 0000000..94ec4c0 --- /dev/null +++ b/docs/stable-configuration-snapshots.md @@ -0,0 +1,28 @@ +# Offline stable configuration snapshot metadata + +The experimental MRFTS003 snapshot format preserves a stable voter set and the +index and term of the configuration entry that selected it. This fills a gap in +MRFTS001: its voter list does not establish the identity needed to validate a +later configuration entry. MRFTS003 stores magic[8], included index[8], included +term[8], voter count[4], application length[8], configuration index[8], +configuration term[8], voter IDs[4 each], application data, and CRC32[4]. The +configuration identity must be nonzero, no later than the snapshot boundary, and +have the same term when both indices coincide. Voters are sorted and bounded. +MRFTS001 and MRFTS002 encoding is unchanged. + +`TConfigurationHistory::FromStableSnapshot` initializes offline history from +this identity. The retained-log scanner checks the baseline identity against +the snapshot before checking subsequent joint/stable entries. These checks do +not prove commitment or authenticate the snapshot: the caller must establish a +trusted snapshot and commit boundary independently. MRFTS003 cannot assert that +there was no earlier configuration entry; an explicit trusted baseline remains +necessary for such histories. + +The fixed-membership runtime, storage installation and durable recovery reject +MRFTS003, and the network receiver rejects it even at an already covered boundary. +The standalone snapshot store supports offline export. Older binaries may falsely +acknowledge covered unknown formats; do not send these images to a mixed-version +cluster. Live configuration activation, rollback, commit rules, joint snapshot +installation and membership recovery remain open. This experimental format may +change before those semantics are implemented. Rebuild dependents because the +in-memory snapshot structure has changed. diff --git a/miniraft/configuration.cpp b/miniraft/configuration.cpp index 171e2b0..0126c29 100644 --- a/miniraft/configuration.cpp +++ b/miniraft/configuration.cpp @@ -84,7 +84,7 @@ TConfigurationHistory TConfigurationHistory::Stable(std::vector voters TConfigurationHistory TConfigurationHistory::FromJointSnapshot(const TSnapshot& snapshot) { ValidateSnapshot(snapshot); - if (!snapshot.IsJoint()) throw std::invalid_argument("Snapshot does not identify a joint configuration"); + if (!snapshot.IsJoint() || snapshot.StableConfiguration) throw std::invalid_argument("Snapshot does not identify a joint configuration"); TConfigurationHistory history; history.Old = snapshot.Voters; history.New = snapshot.JointVoters; history.ConfigIndex = snapshot.ConfigurationIndex; history.ConfigTerm = snapshot.ConfigurationTerm; @@ -92,6 +92,12 @@ TConfigurationHistory TConfigurationHistory::FromJointSnapshot(const TSnapshot& return history; } +TConfigurationHistory TConfigurationHistory::FromStableSnapshot(const TSnapshot& snapshot) { + ValidateSnapshot(snapshot); + if (!snapshot.StableConfiguration) throw std::invalid_argument("Snapshot does not identify a stable configuration"); + return Stable(snapshot.Voters, snapshot.ConfigurationIndex, snapshot.ConfigurationTerm, snapshot.Index, snapshot.Term); +} + TConfigurationHistory TConfigurationHistory::Advance(uint64_t index, TMessageHolder entry) const { const auto record = DecodeConfigurationEntry(std::move(entry)); if (Through >= uint64_t(INT64_MAX) || index != Through + 1 || record.Term < ThroughLogTerm || @@ -150,7 +156,9 @@ TConfigurationScan ScanConfigurationHistory(const IState& state, baseline.ConfigurationIndex() != expected.ConfigurationIndex() || baseline.ConfigurationTerm() != expected.ConfigurationTerm()) throw std::invalid_argument("Joint snapshot baseline mismatch"); - } else if (baseline.IsJoint() || baseline.OldVoters() != snapshot->Voters) { + } else if (baseline.IsJoint() || baseline.OldVoters() != snapshot->Voters || + (snapshot->StableConfiguration && (baseline.ConfigurationIndex() != snapshot->ConfigurationIndex || + baseline.ConfigurationTerm() != snapshot->ConfigurationTerm))) { throw std::invalid_argument("Fixed snapshot baseline mismatch"); } } else if (snapshot || state.SnapshotTerm || baseline.IsJoint()) { diff --git a/miniraft/configuration.h b/miniraft/configuration.h index 9834af9..b673521 100644 --- a/miniraft/configuration.h +++ b/miniraft/configuration.h @@ -30,6 +30,7 @@ class TConfigurationHistory { uint64_t configurationIndex, uint64_t configurationTerm, uint64_t throughIndex, uint64_t throughTerm); static TConfigurationHistory FromJointSnapshot(const TSnapshot& snapshot); + static TConfigurationHistory FromStableSnapshot(const TSnapshot& snapshot); // Visit every retained entry in index order; neither method changes *this. TConfigurationHistory Observe(uint64_t index, TMessageHolder entry) const; // Metadata must come from a trusted, already validated storage backend. diff --git a/miniraft/durable.cpp b/miniraft/durable.cpp index b3ea345..e696c19 100644 --- a/miniraft/durable.cpp +++ b/miniraft/durable.cpp @@ -167,7 +167,7 @@ TDurableState::TDurableState(const std::string& name, uint32_t id, THook hook) if (!length && compacted) throw std::runtime_error("Missing embedded snapshot"); if (length) { auto snapshot = DecodeSnapshot(std::string_view(reinterpret_cast(bytes.data() + position), size_t(length))); - if (snapshot.IsJoint()) throw std::runtime_error("Joint snapshot recovery is unsupported"); + if (snapshot.HasConfigurationMetadata()) throw std::runtime_error("Configuration snapshot recovery is unsupported"); if (snapshot.Term > CurrentTerm) throw std::runtime_error("Future durable snapshot term"); if (configured && snapshot.Voters != Impl->Voters) throw std::runtime_error("Durable snapshot/voter mismatch"); position += size_t(length); @@ -309,7 +309,7 @@ uint64_t TDurableState::RetainedLogFlags(int64_t index) const { std::shared_ptr TDurableState::GetSnapshot() const { return Impl->Snapshot; } void TDurableState::InstallSnapshot(const TSnapshot& snapshot) { - if (snapshot.IsJoint()) throw std::invalid_argument("Joint snapshot installation is unsupported"); + if (snapshot.HasConfigurationMetadata()) throw std::invalid_argument("Configuration snapshot installation is unsupported"); EncodeSnapshot(snapshot); if (!Impl->Voters.empty() && snapshot.Voters != Impl->Voters) throw std::invalid_argument("Snapshot would change bound voters"); diff --git a/miniraft/raft.cpp b/miniraft/raft.cpp index 1ed62e0..b82eed1 100644 --- a/miniraft/raft.cpp +++ b/miniraft/raft.cpp @@ -198,8 +198,8 @@ TRaft::TRaft(std::shared_ptr state, int node, const TNodeDict& nodes, st if (!Learners.empty() && !State->SupportsVoterBinding()) throw std::invalid_argument("Learners require voter-binding-capable storage"); const auto bound = State->GetBoundVoters(); - if (auto snapshot = State->GetSnapshot(); snapshot && snapshot->IsJoint()) - throw std::invalid_argument("Joint snapshot recovery is unsupported"); + if (auto snapshot = State->GetSnapshot(); snapshot && snapshot->HasConfigurationMetadata()) + throw std::invalid_argument("Configuration snapshot recovery is unsupported"); if (State->SnapshotIndex > State->LastLogIndex || State->LastLogIndex > uint64_t(INT64_MAX)) throw std::invalid_argument("Invalid retained log bounds"); // Reject before binding voters or serving traffic. Current Raft cannot @@ -868,7 +868,7 @@ void TRaft::OnInstallSnapshot(ITimeSource::Time now, TMessageHolderSnapshotIndex || snapshot.Term != request->SnapshotTerm || snapshot.Voters != GetVoterIds()) + if (snapshot.HasConfigurationMetadata() || snapshot.Index != request->SnapshotIndex || snapshot.Term != request->SnapshotTerm || snapshot.Voters != GetVoterIds()) throw TMessageError("Snapshot metadata mismatch"); application->ValidateSnapshot(snapshot.Data, snapshot.Index); } catch (const std::exception&) { diff --git a/miniraft/snapshot.cpp b/miniraft/snapshot.cpp index a0ba243..ce68618 100644 --- a/miniraft/snapshot.cpp +++ b/miniraft/snapshot.cpp @@ -16,11 +16,12 @@ namespace { using namespace NSnapshotCodec; constexpr std::string_view Magic = "MRFTS001"; constexpr std::string_view JointMagic = "MRFTS002"; +constexpr std::string_view StableMagic = "MRFTS003"; void Validate(const TSnapshot& snapshot) { if (!snapshot.Index || snapshot.Index > uint64_t(INT64_MAX) || !snapshot.Term || snapshot.Voters.empty() || snapshot.Voters.size() > TSnapshot::MaxVoters || snapshot.JointVoters.size() > TSnapshot::MaxVoters || - snapshot.Data.size() > TSnapshot::MaxBytes - (snapshot.IsJoint() ? 60 : 40) - + snapshot.Data.size() > TSnapshot::MaxBytes - (snapshot.StableConfiguration ? 56 : snapshot.IsJoint() ? 60 : 40) - 4 * (snapshot.Voters.size() + snapshot.JointVoters.size())) throw std::runtime_error("Invalid snapshot metadata/size"); const auto validateVoters = [](const auto& voters) { @@ -31,8 +32,8 @@ void Validate(const TSnapshot& snapshot) { } }; validateVoters(snapshot.Voters); - if (snapshot.IsJoint()) { - if (snapshot.JointVoters.empty() || !snapshot.ConfigurationIndex || + if (snapshot.HasConfigurationMetadata()) { + if ((snapshot.StableConfiguration ? !snapshot.JointVoters.empty() : snapshot.JointVoters.empty()) || !snapshot.ConfigurationIndex || snapshot.ConfigurationIndex > snapshot.Index || !snapshot.ConfigurationTerm || snapshot.ConfigurationTerm > snapshot.Term || (snapshot.ConfigurationIndex == snapshot.Index && snapshot.ConfigurationTerm != snapshot.Term)) @@ -42,7 +43,7 @@ void Validate(const TSnapshot& snapshot) { } void CheckLocalLog(const TSnapshot& snapshot, const TRaft& raft) { Validate(snapshot); - if (snapshot.IsJoint() || snapshot.Voters != raft.GetVoterIds() || snapshot.Index > raft.GetLastIndex() || + if (snapshot.HasConfigurationMetadata() || snapshot.Voters != raft.GetVoterIds() || snapshot.Index > raft.GetLastIndex() || snapshot.Term > raft.GetState()->CurrentTerm || raft.GetState()->LogTerm(snapshot.Index) != snapshot.Term) throw std::runtime_error("Snapshot does not match retained log and fixed configuration"); @@ -70,15 +71,15 @@ void ValidateSnapshot(const TSnapshot& snapshot) { Validate(snapshot); } std::string EncodeSnapshot(const TSnapshot& snapshot) { Validate(snapshot); - std::string out(snapshot.IsJoint() ? JointMagic : Magic); + std::string out(snapshot.StableConfiguration ? StableMagic : snapshot.IsJoint() ? JointMagic : Magic); Put(out, snapshot.Index, 8); Put(out, snapshot.Term, 8); Put(out, snapshot.Voters.size(), 4); Put(out, snapshot.Data.size(), 8); - if (snapshot.IsJoint()) { + if (snapshot.HasConfigurationMetadata()) { Put(out, snapshot.ConfigurationIndex, 8); Put(out, snapshot.ConfigurationTerm, 8); - Put(out, snapshot.JointVoters.size(), 4); + if (!snapshot.StableConfiguration) Put(out, snapshot.JointVoters.size(), 4); } for (auto voter : snapshot.Voters) Put(out, voter, 4); for (auto voter : snapshot.JointVoters) Put(out, voter, 4); @@ -96,17 +97,19 @@ TSnapshot DecodeSnapshot(std::string_view bytes) { TReader reader{bytes.substr(0, bytes.size() - 4)}; const auto magic = reader.Read(Magic.size()); const bool joint = magic == JointMagic; - if (!joint && magic != Magic) throw std::runtime_error("Unknown snapshot format"); + const bool stable = magic == StableMagic; + if (!joint && !stable && magic != Magic) throw std::runtime_error("Unknown snapshot format"); TSnapshot snapshot; + snapshot.StableConfiguration = stable; snapshot.Index = reader.Take(8); snapshot.Term = reader.Take(8); const auto count = reader.Take(4); const auto length = reader.Take(8); uint64_t newCount = 0; - if (joint) { + if (joint || stable) { snapshot.ConfigurationIndex = reader.Take(8); snapshot.ConfigurationTerm = reader.Take(8); - newCount = reader.Take(4); + if (joint) newCount = reader.Take(4); } if (count > TSnapshot::MaxVoters || newCount > TSnapshot::MaxVoters || count + newCount > reader.Input.size() / 4) diff --git a/miniraft/snapshot.h b/miniraft/snapshot.h index 6ae01e2..62f905a 100644 --- a/miniraft/snapshot.h +++ b/miniraft/snapshot.h @@ -22,7 +22,10 @@ struct TSnapshot { // Runtime installation is disabled until membership recovery is implemented. std::vector JointVoters; uint64_t ConfigurationIndex = 0, ConfigurationTerm = 0; - bool IsJoint() const { return !JointVoters.empty() || ConfigurationIndex || ConfigurationTerm; } + // Experimental stable-configuration identity; fixed runtime rejects it. + bool StableConfiguration = false; + bool IsJoint() const { return !JointVoters.empty() || (!StableConfiguration && (ConfigurationIndex || ConfigurationTerm)); } + bool HasConfigurationMetadata() const { return StableConfiguration || IsJoint(); } bool operator==(const TSnapshot&) const = default; }; std::string EncodeSnapshot(const TSnapshot& snapshot); diff --git a/miniraft/state.h b/miniraft/state.h index afebc93..837d5bc 100644 --- a/miniraft/state.h +++ b/miniraft/state.h @@ -70,7 +70,7 @@ inline void ValidateVoterBinding(const IState& state, const std::vectorIsJoint()) throw std::invalid_argument("Joint snapshot recovery is unsupported"); + if (snapshot->HasConfigurationMetadata()) throw std::invalid_argument("Configuration snapshot recovery is unsupported"); if (snapshot->Voters != voters) throw std::invalid_argument("Snapshot voter configuration mismatch"); } else if (state.CurrentTerm != 1 || state.VotedFor || state.LastLogIndex) { throw std::invalid_argument("Unbound nonfresh store needs a verified snapshot before enabling learners"); @@ -149,7 +149,7 @@ struct TState: IState { } std::shared_ptr GetSnapshot() const override { return Snapshot; } void InstallSnapshot(const TSnapshot& snapshot) override { - if (snapshot.IsJoint()) throw std::invalid_argument("Joint snapshot installation is unsupported"); + if (snapshot.HasConfigurationMetadata()) throw std::invalid_argument("Configuration snapshot installation is unsupported"); EncodeSnapshot(snapshot); // Validate before changing any state. if (!BoundVoters.empty() && snapshot.Voters != BoundVoters) throw std::invalid_argument("Snapshot would change bound voters"); diff --git a/test/test_compaction.cpp b/test/test_compaction.cpp index 7562416..849bff3 100644 --- a/test/test_compaction.cpp +++ b/test/test_compaction.cpp @@ -357,6 +357,15 @@ void test_joint_snapshot_is_rejected_even_if_boundary_is_already_covered(void**) assert_true(replies.back()->Success); assert_false(replies.back()->Installed); raft->Process({}, Chunk(joint, bytes, 4, 4)); assert_false(replies.back()->Success); assert_false(replies.back()->Installed); + auto stable = fixed; stable.StableConfiguration = true; stable.ConfigurationIndex = 4; stable.ConfigurationTerm = 1; + auto stableBytes = EncodeSnapshot(stable); + raft->Process({}, Chunk(stable, stableBytes, 0, 4)); + assert_true(replies.back()->Success); assert_false(replies.back()->Installed); + raft->Process({}, Chunk(stable, stableBytes, 4, 4)); + assert_false(replies.back()->Success); assert_false(replies.back()->Installed); + stable.Index = 6; stableBytes = EncodeSnapshot(stable); + raft->Process({}, Chunk(stable, stableBytes, 0, stableBytes.size())); + assert_false(replies.back()->Success); assert_false(replies.back()->Installed); joint.Index = 6; const auto newer = EncodeSnapshot(joint); raft->Process({}, Chunk(joint, newer, 0, newer.size())); diff --git a/test/test_configuration.cpp b/test/test_configuration.cpp index 6572366..7a601f0 100644 --- a/test/test_configuration.cpp +++ b/test/test_configuration.cpp @@ -161,6 +161,31 @@ void test_scan_from_joint_snapshot_and_rejects_mismatched_baselines(void**) { state.LastLogTerm = 2; Reject([&] { ScanConfigurationHistory(state, base, 2); }); } +void test_scan_from_stable_configuration_snapshot(void**) { + TSnapshot snapshot{5, 2, {1, 2}, "image"}; snapshot.StableConfiguration = true; + snapshot.ConfigurationIndex = 4; snapshot.ConfigurationTerm = 2; + auto decoded = DecodeSnapshot(EncodeSnapshot(snapshot)); + auto baseline = TConfigurationHistory::FromStableSnapshot(decoded); + assert_false(baseline.IsJoint()); assert_int_equal(baseline.ConfigurationIndex(), 4); + TState state; state.Snapshot = std::make_shared(decoded); + state.SnapshotIndex = state.LastLogIndex = 5; state.SnapshotTerm = state.LastLogTerm = 2; + state.CurrentTerm = 3; + TConfigurationRecord joint{EConfigurationPhase::Joint, 3, 4, 2, {1, 2}, {2, 3}}; + state.Append(EncodeConfigurationEntry(joint)); + auto scan = ScanConfigurationHistory(state, baseline, 5); + assert_false(scan.Committed.IsJoint()); assert_true(scan.Latest.IsJoint()); + assert_true(scan.HasUncommittedConfiguration()); + auto wrong = TConfigurationHistory::Stable({1, 2}, 0, 0, 5, 2); + Reject([&] { ScanConfigurationHistory(state, wrong, 5); }); + wrong = TConfigurationHistory::Stable({1, 2}, 4, 1, 5, 2); + Reject([&] { ScanConfigurationHistory(state, wrong, 5); }); + Reject([&] { TConfigurationHistory::FromJointSnapshot(decoded); }); + Reject([&] { TConfigurationHistory::FromStableSnapshot(TSnapshot{5, 2, {1, 2}, "legacy"}); }); + Reject([&] { state.InstallSnapshot(snapshot); }); + auto wrongPredecessor = joint; wrongPredecessor.PreviousIndex = 3; + state.RemoveLast(); state.Append(EncodeConfigurationEntry(wrongPredecessor)); + Reject([&] { ScanConfigurationHistory(state, baseline, 5); }); +} void test_scan_fails_closed_on_invalid_suffix(void**) { TState state; state.CurrentTerm = 3; auto base = TConfigurationHistory::Stable({1}, 0, 0, 0, 0); @@ -374,6 +399,7 @@ int main() { cmocka_unit_test(test_history_rejects_skips_wrong_predecessors_and_term_regression), cmocka_unit_test(test_scan_separates_commit_and_replaced_suffix), cmocka_unit_test(test_scan_from_joint_snapshot_and_rejects_mismatched_baselines), + cmocka_unit_test(test_scan_from_stable_configuration_snapshot), cmocka_unit_test(test_scan_fails_closed_on_invalid_suffix), cmocka_unit_test(test_configuration_roundtrips_both_phases_and_bounded_maximum), cmocka_unit_test(test_configuration_rejects_invalid_fields_and_valid_crc_corruption), diff --git a/test/test_snapshots.cpp b/test/test_snapshots.cpp index 7a80623..74697ac 100644 --- a/test/test_snapshots.cpp +++ b/test/test_snapshots.cpp @@ -99,6 +99,43 @@ void test_joint_snapshot_codec_roundtrip_and_corruption(void**) { Reject([&] { DecodeSnapshot(std::string_view(encoded).substr(0, i)); }); } } +void test_stable_configuration_snapshot_codec_and_runtime_guards(void**) { + auto snapshot = Sample(); snapshot.StableConfiguration = true; + snapshot.ConfigurationIndex = 1; snapshot.ConfigurationTerm = 1; + const auto encoded = EncodeSnapshot(snapshot); + assert_true(encoded.substr(0, 8) == "MRFTS003"); + assert_true(DecodeSnapshot(encoded) == snapshot); assert_false(snapshot.IsJoint()); + for (size_t i = 0; i < encoded.size(); ++i) { + auto damaged = encoded; damaged[i] ^= 1; + Reject([&] { DecodeSnapshot(damaged); }); + Reject([&] { DecodeSnapshot(std::string_view(encoded).substr(0, i)); }); + } + for (int mode = 0; mode < 6; ++mode) { + auto bad = snapshot; + if (mode == 0) bad.ConfigurationIndex = 0; + if (mode == 1) bad.ConfigurationIndex = 3; + if (mode == 2) bad.ConfigurationTerm = 0; + if (mode == 3) bad.ConfigurationTerm = 2; + if (mode == 4) { bad.Term = 2; bad.ConfigurationIndex = 2; bad.ConfigurationTerm = 1; } + if (mode == 5) bad.JointVoters = {3, 4}; + Reject([&] { EncodeSnapshot(bad); }); + } + for (size_t offset : {size_t(36), size_t(44)}) { + auto damaged = encoded; + for (size_t i = offset; i < offset + 8; ++i) damaged[i] = char(0xff); + FixChecksum(damaged); Reject([&] { DecodeSnapshot(damaged); }); + } + auto memory = std::make_shared(); + Reject([&] { memory->InstallSnapshot(snapshot); }); + memory->Snapshot = std::make_shared(snapshot); + Reject([&] { TRaft unsupported(memory, 1, {{2, std::make_shared()}, {3, std::make_shared()}}); }); + assert_true(memory->GetBoundVoters().empty()); +#ifdef __linux__ + TFiles files; + { TSnapshotStore store(files.Name, 1); store.Save(snapshot); assert_true(store.Load().value() == snapshot); } + { TDurableState state(files.Name, 1); Reject([&] { state.InstallSnapshot(snapshot); }); } +#endif +} void test_joint_snapshot_metadata_rejects_invalid_boundaries_and_voters(void**) { for (int mode = 0; mode < 10; ++mode) { auto bad = JointSample(); @@ -245,6 +282,25 @@ void test_restore_rejects_mismatched_log_configuration_and_used_application(void assert_true(Read(restored, "a") == "one"); } #ifdef __linux__ +void test_stable_configuration_snapshot_durable_recovery_fails_closed(void**) { + TFiles files; const auto path = files.Name + ".durable.1"; + { auto state = std::make_shared(files.Name, 1); + TRaft raft(state, 1, {{2, std::make_shared()}, {3, std::make_shared()}}); } + std::ifstream input(path, std::ios::binary); std::string bytes(std::istreambuf_iterator(input), {}); input.close(); + assert_true(bytes.substr(0, 8) == "MRFTD003"); + auto snapshot = Sample(); snapshot.StableConfiguration = true; + snapshot.ConfigurationIndex = 1; snapshot.ConfigurationTerm = 1; + const auto image = EncodeSnapshot(snapshot); + for (int i = 0; i < 8; ++i) bytes[56 + i] = char(uint64_t(image.size()) >> (8 * i)); + bytes.insert(64, image); + for (int i = 0; i < 8; ++i) bytes[8 + i] = char(uint64_t(bytes.size() - 20) >> (8 * i)); + const auto crc = NSnapshotCodec::Checksum(bytes.substr(0, 8) + bytes.substr(20)); + for (int i = 0; i < 4; ++i) bytes[16 + i] = char(crc >> (8 * i)); + { std::ofstream output(path, std::ios::binary | std::ios::trunc); output.write(bytes.data(), bytes.size()); } + Reject([&] { TDurableState state(files.Name, 1); }); + std::ifstream after(path, std::ios::binary); + assert_true(std::string(std::istreambuf_iterator(after), {}) == bytes); +} void test_joint_snapshot_export_roundtrips_but_durable_state_rejects_it(void**) { TFiles files; { TSnapshotStore exportStore(files.Name, 1); exportStore.Save(JointSample()); @@ -387,6 +443,7 @@ int main() { const CMUnitTest tests[] = { cmocka_unit_test(test_snapshot_codec_roundtrip_and_all_byte_corruption), cmocka_unit_test(test_joint_snapshot_codec_roundtrip_and_corruption), + cmocka_unit_test(test_stable_configuration_snapshot_codec_and_runtime_guards), cmocka_unit_test(test_joint_snapshot_metadata_rejects_invalid_boundaries_and_voters), cmocka_unit_test(test_joint_snapshot_cannot_be_installed_or_restored_by_fixed_runtime), cmocka_unit_test(test_snapshot_metadata_and_declared_lengths_are_bounded), @@ -395,6 +452,7 @@ int main() { cmocka_unit_test(test_capture_requires_committed_applied_progress), cmocka_unit_test(test_restore_rejects_mismatched_log_configuration_and_used_application), #ifdef __linux__ + cmocka_unit_test(test_stable_configuration_snapshot_durable_recovery_fails_closed), cmocka_unit_test(test_joint_snapshot_export_roundtrips_but_durable_state_rejects_it), cmocka_unit_test(test_snapshot_store_rejects_oversized_files_and_temp_symlinks), cmocka_unit_test(test_snapshot_store_roundtrip_lock_and_corrupt_recovery), From 467bb14ffbd62039795e7738062bbed52e05c0ae Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Fri, 13 Nov 2026 20:34:26 -0500 Subject: [PATCH 35/66] Project offline configuration history into snapshots (#35) --- docs/configuration-history.md | 11 +++++++++++ miniraft/configuration.cpp | 13 +++++++++++++ miniraft/configuration.h | 3 +++ test/test_configuration.cpp | 36 +++++++++++++++++++++++++++++++++++ 4 files changed, 63 insertions(+) diff --git a/docs/configuration-history.md b/docs/configuration-history.md index 1681a85..6e37170 100644 --- a/docs/configuration-history.md +++ b/docs/configuration-history.md @@ -1,5 +1,16 @@ # Offline configuration history validation: v0.1.0-alpha.27 +`TConfigurationHistory::MakeSnapshot` projects the history at its through-index +and term into an offline snapshot. It selects MRFTS002 for a joint configuration, +MRFTS003 for a stable configuration with a known configuration entry, and the +legacy MRFTS001 format when the trusted baseline asserts no earlier configuration +entry. That legacy format does not encode or prove the zero-identity assertion. +The caller must independently establish that the history boundary is committed +and that the supplied application image corresponds exactly to that boundary. +The method validates snapshot metadata and size; it does not capture application +state, compact or mutate storage, establish commitment, or activate voters. +Experimental configuration snapshots remain rejected by the live runtime. + TConfigurationHistory checks a sequence of retained log entries against a trusted configuration baseline. It joins the experimental configuration-entry codec with the joint-snapshot metadata and checks that joint and stable records have the diff --git a/miniraft/configuration.cpp b/miniraft/configuration.cpp index 0126c29..0bd4dfd 100644 --- a/miniraft/configuration.cpp +++ b/miniraft/configuration.cpp @@ -98,6 +98,19 @@ TConfigurationHistory TConfigurationHistory::FromStableSnapshot(const TSnapshot& return Stable(snapshot.Voters, snapshot.ConfigurationIndex, snapshot.ConfigurationTerm, snapshot.Index, snapshot.Term); } +TSnapshot TConfigurationHistory::MakeSnapshot(std::string data) const { + TSnapshot snapshot{Through, ThroughLogTerm, Old, std::move(data)}; + if (IsJoint()) { + snapshot.JointVoters = New; + snapshot.ConfigurationIndex = ConfigIndex; snapshot.ConfigurationTerm = ConfigTerm; + } else if (ConfigIndex) { + snapshot.StableConfiguration = true; + snapshot.ConfigurationIndex = ConfigIndex; snapshot.ConfigurationTerm = ConfigTerm; + } + ValidateSnapshot(snapshot); + return snapshot; +} + TConfigurationHistory TConfigurationHistory::Advance(uint64_t index, TMessageHolder entry) const { const auto record = DecodeConfigurationEntry(std::move(entry)); if (Through >= uint64_t(INT64_MAX) || index != Through + 1 || record.Term < ThroughLogTerm || diff --git a/miniraft/configuration.h b/miniraft/configuration.h index b673521..fb17eda 100644 --- a/miniraft/configuration.h +++ b/miniraft/configuration.h @@ -36,6 +36,9 @@ class TConfigurationHistory { // Metadata must come from a trusted, already validated storage backend. TConfigurationHistory ObserveMetadata(uint64_t index, uint64_t term, uint64_t flags) const; TConfigurationHistory Advance(uint64_t index, TMessageHolder entry) const; + // Offline projection only. The caller must establish that ThroughIndex is + // committed and that data is the application image at exactly that index. + TSnapshot MakeSnapshot(std::string data) const; bool IsJoint() const { return !New.empty(); } const std::vector& OldVoters() const { return Old; } diff --git a/test/test_configuration.cpp b/test/test_configuration.cpp index 7a601f0..34866f0 100644 --- a/test/test_configuration.cpp +++ b/test/test_configuration.cpp @@ -40,6 +40,38 @@ struct TEndpoint : INode { TMessageHolder Ordinary(uint64_t term) { auto entry = NewHoldedMessage(); entry->Term = term; return entry; } +void test_configuration_snapshot_projection_and_history_resume(void**) { + auto initial = TConfigurationHistory::Stable({1, 2, 3}, 0, 0, 0, 0); + Reject([&] { initial.MakeSnapshot("empty boundary"); }); + auto prefix = initial.Observe(1, Ordinary(1)); + const auto bootstrap = prefix.MakeSnapshot("bootstrap"); + assert_true(EncodeSnapshot(bootstrap).substr(0, 8) == "MRFTS001"); + assert_int_equal(bootstrap.Index, 1); assert_int_equal(bootstrap.Term, 1); + assert_false(bootstrap.HasConfigurationMetadata()); + auto joint = Joint(); joint.Term = 2; joint.PreviousIndex = joint.PreviousTerm = 0; + auto pending = prefix.Advance(2, EncodeConfigurationEntry(joint)).Observe(3, Ordinary(2)); + const auto pendingSnapshot = pending.MakeSnapshot(std::string("j\0x", 3)); + const auto jointBytes = EncodeSnapshot(pendingSnapshot); + assert_true(jointBytes.substr(0, 8) == "MRFTS002"); + auto restoredJoint = TConfigurationHistory::FromJointSnapshot(DecodeSnapshot(jointBytes)); + assert_int_equal(restoredJoint.ThroughIndex(), 3); assert_int_equal(restoredJoint.ConfigurationIndex(), 2); + assert_true(restoredJoint.OldVoters() == joint.OldVoters); assert_true(restoredJoint.NewVoters() == joint.NewVoters); + auto final = joint; final.Phase = EConfigurationPhase::Stable; final.Term = 3; + final.PreviousIndex = 2; final.PreviousTerm = 2; + auto stable = restoredJoint.Advance(4, EncodeConfigurationEntry(final)); + const auto stableSnapshot = stable.MakeSnapshot("stable"); + const auto stableBytes = EncodeSnapshot(stableSnapshot); + assert_true(stableBytes.substr(0, 8) == "MRFTS003"); + auto restoredStable = TConfigurationHistory::FromStableSnapshot(DecodeSnapshot(stableBytes)); + assert_int_equal(restoredStable.ConfigurationIndex(), 4); assert_int_equal(restoredStable.ConfigurationTerm(), 3); + assert_true(restoredStable.OldVoters() == joint.NewVoters); assert_false(restoredStable.IsJoint()); + TConfigurationRecord next{EConfigurationPhase::Joint, 3, 4, 3, joint.NewVoters, {3, 4, 6}}; + auto resumed = restoredStable.Advance(5, EncodeConfigurationEntry(next)); + assert_true(resumed.IsJoint()); assert_true(resumed.NewVoters() == next.NewVoters); + assert_true(pending.NewVoters() == joint.NewVoters); assert_false(stable.IsJoint()); + TState state; Reject([&] { state.InstallSnapshot(pendingSnapshot); }); + Reject([&] { state.InstallSnapshot(stableSnapshot); }); +} void test_history_validates_joint_stable_chain_and_independent_rollback(void**) { auto initial = TConfigurationHistory::Stable({1, 2, 3}, 0, 0, 0, 0); auto prefix = initial.Observe(1, Ordinary(1)); @@ -128,6 +160,9 @@ void test_scan_separates_commit_and_replaced_suffix(void**) { assert_true(scan.Committed.IsJoint()); assert_int_equal(scan.Committed.ThroughIndex(), 2); assert_false(scan.Latest.IsJoint()); assert_true(scan.Latest.OldVoters() == final.NewVoters); assert_true(scan.HasUncommittedConfiguration()); assert_int_equal(state.Gets, 2); + const auto committedImage = scan.Committed.MakeSnapshot("committed image"); + assert_true(committedImage.IsJoint()); assert_int_equal(committedImage.Index, 2); + assert_int_equal(committedImage.ConfigurationIndex, 2); scan = ScanConfigurationHistory(state, base, 4); assert_false(scan.HasUncommittedConfiguration()); assert_int_equal(scan.Committed.ThroughIndex(), 4); state.RemoveLast(); @@ -394,6 +429,7 @@ void test_offline_history_scan_after_durable_snapshot_and_restart(void**) { } int main() { const CMUnitTest tests[] = { + cmocka_unit_test(test_configuration_snapshot_projection_and_history_resume), cmocka_unit_test(test_history_validates_joint_stable_chain_and_independent_rollback), cmocka_unit_test(test_history_from_joint_snapshot_requires_a_consistent_identity), cmocka_unit_test(test_history_rejects_skips_wrong_predecessors_and_term_regression), From 42fb6c00dc4d4dd48eaa997c13f6f3cec56b2b88 Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Fri, 13 Nov 2026 20:46:27 -0500 Subject: [PATCH 36/66] Validate snapshot boundaries before Raft recovery (#36) --- docs/compaction.md | 8 ++++++++ miniraft/raft.cpp | 20 +++++++++++++++----- test/test_snapshots.cpp | 36 ++++++++++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 5 deletions(-) diff --git a/docs/compaction.md b/docs/compaction.md index b5d40ce..2082e51 100644 --- a/docs/compaction.md +++ b/docs/compaction.md @@ -34,6 +34,14 @@ the persisted current term or vote. ## Boundaries and atomic persistence +Raft startup validates a recovered snapshot before binding voters: its image, +index and term must agree with the storage boundary, its voter set must match +the fixed configuration, and the last term must be consistent with the +boundary and current term. A nonzero boundary without an image is rejected. +Custom backends remain responsible for validating the retained entries and +supplying a consistent, serialized view; these boundary checks are not a +complete storage audit or proof of snapshot contents. + SnapshotIndex/SnapshotTerm identify the last included entry. Get only returns entries after SnapshotIndex, while LogTerm at the boundary returns SnapshotTerm. LastLogIndex and all RPC/application indices remain absolute. Appends, truncation, diff --git a/miniraft/raft.cpp b/miniraft/raft.cpp index b82eed1..612c54f 100644 --- a/miniraft/raft.cpp +++ b/miniraft/raft.cpp @@ -198,20 +198,30 @@ TRaft::TRaft(std::shared_ptr state, int node, const TNodeDict& nodes, st if (!Learners.empty() && !State->SupportsVoterBinding()) throw std::invalid_argument("Learners require voter-binding-capable storage"); const auto bound = State->GetBoundVoters(); - if (auto snapshot = State->GetSnapshot(); snapshot && snapshot->HasConfigurationMetadata()) - throw std::invalid_argument("Configuration snapshot recovery is unsupported"); if (State->SnapshotIndex > State->LastLogIndex || State->LastLogIndex > uint64_t(INT64_MAX)) throw std::invalid_argument("Invalid retained log bounds"); + const auto snapshot = State->GetSnapshot(); + if (snapshot) { + ValidateSnapshot(*snapshot); + if (snapshot->HasConfigurationMetadata()) + throw std::invalid_argument("Configuration snapshot recovery is unsupported"); + if (snapshot->Index != State->SnapshotIndex || snapshot->Term != State->SnapshotTerm || + snapshot->Term > State->CurrentTerm || snapshot->Voters != Voters || + State->LastLogTerm < State->SnapshotTerm || State->LastLogTerm > State->CurrentTerm || + (State->LastLogIndex == State->SnapshotIndex && State->LastLogTerm != State->SnapshotTerm)) + throw std::invalid_argument("Compacted state snapshot boundary mismatch"); + } else if (State->SnapshotIndex || State->SnapshotTerm) { + throw std::invalid_argument("Compacted state is missing its snapshot"); + } // Reject before binding voters or serving traffic. Current Raft cannot // interpret configuration entries, including uncommitted suffix entries. for (uint64_t index = State->SnapshotIndex + 1; index <= State->LastLogIndex; ++index) if (State->RetainedLogFlags(index) & TLogEntry::EConfiguration) throw std::invalid_argument("Configuration log recovery is unsupported"); const bool fresh = State->CurrentTerm == 1 && !State->VotedFor && !State->LastLogIndex; - if (State->SupportsVoterBinding() && (!bound.empty() || fresh || State->GetSnapshot() || !Learners.empty())) + if (State->SupportsVoterBinding() && (!bound.empty() || fresh || snapshot || !Learners.empty())) State->BindVoters(Voters); - if (auto snapshot = State->GetSnapshot()) { - if (snapshot->Voters != GetVoterIds()) throw std::invalid_argument("Compacted state voter configuration mismatch"); + if (snapshot) { VolatileState->CommitIndex = State->SnapshotIndex; } } diff --git a/test/test_snapshots.cpp b/test/test_snapshots.cpp index 74697ac..8bc7c87 100644 --- a/test/test_snapshots.cpp +++ b/test/test_snapshots.cpp @@ -181,6 +181,41 @@ void test_joint_snapshot_cannot_be_installed_or_restored_by_fixed_runtime(void** Reject([&] { TRaft unsupported(recovered, 1, {{2, std::make_shared()}, {3, std::make_shared()}}); }); assert_true(recovered->GetBoundVoters().empty()); } +void test_startup_rejects_inconsistent_snapshot_boundaries_before_binding(void**) { + for (int mode = 0; mode < 11; ++mode) { + auto state = std::make_shared(); state->CurrentTerm = 2; + state->SnapshotIndex = state->LastLogIndex = 5; + state->SnapshotTerm = state->LastLogTerm = 2; + TSnapshot image{5, 2, {1, 2}, "opaque"}; + if (mode == 1) image.Index = 4; + if (mode == 2) image.Term = 1; + if (mode == 3) state->LastLogTerm = 1; + if (mode == 4) state->CurrentTerm = 1; + if (mode == 5) image.Voters = {1, 1}; + if (mode == 6 || mode == 7) { + state->SnapshotIndex = state->LastLogIndex = 0; + state->SnapshotTerm = mode == 6 ? 2 : 0; state->LastLogTerm = 0; + } + if (mode == 8) image.Voters = {1, 3}; + if (mode != 0 && mode != 6) state->Snapshot = std::make_shared(image); + if (mode == 9 || mode == 10) { + auto entry = NewHoldedMessage(); entry->Term = mode == 9 ? 1 : 3; state->Append(entry); + } + const auto before = static_cast(*state); + Reject([&] { TRaft invalid(state, 1, {{2, std::make_shared()}}); }); + assert_true(static_cast(*state) == before); + assert_true(state->GetBoundVoters().empty()); + } + auto valid = std::make_shared(); valid->CurrentTerm = 2; + valid->SnapshotIndex = valid->LastLogIndex = 5; valid->SnapshotTerm = valid->LastLogTerm = 2; + valid->Snapshot = std::make_shared(TSnapshot{5, 2, {1, 2}, "opaque"}); + TRaft raft(valid, 1, {{2, std::make_shared()}}); + assert_int_equal(raft.GetVolatileState()->CommitIndex, 5); + assert_true(valid->GetBoundVoters() == std::vector({1, 2})); + auto entry = NewHoldedMessage(); entry->Term = 2; valid->Append(entry); + TRaft withSuffix(valid, 1, {{2, std::make_shared()}}); + assert_int_equal(withSuffix.GetVolatileState()->CommitIndex, 5); +} void test_snapshot_metadata_and_declared_lengths_are_bounded(void**) { auto snapshot = Sample(); for (uint64_t index : {uint64_t(0), UINT64_MAX}) { @@ -446,6 +481,7 @@ int main() { cmocka_unit_test(test_stable_configuration_snapshot_codec_and_runtime_guards), cmocka_unit_test(test_joint_snapshot_metadata_rejects_invalid_boundaries_and_voters), cmocka_unit_test(test_joint_snapshot_cannot_be_installed_or_restored_by_fixed_runtime), + cmocka_unit_test(test_startup_rejects_inconsistent_snapshot_boundaries_before_binding), cmocka_unit_test(test_snapshot_metadata_and_declared_lengths_are_bounded), cmocka_unit_test(test_kv_snapshot_is_canonical_and_binary_safe), cmocka_unit_test(test_kv_invalid_snapshot_preserves_existing_state_and_progress), From 91d8a0c4d9a843e5f7e42c5a1541664d270046fd Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Sat, 14 Nov 2026 09:12:35 -0500 Subject: [PATCH 37/66] Validate compacted suffix terms during recovery (#37) --- docs/compaction.md | 11 +++++++---- miniraft/raft.cpp | 12 +++++++++++- test/test_snapshots.cpp | 21 +++++++++++++++++++++ 3 files changed, 39 insertions(+), 5 deletions(-) diff --git a/docs/compaction.md b/docs/compaction.md index 2082e51..56c1e8e 100644 --- a/docs/compaction.md +++ b/docs/compaction.md @@ -37,10 +37,13 @@ the persisted current term or vote. Raft startup validates a recovered snapshot before binding voters: its image, index and term must agree with the storage boundary, its voter set must match the fixed configuration, and the last term must be consistent with the -boundary and current term. A nonzero boundary without an image is rejected. -Custom backends remain responsible for validating the retained entries and -supplying a consistent, serialized view; these boundary checks are not a -complete storage audit or proof of snapshot contents. +boundary and current term. It scans retained terms for nonzero, nondecreasing +values no newer than the current term and verifies the recorded last term. A +nonzero boundary without an image is rejected. Custom backends must report +accurate entry metadata, validate payloads, and supply a consistent, serialized +view; these checks are not a complete storage audit or proof of snapshot contents. +The scan uses `RetainedLogTerm`; backends relying on its default implementation +may read full entry payloads at startup. SnapshotIndex/SnapshotTerm identify the last included entry. Get only returns entries after SnapshotIndex, while LogTerm at the boundary returns SnapshotTerm. diff --git a/miniraft/raft.cpp b/miniraft/raft.cpp index 612c54f..c436fa1 100644 --- a/miniraft/raft.cpp +++ b/miniraft/raft.cpp @@ -215,9 +215,19 @@ TRaft::TRaft(std::shared_ptr state, int node, const TNodeDict& nodes, st } // Reject before binding voters or serving traffic. Current Raft cannot // interpret configuration entries, including uncommitted suffix entries. - for (uint64_t index = State->SnapshotIndex + 1; index <= State->LastLogIndex; ++index) + auto previousTerm = State->SnapshotTerm; + for (uint64_t index = State->SnapshotIndex + 1; index <= State->LastLogIndex; ++index) { if (State->RetainedLogFlags(index) & TLogEntry::EConfiguration) throw std::invalid_argument("Configuration log recovery is unsupported"); + if (snapshot) { + const auto term = State->RetainedLogTerm(index); + if (!term || term < previousTerm || term > State->CurrentTerm) + throw std::invalid_argument("Invalid compacted log term sequence"); + previousTerm = term; + } + } + if (snapshot && previousTerm != State->LastLogTerm) + throw std::invalid_argument("Compacted log last term mismatch"); const bool fresh = State->CurrentTerm == 1 && !State->VotedFor && !State->LastLogIndex; if (State->SupportsVoterBinding() && (!bound.empty() || fresh || snapshot || !Learners.empty())) State->BindVoters(Voters); diff --git a/test/test_snapshots.cpp b/test/test_snapshots.cpp index 8bc7c87..7545dda 100644 --- a/test/test_snapshots.cpp +++ b/test/test_snapshots.cpp @@ -216,6 +216,26 @@ void test_startup_rejects_inconsistent_snapshot_boundaries_before_binding(void** TRaft withSuffix(valid, 1, {{2, std::make_shared()}}); assert_int_equal(withSuffix.GetVolatileState()->CommitIndex, 5); } +void test_startup_rejects_invalid_compacted_suffix_terms(void**) { + for (int mode = 0; mode < 4; ++mode) { + auto state = std::make_shared(); state->CurrentTerm = 3; + state->SnapshotIndex = state->LastLogIndex = 5; state->SnapshotTerm = state->LastLogTerm = 2; + state->Snapshot = std::make_shared(TSnapshot{5, 2, {1, 2}, "opaque"}); + auto first = NewHoldedMessage(); first->Term = mode == 0 ? 1 : mode == 2 ? 0 : mode == 3 ? 4 : 2; + auto last = NewHoldedMessage(); last->Term = mode == 1 ? 3 : 2; + state->Append(first); state->Append(last); + if (mode == 1) state->LastLogTerm = 2; // in range but differs from the actual last entry + const auto before = static_cast(*state); + Reject([&] { TRaft invalid(state, 1, {{2, std::make_shared()}}); }); + assert_true(static_cast(*state) == before); assert_true(state->GetBoundVoters().empty()); + } + auto valid = std::make_shared(); valid->CurrentTerm = 3; + valid->SnapshotIndex = valid->LastLogIndex = 5; valid->SnapshotTerm = valid->LastLogTerm = 2; + valid->Snapshot = std::make_shared(TSnapshot{5, 2, {1, 2}, "opaque"}); + for (uint64_t term : {2, 2, 3}) { auto entry = NewHoldedMessage(); entry->Term = term; valid->Append(entry); } + TRaft raft(valid, 1, {{2, std::make_shared()}}); + assert_int_equal(raft.GetVolatileState()->CommitIndex, 5); +} void test_snapshot_metadata_and_declared_lengths_are_bounded(void**) { auto snapshot = Sample(); for (uint64_t index : {uint64_t(0), UINT64_MAX}) { @@ -482,6 +502,7 @@ int main() { cmocka_unit_test(test_joint_snapshot_metadata_rejects_invalid_boundaries_and_voters), cmocka_unit_test(test_joint_snapshot_cannot_be_installed_or_restored_by_fixed_runtime), cmocka_unit_test(test_startup_rejects_inconsistent_snapshot_boundaries_before_binding), + cmocka_unit_test(test_startup_rejects_invalid_compacted_suffix_terms), cmocka_unit_test(test_snapshot_metadata_and_declared_lengths_are_bounded), cmocka_unit_test(test_kv_snapshot_is_canonical_and_binary_safe), cmocka_unit_test(test_kv_invalid_snapshot_preserves_existing_state_and_progress), From 312649350db212264554941f6b375cd50d9d23e4 Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Sat, 14 Nov 2026 09:18:04 -0500 Subject: [PATCH 38/66] Validate recovered votes against fixed voters (#38) --- docs/learners.md | 6 ++++++ miniraft/raft.cpp | 3 +++ test/test_learners.cpp | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 42 insertions(+) diff --git a/docs/learners.md b/docs/learners.md index dda2e0e..5393e2f 100644 --- a/docs/learners.md +++ b/docs/learners.md @@ -1,5 +1,11 @@ # Non-voting learners: v0.1.0-alpha.17 +Raft startup rejects a zero current term or a persisted vote for an ID outside +the configured fixed voter set, including on unbound nonfresh and legacy stores. +It performs this check before voter binding or serving traffic. This check does +not prove that the full voter set matches a legacy store's original cluster; +legacy storage still has no persisted voter binding. + This milestone separates replication targets from the fixed voting set. Learners receive log entries, install snapshots and apply committed commands, but never campaign, vote, lead, propose entries or contribute to election, commit or strong diff --git a/miniraft/raft.cpp b/miniraft/raft.cpp index c436fa1..b7f7393 100644 --- a/miniraft/raft.cpp +++ b/miniraft/raft.cpp @@ -192,8 +192,11 @@ TRaft::TRaft(std::shared_ptr state, int node, const TNodeDict& nodes, st std::sort(Voters.begin(), Voters.end()); ValidateFixedVoters(Voters); VotingQuorum.emplace(Voters); + if (!State->CurrentTerm) throw std::invalid_argument("Stored Raft term must be positive"); if (!IsVoter(Id) && State->VotedFor) throw std::invalid_argument("A learner store must not contain a vote"); + if (State->VotedFor && !IsVoter(State->VotedFor)) + throw std::invalid_argument("Stored vote does not name a configured voter"); Nservers = Voters.size(); MinVotes = Nservers / 2 + 1; if (!Learners.empty() && !State->SupportsVoterBinding()) throw std::invalid_argument("Learners require voter-binding-capable storage"); diff --git a/test/test_learners.cpp b/test/test_learners.cpp index 5bd42b6..e4087a2 100644 --- a/test/test_learners.cpp +++ b/test/test_learners.cpp @@ -236,6 +236,19 @@ void test_nonfresh_unbound_store_requires_snapshot_proof_for_learners(void**) { TRaft adopted(state, 1, Peers(1), {4}); assert_true(state->GetBoundVoters() == (std::vector{1, 2, 3})); } +void test_recovered_hard_state_must_match_fixed_voters(void**) { + for (int mode = 0; mode < 2; ++mode) { + auto state = std::make_shared(); + state->CurrentTerm = mode == 0 ? 2 : 0; state->VotedFor = mode == 0 ? 4 : 0; + const auto before = static_cast(*state); + Reject([&] { TRaft invalid(state, 1, Peers(1, 3)); }); + assert_true(static_cast(*state) == before); assert_true(state->GetBoundVoters().empty()); + } + for (uint32_t vote : {0u, 1u, 2u}) { + auto state = std::make_shared(); state->CurrentTerm = 2; state->VotedFor = vote; + TRaft valid(state, 1, Peers(1, 3)); assert_int_equal(state->VotedFor, vote); + } +} #ifdef __linux__ struct TFiles { std::filesystem::path Directory; std::string Name; @@ -246,6 +259,24 @@ struct TFiles { } ~TFiles() { std::error_code ignored; std::filesystem::remove_all(Directory, ignored); } }; +void test_legacy_recovered_vote_outside_voters_preserves_storage(void**) { + TFiles files; + { TDiskState state(files.Name, 1); state.CurrentTerm = 2; state.VotedFor = 4; state.Commit(); } + std::vector before; + for (auto suffix : {".entries.1", ".index.1", ".state.1"}) { + std::ifstream input(files.Name + suffix, std::ios::binary); + before.emplace_back(std::istreambuf_iterator(input), std::istreambuf_iterator()); + } + { auto state = std::make_shared(files.Name, 1); + const auto fields = static_cast(*state); + Reject([&] { TRaft invalid(state, 1, Peers(1, 3)); }); + assert_true(static_cast(*state) == fields); } + size_t i = 0; + for (auto suffix : {".entries.1", ".index.1", ".state.1"}) { + std::ifstream input(files.Name + suffix, std::ios::binary); + assert_true(std::string(std::istreambuf_iterator(input), {}) == before[i++]); + } +} void test_durable_voter_binding_is_atomic_before_first_vote(void**) { using TStage = TDurableState::EStage; for (auto stage : {TStage::TempCreated, TStage::Written, TStage::FileSynced, TStage::Renamed, TStage::DirectorySynced}) { @@ -343,7 +374,9 @@ int main() { cmocka_unit_test(test_learner_snapshot_catchup_status_and_restart_preserve_role), cmocka_unit_test(test_restart_cannot_promote_learner_or_change_voters_but_can_add_transport), cmocka_unit_test(test_nonfresh_unbound_store_requires_snapshot_proof_for_learners), + cmocka_unit_test(test_recovered_hard_state_must_match_fixed_voters), #ifdef __linux__ + cmocka_unit_test(test_legacy_recovered_vote_outside_voters_preserves_storage), cmocka_unit_test(test_durable_voter_binding_is_atomic_before_first_vote), cmocka_unit_test(test_durable_learner_snapshot_suffix_and_role_survive_restart), cmocka_unit_test(test_durable_v3_corrupt_roles_fail_without_rewriting), From 25890fea52527298d6e072ab4a1cda52d289315c Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Sat, 14 Nov 2026 09:29:57 -0500 Subject: [PATCH 39/66] Support leader-local online learner staging (#39) --- README.md | 2 + docs/learners.md | 6 +- docs/online-learner-staging.md | 39 +++++++ miniraft/raft.cpp | 37 ++++++ miniraft/raft.h | 7 +- test/test_learners.cpp | 205 ++++++++++++++++++++++++++++++++- 6 files changed, 290 insertions(+), 6 deletions(-) create mode 100644 docs/online-learner-staging.md diff --git a/README.md b/README.md index 5203bc5..c42987e 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,8 @@ The [configuration history validator](docs/configuration-history.md) checks an offline retained-log sequence against a trusted baseline or joint snapshot. The [configuration log scanner](docs/configuration-scan.md) validates a retained suffix at a caller-supplied commit boundary and at its end. +The [online learner staging API](docs/online-learner-staging.md) lets embedders +add and remove leader-local, non-voting replication targets. The [stable configuration snapshot foundation](docs/stable-configuration-snapshots.md) preserves configuration identity for offline recovery validation. The [snapshot term validation fix](docs/snapshot-committed-terms.md) rejects diff --git a/docs/learners.md b/docs/learners.md index 5393e2f..41c6c57 100644 --- a/docs/learners.md +++ b/docs/learners.md @@ -40,9 +40,9 @@ Memory TState supports role binding for library tests/volatile deployments, but only the durable backend preserves that binding across actual process loss. Legacy TDiskState does not support learner mode. -Transport discovery is still static. Adding/removing learner endpoints requires -updating replica startup configuration and restarting those transports while -keeping the bound voter set unchanged. There is no online enrollment API yet. +The KV CLI and built-in server use static transport discovery. Library embedders +can use [online learner staging](online-learner-staging.md) on the current leader; +that local, non-voting replication target is not persisted or cluster membership. Deleting `--learner 4` to turn it into a voter is rejected for bound stores; it is not a membership-change procedure. Voter-set changes remain disabled. diff --git a/docs/online-learner-staging.md b/docs/online-learner-staging.md new file mode 100644 index 0000000..beccedd --- /dev/null +++ b/docs/online-learner-staging.md @@ -0,0 +1,39 @@ +# Online learner replication staging + +`TRaft::AddLearnerTarget(id, endpoint)` lets the current leader add a non-voting +replication target without restarting that Raft object. `RemoveLearnerTarget(id)` +removes a local target and returns its endpoint. The voter set and persisted +binding do not change. Staged learners receive the ordinary log or a snapshot +when needed, but never count toward election, commit, or strong-read quorums. +`IsLearnerCaughtUp` remains a point-in-time log watermark, not an authorization +to promote or a guarantee of application health. + +These are serialized embedding APIs, not a replicated cluster configuration or a +promotion API. The caller must ensure the destination is configured as a learner +with the same voter set, arrange and authenticate its transport, and keep its +endpoint alive while asynchronous operations may still use it. After removal, +use the returned endpoint to coordinate safe transport shutdown; queued traffic +can still exist. The caller must serialize add/remove with Raft processing and +must not re-enter them from transport callbacks. Duplicate IDs, voters, the local +ID, null endpoints and calls on nonleaders are rejected. Staging requires storage +with an already-bound matching voter set; unbound and legacy stores cannot stage. + +Registration is local to this Raft object and is not persisted or propagated to +other voters. The embedding application must reconcile targets on a new leader +and after restart; a previous leader retains its local list if it becomes a +follower. Removing a target drops replication progress and pending RPCs, and a +subsequent add starts a fresh probe. Delayed responses cannot satisfy new RPCs. +The existing KV CLI and `TRaftServer` still use static transport setup and do not +expose these APIs; embedders must arrange endpoint draining and inbound delivery. +Older binaries must not be mixed with learner configurations because they treat +unmarked peers as voters. + +Deterministic tests cover live add, snapshot catch-up, voter partitions and strong +reads, removal and re-addition with delayed append replies, removal during a +multi-chunk snapshot with delayed acknowledgements, invalid calls, and refusal +on unbound or legacy stores. Durable-store tests verify that staging does not +rewrite the binding and is absent on restart. A leader-change test verifies +that an embedding application must re-register the target on the new leader. +Dynamic voter promotion/removal, replicated joint consensus, persistent discovery, +and membership-aware snapshot installation are still required before live voting +membership changes are supported. diff --git a/miniraft/raft.cpp b/miniraft/raft.cpp index b7f7393..067759e 100644 --- a/miniraft/raft.cpp +++ b/miniraft/raft.cpp @@ -746,6 +746,43 @@ std::vector TRaft::GetVoterIds() const { return Voters; } bool TRaft::IsVoter(uint32_t id) const { return std::binary_search(Voters.begin(), Voters.end(), id); } +void TRaft::AddLearnerTarget(uint32_t id, std::shared_ptr node) { + if (StateName != EState::LEADER) throw std::logic_error("Only a leader can stage a learner"); + if (!id || id == Id || !node || Nodes.contains(id) || IsVoter(id) || + std::binary_search(Learners.begin(), Learners.end(), id)) + throw std::invalid_argument("Invalid or duplicate learner target"); + if (!State->SupportsVoterBinding() || State->GetBoundVoters() != Voters) + throw std::logic_error("Learner staging requires a bound voter configuration"); + if (Nodes.size() >= size_t(INT_MAX)) throw std::overflow_error("Too many replication targets"); + // Build all allocating changes before publishing any part of the target. + auto nodes = Nodes; + auto learners = Learners; + auto nextIndex = VolatileState->NextIndex; + nodes.emplace(id, std::move(node)); + learners.insert(std::lower_bound(learners.begin(), learners.end(), id), id); + nextIndex.emplace(id, State->LastLogIndex + 1); + Nodes.swap(nodes); Learners.swap(learners); VolatileState->NextIndex.swap(nextIndex); + Npeers = int(Nodes.size()); +} + +std::shared_ptr TRaft::RemoveLearnerTarget(uint32_t id) { + if (StateName != EState::LEADER) throw std::logic_error("Only a leader can remove a learner target"); + auto learner = std::lower_bound(Learners.begin(), Learners.end(), id); + auto peer = Nodes.find(id); + if (id == Id || learner == Learners.end() || *learner != id || peer == Nodes.end() || IsVoter(id)) + throw std::invalid_argument("Unknown learner target"); + auto node = peer->second; + Nodes.erase(peer); Learners.erase(learner); + VolatileState->NextIndex.erase(id); VolatileState->MatchIndex.erase(id); + VolatileState->ReadAckSeqno.erase(id); VolatileState->LastSentSeqno.erase(id); + VolatileState->PendingAppendRpcs.erase(id); VolatileState->SnapshotRpcs.erase(id); + VolatileState->HeartbeatDue.erase(id); VolatileState->RpcDue.erase(id); + VolatileState->BatchSize.erase(id); VolatileState->BackOff.erase(id); + VolatileState->Votes.erase(id); + Npeers = int(Nodes.size()); + return node; +} + bool TRaft::IsLearnerCaughtUp(uint32_t id) const { if (StateName != EState::LEADER || !std::binary_search(Learners.begin(), Learners.end(), id) || VolatileState->CommitIndex == 0 || State->LogTerm(VolatileState->CommitIndex) != State->CurrentTerm) return false; diff --git a/miniraft/raft.h b/miniraft/raft.h index 816d164..7965309 100644 --- a/miniraft/raft.h +++ b/miniraft/raft.h @@ -114,7 +114,7 @@ enum class EState: int { class TRaft { public: // Nodes contains every replication peer; learners may include the local ID. - // Roles are fixed in this milestone: no voter promotion/reconfiguration API. + // Voter roles remain fixed; learner replication targets can be staged locally. TRaft(std::shared_ptr state, int node, const TNodeDict& nodes, std::vector learners = {}); void Process(ITimeSource::Time now, TMessageHolder message, const std::shared_ptr& replyTo = {}); @@ -130,6 +130,11 @@ class TRaft { bool IsVoter(uint32_t id) const; // Leader-only, point-in-time log catch-up; not permission to promote. bool IsLearnerCaughtUp(uint32_t id) const; + // Leader-local replication staging; not a membership change. Caller owns + // endpoint lifecycle, reconciles targets after failover/restart, and must + // serialize these calls with all Raft processing. + void AddLearnerTarget(uint32_t id, std::shared_ptr node); + std::shared_ptr RemoveLearnerTarget(uint32_t id); // Capture applied state and atomically replace its prefix with a snapshot. TSnapshot Compact(); uint64_t ApproveRead(); diff --git a/test/test_learners.cpp b/test/test_learners.cpp index e4087a2..7783d80 100644 --- a/test/test_learners.cpp +++ b/test/test_learners.cpp @@ -61,15 +61,20 @@ struct TCluster { std::deque> Packets; std::vector VoteTargets; std::set Isolated; - TCluster() { for (uint32_t id = 1; id <= 4; ++id) Build(id, std::make_shared()); } + bool StageVoterLearner = false; + explicit TCluster(bool stageVoterLearner = false) : StageVoterLearner(stageVoterLearner) { + for (uint32_t id = 1; id <= 4; ++id) Build(id, std::make_shared()); + } void Build(uint32_t id, std::shared_ptr state) { auto peers = Peers(id); + if (StageVoterLearner && id != 4) peers.erase(4); for (auto& [_, peer] : peers) peer = std::make_shared([this](auto message) { if (message->Type == uint32_t(EMessageType::REQUEST_VOTE_REQUEST)) VoteTargets.push_back(message.template Cast()->Dst); Packets.push_back(std::move(message)); }); Processors[id - 1].reset(); - Nodes[id - 1] = std::make_shared(state, id, peers, std::vector{4}); + Nodes[id - 1] = std::make_shared(state, id, peers, + StageVoterLearner && id != 4 ? std::vector{} : std::vector{4}); Apps[id - 1] = std::make_shared(); Processors[id - 1] = std::make_unique(Nodes[id - 1], Apps[id - 1], peers); } @@ -236,6 +241,165 @@ void test_nonfresh_unbound_store_requires_snapshot_proof_for_learners(void**) { TRaft adopted(state, 1, Peers(1), {4}); assert_true(state->GetBoundVoters() == (std::vector{1, 2, 3})); } +void test_online_learner_staging_snapshot_catchup_and_quorum_isolation(void**) { + TCluster cluster(true); cluster.Elect(1); + for (const std::string key : {"a", "b", "c"}) cluster.Write(key, std::string(60000, key[0])); + auto leader = cluster.Nodes[0]; const auto fixed = leader->GetVoterIds(); + const auto snapshot = leader->Compact(); assert_true(snapshot.Index > 1); + auto endpoint = std::make_shared([&](auto message) { cluster.Packets.push_back(std::move(message)); }); + const auto fields = static_cast(*leader->GetState()); + Reject([&] { leader->AddLearnerTarget(2, endpoint); }); + leader->AddLearnerTarget(4, endpoint); + assert_true(static_cast(*leader->GetState()) == fields); + assert_true(leader->GetState()->GetBoundVoters() == fixed); + assert_true(leader->GetVoterIds() == fixed); assert_int_equal(leader->GetNservers(), 3); + assert_int_equal(leader->GetNpeers(), 3); assert_false(leader->IsLearnerCaughtUp(4)); + assert_true(leader->GetLearnerIds() == std::vector({4})); + cluster.Pump(1, 120, true); + assert_true(leader->IsLearnerCaughtUp(4)); assert_int_equal(cluster.Apps[3]->Loads, 1); + assert_true(Read(*cluster.Apps[3], "c") == std::string(60000, 'c')); + Reject([&] { TRaft promoted(leader->GetState(), 1, Peers(1)); }); + assert_true(leader->GetState()->GetBoundVoters() == fixed); + const auto before = leader->GetVolatileState()->CommitIndex; + cluster.Isolated = {2, 3}; cluster.Write("uncommitted", "value"); + assert_true(leader->GetVolatileState()->MatchIndex.at(4) > before); + assert_int_equal(leader->GetVolatileState()->CommitIndex, before); + assert_true(Read(*cluster.Apps[0], "uncommitted").empty()); + size_t replies = 0; + auto client = std::make_shared([&](auto message) { + assert_int_equal(message.template Cast()->ErrorCode, 0); ++replies; + }); + auto read = NewHoldedMessage(sizeof(TReadKv) + 1); + read->Flags = TCommandRequest::EConsistent; read->KeySize = 1; read->Data[0] = 'a'; + cluster.Processors[0]->OnCommandRequest(read, client); cluster.Pump(); + assert_int_equal(replies, 0); + cluster.Isolated = {3}; cluster.Pump(1, 80); + assert_int_equal(replies, 1); + assert_true(Read(*cluster.Apps[0], "uncommitted") == "value"); + assert_true(leader->IsLearnerCaughtUp(4)); +} +void test_online_learner_removal_and_stale_reply_after_readdition(void**) { + TCluster cluster(true); cluster.Elect(1); cluster.Pump(); + auto leader = cluster.Nodes[0]; + auto endpoint = std::make_shared([&](auto message) { cluster.Packets.push_back(std::move(message)); }); + leader->AddLearnerTarget(4, endpoint); cluster.Pump(); assert_true(leader->IsLearnerCaughtUp(4)); + leader->ApproveRead(); + TMessageHolder request; + for (auto it = cluster.Packets.begin(); it != cluster.Packets.end(); ++it) { + auto header = it->Cast(); + if (header->Src == 1 && header->Dst == 4 && header->Type == uint32_t(EMessageType::APPEND_ENTRIES_REQUEST)) { + request = *it; cluster.Packets.erase(it); break; + } + } + assert_true(bool(request)); + cluster.Nodes[3]->Process(cluster.Now, request); + TMessageHolder delayed; + for (auto it = cluster.Packets.begin(); it != cluster.Packets.end(); ++it) { + auto header = it->Cast(); + if (header->Src == 4 && header->Dst == 1 && header->Type == uint32_t(EMessageType::APPEND_ENTRIES_RESPONSE)) { + delayed = *it; cluster.Packets.erase(it); break; + } + } + assert_true(bool(delayed)); cluster.Packets.clear(); + assert_true(leader->RemoveLearnerTarget(4) == endpoint); + assert_true(leader->GetLearnerIds().empty()); assert_int_equal(leader->GetNpeers(), 2); + assert_false(leader->IsLearnerCaughtUp(4)); + leader->Process(cluster.Now, delayed); // no longer a configured peer + leader->AddLearnerTarget(4, endpoint); + cluster.Now += std::chrono::seconds(3); leader->ProcessTimeout(cluster.Now); + leader->Process(cluster.Now, delayed); // old RPC identity cannot confirm the new target + assert_false(leader->IsLearnerCaughtUp(4)); + assert_int_equal(leader->GetVolatileState()->MatchIndex.at(4), 0); + cluster.Drain(); cluster.Pump(); assert_true(leader->IsLearnerCaughtUp(4)); + assert_true(leader->GetVoterIds() == std::vector({1, 2, 3})); +} +void test_online_learner_removal_during_snapshot_and_delayed_ack(void**) { + TCluster cluster(true); cluster.Elect(1); + for (const std::string key : {"a", "b", "c"}) cluster.Write(key, std::string(60000, key[0])); + auto leader = cluster.Nodes[0]; leader->Compact(); + auto endpoint = std::make_shared([&](auto message) { cluster.Packets.push_back(std::move(message)); }); + const auto nextSnapshot = [&]() -> TMessageHolder { + for (int attempt = 0; attempt < 40; ++attempt) { + cluster.Now += std::chrono::seconds(3); leader->ProcessTimeout(cluster.Now); + size_t budget = 1000; + while (!cluster.Packets.empty()) { + assert_true(budget-- > 0); + auto message = cluster.Packets.front(); cluster.Packets.pop_front(); + auto header = message.Cast(); + if (header->Src == 1 && header->Dst == 4 && header->Type == uint32_t(EMessageType::INSTALL_SNAPSHOT_REQUEST)) + return message; + cluster.Nodes[header->Dst - 1]->Process(cluster.Now, message); + cluster.Processors[header->Dst - 1]->ProcessCommitted(); + } + } + fail_msg("Learner did not enter snapshot transfer"); return {}; + }; + leader->AddLearnerTarget(4, endpoint); + auto first = nextSnapshot(); const auto firstSeq = first.Cast()->Seqno; + cluster.Nodes[3]->Process(cluster.Now, first); + TMessageHolder delayed; + for (auto it = cluster.Packets.begin(); it != cluster.Packets.end(); ++it) { + auto header = it->Cast(); + if (header->Src == 4 && header->Dst == 1 && header->Type == uint32_t(EMessageType::INSTALL_SNAPSHOT_RESPONSE)) { + delayed = *it; cluster.Packets.erase(it); break; + } + } + assert_true(bool(delayed)); assert_false(delayed.Cast()->Installed); + assert_true(leader->GetVolatileState()->SnapshotRpcs.contains(4)); + assert_true(leader->RemoveLearnerTarget(4) == endpoint); + assert_false(leader->GetVolatileState()->SnapshotRpcs.contains(4)); cluster.Packets.clear(); + leader->AddLearnerTarget(4, endpoint); + auto replacement = nextSnapshot(); const auto replacementSeq = replacement.Cast()->Seqno; + assert_true(replacementSeq != firstSeq); + leader->Process(cluster.Now, delayed); + const auto& pending = leader->GetVolatileState()->SnapshotRpcs.at(4); + assert_int_equal(pending.Seqno, replacementSeq); assert_int_equal(pending.Offset, 0); + cluster.Nodes[3]->Process(cluster.Now, replacement); cluster.Drain(); cluster.Pump(1, 120, true); + assert_true(leader->IsLearnerCaughtUp(4)); assert_int_equal(cluster.Apps[3]->Loads, 1); + assert_true(Read(*cluster.Apps[3], "c") == std::string(60000, 'c')); +} +void test_online_learner_requires_reconciliation_after_leader_change(void**) { + TCluster cluster(true); cluster.Elect(1); + auto endpoint = std::make_shared([&](auto message) { cluster.Packets.push_back(std::move(message)); }); + cluster.Nodes[0]->AddLearnerTarget(4, endpoint); cluster.Pump(); + assert_true(cluster.Nodes[0]->IsLearnerCaughtUp(4)); + cluster.Isolated = {1}; cluster.Elect(2); cluster.Pump(2); + assert_true(cluster.Nodes[1]->GetLearnerIds().empty()); + assert_false(cluster.Nodes[1]->IsLearnerCaughtUp(4)); + cluster.Nodes[1]->AddLearnerTarget(4, endpoint); cluster.Pump(2); + assert_true(cluster.Nodes[1]->IsLearnerCaughtUp(4)); + cluster.Processors[1]->OnCommandRequest(Command("after", "failover"), {}); cluster.Pump(2, 80); + assert_true(Read(*cluster.Apps[3], "after") == "failover"); + assert_true(cluster.Nodes[1]->GetVoterIds() == std::vector({1, 2, 3})); + cluster.Isolated.clear(); cluster.Pump(2, 80); + assert_int_equal(cluster.Nodes[0]->CurrentStateName(), EState::FOLLOWER); + assert_true(cluster.Nodes[0]->GetLearnerIds() == std::vector({4})); +} +void test_online_learner_staging_rejects_invalid_calls(void**) { + auto state = std::make_shared(); auto raft = std::make_shared(state, 1, Peers(1, 3)); + auto endpoint = std::make_shared(); + Reject([&] { raft->AddLearnerTarget(4, endpoint); }); + Reject([&] { raft->RemoveLearnerTarget(2); }); + raft->ProcessTimeout({}); // candidate; no remote votes + Reject([&] { raft->AddLearnerTarget(4, endpoint); }); + auto single = std::make_shared(std::make_shared(), 1, TNodeDict{}); + single->ProcessTimeout({}); assert_int_equal(single->CurrentStateName(), EState::LEADER); + for (uint32_t id : {0u, 1u}) Reject([&] { single->AddLearnerTarget(id, endpoint); }); + Reject([&] { single->AddLearnerTarget(2, {}); }); + single->AddLearnerTarget(2, endpoint); + Reject([&] { single->AddLearnerTarget(2, endpoint); }); + Reject([&] { single->RemoveLearnerTarget(1); }); + Reject([&] { single->RemoveLearnerTarget(3); }); + assert_true(single->GetVoterIds() == std::vector({1})); + assert_int_equal(single->GetNservers(), 1); assert_int_equal(single->GetNpeers(), 1); + auto unbound = std::make_shared(); unbound->CurrentTerm = 2; + unbound->Append(NewHoldedMessage()); + TRaft oldStore(unbound, 1, {}); oldStore.ProcessTimeout({}); + assert_int_equal(oldStore.CurrentStateName(), EState::LEADER); + assert_true(unbound->GetBoundVoters().empty()); + Reject([&] { oldStore.AddLearnerTarget(2, endpoint); }); + assert_true(oldStore.GetLearnerIds().empty()); assert_int_equal(oldStore.GetNpeers(), 0); +} void test_recovered_hard_state_must_match_fixed_voters(void**) { for (int mode = 0; mode < 2; ++mode) { auto state = std::make_shared(); @@ -360,6 +524,36 @@ void test_legacy_disk_backend_cannot_enable_learner_roles(void**) { Reject([&] { TRaft raft(state, 4, Peers(4), {4}); }); assert_int_equal(state->CurrentTerm, 1); assert_int_equal(state->VotedFor, 0); } +void test_legacy_leader_cannot_stage_online_learner(void**) { + TFiles files; auto state = std::make_shared(files.Name, 1); + TRaft raft(state, 1, {}); raft.ProcessTimeout({}); + assert_int_equal(raft.CurrentStateName(), EState::LEADER); + const auto fields = static_cast(*state); + Reject([&] { raft.AddLearnerTarget(2, std::make_shared()); }); + assert_true(static_cast(*state) == fields); + assert_true(raft.GetLearnerIds().empty()); assert_int_equal(raft.GetNpeers(), 0); +} +void test_online_learner_staging_does_not_change_durable_binding(void**) { + TFiles files; const auto path = files.Name + ".durable.1"; + { + auto state = std::make_shared(files.Name, 1); TRaft raft(state, 1, {}); + raft.ProcessTimeout({}); assert_int_equal(raft.CurrentStateName(), EState::LEADER); + std::ifstream input(path, std::ios::binary); std::string before(std::istreambuf_iterator(input), {}); input.close(); + auto endpoint = std::make_shared(); raft.AddLearnerTarget(2, endpoint); + assert_true(state->GetBoundVoters() == std::vector({1})); + std::ifstream afterAdd(path, std::ios::binary); + assert_true(std::string(std::istreambuf_iterator(afterAdd), {}) == before); + assert_true(raft.RemoveLearnerTarget(2) == endpoint); + std::ifstream afterRemove(path, std::ios::binary); + assert_true(std::string(std::istreambuf_iterator(afterRemove), {}) == before); + raft.AddLearnerTarget(2, endpoint); // Leave staged at process shutdown. + std::ifstream afterReadd(path, std::ios::binary); + assert_true(std::string(std::istreambuf_iterator(afterReadd), {}) == before); + } + auto recovered = std::make_shared(files.Name, 1); + TRaft raft(recovered, 1, {}); + assert_true(raft.GetLearnerIds().empty()); assert_true(recovered->GetBoundVoters() == std::vector({1})); +} #endif } int main() { @@ -374,6 +568,11 @@ int main() { cmocka_unit_test(test_learner_snapshot_catchup_status_and_restart_preserve_role), cmocka_unit_test(test_restart_cannot_promote_learner_or_change_voters_but_can_add_transport), cmocka_unit_test(test_nonfresh_unbound_store_requires_snapshot_proof_for_learners), + cmocka_unit_test(test_online_learner_staging_snapshot_catchup_and_quorum_isolation), + cmocka_unit_test(test_online_learner_removal_and_stale_reply_after_readdition), + cmocka_unit_test(test_online_learner_removal_during_snapshot_and_delayed_ack), + cmocka_unit_test(test_online_learner_requires_reconciliation_after_leader_change), + cmocka_unit_test(test_online_learner_staging_rejects_invalid_calls), cmocka_unit_test(test_recovered_hard_state_must_match_fixed_voters), #ifdef __linux__ cmocka_unit_test(test_legacy_recovered_vote_outside_voters_preserves_storage), @@ -382,6 +581,8 @@ int main() { cmocka_unit_test(test_durable_v3_corrupt_roles_fail_without_rewriting), cmocka_unit_test(test_durable_v2_snapshot_adoption_preserves_vote_log_and_application), cmocka_unit_test(test_legacy_disk_backend_cannot_enable_learner_roles), + cmocka_unit_test(test_legacy_leader_cannot_stage_online_learner), + cmocka_unit_test(test_online_learner_staging_does_not_change_durable_binding), #endif }; return cmocka_run_group_tests(tests, nullptr, nullptr); From c9127880871329aaece66543299863b2bd4512ce Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Sat, 14 Nov 2026 14:10:17 -0500 Subject: [PATCH 40/66] Integrate staged learners with the network server (#40) Add runtime learner target management to TRaftServer, connect staged targets to server draining, and document endpoint lifecycle and remaining membership limits. --- README.md | 2 ++ docs/learners.md | 8 +++-- docs/online-learner-network.md | 35 +++++++++++++++++++++ docs/online-learner-staging.md | 6 ++-- miniraft/net/server.cpp | 41 ++++++++++++++++++++++--- miniraft/net/server.h | 12 +++++++- test/test_read_write.cpp | 56 ++++++++++++++++++++++++++++++++++ 7 files changed, 148 insertions(+), 12 deletions(-) create mode 100644 docs/online-learner-network.md diff --git a/README.md b/README.md index c42987e..0141051 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,8 @@ The [configuration log scanner](docs/configuration-scan.md) validates a retained suffix at a caller-supplied commit boundary and at its end. The [online learner staging API](docs/online-learner-staging.md) lets embedders add and remove leader-local, non-voting replication targets. +The [network learner follow-up](docs/online-learner-network.md) connects that +API to the built-in server's runtime drain loop. The [stable configuration snapshot foundation](docs/stable-configuration-snapshots.md) preserves configuration identity for offline recovery validation. The [snapshot term validation fix](docs/snapshot-committed-terms.md) rejects diff --git a/docs/learners.md b/docs/learners.md index 41c6c57..2f59618 100644 --- a/docs/learners.md +++ b/docs/learners.md @@ -40,9 +40,11 @@ Memory TState supports role binding for library tests/volatile deployments, but only the durable backend preserves that binding across actual process loss. Legacy TDiskState does not support learner mode. -The KV CLI and built-in server use static transport discovery. Library embedders -can use [online learner staging](online-learner-staging.md) on the current leader; -that local, non-voting replication target is not persisted or cluster membership. +The KV CLI uses static transport discovery. Library embedders can use +[online learner staging](online-learner-staging.md) on the current leader, and +`TRaftServer` can attach its `TNode` endpoint through +[network learner staging](online-learner-network.md). That local, non-voting +replication target is not persisted or cluster membership. Deleting `--learner 4` to turn it into a voter is rejected for bound stores; it is not a membership-change procedure. Voter-set changes remain disabled. diff --git a/docs/online-learner-network.md b/docs/online-learner-network.md new file mode 100644 index 0000000..2fdadaa --- /dev/null +++ b/docs/online-learner-network.md @@ -0,0 +1,35 @@ +# Network server learner staging: v0.1.0-alpha.40 + +`TRaftServer` now exposes `AddLearnerTarget(id, endpoint)` and +`RemoveLearnerTarget(id)`, using a `shared_ptr>` endpoint. Adding +a learner registers it with Raft and the server's drain set, so the normal +server loop can send its replication and snapshot traffic without restarting. +Removing it drops its Raft progress and removes it from future server drains; +the method returns the endpoint so the embedding application can manage its +transport lifetime. The voter list and quorum calculations remain unchanged. + +Construct the remote node with the right address and socket factory, and make +sure its local Raft instance is configured as a learner with the same voter set. +The storage on the current leader must have a bound voter list. The learner +still must have a configured return path to the voters. The server does not +discover, authenticate or configure the remote process. + +These calls must run on the same event-loop thread as `TRaftServer` and be +serialized with Raft processing. They may be called before `Serve()` or from a +serialized callback after serving begins. Calls from other threads and +reentrant calls from transport or application callbacks are unsupported. +After removal, a write already in progress or data already queued in the +endpoint may remain. Keep the returned endpoint alive while that I/O can still +use it, then shut it down according to the embedding application's transport +lifecycle. + +The KV CLI still discovers peers at startup. Staging is leader-local and is +lost on restart or failover; the new leader and the application must reconcile +it. This adds network transport support for non-voting catch-up only. Voter +promotion/removal, replicated joint consensus, membership-aware snapshot +installation and durable configuration recovery remain unsupported. + +The deterministic server test checks target registration, voter/quorum +invariance, removal and rollback after a nonleader rejection. The existing +learner tests cover snapshot catch-up and stale responses after removal. No +independent reviewer approved this milestone. diff --git a/docs/online-learner-staging.md b/docs/online-learner-staging.md index beccedd..9e7702b 100644 --- a/docs/online-learner-staging.md +++ b/docs/online-learner-staging.md @@ -23,9 +23,9 @@ other voters. The embedding application must reconcile targets on a new leader and after restart; a previous leader retains its local list if it becomes a follower. Removing a target drops replication progress and pending RPCs, and a subsequent add starts a fresh probe. Delayed responses cannot satisfy new RPCs. -The existing KV CLI and `TRaftServer` still use static transport setup and do not -expose these APIs; embedders must arrange endpoint draining and inbound delivery. -Older binaries must not be mixed with learner configurations because they treat +The KV CLI still uses static transport setup. `TRaftServer` runtime integration +is described in [network learner staging](online-learner-network.md). Older +binaries must not be mixed with learner configurations because they treat unmarked peers as voters. Deterministic tests cover live add, snapshot catch-up, voter partitions and strong diff --git a/miniraft/net/server.cpp b/miniraft/net/server.cpp index bc31336..6af0d10 100644 --- a/miniraft/net/server.cpp +++ b/miniraft/net/server.cpp @@ -43,17 +43,48 @@ NNet::TVoidTask TRaftServer::InboundConnection(TSocket socket) { template void TRaftServer::Serve() { + if (Serving) throw std::logic_error("Raft server is already serving"); + Serving = true; Idle(); InboundServe(); std::vector tasks; - for (auto& node : Nodes) { - auto realNode = std::dynamic_pointer_cast>(node); - if (realNode) { - tasks.emplace_back(OutboundServe(realNode)); - } + for (auto& [id, node] : PeerNodes) { + // Learners only receive replication and reply through their own + // configured voter connections. They cannot receive forwarded client + // commands, so no outbound response reader is needed for them. + if (Raft->IsVoter(id)) tasks.emplace_back(OutboundServe(node)); + } +} + +template +void TRaftServer::AddLearnerTarget(uint32_t id, std::shared_ptr> node) { + if (!node) throw std::invalid_argument("Learner target endpoint is null"); + auto [peer, insertedPeer] = PeerNodes.emplace(id, node); + if (!insertedPeer) throw std::invalid_argument("Duplicate network peer ID"); + + bool insertedNode = false; + try { + insertedNode = Nodes.insert(node).second; + if (!insertedNode) throw std::invalid_argument("Learner endpoint is already registered"); + Raft->AddLearnerTarget(id, node); + } catch (...) { + if (insertedNode) Nodes.erase(node); + PeerNodes.erase(peer); + throw; } } +template +std::shared_ptr> TRaftServer::RemoveLearnerTarget(uint32_t id) { + auto peer = PeerNodes.find(id); + if (peer == PeerNodes.end()) throw std::invalid_argument("Unknown network learner target"); + auto endpoint = peer->second; + Raft->RemoveLearnerTarget(id); // Rejects voters and nonleaders before server state changes. + Nodes.erase(endpoint); + PeerNodes.erase(peer); + return endpoint; +} + template void TRaftServer::DrainNodes() { for (const auto& node : Nodes) { diff --git a/miniraft/net/server.h b/miniraft/net/server.h index dd9af49..37edcff 100644 --- a/miniraft/net/server.h +++ b/miniraft/net/server.h @@ -167,13 +167,21 @@ class TRaftServer { , RequestProcessor(std::make_shared(raft, rsm, nodes)) , TimeSource(ts) { - for (const auto& [_, node] : nodes) { + for (const auto& [id, node] : nodes) { Nodes.insert(node); + if (auto realNode = std::dynamic_pointer_cast>(node)) { + PeerNodes.emplace(id, std::move(realNode)); + } } } void Serve(); + // Runtime learner targets are replication-only endpoints. Calls must be + // serialized with the poller loop and other Raft processing. + void AddLearnerTarget(uint32_t id, std::shared_ptr> node); + std::shared_ptr> RemoveLearnerTarget(uint32_t id); + private: NNet::TVoidTask InboundServe(); NNet::TVoidTask InboundConnection(TSocket socket); @@ -187,7 +195,9 @@ class TRaftServer { std::shared_ptr Raft; std::shared_ptr RequestProcessor; std::unordered_set> Nodes; + std::unordered_map>> PeerNodes; std::shared_ptr TimeSource; + bool Serving = false; // for debug TStateFields PersistentFields; diff --git a/test/test_read_write.cpp b/test/test_read_write.cpp index b600ed6..c40e921 100644 --- a/test/test_read_write.cpp +++ b/test/test_read_write.cpp @@ -4,6 +4,8 @@ #include #include #include +#include +#include #include #include @@ -153,11 +155,65 @@ void test_read_write_payload(void**) { h1.destroy(); h2.destroy(); } +void test_server_stages_and_removes_runtime_learner(void**) { + NNet::TLoop loop; + auto timeSource = std::make_shared(); + auto raft = std::make_shared(std::make_shared(), 1, TNodeDict{}); + raft->Become(EState::LEADER); + NNet::TSocket listener(NNet::TAddress{"127.0.0.1", 0}, loop.Poller()); + TRaftServer server( + loop.Poller(), std::move(listener), raft, std::make_shared(), TNodeDict{}, timeSource); + auto makeEndpoint = [&] { + return std::make_shared>( + [&](const NNet::TAddress& addr) { return NNet::TSocket(addr, loop.Poller()); }, + "learner", NNet::TAddress{"127.0.0.1", 65534}, timeSource); + }; + auto endpoint = makeEndpoint(); + + server.AddLearnerTarget(2, endpoint); + assert_true(raft->GetLearnerIds() == std::vector({2})); + assert_int_equal(raft->GetNpeers(), 1); + assert_int_equal(raft->GetNservers(), 1); + + bool duplicateRejected = false; + try { + server.AddLearnerTarget(2, makeEndpoint()); + } catch (const std::invalid_argument&) { + duplicateRejected = true; + } + assert_true(duplicateRejected); + assert_true(raft->GetLearnerIds() == std::vector({2})); + + auto removed = server.RemoveLearnerTarget(2); + assert_true(removed == endpoint); + assert_true(raft->GetLearnerIds().empty()); + assert_int_equal(raft->GetNpeers(), 0); + + raft->Become(EState::FOLLOWER); + bool nonleaderRejected = false; + try { + server.AddLearnerTarget(3, makeEndpoint()); + } catch (const std::logic_error&) { + nonleaderRejected = true; + } + assert_true(nonleaderRejected); + assert_true(raft->GetLearnerIds().empty()); + assert_int_equal(raft->GetNpeers(), 0); + + raft->Become(EState::LEADER); + auto retryEndpoint = makeEndpoint(); + server.AddLearnerTarget(3, retryEndpoint); + assert_true(raft->GetLearnerIds() == std::vector({3})); + assert_true(server.RemoveLearnerTarget(3) == retryEndpoint); + assert_true(raft->GetLearnerIds().empty()); +} + int main() { const struct CMUnitTest tests[] = { cmocka_unit_test(test_message_send_recv), cmocka_unit_test(test_read_write), cmocka_unit_test(test_read_write_payload), + cmocka_unit_test(test_server_stages_and_removes_runtime_learner), }; return cmocka_run_group_tests(tests, NULL, NULL); }; From 3010108e586f16099913ae37de411be02173bf3d Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Sat, 14 Nov 2026 14:22:45 -0500 Subject: [PATCH 41/66] Validate covered snapshot identity before ACK (#41) Check the embedded boundary, declared size, and voter list before a covered InstallSnapshot fast-path acknowledgement. --- README.md | 2 ++ docs/covered-snapshot-identity.md | 23 ++++++++++++ miniraft/raft.cpp | 58 +++++++++++++++++++++++++++---- test/test_compaction.cpp | 31 ++++++++++++++--- 4 files changed, 103 insertions(+), 11 deletions(-) create mode 100644 docs/covered-snapshot-identity.md diff --git a/README.md b/README.md index 0141051..0cc3276 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,8 @@ The [online learner staging API](docs/online-learner-staging.md) lets embedders add and remove leader-local, non-voting replication targets. The [network learner follow-up](docs/online-learner-network.md) connects that API to the built-in server's runtime drain loop. +The [covered-snapshot follow-up](docs/covered-snapshot-identity.md) validates +snapshot boundary and voter metadata before acknowledging an already-covered image. The [stable configuration snapshot foundation](docs/stable-configuration-snapshots.md) preserves configuration identity for offline recovery validation. The [snapshot term validation fix](docs/snapshot-committed-terms.md) rejects diff --git a/docs/covered-snapshot-identity.md b/docs/covered-snapshot-identity.md new file mode 100644 index 0000000..893c86a --- /dev/null +++ b/docs/covered-snapshot-identity.md @@ -0,0 +1,23 @@ +# Covered snapshot identity checks: v0.1.0-alpha.41 + +When a follower already has a committed prefix covering an InstallSnapshot +boundary, it does not need to install the image. Previously the fast path +recognized only the `MRFTS001` magic, then acknowledged the full snapshot as +installed. It skipped checking the snapshot's embedded index, term and voter +IDs against the RPC and the receiver's fixed voter set. + +The fast path now waits for and parses the bounded fixed-format metadata prefix. +It acknowledges coverage only when the embedded boundary and sorted voter list +match. It also checks the encoded application length against the declared total +size. It does not read the application body or verify its CRC because it never +uses that image; the receiver's already committed prefix is the evidence for +coverage. Actual installations continue to validate the complete image, CRC, +metadata, voter set and application payload before replacing state. + +Deterministic regression cases cover mismatched embedded voters, a snapshot +boundary that disagrees with the request, and a valid metadata prefix split +across chunks. Rejected images do not change durable Raft or application state. +Joint and stable configuration snapshots remain unsupported by the live +runtime, and live voter changes remain disabled. Self-review completed; no +independent reviewer approval is claimed. Network-enabled Release and ASan/UBSan +CTest suites both pass all 20 targets. diff --git a/miniraft/raft.cpp b/miniraft/raft.cpp index 067759e..c5ae052 100644 --- a/miniraft/raft.cpp +++ b/miniraft/raft.cpp @@ -23,6 +23,51 @@ static uint32_t rand_(uint32_t* seed) { return *seed; } +struct TCoveredSnapshotProbe { + bool Complete = false; + bool Supported = false; + size_t Need = 8; +}; + +bool ReadSnapshotInteger(std::string_view bytes, size_t offset, unsigned width, uint64_t& value) { + if (offset > bytes.size() || width > bytes.size() - offset) return false; + value = 0; + for (unsigned i = 0; i < width; ++i) + value |= uint64_t(uint8_t(bytes[offset + i])) << (8 * i); + return true; +} + +TCoveredSnapshotProbe ProbeCoveredFixedSnapshot(std::string_view prefix, uint64_t totalBytes, + uint64_t index, uint64_t term, const std::vector& voters) { + if (prefix.size() < 8) return {false, false, 8}; + if (prefix.substr(0, 8) != "MRFTS001") return {true, false, 0}; + + // A covered snapshot is not installed, but its identity must still agree + // with the RPC and the receiver's fixed membership before it can count as + // replication progress. The application payload and CRC are unnecessary + // because the receiver already has this committed log prefix. + if (prefix.size() < 36) return {false, false, 36}; + uint64_t embeddedIndex = 0, embeddedTerm = 0, voterCount = 0, dataBytes = 0; + if (!ReadSnapshotInteger(prefix, 8, 8, embeddedIndex) || + !ReadSnapshotInteger(prefix, 16, 8, embeddedTerm) || + !ReadSnapshotInteger(prefix, 24, 4, voterCount) || + !ReadSnapshotInteger(prefix, 28, 8, dataBytes) || + embeddedIndex != index || embeddedTerm != term || voterCount != voters.size()) + return {true, false, 0}; + + const uint64_t voterBytes = voterCount * 4; + const uint64_t headerBytes = 36 + voterBytes; + if (headerBytes + 4 > totalBytes || dataBytes != totalBytes - headerBytes - 4) + return {true, false, 0}; + if (headerBytes > prefix.size()) return {false, false, size_t(headerBytes)}; + for (size_t i = 0; i < voters.size(); ++i) { + uint64_t voter = 0; + if (!ReadSnapshotInteger(prefix, 36 + i * 4, 4, voter) || voter != voters[i]) + return {true, false, 0}; + } + return {true, true, size_t(headerBytes)}; +} + } // namespace void TDummyRsm::Validate(TMessageHolder message) const { @@ -874,9 +919,9 @@ void TRaft::OnInstallSnapshot(ITimeSource::Time now, TMessageHolderLogTerm(request->SnapshotIndex); const bool covered = request->SnapshotIndex < State->SnapshotIndex || term == request->SnapshotTerm; if (!covered) { reply(false, false, 0); return; } - // A covered image needs no application replacement, but must first be - // identified as a supported fixed-membership format. Accumulate a short - // prefix when a peer splits the magic across multiple valid chunks. + // A covered image needs no application replacement, but its snapshot + // boundary and fixed voter set must match before it can count as + // replicated. Accumulate only the bounded metadata prefix. const bool same = IncomingSnapshot && IncomingSnapshot->Term == request->Term && IncomingSnapshot->Leader == request->LeaderId && IncomingSnapshot->Index == request->SnapshotIndex && IncomingSnapshot->BoundaryTerm == request->SnapshotTerm && IncomingSnapshot->Total == request->TotalBytes; @@ -895,10 +940,11 @@ void TRaft::OnInstallSnapshot(ITimeSource::Time now, TMessageHolderData, count); - if (prefix.size() < 8) { reply(true, false, prefix.size()); return; } - const bool supported = std::string_view(prefix).substr(0, 8) == "MRFTS001"; + const auto probe = ProbeCoveredFixedSnapshot(prefix, request->TotalBytes, + request->SnapshotIndex, request->SnapshotTerm, GetVoterIds()); + if (!probe.Complete) { reply(true, false, prefix.size()); return; } IncomingSnapshot.reset(); - reply(supported, supported, supported ? request->TotalBytes : 0); + reply(probe.Supported, probe.Supported, probe.Supported ? request->TotalBytes : 0); return; } auto application = SnapshotApplication.lock(); diff --git a/test/test_compaction.cpp b/test/test_compaction.cpp index 849bff3..feb0a2f 100644 --- a/test/test_compaction.cpp +++ b/test/test_compaction.cpp @@ -3,6 +3,7 @@ #include #include "../examples/kv.h" #include "../examples/kv_messages.h" +#include #include #include #include @@ -346,11 +347,31 @@ void test_joint_snapshot_is_rejected_even_if_boundary_is_already_covered(void**) raft->Process({}, Chunk(joint, bytes, 0, bytes.size())); assert_false(replies.back()->Success); assert_false(replies.back()->Installed); assert_true(static_cast(*state) == before); assert_int_equal(app->Loads, 1); - // A supported covered snapshot can also identify its magic across chunks. - raft->Process({}, Chunk(fixed, fixedBytes, 0, 4)); - assert_true(replies.back()->Success); assert_false(replies.back()->Installed); - assert_int_equal(replies.back()->NextOffset, 4); - raft->Process({}, Chunk(fixed, fixedBytes, 4, 4)); + + // Covered fast-path ACKs still require the embedded boundary and voter set + // to match the request and this node's fixed membership. + auto wrongVoters = fixed; wrongVoters.Voters = {1, 3}; + auto wrongVoterBytes = EncodeSnapshot(wrongVoters); + raft->Process({}, Chunk(wrongVoters, wrongVoterBytes, 0, wrongVoterBytes.size())); + assert_false(replies.back()->Success); assert_false(replies.back()->Installed); + assert_true(static_cast(*state) == before); assert_int_equal(app->Loads, 1); + + auto wrongBoundary = fixed; wrongBoundary.Index = 4; + auto wrongBoundaryBytes = EncodeSnapshot(wrongBoundary); + raft->Process({}, Chunk(fixed, wrongBoundaryBytes, 0, wrongBoundaryBytes.size())); + assert_false(replies.back()->Success); assert_false(replies.back()->Installed); + assert_true(static_cast(*state) == before); assert_int_equal(app->Loads, 1); + + // The metadata-only probe remains bounded and handles a split prefix. + const size_t metadataBytes = 36 + 4 * fixed.Voters.size(); + for (size_t offset = 0; offset < metadataBytes; offset += 4) { + const auto count = std::min(4, metadataBytes - offset); + raft->Process({}, Chunk(fixed, fixedBytes, offset, static_cast(count))); + if (offset + count < metadataBytes) { + assert_true(replies.back()->Success); assert_false(replies.back()->Installed); + assert_int_equal(replies.back()->NextOffset, offset + count); + } + } assert_true(replies.back()->Success); assert_true(replies.back()->Installed); assert_int_equal(replies.back()->NextOffset, fixedBytes.size()); raft->Process({}, Chunk(joint, bytes, 0, 4)); From 53e1a0e118abd474c21a57af5baf46e3c59a811c Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Sat, 14 Nov 2026 14:52:45 -0500 Subject: [PATCH 42/66] Expire idle partial snapshot transfers (#42) --- README.md | 3 ++ docs/snapshot-transfer-expiry.md | 29 +++++++++++++++++++ miniraft/messages.h | 3 ++ miniraft/raft.cpp | 12 ++++++-- miniraft/raft.h | 1 + test/test_compaction.cpp | 48 ++++++++++++++++++++++++++++++++ 6 files changed, 94 insertions(+), 2 deletions(-) create mode 100644 docs/snapshot-transfer-expiry.md diff --git a/README.md b/README.md index 0cc3276..a9c1f64 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,9 @@ The [network learner follow-up](docs/online-learner-network.md) connects that API to the built-in server's runtime drain loop. The [covered-snapshot follow-up](docs/covered-snapshot-identity.md) validates snapshot boundary and voter metadata before acknowledging an already-covered image. +The [snapshot transfer expiry follow-up](docs/snapshot-transfer-expiry.md) +releases incomplete incoming images after 30 seconds without a valid chunk, +including on learners. The [stable configuration snapshot foundation](docs/stable-configuration-snapshots.md) preserves configuration identity for offline recovery validation. The [snapshot term validation fix](docs/snapshot-committed-terms.md) rejects diff --git a/docs/snapshot-transfer-expiry.md b/docs/snapshot-transfer-expiry.md new file mode 100644 index 0000000..a42fc55 --- /dev/null +++ b/docs/snapshot-transfer-expiry.md @@ -0,0 +1,29 @@ +# Idle InstallSnapshot transfer expiry: v0.1.0-alpha.42 + +InstallSnapshot is chunked and bounded to 64 MiB per image, but a receiver held +the accumulated partial image until a replacement transfer, a completed +transfer, or a voter election timeout. Learners do not campaign, so a learner +could retain that allocation indefinitely after an interrupted transfer. Normal +heartbeats could also keep a voter from reaching its election timeout while an +image remained abandoned. + +Incoming snapshot state now records the time of the last valid chunk. The +receiver discards an incomplete image after 30 seconds without a valid chunk, +regardless of role or heartbeat traffic. Matching retries count as activity so +lost acknowledgements remain safe; malformed, conflicting, and out-of-order +chunks do not extend the deadline. A continuation after expiry receives a +failure with offset zero, which tells the sender to restart the pinned image +from its first chunk. Completed installations and the covered-prefix metadata +fast path release the partial state as before. + +A deterministic learner test starts a partial transfer, continues ordinary +leader heartbeats, advances the injected clock beyond the idle limit, and +verifies that the old continuation is rejected and a fresh transfer succeeds. +The test also verifies the application and Raft snapshot boundary are unchanged +until the fresh image completes. + +Self-review checked both receiving paths, duplicate chunk handling, role +independence and restart-from-zero behavior. This bounds the lifetime of one +partial in-memory image; it does not impose a cluster-wide memory quota or stream +snapshot content to disk. Live voter reconfiguration and joint snapshot adoption +remain separate open features. diff --git a/miniraft/messages.h b/miniraft/messages.h index 8bbd6de..625dd04 100644 --- a/miniraft/messages.h +++ b/miniraft/messages.h @@ -151,6 +151,9 @@ struct TTimeout { static constexpr std::chrono::milliseconds Election = std::chrono::milliseconds(5000); static constexpr std::chrono::milliseconds Heartbeat = std::chrono::milliseconds(1000); static constexpr std::chrono::milliseconds Rpc = std::chrono::milliseconds(10000); + // Keep partial snapshot transfers across normal RPC retries, but bound how + // long an idle receiver retains the accumulated image. + static constexpr std::chrono::milliseconds SnapshotTransferIdle = std::chrono::seconds(30); }; template diff --git a/miniraft/raft.cpp b/miniraft/raft.cpp index c5ae052..d033b3b 100644 --- a/miniraft/raft.cpp +++ b/miniraft/raft.cpp @@ -696,6 +696,12 @@ uint64_t TRaft::NextSeqno() { } void TRaft::ProcessTimeout(ITimeSource::Time now) { + // Learners do not campaign, so their election timeout cannot be used to + // reclaim an interrupted InstallSnapshot transfer. Expire partial images + // independently of role and ordinary heartbeat activity. + if (IncomingSnapshot && now >= IncomingSnapshot->LastProgress && + now - IncomingSnapshot->LastProgress >= TTimeout::SnapshotTransferIdle) + IncomingSnapshot.reset(); if (!IsVoter(Id)) return; if (StateName == EState::CANDIDATE || StateName == EState::FOLLOWER) { if (VolatileState->ElectionDue <= now) { @@ -929,7 +935,7 @@ void TRaft::OnInstallSnapshot(ITimeSource::Time now, TMessageHolderOffset != 0 || (IncomingSnapshot && IncomingSnapshot->Term == request->Term && IncomingSnapshot->Index > request->SnapshotIndex)) { reply(false, false, 0); return; } IncomingSnapshot = TIncomingSnapshot{request->Term, request->SnapshotIndex, request->SnapshotTerm, - request->TotalBytes, request->LeaderId, {}}; + request->TotalBytes, request->LeaderId, now, {}}; } auto& prefix = IncomingSnapshot->Bytes; const auto count = request->Len - sizeof(TInstallSnapshotRequest); @@ -940,6 +946,7 @@ void TRaft::OnInstallSnapshot(ITimeSource::Time now, TMessageHolderData, count); + IncomingSnapshot->LastProgress = now; const auto probe = ProbeCoveredFixedSnapshot(prefix, request->TotalBytes, request->SnapshotIndex, request->SnapshotTerm, GetVoterIds()); if (!probe.Complete) { reply(true, false, prefix.size()); return; } @@ -956,7 +963,7 @@ void TRaft::OnInstallSnapshot(ITimeSource::Time now, TMessageHolderOffset != 0 || (IncomingSnapshot && IncomingSnapshot->Term == request->Term && IncomingSnapshot->Index > request->SnapshotIndex)) { reply(false, false, 0); return; } IncomingSnapshot = TIncomingSnapshot{request->Term, request->SnapshotIndex, request->SnapshotTerm, - request->TotalBytes, request->LeaderId, {}}; + request->TotalBytes, request->LeaderId, now, {}}; } auto& bytes = IncomingSnapshot->Bytes; const auto count = request->Len - sizeof(TInstallSnapshotRequest); @@ -967,6 +974,7 @@ void TRaft::OnInstallSnapshot(ITimeSource::Time now, TMessageHolderData, count); + IncomingSnapshot->LastProgress = now; if (bytes.size() >= 8 && std::string_view(bytes).substr(0, 8) != "MRFTS001") { IncomingSnapshot.reset(); reply(false, false, 0); return; } diff --git a/miniraft/raft.h b/miniraft/raft.h index 7965309..a0ff81f 100644 --- a/miniraft/raft.h +++ b/miniraft/raft.h @@ -213,6 +213,7 @@ class TRaft { struct TIncomingSnapshot { uint64_t Term, Index, BoundaryTerm, Total; uint32_t Leader; + ITimeSource::Time LastProgress; std::string Bytes; }; std::optional IncomingSnapshot; diff --git a/test/test_compaction.cpp b/test/test_compaction.cpp index feb0a2f..de1e51a 100644 --- a/test/test_compaction.cpp +++ b/test/test_compaction.cpp @@ -394,6 +394,53 @@ void test_joint_snapshot_is_rejected_even_if_boundary_is_already_covered(void**) assert_true(static_cast(*state) == before); assert_int_equal(app->Loads, 1); assert_true(Read(*app, "a") == "old"); } +void test_idle_partial_snapshot_expires_on_learner_despite_heartbeats(void**) { + auto state = std::make_shared(); + state->BindVoters({1}); + auto entry = NewHoldedMessage(); entry->Term = 1; state->Append(entry); + std::vector> replies; + auto peer = std::make_shared([&](auto message) { + if (message->Type == uint32_t(EMessageType::INSTALL_SNAPSHOT_RESPONSE)) + replies.push_back(message.template Cast()); + }); + auto raft = std::make_shared(state, 2, TNodeDict{{1, peer}}, std::vector{2}); + auto app = std::make_shared(); TRequestProcessor processor(raft, app, {}); + TKv source; source.Write(source.Prepare(Command("key", "value")), 1); + TSnapshot snapshot{1, 1, {1}, source.SaveSnapshot()}; + const auto bytes = EncodeSnapshot(snapshot); + + raft->Process({}, Chunk(snapshot, bytes, 0, 8)); + assert_true(replies.back()->Success); assert_false(replies.back()->Installed); + assert_int_equal(replies.back()->NextOffset, 8); assert_int_equal(app->Loads, 0); + + // Ordinary leader contact keeps Raft's election timer fresh, but must not + // keep an abandoned multi-chunk snapshot allocation alive. + const auto heartbeatAt = ITimeSource::Time{} + std::chrono::seconds(20); + auto heartbeat = NewHoldedMessage(TMessageEx{.Src = 1, .Dst = 2, .Term = 3}, + TAppendEntriesRequest{.PrevLogIndex = 1, .PrevLogTerm = 1, .LeaderCommit = 0, .LeaderId = 1}); + raft->Process(heartbeatAt, heartbeat); + // A matching retry is valid transfer activity and refreshes the deadline. + raft->Process(heartbeatAt, Chunk(snapshot, bytes, 0, 8)); + assert_true(replies.back()->Success); assert_int_equal(replies.back()->NextOffset, 8); + const auto nextChunkAt = heartbeatAt + std::chrono::seconds(10); + raft->ProcessTimeout(nextChunkAt); + raft->Process(nextChunkAt, Chunk(snapshot, bytes, 8, 8)); + assert_true(replies.back()->Success); assert_int_equal(replies.back()->NextOffset, 16); + + // A continuation after expiry is rejected with offset zero so the leader + // restarts from the beginning instead of extending a discarded image. + const auto expiredAt = nextChunkAt + TTimeout::SnapshotTransferIdle + std::chrono::milliseconds(1); + raft->ProcessTimeout(expiredAt); + raft->Process(expiredAt, Chunk(snapshot, bytes, 16, 8)); + assert_false(replies.back()->Success); assert_false(replies.back()->Installed); + assert_int_equal(replies.back()->NextOffset, 0); assert_int_equal(app->Loads, 0); + assert_int_equal(app->LastAppliedIndex, 0); assert_int_equal(state->SnapshotIndex, 0); + + raft->Process(expiredAt, Chunk(snapshot, bytes, 0, static_cast(bytes.size()))); + assert_true(replies.back()->Success); assert_true(replies.back()->Installed); + assert_int_equal(app->Loads, 1); assert_int_equal(app->LastAppliedIndex, 1); + assert_true(Read(*app, "key") == "value"); +} void test_corrupt_snapshot_and_invalid_headers_cannot_mutate_application(void**) { TKv source; source.Write(source.Prepare(Command("a", "new")), 5); TSnapshot snapshot{5, 1, {1, 2}, source.SaveSnapshot()}; @@ -754,6 +801,7 @@ int main() { cmocka_unit_test(test_snapshot_is_rechecked_when_commit_advances_mid_transfer), cmocka_unit_test(test_memory_snapshot_storage_rejects_term_regression), cmocka_unit_test(test_joint_snapshot_is_rejected_even_if_boundary_is_already_covered), + cmocka_unit_test(test_idle_partial_snapshot_expires_on_learner_despite_heartbeats), cmocka_unit_test(test_remote_snapshot_retains_suffix_without_applying_it_before_commit), cmocka_unit_test(test_snapshot_boundary_index_exhaustion_does_not_append_wrapped_noop), cmocka_unit_test(test_memory_storage_rejects_index_exhaustion), From c52eb0aad426e04078e3f211c901babb7147c592 Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Sat, 14 Nov 2026 15:20:38 -0500 Subject: [PATCH 43/66] Bound pinned snapshot generations during compaction (#43) --- README.md | 2 ++ docs/snapshot-pin-budget.md | 34 +++++++++++++++++++++++++++ docs/snapshot-pinning.md | 9 +++---- miniraft/raft.cpp | 11 +++++++++ miniraft/raft.h | 5 ++++ test/test_compaction.cpp | 47 +++++++++++++++++++++++++++++++++++++ 6 files changed, 104 insertions(+), 4 deletions(-) create mode 100644 docs/snapshot-pin-budget.md diff --git a/README.md b/README.md index a9c1f64..47eb6e7 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,8 @@ snapshot boundary and voter metadata before acknowledging an already-covered ima The [snapshot transfer expiry follow-up](docs/snapshot-transfer-expiry.md) releases incomplete incoming images after 30 seconds without a valid chunk, including on learners. +The [snapshot pin budget follow-up](docs/snapshot-pin-budget.md) bounds outgoing +transfer retention to two image generations and defers local compaction at the cap. The [stable configuration snapshot foundation](docs/stable-configuration-snapshots.md) preserves configuration identity for offline recovery validation. The [snapshot term validation fix](docs/snapshot-committed-terms.md) rejects diff --git a/docs/snapshot-pin-budget.md b/docs/snapshot-pin-budget.md new file mode 100644 index 0000000..cdde3ba --- /dev/null +++ b/docs/snapshot-pin-budget.md @@ -0,0 +1,34 @@ +# Bound pinned outgoing snapshot generations: v0.1.0-alpha.43 + +Per-peer transfer pinning guarantees that periodic compaction cannot continually +restart an in-flight InstallSnapshot. However, different peers can be transferring +different generations. A lagging peer can retain an older image while another +peer pins a newer one, and unbounded compaction can keep adding pinned generations. + +The leader now allows at most two distinct snapshot generations to remain pinned +by outgoing transfers. `TRaft::Compact` checks this before capturing or changing +storage. At the cap it throws a transient `std::runtime_error` without changing +the application, log, snapshot boundary or active transfers. Callers can retry +after a transfer completes; the KV checkpoint loop already retries failed +compaction attempts while the applied-index threshold remains met. Term changes +and target removal continue to release their corresponding transfer contexts. + +After a successful compaction, the leader drops its standalone encoded-snapshot +cache. Active transfers keep their own shared reference to the immutable bytes, +so they continue unchanged; a later transfer encodes the newer local image. +Transfers of the same image still share the same snapshot and encoding. + +A deterministic test pins two different generations, verifies that compaction +fails without mutation, completes one transfer and verifies that compaction then +succeeds while the other peer still owns its older image. Existing tests continue +to verify catch-up progress under compaction during every chunk. + +This caps the number of distinct active transfer generations, not total process +heap usage. Current application/log state, snapshot capture and installation, +transport buffers and other allocations remain outside this limit; a global byte +quota and disk-streamed snapshot transfer remain open. Voter membership changes +and joint snapshot installation are also still unsupported. + +Self-review checked the two-generation accounting, exception ordering before +state mutation, cache ownership and retry path. No independent reviewer approval +is claimed. diff --git a/docs/snapshot-pinning.md b/docs/snapshot-pinning.md index c7e3638..c30abf3 100644 --- a/docs/snapshot-pinning.md +++ b/docs/snapshot-pinning.md @@ -16,10 +16,11 @@ prefix. Snapshot ACKs still cannot confirm strong-read barriers. Failed chunks restart the pinned image from offset zero. Leadership changes still discard outgoing transfer contexts. One image/encoding pair is retained per -active peer transfer, with sharing between transfers of the same image. A stalled -peer may keep an older image alive while the store compacts; total snapshot memory -is not independent of the configured peer count. This does not introduce a -resource quota or general transfer cancellation API. +active peer transfer, with sharing between transfers of the same image. The +[alpha.43 pin budget](snapshot-pin-budget.md) later capped active retention to two +distinct image generations and defers further compaction until a transfer +completes. It does not add a byte quota for all Raft, application, transport and +temporary snapshot memory, or a general transfer cancellation API. A deterministic regression commits and compacts new data before every chunk ACK, including the final ACK, for both voter and learner receivers. The old code fails diff --git a/miniraft/raft.cpp b/miniraft/raft.cpp index d033b3b..806d804 100644 --- a/miniraft/raft.cpp +++ b/miniraft/raft.cpp @@ -542,6 +542,8 @@ void TRaft::Process(ITimeSource::Time now, TMessageHolder message, con StateName = EState::FOLLOWER; IncomingSnapshot.reset(); VolatileState->SnapshotRpcs.clear(); + EncodedSnapshot.reset(); + EncodedSnapshotIndex = 0; VolatileState->LeaderId = 0; if (VolatileState->ElectionDue <= now || VolatileState->ElectionDue == ITimeSource::Max) { VolatileState->ElectionDue = MakeElection(now); @@ -846,9 +848,18 @@ TSnapshot TRaft::Compact() { auto application = SnapshotApplication.lock(); if (!application) throw std::logic_error("Compaction requires a bound application"); if (application->LastAppliedIndex == State->SnapshotIndex && State->GetSnapshot()) return *State->GetSnapshot(); + std::unordered_set pinnedGenerations; + for (const auto& [_, rpc] : VolatileState->SnapshotRpcs) + if (rpc.Snapshot) pinnedGenerations.insert(rpc.Snapshot->Index); + if (pinnedGenerations.size() >= TVolatileState::MaxPinnedSnapshotGenerations) + throw std::runtime_error("Compaction deferred while snapshot transfers pin multiple generations"); auto snapshot = CaptureSnapshot(*this, *application); application->ValidateSnapshot(snapshot.Data, snapshot.Index); State->InstallSnapshot(snapshot); + // Outstanding transfers own their immutable encoded bytes; discard any + // otherwise-retained encoding of the old local snapshot. + EncodedSnapshot.reset(); + EncodedSnapshotIndex = 0; return snapshot; } diff --git a/miniraft/raft.h b/miniraft/raft.h index a0ff81f..b69c57d 100644 --- a/miniraft/raft.h +++ b/miniraft/raft.h @@ -56,6 +56,9 @@ using TNodeDict = std::unordered_map>; struct TVolatileState { struct TAppendRpc { uint64_t Seqno, LastIndex; }; static constexpr size_t MaxPendingAppendRpcs = 256; + // Distinct snapshot images retained by active peer transfers. New + // compaction is deferred when this many generations are already pinned. + static constexpr size_t MaxPinnedSnapshotGenerations = 2; uint64_t CommitIndex = 0; // Successful current-term RPC acknowledgements, separately from log indices. std::unordered_map ReadAckSeqno; @@ -136,6 +139,8 @@ class TRaft { void AddLearnerTarget(uint32_t id, std::shared_ptr node); std::shared_ptr RemoveLearnerTarget(uint32_t id); // Capture applied state and atomically replace its prefix with a snapshot. + // Throws without mutation while two distinct generations remain pinned by + // outgoing peer transfers; callers may retry after a transfer completes. TSnapshot Compact(); uint64_t ApproveRead(); // Highest barrier confirmed by a fresh majority in this leadership term. diff --git a/test/test_compaction.cpp b/test/test_compaction.cpp index de1e51a..13af022 100644 --- a/test/test_compaction.cpp +++ b/test/test_compaction.cpp @@ -278,6 +278,52 @@ void test_compaction_during_each_chunk_cannot_restart_snapshot_catchup(void**) { if (!learners.empty()) assert_true(cluster.Nodes[0]->IsLearnerCaughtUp(3)); } } +void test_compaction_is_deferred_until_one_pinned_generation_finishes(void**) { + TCluster cluster({3}); cluster.Prepare(); + auto firstChunk = cluster.SendChunk(); + const auto firstIndex = firstChunk->SnapshotIndex; + cluster.Deliver(firstChunk); // Lose the ACK; peer 3 keeps the first image pinned. + cluster.Packets.clear(); + auto* progress = const_cast(cluster.Nodes[0]->GetVolatileState()); + assert_true(progress->SnapshotRpcs.contains(3)); + + cluster.Isolated = {3}; + cluster.Write(1, "generation", "two"); + auto second = cluster.Nodes[0]->Compact(); + assert_true(second.Index > firstIndex); + cluster.Write(1, "generation", "three"); + assert_true(cluster.Apps[0]->LastAppliedIndex > second.Index); + + // Force a second peer to begin a transfer from the newer generation. + progress->NextIndex[2] = 1; + cluster.Packets.clear(); + cluster.Nodes[0]->ApproveRead(); + auto secondChunk = cluster.Take(1, 2, EMessageType::INSTALL_SNAPSHOT_REQUEST).Cast(); + assert_int_equal(secondChunk->SnapshotIndex, second.Index); + assert_int_equal(progress->SnapshotRpcs.at(3).Snapshot->Index, firstIndex); + assert_int_equal(progress->SnapshotRpcs.at(2).Snapshot->Index, second.Index); + + const auto storedBoundary = cluster.Nodes[0]->GetState()->SnapshotIndex; + const auto storedLastIndex = cluster.Nodes[0]->GetState()->LastLogIndex; + const auto appliedIndex = cluster.Apps[0]->LastAppliedIndex; + Reject([&] { cluster.Nodes[0]->Compact(); }); + assert_int_equal(cluster.Nodes[0]->GetState()->SnapshotIndex, storedBoundary); + assert_int_equal(cluster.Nodes[0]->GetState()->LastLogIndex, storedLastIndex); + assert_int_equal(cluster.Apps[0]->LastAppliedIndex, appliedIndex); + assert_true(progress->SnapshotRpcs.contains(2)); assert_true(progress->SnapshotRpcs.contains(3)); + + // Once peer 2 completes, compaction can advance while peer 3 still owns + // its older immutable image. + cluster.Deliver(secondChunk); + auto ack = cluster.Take(2, 1, EMessageType::INSTALL_SNAPSHOT_RESPONSE).Cast(); + cluster.Deliver(ack); + assert_true(ack->Installed); + assert_false(progress->SnapshotRpcs.contains(2)); assert_true(progress->SnapshotRpcs.contains(3)); + cluster.Packets.clear(); + auto latest = cluster.Nodes[0]->Compact(); + assert_true(latest.Index > storedBoundary); + assert_int_equal(cluster.Nodes[0]->GetState()->SnapshotIndex, latest.Index); +} void test_leader_change_mid_snapshot_rejects_old_chunks(void**) { TCluster cluster; cluster.Prepare(); auto old = cluster.SendChunk(); cluster.AckChunk(old); @@ -792,6 +838,7 @@ int main() { cmocka_unit_test(test_snapshot_timeout_retries_do_not_starve_delayed_acks), cmocka_unit_test(test_snapshot_read_traffic_does_not_starve_delayed_acks), cmocka_unit_test(test_compaction_during_each_chunk_cannot_restart_snapshot_catchup), + cmocka_unit_test(test_compaction_is_deferred_until_one_pinned_generation_finishes), cmocka_unit_test(test_leader_change_mid_snapshot_rejects_old_chunks), cmocka_unit_test(test_partial_and_installed_snapshot_acks_do_not_confirm_strong_reads), cmocka_unit_test(test_compacted_follower_hint_requires_boundary_probe_before_match), From 06d936ae4ffd1f389c319675296462efc66de1f0 Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Sat, 14 Nov 2026 15:56:31 -0500 Subject: [PATCH 44/66] Add SQL snapshot capture and recovery (#44) --- README.md | 3 + docs/compaction.md | 16 ++-- docs/snapshots.md | 15 ++-- docs/sql-snapshots.md | 74 +++++++++++++++++ examples/sql.cpp | 39 ++++++++- examples/sql_rsm.cpp | 160 ++++++++++++++++++++++++++++++++++++ examples/sql_rsm.h | 3 + miniraft/raft.cpp | 2 +- miniraft/snapshot.cpp | 4 +- miniraft/snapshot.h | 4 +- test/CMakeLists.txt | 5 ++ test/smoke_sql_snapshots.py | 104 +++++++++++++++++++++++ test/test_snapshots.cpp | 16 ++++ test/test_sql_payloads.cpp | 118 ++++++++++++++++++++++++++ 14 files changed, 545 insertions(+), 18 deletions(-) create mode 100644 docs/sql-snapshots.md create mode 100644 test/smoke_sql_snapshots.py diff --git a/README.md b/README.md index 47eb6e7..3456f4f 100644 --- a/README.md +++ b/README.md @@ -82,6 +82,9 @@ The [snapshot term validation fix](docs/snapshot-committed-terms.md) rejects snapshot boundaries that contradict the follower's committed prefix. The [message buffer follow-up](docs/message-extents.md) checks allocated buffer and batch extents before accepting locally constructed frames. +The [SQL snapshot follow-up](docs/sql-snapshots.md) adds bounded SQLite snapshot +capture, validated restore and recovery from an application checkpoint left +behind a durably installed Raft snapshot. [![Number of GitHub stars](https://img.shields.io/github/stars/resetius/miniraft-cpp)](https://github.com/resetius/miniraft-cpp/stargazers) ![GitHub commit activity](https://img.shields.io/github/commit-activity/m/resetius/miniraft-cpp) diff --git a/docs/compaction.md b/docs/compaction.md index 56c1e8e..7ebdf41 100644 --- a/docs/compaction.md +++ b/docs/compaction.md @@ -5,8 +5,9 @@ voter binding while retaining backward decoding of the v1/v2 formats below. This milestone adds prefix deletion and follower state transfer to the alpha.15 KV checkpoint foundation. TState and Linux TDurableState support compacted logs; -legacy TDiskState does not. KV implements snapshot validation/restoration. SQL and -dummy state machines remain unsupported for compaction and installation. +legacy TDiskState does not. KV implements snapshot validation/restoration. SQL +support is described in [the SQL snapshot follow-up](sql-snapshots.md); the dummy +state machine remains unsupported for compaction and installation. ## Operation and compatibility @@ -25,8 +26,9 @@ remote installation writes v2; downgrading that store is unsupported. Fixed vote IDs must match the embedded snapshot. Static `--node` edits remain unsafe as a membership-change mechanism. -Startup restores a fresh application from the embedded snapshot before request -processing, even when periodic snapshots are disabled. A newer standalone +Startup restores an application behind the embedded snapshot before request +processing, including after a crash between durable publication and application +replacement. A newer standalone checkpoint can restore progress within the retained suffix; an older one never replaces embedded state. With periodic snapshots enabled, an existing corrupt standalone file is still rejected conservatively. Snapshot recovery never resets @@ -142,8 +144,8 @@ Publication-stage process crashes do not replace hardware power-cut testing. Membership remains fixed. Learners, catch-up gates, replicated joint-consensus configuration entries, leader removal and corresponding quorum/read changes are next. All participating applications/backends must support snapshots before -compaction is enabled; no capability negotiation is implemented. SQL snapshots, -asynchronous/COW capture, disk-streamed transfer, bounded transport queues, total -heap quotas, cluster/store authentication and independent review remain open. +compaction is enabled; no capability negotiation is implemented. Asynchronous/COW +capture, disk-streamed transfer, bounded transport queues, total heap quotas, +cluster/store authentication and independent review remain open. The conservative backend still rewrites the entire snapshot and retained suffix on each update, and capture/publication runs synchronously on the event loop. diff --git a/docs/snapshots.md b/docs/snapshots.md index e1ac824..1c7b13c 100644 --- a/docs/snapshots.md +++ b/docs/snapshots.md @@ -38,14 +38,17 @@ IRsm gains optional SaveSnapshot/LoadSnapshot callbacks; the defaults throw an unsupported-operation error. KV implements canonical, sorted, binary-safe state encoding. Restore validates the entire image into a temporary map before swapping state and progress. Key/value lengths retain the KV wire format's 65,535-byte -limits. SQL and the dummy application do not implement snapshots in this release. +limits. SQL snapshot support is described in [its follow-up](sql-snapshots.md); +the dummy application still does not implement snapshots. CaptureSnapshot requires a nonzero applied index no greater than the known commit index or retained log end. Metadata records its index, term and sorted fixed voter -IDs. RestoreSnapshot requires a fresh application and an exact voter-set match; +IDs. RestoreSnapshot requires an application behind the snapshot and an exact voter-set match; the boundary index must exist in the local log with the same term, not newer than -CurrentTerm. Restore does not overwrite current term or votedFor. These APIs must -run serialized, with restore only during startup before request processing. +CurrentTerm. Startup restore can recover a process stopped after durable Raft +snapshot publication but before application replacement. Restore does not +overwrite current term or votedFor. These APIs must run serialized, with restore +only during startup before request processing. The versioned image contains little-endian fields in this order: @@ -119,5 +122,5 @@ These follow the snapshot metadata/state-transfer requirements in [Raft section 7](https://raft.github.io/raft.pdf) and the overlapping-quorum configuration protocol in section 6. Editing the static `--node` list is not a membership-change protocol. Snapshots do not make such edits safe. This release -keeps fixed membership and leaves full compaction, transfer, dynamic membership, -background capture, SQL snapshots and independent review as open release gates. +keeps fixed membership. Background capture and independent review remain open +release gates. diff --git a/docs/sql-snapshots.md b/docs/sql-snapshots.md new file mode 100644 index 0000000..fcc07d5 --- /dev/null +++ b/docs/sql-snapshots.md @@ -0,0 +1,74 @@ +# SQL application snapshots: v0.1.0-alpha.44 + +The [snapshot foundation](snapshots.md) and [compaction follow-up](compaction.md) +initially supported only the KV example. SQL now supplies the same application +snapshot callbacks, so a Linux SQL server can use the durable backend's +compaction and InstallSnapshot paths. + +## Use + +Start each SQL server with its unchanged fixed voter list and add +`--durable --snapshots N`: + +```sh +./build/full/examples/sql --server --id 1 --node 127.0.0.1:8001:1 \ + --durable --snapshots 1000 +``` + +`N` must be positive. The SQL CLI requires this option pair for periodic +compaction; snapshots remain off otherwise. Each node uses its own +`sql_state.durable.` Raft store and `sql_file.db.` SQLite database. +Every voter must run a binary that understands compacted durable images and +InstallSnapshot. Existing SQL and legacy Raft files remain in place, but old +binaries cannot read the compacted store. There is no automatic format +migration or mixed-version support. + +## Image and recovery contract + +The application image is captured with SQLite's backup API into an in-memory +database and then serialized. This includes committed pages still present in a +WAL file. Save requires a committed `LastAppliedIndex` marker and no open +transaction. The SQL payload is capped below 64 MiB to leave room for the Raft +snapshot header and maximum voter metadata. The decoder adds SQLite's recommended +read slack around malformed untrusted images. Before restore or installation, +SQL checks that the image can be opened, +passes `PRAGMA quick_check(1)`, and has exactly one valid progress marker equal +to the Raft snapshot boundary. WAL-mode backup bytes are normalized to SQLite's +standalone rollback-mode format before deserialization. The receiver also +requires the target database page size to match before Raft publishes the +snapshot durably. Invalid or incompatible images do not replace the live +database. + +Restore uses SQLite's backup API to copy the validated in-memory image into the +existing database connection. The application progress marker in memory moves +only after the backup completes. A failed backup leaves the destination +unchanged according to SQLite's backup transaction contract. Snapshot +installation first publishes the Raft snapshot and retained suffix durably, +then replaces the application database, then acknowledges installation. If +the process stops between the first two steps, startup replaces any SQL +checkpoint behind the durable Raft snapshot before request processing begins. + +This recovery only applies when the saved application index is below the +snapshot boundary. An application at or beyond the boundary is preserved; an +application behind the compacted boundary with no matching snapshot still +fails startup. The image does not make SQL files portable across arbitrary +SQLite versions or extensions. Use the same application schema, SQLite build +and database page size across a cluster. + +Serialization runs synchronously on the event loop and can pause request and +replication processing for large databases. This feature does not provide +background or copy-on-write capture, application database quotas below the +64 MiB image cap, or live voter changes. Fixed voter IDs must still match on +every node. + +## Validation + +Deterministic tests cover round-trip capture and restore, malformed/corrupt +images, wrong progress boundaries, preservation of existing state after +validation or backup failure, and restart recovery when durable Raft state is +ahead of the SQL database. The regular release and ASan/UBSan CTest suites +also exercise the library and examples. + +SQLite API details: [serialize](https://www.sqlite.org/c3ref/serialize.html), +[deserialize, including the WAL-mode restriction](https://www.sqlite.org/c3ref/deserialize.html), and +[backup completion](https://www.sqlite.org/c3ref/backup_finish.html). diff --git a/examples/sql.cpp b/examples/sql.cpp index 930d9d9..09a5a8f 100644 --- a/examples/sql.cpp +++ b/examples/sql.cpp @@ -1,8 +1,10 @@ #include +#include #include #include #include +#include #include #include "payload_validation.h" @@ -67,8 +69,22 @@ NNet::TFuture Client(TPoller& poller, TSocket socket) { co_return; } +template +NNet::TVoidTask CompactLoop(TPoller& poller, std::shared_ptr raft, + std::shared_ptr rsm, uint64_t interval, uint64_t saved) { + while (true) { + co_await poller.Sleep(std::chrono::seconds(1)); + if (rsm->LastAppliedIndex <= saved || rsm->LastAppliedIndex - saved < interval) continue; + try { + saved = raft->Compact().Index; + } catch (const std::exception& error) { + std::cerr << "SQL compaction deferred/failed: " << error.what() << "\n"; + } + } +} + void usage(const char* prog) { - std::cerr << prog << "--client|--server --id myid --node ip:port:id [--node ip:port:id ...]" << "\n"; + std::cerr << prog << "--client|--server --id myid --node ip:port:id [--node ip:port:id ...] [--durable] [--snapshots entries]" << "\n"; exit(0); } @@ -80,6 +96,8 @@ int main(int argc, char** argv) TNodeDict nodes; uint32_t id = 0; bool server = false; + bool durable = false; + uint64_t snapshotInterval = 0; for (int i = 1; i < argc; i++) { if (!strcmp(argv[i], "--server")) { server = true; @@ -88,11 +106,24 @@ int main(int argc, char** argv) hosts.push_back(THost{argv[++i]}); } else if (!strcmp(argv[i], "--id") && i < argc - 1) { id = atoi(argv[++i]); + } else if (!strcmp(argv[i], "--durable")) { + durable = true; + } else if (!strcmp(argv[i], "--snapshots")) { + if (i + 1 >= argc) { std::cerr << "--snapshots requires a positive entry interval\n"; return 1; } + const std::string_view value(argv[++i]); + const auto result = std::from_chars(value.data(), value.data() + value.size(), snapshotInterval); + if (result.ec != std::errc{} || result.ptr != value.data() + value.size() || !snapshotInterval) { + std::cerr << "--snapshots requires a positive entry interval\n"; return 1; + } } else if (!strcmp(argv[i], "--help")) { usage(argv[0]); } } + if (snapshotInterval && (!server || !durable)) { + std::cerr << "--snapshots requires --server --durable\n"; return 1; + } + using TPoller = NNet::TDefaultPoller; std::shared_ptr timeSource = std::make_shared(); NNet::TLoop loop; @@ -118,13 +149,17 @@ int main(int argc, char** argv) } std::shared_ptr rsm = std::make_shared("sql_file.db", myHost.Id); - auto state = std::make_shared("sql_state", myHost.Id); + std::shared_ptr state = durable + ? std::static_pointer_cast(std::make_shared("sql_state", myHost.Id)) + : std::static_pointer_cast(std::make_shared("sql_state", myHost.Id)); auto raft = std::make_shared(state, myHost.Id, nodes); + const auto savedSnapshot = state->SnapshotIndex; TPoller::TSocket socket(NNet::TAddress{myHost.Address, myHost.Port}, loop.Poller()); socket.Bind(); socket.Listen(); TRaftServer server(loop.Poller(), std::move(socket), raft, rsm, nodes, timeSource); server.Serve(); + if (snapshotInterval) CompactLoop(loop.Poller(), raft, rsm, snapshotInterval, savedSnapshot); loop.Loop(); } else { NNet::TAddress addr{hosts[0].Address, hosts[0].Port}; diff --git a/examples/sql_rsm.cpp b/examples/sql_rsm.cpp index f6da187..3f514a1 100644 --- a/examples/sql_rsm.cpp +++ b/examples/sql_rsm.cpp @@ -74,6 +74,123 @@ struct TReadBudget { } }; using TStatement = std::unique_ptr; +constexpr size_t MaxSqlSnapshotBytes = TSnapshot::MaxBytes - 40 - 4 * TSnapshot::MaxVoters; + +struct TSnapshotDatabase { + // Keep the deserialize buffer alive until after the connection closes. + std::unique_ptr Buffer{nullptr, sqlite3_free}; + std::unique_ptr Db{nullptr, sqlite3_close_v2}; +}; + +TSnapshotDatabase OpenSnapshotDatabase(std::string_view bytes) { + if (bytes.size() < 100 || bytes.size() > MaxSqlSnapshotBytes) + throw TMessageError("Invalid SQL snapshot size"); + TSnapshotDatabase result; + constexpr size_t MalformedImageReadSlack = 20; + result.Buffer.reset(static_cast(sqlite3_malloc64(bytes.size() + MalformedImageReadSlack))); + if (!result.Buffer) throw std::bad_alloc(); + memcpy(result.Buffer.get(), bytes.data(), bytes.size()); + memset(result.Buffer.get() + bytes.size(), 0, MalformedImageReadSlack); + if (result.Buffer.get()[18] != 1 || result.Buffer.get()[19] != 1) + throw TMessageError("SQL snapshot is not a standalone rollback-mode image"); + + sqlite3* raw = nullptr; + const int opened = sqlite3_open_v2(":memory:", &raw, + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_MEMORY, nullptr); + result.Db.reset(raw); + if (opened != SQLITE_OK) throw TMessageError("Cannot open SQL snapshot database"); + const int rc = sqlite3_deserialize(result.Db.get(), "main", result.Buffer.get(), + sqlite3_int64(bytes.size()), sqlite3_int64(bytes.size()), SQLITE_DESERIALIZE_READONLY); + if (rc != SQLITE_OK) throw TMessageError("Invalid SQL snapshot database image"); + return result; +} + +uint64_t SnapshotAppliedIndex(sqlite3* db) { + sqlite3_stmt* raw = nullptr; + const int prepared = sqlite3_prepare_v2(db, + "SELECT value FROM raft_metadata_ WHERE key='LastAppliedIndex'", -1, &raw, nullptr); + TStatement statement(raw, sqlite3_finalize); + if (prepared != SQLITE_OK || !statement) throw TMessageError("SQL snapshot has no progress metadata"); + int rc = sqlite3_step(statement.get()); + if (rc != SQLITE_ROW || sqlite3_column_type(statement.get(), 0) != SQLITE_TEXT) + throw TMessageError("Invalid SQL snapshot progress metadata"); + const auto* text = reinterpret_cast(sqlite3_column_text(statement.get(), 0)); + const auto length = sqlite3_column_bytes(statement.get(), 0); + if (!text || length <= 0) throw TMessageError("Invalid SQL snapshot progress metadata"); + uint64_t index = 0; + const auto parsed = std::from_chars(text, text + length, index); + if (parsed.ec != std::errc{} || parsed.ptr != text + length || !index || index > uint64_t(INT64_MAX)) + throw TMessageError("Invalid SQL snapshot progress metadata"); + if (sqlite3_step(statement.get()) != SQLITE_DONE) + throw TMessageError("Duplicate SQL snapshot progress metadata"); + return index; +} + +void ValidateSnapshotDatabase(sqlite3* db, uint64_t index) { + if (!index || index > uint64_t(INT64_MAX)) throw TMessageError("Invalid SQL snapshot index"); + sqlite3_stmt* raw = nullptr; + const int prepared = sqlite3_prepare_v2(db, "PRAGMA quick_check(1)", -1, &raw, nullptr); + TStatement statement(raw, sqlite3_finalize); + if (prepared != SQLITE_OK || !statement) + throw TMessageError(std::string("Cannot run SQL snapshot integrity check: ") + sqlite3_errmsg(db)); + const int check = sqlite3_step(statement.get()); + if (check != SQLITE_ROW || sqlite3_column_type(statement.get(), 0) != SQLITE_TEXT) + throw TMessageError(std::string("Cannot read SQL snapshot integrity check: ") + sqlite3_errmsg(db)); + const auto* result = reinterpret_cast(sqlite3_column_text(statement.get(), 0)); + const auto length = sqlite3_column_bytes(statement.get(), 0); + if (!result || std::string_view(result, size_t(length)) != "ok") + throw TMessageError("SQL snapshot integrity check failed: " + + std::string(result ? result : "", result ? size_t(length) : 6)); + if (sqlite3_step(statement.get()) != SQLITE_DONE) + throw TMessageError("SQL snapshot integrity check failed"); + if (SnapshotAppliedIndex(db) != index) + throw TMessageError("SQL snapshot application progress does not match its boundary"); +} + +uint64_t ReadPragmaInteger(sqlite3* db, const char* pragma) { + sqlite3_stmt* raw = nullptr; + const std::string query = std::string("PRAGMA ") + pragma; + const int prepared = sqlite3_prepare_v2(db, query.c_str(), -1, &raw, nullptr); + TStatement statement(raw, sqlite3_finalize); + if (prepared != SQLITE_OK || !statement || sqlite3_step(statement.get()) != SQLITE_ROW || + sqlite3_column_type(statement.get(), 0) != SQLITE_INTEGER) + throw TMessageError("Cannot inspect SQL snapshot size"); + const auto value = sqlite3_column_int64(statement.get(), 0); + if (value <= 0 || sqlite3_step(statement.get()) != SQLITE_DONE) + throw TMessageError("Invalid SQL snapshot size metadata"); + return uint64_t(value); +} + +void ValidateSnapshotForTarget(sqlite3* snapshot, sqlite3* target, uint64_t index) { + ValidateSnapshotDatabase(snapshot, index); + if (!sqlite3_get_autocommit(target)) + throw TMessageError("Cannot replace SQL state during a transaction"); + if (ReadPragmaInteger(snapshot, "page_size") != ReadPragmaInteger(target, "page_size")) + throw TMessageError("SQL snapshot page size does not match the target database"); +} + +TSnapshotDatabase CopyDatabaseToMemory(sqlite3* source, uint64_t pageSize) { + TSnapshotDatabase result; + sqlite3* raw = nullptr; + const int opened = sqlite3_open_v2(":memory:", &raw, + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_MEMORY, nullptr); + result.Db.reset(raw); + if (opened != SQLITE_OK) throw TMessageError("Cannot open SQL snapshot database"); + + // SQLite's backup API requires matching page sizes for an in-memory + // destination. Set it before copying any pages. + const auto setPageSize = std::string("PRAGMA page_size = ") + std::to_string(pageSize); + if (sqlite3_exec(result.Db.get(), setPageSize.c_str(), nullptr, nullptr, nullptr) != SQLITE_OK) + throw TMessageError("Cannot configure SQL snapshot page size"); + + sqlite3_backup* backup = sqlite3_backup_init(result.Db.get(), "main", source, "main"); + if (!backup) throw TMessageError("Cannot begin SQL snapshot capture"); + const int copied = sqlite3_backup_step(backup, -1); + const int finished = sqlite3_backup_finish(backup); + if (copied != SQLITE_DONE || finished != SQLITE_OK) + throw TMessageError("Cannot complete SQL snapshot capture"); + return result; +} } std::string TSql::ExecuteRead(const std::string& q) { @@ -256,3 +373,46 @@ TMessageHolder TSql::Prepare(TMessageHolder command) return entry; } +std::string TSql::SaveSnapshot() const { + if (!LastAppliedIndex || LastAppliedIndex > uint64_t(INT64_MAX) || !sqlite3_get_autocommit(Db)) + throw TMessageError("SQL snapshot requires committed application progress"); + const auto pages = ReadPragmaInteger(Db, "page_count"); + const auto pageSize = ReadPragmaInteger(Db, "page_size"); + if (pages > MaxSqlSnapshotBytes / pageSize) + throw TMessageError("SQL snapshot exceeds the maximum image size"); + auto consistent = CopyDatabaseToMemory(Db, pageSize); + sqlite3_int64 size = 0; + auto* bytes = sqlite3_serialize(consistent.Db.get(), "main", &size, 0); + std::unique_ptr image(bytes, sqlite3_free); + if (!bytes || size <= 0 || uint64_t(size) > MaxSqlSnapshotBytes) + throw TMessageError("Cannot serialize SQL snapshot within the size limit"); + std::string result(reinterpret_cast(bytes), size_t(size)); + if (result.size() < 100 || + !((static_cast(result[18]) == 1 && static_cast(result[19]) == 1) || + (static_cast(result[18]) == 2 && static_cast(result[19]) == 2))) + throw TMessageError("Unsupported SQL snapshot database format"); + // SQLite documents this conversion for deserializing a complete database + // image. Backup copied all WAL-visible pages above, so rollback mode is now + // the correct self-contained representation. + result[18] = 1; + result[19] = 1; + return result; +} + +void TSql::ValidateSnapshot(std::string_view bytes, uint64_t index) const { + auto image = OpenSnapshotDatabase(bytes); + ValidateSnapshotForTarget(image.Db.get(), Db, index); +} + +void TSql::LoadSnapshot(std::string_view bytes, uint64_t index) { + auto image = OpenSnapshotDatabase(bytes); + ValidateSnapshotForTarget(image.Db.get(), Db, index); + sqlite3_backup* backup = sqlite3_backup_init(Db, "main", image.Db.get(), "main"); + if (!backup) throw TMessageError("Cannot begin SQL snapshot installation"); + const int copied = sqlite3_backup_step(backup, -1); + const int finished = sqlite3_backup_finish(backup); + if (copied != SQLITE_DONE || finished != SQLITE_OK) + throw TMessageError("Cannot atomically install SQL snapshot"); + LastAppliedIndex = index; +} + diff --git a/examples/sql_rsm.h b/examples/sql_rsm.h index e7f72b6..b2a6f54 100644 --- a/examples/sql_rsm.h +++ b/examples/sql_rsm.h @@ -37,6 +37,9 @@ class TSql: public IRsm { TMessageHolder Write(TMessageHolder message, uint64_t index) override; // convert request to log message TMessageHolder Prepare(TMessageHolder message) override; + std::string SaveSnapshot() const override; + void ValidateSnapshot(std::string_view bytes, uint64_t index) const override; + void LoadSnapshot(std::string_view bytes, uint64_t index) override; private: int Execute(const std::string& q); diff --git a/miniraft/raft.cpp b/miniraft/raft.cpp index 806d804..50c42b8 100644 --- a/miniraft/raft.cpp +++ b/miniraft/raft.cpp @@ -1021,7 +1021,7 @@ TRequestProcessor::TRequestProcessor(std::shared_ptr raft, std::shared_pt : Raft(std::move(raft)), Rsm(std::move(rsm)), Nodes(nodes) { if (!Raft || !Rsm) throw std::invalid_argument("Request processor requires Raft and application state"); - if (auto snapshot = Raft->GetState()->GetSnapshot(); snapshot && Rsm->LastAppliedIndex == 0) + if (auto snapshot = Raft->GetState()->GetSnapshot(); snapshot && Rsm->LastAppliedIndex < snapshot->Index) RestoreSnapshot(*snapshot, *Raft, *Rsm); if (Rsm->LastAppliedIndex < Raft->GetState()->SnapshotIndex) throw std::invalid_argument("Application progress precedes compacted log"); diff --git a/miniraft/snapshot.cpp b/miniraft/snapshot.cpp index ce68618..dd9be94 100644 --- a/miniraft/snapshot.cpp +++ b/miniraft/snapshot.cpp @@ -137,7 +137,9 @@ TSnapshot CaptureSnapshot(const TRaft& raft, const IRsm& rsm) { void RestoreSnapshot(const TSnapshot& snapshot, const TRaft& raft, IRsm& rsm) { CheckLocalLog(snapshot, raft); - if (rsm.LastAppliedIndex != 0) throw std::runtime_error("Snapshot restore requires a fresh application"); + if (rsm.LastAppliedIndex >= snapshot.Index) + throw std::runtime_error("Application is not behind the snapshot"); + rsm.ValidateSnapshot(snapshot.Data, snapshot.Index); rsm.LoadSnapshot(snapshot.Data, snapshot.Index); if (rsm.LastAppliedIndex != snapshot.Index) throw std::runtime_error("Application failed to restore snapshot progress"); } diff --git a/miniraft/snapshot.h b/miniraft/snapshot.h index 62f905a..b5baf32 100644 --- a/miniraft/snapshot.h +++ b/miniraft/snapshot.h @@ -33,7 +33,9 @@ TSnapshot DecodeSnapshot(std::string_view bytes); // Validate metadata and size without copying the application image. void ValidateSnapshot(const TSnapshot& snapshot); TSnapshot CaptureSnapshot(const TRaft& raft, const IRsm& rsm); -// Startup only, with a fresh application and a matching local boundary/log. +// Startup only, with an application behind the snapshot and +// a matching local boundary/log. This can recover an app left behind by a crash +// after durable Raft snapshot publication. void RestoreSnapshot(const TSnapshot& snapshot, const TRaft& raft, IRsm& rsm); class TSnapshotStore { diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 7ec08da..9dfb524 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -48,6 +48,11 @@ set_tests_properties(test_install_snapshot_cli PROPERTIES TIMEOUT 120) add_test(NAME test_learner_cli COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/smoke_install_snapshot.py $ learner) set_tests_properties(test_learner_cli PROPERTIES TIMEOUT 120) +if(SQLite3_FOUND) +add_test(NAME test_sql_snapshots_cli COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/smoke_sql_snapshots.py $) +set_tests_properties(test_sql_snapshots_cli PROPERTIES TIMEOUT 60) +endif() endif() endif() endif() diff --git a/test/smoke_sql_snapshots.py b/test/smoke_sql_snapshots.py new file mode 100644 index 0000000..4f2d2b4 --- /dev/null +++ b/test/smoke_sql_snapshots.py @@ -0,0 +1,104 @@ +"""SQL CLI compaction and restart smoke; fault schedules live in C++ tests.""" +import pathlib +import re +import socket +import subprocess +import sys +import tempfile +import time +sys.dont_write_bytecode = True +from smoke_kv_checkpoints import no_core, port, snapshot_boundary + + +def main(): + binary = str(pathlib.Path(sys.argv[1]).resolve()) + with tempfile.TemporaryDirectory(prefix="miniraft-sql-snapshot-cli-") as temporary: + root = pathlib.Path(temporary) + endpoint = port() + node = f"127.0.0.1:{endpoint}:1" + args = [binary, "--server", "--id", "1", "--node", node, "--durable"] + log_path = root / "server.log" + process = None + with log_path.open("w+") as log: + def start(periodic): + extra = ["--snapshots", "1"] if periodic else [] + return subprocess.Popen([*args, *extra], cwd=root, stdout=log, stderr=log, preexec_fn=no_core) + + def stop(child): + child.terminate() + child.wait(timeout=5) + + def wait_ready(child): + deadline = time.monotonic() + 10 + while time.monotonic() < deadline: + assert child.poll() is None, "SQL server exited during startup" + try: + with socket.create_connection(("127.0.0.1", endpoint), timeout=0.1): + return + except OSError: + time.sleep(0.05) + raise AssertionError("SQL server did not listen") + + def run_client(line): + return subprocess.run([binary, "--client", "--node", node], cwd=root, + input=line + "\n", text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + timeout=15, preexec_fn=no_core) + + try: + for options in (["--server", "--snapshots", "1"], + ["--server", "--durable", "--snapshots", "0"], + ["--server", "--durable", "--snapshots"]): + rejected = subprocess.run([binary, *options], cwd=root, stdout=log, + stderr=log, timeout=5, preexec_fn=no_core) + assert rejected.returncode != 0, options + + process = start(periodic=True) + wait_ready(process) + write = "CREATE TABLE IF NOT EXISTS t (value TEXT); DELETE FROM t; INSERT INTO t VALUES ('persisted');" + write_index = None + deadline = time.monotonic() + 35 + while time.monotonic() < deadline: + assert process.poll() is None, "SQL server exited before accepting the write" + result = run_client(write) + match = re.search(r"commitIndex: (\d+)", result.stdout) + if match: + write_index = int(match.group(1)) + break + time.sleep(0.1) + assert write_index is not None, "SQL write did not commit" + + durable = root / "sql_state.durable.1" + deadline = time.monotonic() + 12 + while time.monotonic() < deadline: + if durable.exists() and snapshot_boundary(durable) >= write_index: + break + assert process.poll() is None, "SQL server exited during compaction" + time.sleep(0.05) + else: + raise AssertionError("SQL snapshot did not compact through the committed write") + + stop(process) + process = start(periodic=False) + wait_ready(process) + result = run_client("select value from t") + assert "value\npersisted\n" in result.stdout, result.stdout + stop(process) + + log.flush() + log.seek(0) + output = log.read() + assert "ERROR: AddressSanitizer" not in output and "runtime error:" not in output, output + print("SQL snapshot CLI: option checks, compaction and durable restart/read passed") + except Exception: + log.flush() + log.seek(0) + print(log.read(), file=sys.stderr) + raise + finally: + if process is not None and process.poll() is None: + process.kill() + process.wait(timeout=5) + + +if __name__ == "__main__": + main() diff --git a/test/test_snapshots.cpp b/test/test_snapshots.cpp index 7545dda..8d6f5fa 100644 --- a/test/test_snapshots.cpp +++ b/test/test_snapshots.cpp @@ -336,6 +336,21 @@ void test_restore_rejects_mismatched_log_configuration_and_used_application(void Reject([&] { RestoreSnapshot(snapshot, raft, restored); }); assert_true(Read(restored, "a") == "one"); } +void test_startup_restores_stale_application_from_durable_snapshot_boundary(void**) { + auto state = std::make_shared(); + TKv source; Apply(source, "snapshot", "committed", 2); + TSnapshot snapshot{2, 1, {1}, source.SaveSnapshot()}; + state->InstallSnapshot(snapshot); + auto raft = std::make_shared(state, 1, TNodeDict{}); + + auto stale = std::make_shared(); Apply(*stale, "stale", "must be replaced", 1); + assert_int_equal(stale->LastAppliedIndex, 1); + TRequestProcessor processor(raft, stale, {}); + assert_int_equal(stale->LastAppliedIndex, 2); + assert_true(Read(*stale, "snapshot") == "committed"); + assert_true(Read(*stale, "stale").empty()); + assert_int_equal(raft->GetVolatileState()->CommitIndex, 2); +} #ifdef __linux__ void test_stable_configuration_snapshot_durable_recovery_fails_closed(void**) { TFiles files; const auto path = files.Name + ".durable.1"; @@ -508,6 +523,7 @@ int main() { cmocka_unit_test(test_kv_invalid_snapshot_preserves_existing_state_and_progress), cmocka_unit_test(test_capture_requires_committed_applied_progress), cmocka_unit_test(test_restore_rejects_mismatched_log_configuration_and_used_application), + cmocka_unit_test(test_startup_restores_stale_application_from_durable_snapshot_boundary), #ifdef __linux__ cmocka_unit_test(test_stable_configuration_snapshot_durable_recovery_fails_closed), cmocka_unit_test(test_joint_snapshot_export_roundtrips_but_durable_state_rejects_it), diff --git a/test/test_sql_payloads.cpp b/test/test_sql_payloads.cpp index a724ea3..1644afd 100644 --- a/test/test_sql_payloads.cpp +++ b/test/test_sql_payloads.cpp @@ -1,4 +1,5 @@ #include "../examples/sql_rsm.h" +#include #include #include #include @@ -153,6 +154,118 @@ struct TDatabase { assert_int_equal(rc, SQLITE_OK); } }; +void test_sql_snapshot_roundtrip_and_rejection_are_atomic(void**) { + TDatabase sourceFile, targetFile; + std::string image; + { + TSql source(sourceFile.Path, 1); + Seed(source); + assert_true(Apply(source, "INSERT INTO t VALUES ('second')", 2).empty()); + image = source.SaveSnapshot(); + assert_true(image.substr(0, 16) == std::string("SQLite format 3\0", 16)); + source.ValidateSnapshot(image, 2); + Reject([&] { source.ValidateSnapshot(image, 3); }); + auto corrupt = image; corrupt[0] ^= 1; + Reject([&] { source.ValidateSnapshot(corrupt, 2); }); + } + { + TSql target(targetFile.Path, 1); + Seed(target); + auto corrupt = image; corrupt[0] ^= 1; + Reject([&] { target.LoadSnapshot(corrupt, 2); }); + Reject([&] { target.LoadSnapshot(image, 3); }); + assert_int_equal(target.LastAppliedIndex, 1); + assert_true(ReadText(target, "SELECT value FROM t") == "value\noriginal\n"); + + target.LoadSnapshot(image, 2); + assert_int_equal(target.LastAppliedIndex, 2); + assert_true(ReadText(target, "SELECT value FROM t ORDER BY rowid") == "value\noriginal\nsecond\n"); + } + { + TSql restored(targetFile.Path, 1); + assert_int_equal(restored.LastAppliedIndex, 2); + assert_true(ReadText(restored, "SELECT count(*) FROM t") == "count(*)\n2\n"); + } +} +void test_sql_snapshot_installation_busy_failure_preserves_database(void**) { + TDatabase sourceFile, targetFile; + TSql source(sourceFile.Path, 1); Seed(source); + assert_true(Apply(source, "INSERT INTO t VALUES ('snapshot')", 2).empty()); + const auto image = source.SaveSnapshot(); + TSql target(targetFile.Path, 1); Seed(target); + + sqlite3* other = nullptr; + assert_int_equal(sqlite3_open((targetFile.Path + ".1").c_str(), &other), SQLITE_OK); + assert_int_equal(sqlite3_exec(other, "BEGIN IMMEDIATE", nullptr, nullptr, nullptr), SQLITE_OK); + Reject([&] { target.LoadSnapshot(image, 2); }); + assert_int_equal(target.LastAppliedIndex, 1); + assert_true(ReadText(target, "SELECT value FROM t") == "value\noriginal\n"); + assert_int_equal(sqlite3_exec(other, "ROLLBACK", nullptr, nullptr, nullptr), SQLITE_OK); + assert_int_equal(sqlite3_close(other), SQLITE_OK); + + target.LoadSnapshot(image, 2); + assert_int_equal(target.LastAppliedIndex, 2); + assert_true(ReadText(target, "SELECT value FROM t") == "value\noriginal\nsnapshot\n"); +} +void test_sql_snapshot_includes_committed_wal_pages(void**) { + TDatabase file; + TSql source(file.Path, 1); + Seed(source); + + sqlite3* wal = nullptr; + assert_int_equal(sqlite3_open((file.Path + ".1").c_str(), &wal), SQLITE_OK); + assert_int_equal(sqlite3_exec(wal, "PRAGMA journal_mode=WAL", nullptr, nullptr, nullptr), SQLITE_OK); + assert_int_equal(sqlite3_close(wal), SQLITE_OK); + assert_true(Apply(source, "INSERT INTO t VALUES ('committed in WAL')", 2).empty()); + + const auto image = source.SaveSnapshot(); + assert_int_equal(static_cast(image[18]), 1); + assert_int_equal(static_cast(image[19]), 1); + TSql restored(":memory:", 2); + restored.LoadSnapshot(image, 2); + assert_true(ReadText(restored, "SELECT value FROM t ORDER BY rowid") == + "value\noriginal\ncommitted in WAL\n"); +} +void test_sql_snapshot_page_size_mismatch_rejects_before_replacement(void**) { + TDatabase sourceFile, targetFile; + sourceFile.External("PRAGMA page_size=8192; VACUUM"); + TSql source(sourceFile.Path, 1); + Seed(source); + const auto image = source.SaveSnapshot(); + TSql target(targetFile.Path, 1); + Seed(target); + + Reject([&] { target.ValidateSnapshot(image, 1); }); + assert_int_equal(target.LastAppliedIndex, 1); + assert_true(ReadText(target, "SELECT value FROM t") == "value\noriginal\n"); +} +void test_sql_startup_restores_application_left_behind_raft_snapshot(void**) { + TDatabase sourceFile, targetFile; + TSql source(sourceFile.Path, 1); Seed(source); + assert_true(Apply(source, "INSERT INTO t VALUES ('committed')", 2).empty()); + TSnapshot snapshot{2, 1, {1}, source.SaveSnapshot()}; + { TSql stale(targetFile.Path, 1); Seed(stale); } + const auto raftPath = (targetFile.Directory / "raft").string(); + { TDurableState state(raftPath, 1); state.InstallSnapshot(snapshot); } + + { + auto state = std::make_shared(raftPath, 1); + auto raft = std::make_shared(state, 1, TNodeDict{}); + auto sql = std::make_shared(targetFile.Path, 1); + assert_int_equal(sql->LastAppliedIndex, 1); + TRequestProcessor processor(raft, sql, {}); + assert_int_equal(sql->LastAppliedIndex, 2); + assert_true(ReadText(*sql, "SELECT value FROM t ORDER BY rowid") == "value\noriginal\ncommitted\n"); + } + { + auto state = std::make_shared(raftPath, 1); + auto raft = std::make_shared(state, 1, TNodeDict{}); + auto sql = std::make_shared(targetFile.Path, 1); + TRequestProcessor processor(raft, sql, {}); + assert_int_equal(sql->LastAppliedIndex, 2); + assert_true(ReadText(*sql, "SELECT count(*) FROM t") == "count(*)\n2\n"); + } +} void test_sql_write_cannot_escape_managed_transaction(void**) { for (const std::string control : {"COMMIT", "END", "ROLLBACK", "BEGIN", "SAVEPOINT x", "RELEASE x", "PRAGMA user_version=7", "ATTACH ':memory:' AS other"}) { @@ -434,6 +547,11 @@ int main() { cmocka_unit_test(test_sql_write_handles_trailing_comment_without_losing_progress), cmocka_unit_test(test_sql_automatic_rollback_is_consumed_without_partial_data), cmocka_unit_test(test_sql_metadata_failure_rolls_back_data_and_does_not_advance), + cmocka_unit_test(test_sql_snapshot_roundtrip_and_rejection_are_atomic), + cmocka_unit_test(test_sql_snapshot_installation_busy_failure_preserves_database), + cmocka_unit_test(test_sql_snapshot_includes_committed_wal_pages), + cmocka_unit_test(test_sql_snapshot_page_size_mismatch_rejects_before_replacement), + cmocka_unit_test(test_sql_startup_restores_application_left_behind_raft_snapshot), cmocka_unit_test(test_sql_progress_restores_full_uint64_range), cmocka_unit_test(test_sql_invalid_progress_fails_closed), cmocka_unit_test(test_sql_busy_write_does_not_consume_entry), From 0fea03f6125cf78428b4fe71779e32860bb0dc69 Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Sat, 14 Nov 2026 16:20:10 -0500 Subject: [PATCH 45/66] Persist bound configuration snapshots for offline recovery (#45) --- README.md | 6 +- docs/configuration-history.md | 11 ++- docs/configuration-scan.md | 13 ++-- docs/durable-configuration-snapshots.md | 26 ++++++++ docs/joint-snapshots.md | 27 ++++---- docs/stable-configuration-snapshots.md | 17 ++--- miniraft/configuration.cpp | 8 +-- miniraft/durable.cpp | 6 +- miniraft/state.h | 6 +- test/test_configuration.cpp | 20 +++++- test/test_snapshots.cpp | 89 +++++++++++++++++-------- 11 files changed, 158 insertions(+), 71 deletions(-) create mode 100644 docs/durable-configuration-snapshots.md diff --git a/README.md b/README.md index 3456f4f..0351b29 100644 --- a/README.md +++ b/README.md @@ -77,7 +77,11 @@ including on learners. The [snapshot pin budget follow-up](docs/snapshot-pin-budget.md) bounds outgoing transfer retention to two image generations and defers local compaction at the cap. The [stable configuration snapshot foundation](docs/stable-configuration-snapshots.md) -preserves configuration identity for offline recovery validation. +preserves configuration identity for offline recovery validation. Bound storage +can persist these metadata snapshots across restart for offline scanning; Raft +still rejects runtime adoption and membership changes remain disabled. +The [durable configuration snapshot follow-up](docs/durable-configuration-snapshots.md) +documents its voter-binding requirement and recovery boundary. The [snapshot term validation fix](docs/snapshot-committed-terms.md) rejects snapshot boundaries that contradict the follower's committed prefix. The [message buffer follow-up](docs/message-extents.md) checks allocated buffer diff --git a/docs/configuration-history.md b/docs/configuration-history.md index 6e37170..1d98cfd 100644 --- a/docs/configuration-history.md +++ b/docs/configuration-history.md @@ -10,6 +10,9 @@ and that the supplied application image corresponds exactly to that boundary. The method validates snapshot metadata and size; it does not capture application state, compact or mutate storage, establish commitment, or activate voters. Experimental configuration snapshots remain rejected by the live runtime. +Bound storage can retain and recover stable or joint metadata snapshots for +offline validation, provided the snapshot voter list matches the store's existing +voter binding. This does not activate the configuration after restart. TConfigurationHistory checks a sequence of retained log entries against a trusted configuration baseline. It joins the experimental configuration-entry codec with @@ -51,9 +54,11 @@ restart, confirms the fixture's known bootstrap baseline, and verifies that Raft still rejects the unsupported entries without rewriting storage. Full tests also cover existing snapshot installation and learner catch-up. -This is offline validation, not live reconfiguration. Joint entry activation and -rollback, durable configuration recovery, snapshot installation across membership -transitions, promotion gates and transport discovery remain open. Self-review +This is offline validation, not live reconfiguration. Storage can now preserve +configuration snapshot metadata across restart, but durable recovery into an +active configuration remains open. Joint entry activation and rollback, snapshot +installation across membership transitions, promotion gates and transport +discovery remain open. Self-review checked contiguous scanning, term and predecessor relationships, immutable branch results, snapshot boundary consistency and the unknown-v1-identity case. No independent reviewer approval is claimed. diff --git a/docs/configuration-scan.md b/docs/configuration-scan.md index ebe7d1d..550dfb1 100644 --- a/docs/configuration-scan.md +++ b/docs/configuration-scan.md @@ -23,8 +23,11 @@ backend, which must validate its own records. Memory and durable backends avoid copying ordinary payloads; external backends using default metadata accessors may read full entries. Configuration records are read and validated in full. -Joint snapshots and configuration entries remain rejected by the live runtime and -its storage installation paths. Tests use a synthetic joint snapshot for offline -validation and a real fixed snapshot across durable restart. Live membership -activation, commit rules, snapshot installation across transitions, and recovery -of a proven configuration remain future work. +Configuration entries and configuration snapshots remain rejected by the live +runtime. Bound in-memory and durable stores can retain stable or joint metadata +snapshots for offline validation; `snapshot.Voters` must match the persisted +binding. For joint snapshots, the scanner also requires the baseline's old voter +set to match that binding. Tests cover durable restart and reject a baseline that +differs from the persisted binding. Live membership activation, commit rules, +snapshot installation across transitions, and recovery into an active Raft +configuration remain future work. diff --git a/docs/durable-configuration-snapshots.md b/docs/durable-configuration-snapshots.md new file mode 100644 index 0000000..5caad3d --- /dev/null +++ b/docs/durable-configuration-snapshots.md @@ -0,0 +1,26 @@ +# Durable configuration snapshot metadata: v0.1.0-alpha.45 + +Bound `TState` and `TDurableState` instances can install and recover MRFTS002 +joint and MRFTS003 stable snapshots so offline tools can validate configuration +history after restart. A store must already have a voter binding, and +`snapshot.Voters` must equal that binding. Durable recovery checks the same +invariant before exposing the image. An unbound store cannot infer its original +voter set from configuration metadata and continues to reject the snapshot. + +The durable file layout is unchanged. The backend persists the snapshot together +with its existing voter binding; it does not reinterpret that binding from the +metadata. The history scanner also verifies that a joint baseline's old voters +match the bound set. + +This is persistence for offline validation, not runtime configuration recovery. +`TRaft` still rejects configuration metadata snapshots on startup, the network +receiver rejects them, configuration log entries remain disabled, and compaction +continues to produce fixed-voter snapshots. A stable snapshot whose voter list +differs from the existing binding is rejected. Live transition semantics must +define how the binding changes before such snapshots can be used by Raft. + +Tests cover in-memory installation, durable joint and stable round-trips, bound +voter mismatch rejection without rewriting storage, offline scan baseline checks, +and runtime startup rejection without rewriting the recovered image. Self-review +checked that recovery requires an existing binding and that no live Raft path +adopts the experimental metadata. No independent reviewer approval is claimed. diff --git a/docs/joint-snapshots.md b/docs/joint-snapshots.md index 3afe7f9..577f1c5 100644 --- a/docs/joint-snapshots.md +++ b/docs/joint-snapshots.md @@ -19,16 +19,17 @@ bounded before allocation. Fixed snapshots continue to encode as MRFTS001 with exactly the previous bytes; a golden v1 image tests compatibility. The standalone snapshot store can export and load v2 images for testing or offline tooling. -This is a format foundation, not live joint consensus or complete configuration -history. Current Raft nodes reject joint snapshot restore, installation and -durable recovery; compaction still creates only fixed-voter snapshots. A covered -snapshot is acknowledged only after its prefix identifies the supported v1 -format, including when the prefix arrives in several chunks. Unsupported joint -images are rejected without replacing application data or publishing a joint -snapshot. A valid higher-term RPC header can still update durable hard state -before its unsupported snapshot contents are recognized. -These rules avoid falsely acknowledging a joint image merely because its index -is already committed locally. +This is a format and offline persistence foundation, not live joint consensus or +complete configuration history. A bound memory or durable store can install and +recover a joint snapshot when its persisted voter list matches the snapshot's old +voter set. Raft startup and RPC paths still reject joint snapshots, and compaction +still creates only fixed-voter snapshots. A covered snapshot is acknowledged only +after its prefix identifies the supported v1 format, including when the prefix +arrives in several chunks. Unsupported joint images are rejected without +replacing application data or publishing a joint snapshot. A valid higher-term +RPC header can still update durable hard state before its unsupported snapshot +contents are recognized. These rules avoid falsely acknowledging a joint image +merely because its index is already committed locally. Older binaries do not understand MRFTS002 and may acknowledge an already-covered image without checking its version. Do not send or activate joint snapshots in a @@ -41,9 +42,9 @@ configuration is committed and is not an authoritative Raft checkpoint. Tests cover a golden v1 image, joint round-trip, every-byte corruption and truncation, semantic corruption with valid CRC, bounds and ordering, standalone -publication, rejected memory/durable installation and durable recovery without -rewriting the image, and network rejection both above and below the commit boundary. -They also cover a supported v1 prefix split across chunks. The snapshot struct's +publication, bound durable round-trip, runtime rejection without rewriting the +image, and network rejection both above and below the commit boundary. They also +cover a supported v1 prefix split across chunks. The snapshot struct's in-memory layout changes; rebuild the library and dependents together. Self-review checked version dispatch, bounds, immutable voter binding, covered acknowledgments and fail-closed recovery. No independent reviewer approval is claimed. diff --git a/docs/stable-configuration-snapshots.md b/docs/stable-configuration-snapshots.md index 94ec4c0..ff01c7f 100644 --- a/docs/stable-configuration-snapshots.md +++ b/docs/stable-configuration-snapshots.md @@ -18,11 +18,12 @@ trusted snapshot and commit boundary independently. MRFTS003 cannot assert that there was no earlier configuration entry; an explicit trusted baseline remains necessary for such histories. -The fixed-membership runtime, storage installation and durable recovery reject -MRFTS003, and the network receiver rejects it even at an already covered boundary. -The standalone snapshot store supports offline export. Older binaries may falsely -acknowledge covered unknown formats; do not send these images to a mixed-version -cluster. Live configuration activation, rollback, commit rules, joint snapshot -installation and membership recovery remain open. This experimental format may -change before those semantics are implemented. Rebuild dependents because the -in-memory snapshot structure has changed. +The live runtime rejects MRFTS003, and the network receiver rejects it even at an +already covered boundary. Bound memory and durable stores can retain and recover +the image for offline validation only when its voter list matches the persisted +binding. Storage does not activate a changed voter set from snapshot metadata. +Older binaries may falsely acknowledge covered unknown formats; do not send these +images to a mixed-version cluster. Live configuration activation, rollback, +commit rules, joint snapshot installation and membership recovery remain open. +This experimental format may change before those semantics are implemented. +Rebuild dependents because the in-memory snapshot structure has changed. diff --git a/miniraft/configuration.cpp b/miniraft/configuration.cpp index 0bd4dfd..f243562 100644 --- a/miniraft/configuration.cpp +++ b/miniraft/configuration.cpp @@ -177,11 +177,9 @@ TConfigurationScan ScanConfigurationHistory(const IState& state, } else if (snapshot || state.SnapshotTerm || baseline.IsJoint()) { throw std::invalid_argument("Invalid empty snapshot boundary"); } - if (!baseline.IsJoint()) { - const auto bound = state.GetBoundVoters(); - if (!bound.empty() && bound != baseline.OldVoters()) - throw std::invalid_argument("Configuration baseline differs from bound voters"); - } + const auto bound = state.GetBoundVoters(); + if (!bound.empty() && bound != baseline.OldVoters()) + throw std::invalid_argument("Configuration baseline differs from bound voters"); auto latest = baseline; auto committed = baseline; diff --git a/miniraft/durable.cpp b/miniraft/durable.cpp index e696c19..764fe79 100644 --- a/miniraft/durable.cpp +++ b/miniraft/durable.cpp @@ -167,7 +167,8 @@ TDurableState::TDurableState(const std::string& name, uint32_t id, THook hook) if (!length && compacted) throw std::runtime_error("Missing embedded snapshot"); if (length) { auto snapshot = DecodeSnapshot(std::string_view(reinterpret_cast(bytes.data() + position), size_t(length))); - if (snapshot.HasConfigurationMetadata()) throw std::runtime_error("Configuration snapshot recovery is unsupported"); + if (snapshot.HasConfigurationMetadata() && !configured) + throw std::runtime_error("Configuration snapshot recovery requires a bound voter set"); if (snapshot.Term > CurrentTerm) throw std::runtime_error("Future durable snapshot term"); if (configured && snapshot.Voters != Impl->Voters) throw std::runtime_error("Durable snapshot/voter mismatch"); position += size_t(length); @@ -309,8 +310,9 @@ uint64_t TDurableState::RetainedLogFlags(int64_t index) const { std::shared_ptr TDurableState::GetSnapshot() const { return Impl->Snapshot; } void TDurableState::InstallSnapshot(const TSnapshot& snapshot) { - if (snapshot.HasConfigurationMetadata()) throw std::invalid_argument("Configuration snapshot installation is unsupported"); EncodeSnapshot(snapshot); + if (snapshot.HasConfigurationMetadata() && Impl->Voters.empty()) + throw std::invalid_argument("Configuration snapshots require a bound voter set"); if (!Impl->Voters.empty() && snapshot.Voters != Impl->Voters) throw std::invalid_argument("Snapshot would change bound voters"); if (snapshot.Index <= SnapshotIndex || snapshot.Term < SnapshotTerm || snapshot.Term > CurrentTerm) diff --git a/miniraft/state.h b/miniraft/state.h index 837d5bc..b2d90fc 100644 --- a/miniraft/state.h +++ b/miniraft/state.h @@ -70,7 +70,8 @@ inline void ValidateVoterBinding(const IState& state, const std::vectorHasConfigurationMetadata()) throw std::invalid_argument("Configuration snapshot recovery is unsupported"); + if (snapshot->HasConfigurationMetadata()) + throw std::invalid_argument("Configuration snapshot recovery requires an existing voter binding"); if (snapshot->Voters != voters) throw std::invalid_argument("Snapshot voter configuration mismatch"); } else if (state.CurrentTerm != 1 || state.VotedFor || state.LastLogIndex) { throw std::invalid_argument("Unbound nonfresh store needs a verified snapshot before enabling learners"); @@ -149,8 +150,9 @@ struct TState: IState { } std::shared_ptr GetSnapshot() const override { return Snapshot; } void InstallSnapshot(const TSnapshot& snapshot) override { - if (snapshot.HasConfigurationMetadata()) throw std::invalid_argument("Configuration snapshot installation is unsupported"); EncodeSnapshot(snapshot); // Validate before changing any state. + if (snapshot.HasConfigurationMetadata() && BoundVoters.empty()) + throw std::invalid_argument("Configuration snapshots require a bound voter set"); if (!BoundVoters.empty() && snapshot.Voters != BoundVoters) throw std::invalid_argument("Snapshot would change bound voters"); if (snapshot.Index <= SnapshotIndex || snapshot.Term < SnapshotTerm || snapshot.Term > CurrentTerm) diff --git a/test/test_configuration.cpp b/test/test_configuration.cpp index 34866f0..4d49baa 100644 --- a/test/test_configuration.cpp +++ b/test/test_configuration.cpp @@ -69,8 +69,18 @@ void test_configuration_snapshot_projection_and_history_resume(void**) { auto resumed = restoredStable.Advance(5, EncodeConfigurationEntry(next)); assert_true(resumed.IsJoint()); assert_true(resumed.NewVoters() == next.NewVoters); assert_true(pending.NewVoters() == joint.NewVoters); assert_false(stable.IsJoint()); - TState state; Reject([&] { state.InstallSnapshot(pendingSnapshot); }); - Reject([&] { state.InstallSnapshot(stableSnapshot); }); + TState jointState; jointState.BindVoters(pendingSnapshot.Voters); + jointState.CurrentTerm = pendingSnapshot.Term; + jointState.InstallSnapshot(pendingSnapshot); + assert_true(*jointState.GetSnapshot() == pendingSnapshot); + TState stableState; stableState.BindVoters(stableSnapshot.Voters); + stableState.CurrentTerm = stableSnapshot.Term; + stableState.InstallSnapshot(stableSnapshot); + assert_true(*stableState.GetSnapshot() == stableSnapshot); + TState unbound; unbound.CurrentTerm = pendingSnapshot.Term; + Reject([&] { unbound.InstallSnapshot(pendingSnapshot); }); + unbound.Snapshot = std::make_shared(pendingSnapshot); + Reject([&] { unbound.BindVoters(pendingSnapshot.Voters); }); } void test_history_validates_joint_stable_chain_and_independent_rollback(void**) { auto initial = TConfigurationHistory::Stable({1, 2, 3}, 0, 0, 0, 0); @@ -177,9 +187,10 @@ void test_scan_separates_commit_and_replaced_suffix(void**) { void test_scan_from_joint_snapshot_and_rejects_mismatched_baselines(void**) { TSnapshot snapshot{2, 2, {1, 2}, "image"}; snapshot.JointVoters = {2, 3}; snapshot.ConfigurationIndex = 2; snapshot.ConfigurationTerm = 2; - // Synthetic offline state: runtime storage deliberately rejects joint snapshots. + // Synthetic offline state with the original voter set durably bound. TState state; state.Snapshot = std::make_shared(snapshot); state.SnapshotIndex = state.LastLogIndex = 2; state.SnapshotTerm = state.LastLogTerm = 2; + state.BoundVoters = snapshot.Voters; state.CurrentTerm = 3; state.Append(Ordinary(2)); TConfigurationRecord final{EConfigurationPhase::Stable, 3, 2, 2, {1, 2}, {2, 3}}; state.Append(EncodeConfigurationEntry(final)); @@ -193,6 +204,9 @@ void test_scan_from_joint_snapshot_and_rejects_mismatched_baselines(void**) { Reject([&] { ScanConfigurationHistory(state, TConfigurationHistory::Stable({1, 2}, 0, 0, 2, 2), 2); }); auto wrong = snapshot; wrong.JointVoters = {2, 4}; Reject([&] { ScanConfigurationHistory(state, TConfigurationHistory::FromJointSnapshot(wrong), 2); }); + state.BoundVoters = {1, 2, 4}; + Reject([&] { ScanConfigurationHistory(state, base, 2); }); + state.BoundVoters = snapshot.Voters; state.LastLogTerm = 2; Reject([&] { ScanConfigurationHistory(state, base, 2); }); } diff --git a/test/test_snapshots.cpp b/test/test_snapshots.cpp index 8d6f5fa..a1e4533 100644 --- a/test/test_snapshots.cpp +++ b/test/test_snapshots.cpp @@ -352,17 +352,47 @@ void test_startup_restores_stale_application_from_durable_snapshot_boundary(void assert_int_equal(raft->GetVolatileState()->CommitIndex, 2); } #ifdef __linux__ -void test_stable_configuration_snapshot_durable_recovery_fails_closed(void**) { +void test_stable_configuration_snapshot_durable_roundtrip_but_runtime_rejects(void**) { TFiles files; const auto path = files.Name + ".durable.1"; - { auto state = std::make_shared(files.Name, 1); - TRaft raft(state, 1, {{2, std::make_shared()}, {3, std::make_shared()}}); } - std::ifstream input(path, std::ios::binary); std::string bytes(std::istreambuf_iterator(input), {}); input.close(); - assert_true(bytes.substr(0, 8) == "MRFTD003"); + auto snapshot = Sample(); snapshot.StableConfiguration = true; + snapshot.ConfigurationIndex = 1; snapshot.ConfigurationTerm = 1; + { + TDurableState state(files.Name, 1); + state.BindVoters(snapshot.Voters); + state.InstallSnapshot(snapshot); + } + std::string bytes; + { + TDurableState state(files.Name, 1); + assert_true(state.GetBoundVoters() == snapshot.Voters); + assert_true(*state.GetSnapshot() == snapshot); + std::ifstream input(path, std::ios::binary); + bytes.assign(std::istreambuf_iterator(input), {}); + } + Reject([&] { TRaft raft(std::make_shared(files.Name, 1), 1, + {{2, std::make_shared()}, {3, std::make_shared()}}); }); + std::ifstream after(path, std::ios::binary); + assert_true(std::string(std::istreambuf_iterator(after), {}) == bytes); +} +void test_unbound_durable_configuration_snapshot_recovery_fails_closed(void**) { + TFiles files; const auto path = files.Name + ".durable.1"; + { + TDurableState state(files.Name, 1); + state.InstallSnapshot(Sample()); + } + std::ifstream input(path, std::ios::binary); + std::string bytes(std::istreambuf_iterator(input), {}); input.close(); + assert_true(bytes.substr(0, 8) == "MRFTD002"); auto snapshot = Sample(); snapshot.StableConfiguration = true; snapshot.ConfigurationIndex = 1; snapshot.ConfigurationTerm = 1; const auto image = EncodeSnapshot(snapshot); - for (int i = 0; i < 8; ++i) bytes[56 + i] = char(uint64_t(image.size()) >> (8 * i)); - bytes.insert(64, image); + constexpr size_t imageLengthOffset = 40; + constexpr size_t imageOffset = 48; + uint64_t oldLength = 0; + for (int i = 0; i < 8; ++i) oldLength |= uint64_t(uint8_t(bytes[imageLengthOffset + i])) << (8 * i); + assert_true(oldLength == bytes.size() - imageOffset); + bytes.replace(imageOffset, size_t(oldLength), image); + for (int i = 0; i < 8; ++i) bytes[imageLengthOffset + i] = char(uint64_t(image.size()) >> (8 * i)); for (int i = 0; i < 8; ++i) bytes[8 + i] = char(uint64_t(bytes.size() - 20) >> (8 * i)); const auto crc = NSnapshotCodec::Checksum(bytes.substr(0, 8) + bytes.substr(20)); for (int i = 0; i < 4; ++i) bytes[16 + i] = char(crc >> (8 * i)); @@ -371,32 +401,32 @@ void test_stable_configuration_snapshot_durable_recovery_fails_closed(void**) { std::ifstream after(path, std::ios::binary); assert_true(std::string(std::istreambuf_iterator(after), {}) == bytes); } -void test_joint_snapshot_export_roundtrips_but_durable_state_rejects_it(void**) { +void test_joint_snapshot_export_and_durable_roundtrip_but_runtime_rejects(void**) { TFiles files; { TSnapshotStore exportStore(files.Name, 1); exportStore.Save(JointSample()); assert_true(exportStore.Load().value() == JointSample()); } const auto path = files.Name + ".durable.1"; { - auto state = std::make_shared(files.Name, 1); - TRaft raft(state, 1, {{2, std::make_shared()}, {3, std::make_shared()}}); - const auto before = static_cast(*state); - std::ifstream input(path, std::ios::binary); std::string bytes(std::istreambuf_iterator(input), {}); input.close(); - Reject([&] { state->InstallSnapshot(JointSample()); }); - assert_true(static_cast(*state) == before); - std::ifstream after(path, std::ios::binary); - assert_true(std::string(std::istreambuf_iterator(after), {}) == bytes); + TDurableState state(files.Name, 1); + state.BindVoters(JointSample().Voters); + state.InstallSnapshot(JointSample()); } - // Synthesize a well-checksummed v3 store containing the unsupported image. - std::ifstream input(path, std::ios::binary); std::string bytes(std::istreambuf_iterator(input), {}); input.close(); - assert_true(bytes.substr(0, 8) == "MRFTD003"); - const auto image = EncodeSnapshot(JointSample()); - for (int i = 0; i < 8; ++i) bytes[56 + i] = char(uint64_t(image.size()) >> (8 * i)); - bytes.insert(64, image); - for (int i = 0; i < 8; ++i) bytes[8 + i] = char(uint64_t(bytes.size() - 20) >> (8 * i)); - const auto crc = NSnapshotCodec::Checksum(bytes.substr(0, 8) + bytes.substr(20)); - for (int i = 0; i < 4; ++i) bytes[16 + i] = char(crc >> (8 * i)); - { std::ofstream output(path, std::ios::binary | std::ios::trunc); output.write(bytes.data(), bytes.size()); } - Reject([&] { TDurableState state(files.Name, 1); }); + std::string bytes; + { + TDurableState state(files.Name, 1); + assert_true(state.GetBoundVoters() == JointSample().Voters); + assert_true(*state.GetSnapshot() == JointSample()); + const auto before = static_cast(state); + auto mismatched = JointSample(); mismatched.Voters = {1, 2, 4}; + std::ifstream input(path, std::ios::binary); + bytes.assign(std::istreambuf_iterator(input), {}); + Reject([&] { state.InstallSnapshot(mismatched); }); + assert_true(static_cast(state) == before); + std::ifstream unchanged(path, std::ios::binary); + assert_true(std::string(std::istreambuf_iterator(unchanged), {}) == bytes); + } + Reject([&] { TRaft raft(std::make_shared(files.Name, 1), 1, + {{2, std::make_shared()}, {3, std::make_shared()}}); }); std::ifstream after(path, std::ios::binary); assert_true(std::string(std::istreambuf_iterator(after), {}) == bytes); } @@ -525,8 +555,9 @@ int main() { cmocka_unit_test(test_restore_rejects_mismatched_log_configuration_and_used_application), cmocka_unit_test(test_startup_restores_stale_application_from_durable_snapshot_boundary), #ifdef __linux__ - cmocka_unit_test(test_stable_configuration_snapshot_durable_recovery_fails_closed), - cmocka_unit_test(test_joint_snapshot_export_roundtrips_but_durable_state_rejects_it), + cmocka_unit_test(test_stable_configuration_snapshot_durable_roundtrip_but_runtime_rejects), + cmocka_unit_test(test_unbound_durable_configuration_snapshot_recovery_fails_closed), + cmocka_unit_test(test_joint_snapshot_export_and_durable_roundtrip_but_runtime_rejects), cmocka_unit_test(test_snapshot_store_rejects_oversized_files_and_temp_symlinks), cmocka_unit_test(test_snapshot_store_roundtrip_lock_and_corrupt_recovery), cmocka_unit_test(test_snapshot_crash_at_each_publication_boundary), From a2cd136bfa3e6fbe30c6cb80d35765bed1032341 Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Sat, 14 Nov 2026 16:39:05 -0500 Subject: [PATCH 46/66] Enable fixed-membership stable snapshot recovery (#46) --- README.md | 8 ++- docs/configuration-history.md | 15 ++--- docs/configuration-scan.md | 15 +++-- docs/covered-snapshot-identity.md | 25 ++++---- docs/durable-configuration-snapshots.md | 18 +++--- docs/joint-snapshots.md | 11 ++-- docs/learners.md | 3 +- docs/online-learner-network.md | 4 +- docs/stable-configuration-snapshot-runtime.md | 24 ++++++++ docs/stable-configuration-snapshots.md | 19 +++--- miniraft/raft.cpp | 39 ++++++++---- miniraft/snapshot.cpp | 9 ++- miniraft/snapshot.h | 3 +- test/test_compaction.cpp | 56 +++++++++++++++-- test/test_snapshots.cpp | 60 ++++++++++++++----- 15 files changed, 219 insertions(+), 90 deletions(-) create mode 100644 docs/stable-configuration-snapshot-runtime.md diff --git a/README.md b/README.md index 0351b29..20dc66f 100644 --- a/README.md +++ b/README.md @@ -77,11 +77,13 @@ including on learners. The [snapshot pin budget follow-up](docs/snapshot-pin-budget.md) bounds outgoing transfer retention to two image generations and defers local compaction at the cap. The [stable configuration snapshot foundation](docs/stable-configuration-snapshots.md) -preserves configuration identity for offline recovery validation. Bound storage -can persist these metadata snapshots across restart for offline scanning; Raft -still rejects runtime adoption and membership changes remain disabled. +preserves configuration identity. Raft can recover and install stable metadata +snapshots when their voter list matches the node's existing fixed membership; +joint snapshots and live voter changes remain disabled. The [durable configuration snapshot follow-up](docs/durable-configuration-snapshots.md) documents its voter-binding requirement and recovery boundary. +The [stable snapshot runtime follow-up](docs/stable-configuration-snapshot-runtime.md) +carries that identity through restore, InstallSnapshot, and later compaction. The [snapshot term validation fix](docs/snapshot-committed-terms.md) rejects snapshot boundaries that contradict the follower's committed prefix. The [message buffer follow-up](docs/message-extents.md) checks allocated buffer diff --git a/docs/configuration-history.md b/docs/configuration-history.md index 1d98cfd..1c71a6a 100644 --- a/docs/configuration-history.md +++ b/docs/configuration-history.md @@ -9,10 +9,12 @@ The caller must independently establish that the history boundary is committed and that the supplied application image corresponds exactly to that boundary. The method validates snapshot metadata and size; it does not capture application state, compact or mutate storage, establish commitment, or activate voters. -Experimental configuration snapshots remain rejected by the live runtime. +Joint configuration snapshots remain rejected by the live runtime. Stable +metadata snapshots can be recovered and installed when they match the node's +existing fixed voter binding; this does not enable voter changes. Bound storage can retain and recover stable or joint metadata snapshots for offline validation, provided the snapshot voter list matches the store's existing -voter binding. This does not activate the configuration after restart. +voter binding. TConfigurationHistory checks a sequence of retained log entries against a trusted configuration baseline. It joins the experimental configuration-entry codec with @@ -54,11 +56,10 @@ restart, confirms the fixture's known bootstrap baseline, and verifies that Raft still rejects the unsupported entries without rewriting storage. Full tests also cover existing snapshot installation and learner catch-up. -This is offline validation, not live reconfiguration. Storage can now preserve -configuration snapshot metadata across restart, but durable recovery into an -active configuration remains open. Joint entry activation and rollback, snapshot -installation across membership transitions, promotion gates and transport -discovery remain open. Self-review +This is configuration history validation for fixed membership, not live +reconfiguration. Joint entry activation and rollback, snapshot installation +across membership transitions, promotion gates and transport discovery remain +open. Self-review checked contiguous scanning, term and predecessor relationships, immutable branch results, snapshot boundary consistency and the unknown-v1-identity case. No independent reviewer approval is claimed. diff --git a/docs/configuration-scan.md b/docs/configuration-scan.md index 550dfb1..99a11e0 100644 --- a/docs/configuration-scan.md +++ b/docs/configuration-scan.md @@ -23,11 +23,10 @@ backend, which must validate its own records. Memory and durable backends avoid copying ordinary payloads; external backends using default metadata accessors may read full entries. Configuration records are read and validated in full. -Configuration entries and configuration snapshots remain rejected by the live -runtime. Bound in-memory and durable stores can retain stable or joint metadata -snapshots for offline validation; `snapshot.Voters` must match the persisted -binding. For joint snapshots, the scanner also requires the baseline's old voter -set to match that binding. Tests cover durable restart and reject a baseline that -differs from the persisted binding. Live membership activation, commit rules, -snapshot installation across transitions, and recovery into an active Raft -configuration remain future work. +Configuration entries and joint snapshots remain rejected by the live runtime. +Stable metadata snapshots can be recovered and installed if their voter list +matches the persisted fixed membership. Bound in-memory and durable stores can +also retain joint metadata for offline validation; for joint snapshots, the +scanner requires the baseline's old voter set to match that binding. Live +membership activation, commit rules, and snapshot installation across membership +transitions remain future work. diff --git a/docs/covered-snapshot-identity.md b/docs/covered-snapshot-identity.md index 893c86a..9e838a7 100644 --- a/docs/covered-snapshot-identity.md +++ b/docs/covered-snapshot-identity.md @@ -6,18 +6,17 @@ recognized only the `MRFTS001` magic, then acknowledged the full snapshot as installed. It skipped checking the snapshot's embedded index, term and voter IDs against the RPC and the receiver's fixed voter set. -The fast path now waits for and parses the bounded fixed-format metadata prefix. -It acknowledges coverage only when the embedded boundary and sorted voter list -match. It also checks the encoded application length against the declared total -size. It does not read the application body or verify its CRC because it never -uses that image; the receiver's already committed prefix is the evidence for -coverage. Actual installations continue to validate the complete image, CRC, -metadata, voter set and application payload before replacing state. +The fast path waits for and parses the bounded metadata prefix for fixed v1 and +stable v3 snapshots. It acknowledges coverage only when the embedded boundary, +stable configuration identity (when present), and sorted voter list match. It +also checks the encoded application length against the declared total size. It +does not read the application body or verify its CRC because it never uses that +image; the receiver's already committed prefix is the evidence for coverage. +Actual installations validate the complete image, CRC, metadata, voter set and +application payload before replacing state. Deterministic regression cases cover mismatched embedded voters, a snapshot -boundary that disagrees with the request, and a valid metadata prefix split -across chunks. Rejected images do not change durable Raft or application state. -Joint and stable configuration snapshots remain unsupported by the live -runtime, and live voter changes remain disabled. Self-review completed; no -independent reviewer approval is claimed. Network-enabled Release and ASan/UBSan -CTest suites both pass all 20 targets. +boundary that disagrees with the request, invalid stable metadata, and valid +metadata prefixes split across chunks. Rejected images do not change durable Raft +or application state. Joint snapshots and live voter changes remain disabled. +Self-review completed; no independent reviewer approval is claimed. diff --git a/docs/durable-configuration-snapshots.md b/docs/durable-configuration-snapshots.md index 5caad3d..20c9361 100644 --- a/docs/durable-configuration-snapshots.md +++ b/docs/durable-configuration-snapshots.md @@ -12,15 +12,15 @@ with its existing voter binding; it does not reinterpret that binding from the metadata. The history scanner also verifies that a joint baseline's old voters match the bound set. -This is persistence for offline validation, not runtime configuration recovery. -`TRaft` still rejects configuration metadata snapshots on startup, the network -receiver rejects them, configuration log entries remain disabled, and compaction -continues to produce fixed-voter snapshots. A stable snapshot whose voter list -differs from the existing binding is rejected. Live transition semantics must -define how the binding changes before such snapshots can be used by Raft. +In alpha.45 these images were persistence for offline validation only. The +alpha.46 follow-up enables runtime recovery and transfer of MRFTS003 stable +snapshots when the voter list equals the node's existing fixed membership. +Compaction carries forward that stable configuration identity. MRFTS002 joint +snapshots and configuration log entries remain disabled in the live runtime. +Stable snapshots that claim another voter set are still rejected because +membership transition semantics are not implemented. Tests cover in-memory installation, durable joint and stable round-trips, bound voter mismatch rejection without rewriting storage, offline scan baseline checks, -and runtime startup rejection without rewriting the recovered image. Self-review -checked that recovery requires an existing binding and that no live Raft path -adopts the experimental metadata. No independent reviewer approval is claimed. +stable runtime restore/transfer/compaction, and continued joint snapshot rejection. +No independent reviewer approval is claimed. diff --git a/docs/joint-snapshots.md b/docs/joint-snapshots.md index 577f1c5..02d7368 100644 --- a/docs/joint-snapshots.md +++ b/docs/joint-snapshots.md @@ -21,11 +21,12 @@ snapshot store can export and load v2 images for testing or offline tooling. This is a format and offline persistence foundation, not live joint consensus or complete configuration history. A bound memory or durable store can install and -recover a joint snapshot when its persisted voter list matches the snapshot's old -voter set. Raft startup and RPC paths still reject joint snapshots, and compaction -still creates only fixed-voter snapshots. A covered snapshot is acknowledged only -after its prefix identifies the supported v1 format, including when the prefix -arrives in several chunks. Unsupported joint images are rejected without +recover a joint snapshot for offline validation when its persisted voter list +matches the snapshot's old voter set. Raft startup and RPC paths still reject +joint snapshots, and compaction only creates fixed-voter snapshots. A covered +snapshot is acknowledged only after its prefix identifies supported fixed v1 or +stable v3 metadata, including when the prefix arrives in several chunks. +Unsupported joint images are rejected without replacing application data or publishing a joint snapshot. A valid higher-term RPC header can still update durable hard state before its unsupported snapshot contents are recognized. These rules avoid falsely acknowledging a joint image diff --git a/docs/learners.md b/docs/learners.md index 2f59618..222932a 100644 --- a/docs/learners.md +++ b/docs/learners.md @@ -92,7 +92,8 @@ Never delete voter state or edit votes/terms to work around this validation. The new durable format `MRFTD003` retains the 20-byte header and stores: CurrentTerm[8], VotedFor[4], retained-log count[8], voter count[4], sorted voter -IDs[4 each], snapshot length[8], optional MRFTS001 image, then retained log records. +IDs[4 each], snapshot length[8], optional MRFTS001, MRFTS002, or MRFTS003 image, +then retained log records. An absent snapshot uses length zero. The CRC covers version magic plus payload. Recovery bounds and validates the voter list and requires any snapshot to name the same voters. Both v1 and v2 remain readable; the first binding writes v3 using diff --git a/docs/online-learner-network.md b/docs/online-learner-network.md index 2fdadaa..dd00ea0 100644 --- a/docs/online-learner-network.md +++ b/docs/online-learner-network.md @@ -26,8 +26,8 @@ lifecycle. The KV CLI still discovers peers at startup. Staging is leader-local and is lost on restart or failover; the new leader and the application must reconcile it. This adds network transport support for non-voting catch-up only. Voter -promotion/removal, replicated joint consensus, membership-aware snapshot -installation and durable configuration recovery remain unsupported. +promotion/removal, replicated joint consensus and snapshot installation across +membership transitions remain unsupported. The deterministic server test checks target registration, voter/quorum invariance, removal and rollback after a nonleader rejection. The existing diff --git a/docs/stable-configuration-snapshot-runtime.md b/docs/stable-configuration-snapshot-runtime.md new file mode 100644 index 0000000..5d0a200 --- /dev/null +++ b/docs/stable-configuration-snapshot-runtime.md @@ -0,0 +1,24 @@ +# Stable configuration snapshot runtime support: v0.1.0-alpha.46 + +The runtime now accepts MRFTS003 snapshots for the node's already configured +fixed voter set. Startup requires the snapshot voter list, supplied Raft peers, +and persisted voter binding to agree. Joint MRFTS002 snapshots remain rejected. + +With that same-set check, the existing snapshot lifecycle can restore a durable +snapshot into the application, receive and persist it over InstallSnapshot, and +compact later application progress. Compaction carries the prior stable +configuration entry index and term into the new image. A follower whose committed +prefix already covers a stable snapshot validates the RPC boundary, voter list, +configuration identity and encoded data length before acknowledging it. + +This does not implement membership changes. Configuration log entries remain +disabled, the voter binding cannot change, and a stable snapshot claiming a +different voter list is rejected. Joint consensus, transition rollback and +snapshot installation across a membership transition remain open. Upgrade all +voters before sending MRFTS003 images because older binaries may acknowledge an +already-covered unknown snapshot format. + +Tests cover durable application restore, identity preservation through +compaction and restart, chunked InstallSnapshot, covered-prefix validation of +stable metadata, and rejection of joint snapshots. Self-review completed; no +independent reviewer approval is claimed. diff --git a/docs/stable-configuration-snapshots.md b/docs/stable-configuration-snapshots.md index ff01c7f..c44af17 100644 --- a/docs/stable-configuration-snapshots.md +++ b/docs/stable-configuration-snapshots.md @@ -1,4 +1,4 @@ -# Offline stable configuration snapshot metadata +# Stable configuration snapshot metadata The experimental MRFTS003 snapshot format preserves a stable voter set and the index and term of the configuration entry that selected it. This fills a gap in @@ -18,12 +18,15 @@ trusted snapshot and commit boundary independently. MRFTS003 cannot assert that there was no earlier configuration entry; an explicit trusted baseline remains necessary for such histories. -The live runtime rejects MRFTS003, and the network receiver rejects it even at an -already covered boundary. Bound memory and durable stores can retain and recover -the image for offline validation only when its voter list matches the persisted -binding. Storage does not activate a changed voter set from snapshot metadata. +The runtime accepts MRFTS003 only when its voter list matches the node's existing +fixed voter binding. It can recover the image at startup, restore the application, +install it from a peer, and preserve its configuration identity through later +compaction. The covered-snapshot path checks the stable configuration identity +and voter list before acknowledging an already-covered boundary. These paths do +not activate a changed voter set or enable joint consensus. + Older binaries may falsely acknowledge covered unknown formats; do not send these -images to a mixed-version cluster. Live configuration activation, rollback, -commit rules, joint snapshot installation and membership recovery remain open. -This experimental format may change before those semantics are implemented. +images to a mixed-version cluster. Joint snapshot installation, live +configuration activation, rollback, commit rules and membership recovery remain +open. This experimental format may change before those semantics are implemented. Rebuild dependents because the in-memory snapshot structure has changed. diff --git a/miniraft/raft.cpp b/miniraft/raft.cpp index 50c42b8..2183a3c 100644 --- a/miniraft/raft.cpp +++ b/miniraft/raft.cpp @@ -37,16 +37,20 @@ bool ReadSnapshotInteger(std::string_view bytes, size_t offset, unsigned width, return true; } -TCoveredSnapshotProbe ProbeCoveredFixedSnapshot(std::string_view prefix, uint64_t totalBytes, +TCoveredSnapshotProbe ProbeCoveredSnapshot(std::string_view prefix, uint64_t totalBytes, uint64_t index, uint64_t term, const std::vector& voters) { if (prefix.size() < 8) return {false, false, 8}; - if (prefix.substr(0, 8) != "MRFTS001") return {true, false, 0}; + const auto magic = prefix.substr(0, 8); + const bool stable = magic == "MRFTS003"; + if (!stable && magic != "MRFTS001") return {true, false, 0}; // A covered snapshot is not installed, but its identity must still agree // with the RPC and the receiver's fixed membership before it can count as - // replication progress. The application payload and CRC are unnecessary - // because the receiver already has this committed log prefix. - if (prefix.size() < 36) return {false, false, 36}; + // replication progress. Stable metadata identity is checked as well. The + // application payload and CRC are unnecessary because the receiver already + // has this committed log prefix. + const size_t fixedHeaderBytes = stable ? 52 : 36; + if (prefix.size() < fixedHeaderBytes) return {false, false, fixedHeaderBytes}; uint64_t embeddedIndex = 0, embeddedTerm = 0, voterCount = 0, dataBytes = 0; if (!ReadSnapshotInteger(prefix, 8, 8, embeddedIndex) || !ReadSnapshotInteger(prefix, 16, 8, embeddedTerm) || @@ -54,15 +58,23 @@ TCoveredSnapshotProbe ProbeCoveredFixedSnapshot(std::string_view prefix, uint64_ !ReadSnapshotInteger(prefix, 28, 8, dataBytes) || embeddedIndex != index || embeddedTerm != term || voterCount != voters.size()) return {true, false, 0}; + if (stable) { + uint64_t configurationIndex = 0, configurationTerm = 0; + if (!ReadSnapshotInteger(prefix, 36, 8, configurationIndex) || + !ReadSnapshotInteger(prefix, 44, 8, configurationTerm) || !configurationIndex || + configurationIndex > embeddedIndex || !configurationTerm || configurationTerm > embeddedTerm || + (configurationIndex == embeddedIndex && configurationTerm != embeddedTerm)) + return {true, false, 0}; + } const uint64_t voterBytes = voterCount * 4; - const uint64_t headerBytes = 36 + voterBytes; + const uint64_t headerBytes = fixedHeaderBytes + voterBytes; if (headerBytes + 4 > totalBytes || dataBytes != totalBytes - headerBytes - 4) return {true, false, 0}; if (headerBytes > prefix.size()) return {false, false, size_t(headerBytes)}; for (size_t i = 0; i < voters.size(); ++i) { uint64_t voter = 0; - if (!ReadSnapshotInteger(prefix, 36 + i * 4, 4, voter) || voter != voters[i]) + if (!ReadSnapshotInteger(prefix, fixedHeaderBytes + i * 4, 4, voter) || voter != voters[i]) return {true, false, 0}; } return {true, true, size_t(headerBytes)}; @@ -251,8 +263,10 @@ TRaft::TRaft(std::shared_ptr state, int node, const TNodeDict& nodes, st const auto snapshot = State->GetSnapshot(); if (snapshot) { ValidateSnapshot(*snapshot); - if (snapshot->HasConfigurationMetadata()) - throw std::invalid_argument("Configuration snapshot recovery is unsupported"); + if (snapshot->IsJoint()) + throw std::invalid_argument("Joint snapshot recovery is unsupported"); + if (snapshot->StableConfiguration && bound != Voters) + throw std::invalid_argument("Stable configuration snapshot requires an existing voter binding"); if (snapshot->Index != State->SnapshotIndex || snapshot->Term != State->SnapshotTerm || snapshot->Term > State->CurrentTerm || snapshot->Voters != Voters || State->LastLogTerm < State->SnapshotTerm || State->LastLogTerm > State->CurrentTerm || @@ -958,7 +972,7 @@ void TRaft::OnInstallSnapshot(ITimeSource::Time now, TMessageHolderData, count); IncomingSnapshot->LastProgress = now; - const auto probe = ProbeCoveredFixedSnapshot(prefix, request->TotalBytes, + const auto probe = ProbeCoveredSnapshot(prefix, request->TotalBytes, request->SnapshotIndex, request->SnapshotTerm, GetVoterIds()); if (!probe.Complete) { reply(true, false, prefix.size()); return; } IncomingSnapshot.reset(); @@ -986,14 +1000,15 @@ void TRaft::OnInstallSnapshot(ITimeSource::Time now, TMessageHolderData, count); IncomingSnapshot->LastProgress = now; - if (bytes.size() >= 8 && std::string_view(bytes).substr(0, 8) != "MRFTS001") { + if (bytes.size() >= 8 && std::string_view(bytes).substr(0, 8) != "MRFTS001" && + std::string_view(bytes).substr(0, 8) != "MRFTS003") { IncomingSnapshot.reset(); reply(false, false, 0); return; } if (bytes.size() < request->TotalBytes) { reply(true, false, bytes.size()); return; } TSnapshot snapshot; try { snapshot = DecodeSnapshot(bytes); - if (snapshot.HasConfigurationMetadata() || snapshot.Index != request->SnapshotIndex || snapshot.Term != request->SnapshotTerm || snapshot.Voters != GetVoterIds()) + if (snapshot.IsJoint() || snapshot.Index != request->SnapshotIndex || snapshot.Term != request->SnapshotTerm || snapshot.Voters != GetVoterIds()) throw TMessageError("Snapshot metadata mismatch"); application->ValidateSnapshot(snapshot.Data, snapshot.Index); } catch (const std::exception&) { diff --git a/miniraft/snapshot.cpp b/miniraft/snapshot.cpp index dd9be94..f58b4b0 100644 --- a/miniraft/snapshot.cpp +++ b/miniraft/snapshot.cpp @@ -43,10 +43,10 @@ void Validate(const TSnapshot& snapshot) { } void CheckLocalLog(const TSnapshot& snapshot, const TRaft& raft) { Validate(snapshot); - if (snapshot.HasConfigurationMetadata() || snapshot.Voters != raft.GetVoterIds() || snapshot.Index > raft.GetLastIndex() || + if (snapshot.IsJoint() || snapshot.Voters != raft.GetVoterIds() || snapshot.Index > raft.GetLastIndex() || snapshot.Term > raft.GetState()->CurrentTerm || raft.GetState()->LogTerm(snapshot.Index) != snapshot.Term) - throw std::runtime_error("Snapshot does not match retained log and fixed configuration"); + throw std::runtime_error("Snapshot does not match retained log and active configuration"); } #ifdef __linux__ struct TFd { @@ -129,6 +129,11 @@ TSnapshot CaptureSnapshot(const TRaft& raft, const IRsm& rsm) { if (!index || index > raft.GetVolatileState()->CommitIndex || index > raft.GetLastIndex()) throw std::runtime_error("Snapshot requires applied committed progress"); TSnapshot snapshot{index, raft.GetState()->LogTerm(index), raft.GetVoterIds(), {}}; + if (const auto previous = raft.GetState()->GetSnapshot(); previous && previous->StableConfiguration) { + snapshot.StableConfiguration = true; + snapshot.ConfigurationIndex = previous->ConfigurationIndex; + snapshot.ConfigurationTerm = previous->ConfigurationTerm; + } CheckLocalLog(snapshot, raft); snapshot.Data = rsm.SaveSnapshot(); Validate(snapshot); diff --git a/miniraft/snapshot.h b/miniraft/snapshot.h index b5baf32..b64e239 100644 --- a/miniraft/snapshot.h +++ b/miniraft/snapshot.h @@ -22,7 +22,8 @@ struct TSnapshot { // Runtime installation is disabled until membership recovery is implemented. std::vector JointVoters; uint64_t ConfigurationIndex = 0, ConfigurationTerm = 0; - // Experimental stable-configuration identity; fixed runtime rejects it. + // Experimental stable-configuration identity; runtime requires the same + // persisted fixed voter binding. bool StableConfiguration = false; bool IsJoint() const { return !JointVoters.empty() || (!StableConfiguration && (ConfigurationIndex || ConfigurationTerm)); } bool HasConfigurationMetadata() const { return StableConfiguration || IsJoint(); } diff --git a/test/test_compaction.cpp b/test/test_compaction.cpp index 13af022..895411e 100644 --- a/test/test_compaction.cpp +++ b/test/test_compaction.cpp @@ -426,12 +426,30 @@ void test_joint_snapshot_is_rejected_even_if_boundary_is_already_covered(void**) assert_false(replies.back()->Success); assert_false(replies.back()->Installed); auto stable = fixed; stable.StableConfiguration = true; stable.ConfigurationIndex = 4; stable.ConfigurationTerm = 1; auto stableBytes = EncodeSnapshot(stable); - raft->Process({}, Chunk(stable, stableBytes, 0, 4)); - assert_true(replies.back()->Success); assert_false(replies.back()->Installed); - raft->Process({}, Chunk(stable, stableBytes, 4, 4)); - assert_false(replies.back()->Success); assert_false(replies.back()->Installed); - stable.Index = 6; stableBytes = EncodeSnapshot(stable); raft->Process({}, Chunk(stable, stableBytes, 0, stableBytes.size())); + assert_true(replies.back()->Success); assert_true(replies.back()->Installed); + assert_true(static_cast(*state) == before); assert_int_equal(app->Loads, 1); + + // The stable-format covered probe validates the config identity and voter + // set from a split metadata prefix before crediting the already-held index. + const size_t stableMetadataBytes = 52 + 4 * stable.Voters.size(); + for (size_t offset = 0; offset < stableMetadataBytes; offset += 4) { + const auto count = std::min(4, stableMetadataBytes - offset); + raft->Process({}, Chunk(stable, stableBytes, offset, static_cast(count))); + if (offset + count < stableMetadataBytes) { + assert_true(replies.back()->Success); assert_false(replies.back()->Installed); + assert_int_equal(replies.back()->NextOffset, offset + count); + } + } + assert_true(replies.back()->Success); assert_true(replies.back()->Installed); + + auto stableWrongVoters = stable; stableWrongVoters.Voters = {1, 3}; + auto stableWrongVotersBytes = EncodeSnapshot(stableWrongVoters); + raft->Process({}, Chunk(stableWrongVoters, stableWrongVotersBytes, 0, stableWrongVotersBytes.size())); + assert_false(replies.back()->Success); assert_false(replies.back()->Installed); + auto invalidStableIdentity = stableBytes; + for (size_t i = 36; i < 44; ++i) invalidStableIdentity[i] = 0; + raft->Process({}, Chunk(stable, invalidStableIdentity, 0, invalidStableIdentity.size())); assert_false(replies.back()->Success); assert_false(replies.back()->Installed); joint.Index = 6; const auto newer = EncodeSnapshot(joint); @@ -440,6 +458,33 @@ void test_joint_snapshot_is_rejected_even_if_boundary_is_already_covered(void**) assert_true(static_cast(*state) == before); assert_int_equal(app->Loads, 1); assert_true(Read(*app, "a") == "old"); } +void test_stable_configuration_snapshot_installs_and_recovers_with_fixed_membership(void**) { + TKv source; source.Write(source.Prepare(Command("a", "stable snapshot")), 5); + TSnapshot snapshot{5, 1, {1, 2}, source.SaveSnapshot()}; + snapshot.StableConfiguration = true; + snapshot.ConfigurationIndex = 4; snapshot.ConfigurationTerm = 1; + const auto bytes = EncodeSnapshot(snapshot); + std::vector> replies; + auto peer = std::make_shared([&](auto message) { replies.push_back(message.template Cast()); }); + auto state = std::make_shared(); + { + auto raft = std::make_shared(state, 2, TNodeDict{{1, peer}}); + auto app = std::make_shared(); TRequestProcessor processor(raft, app, {}); + raft->Process({}, Chunk(snapshot, bytes, 0, 8)); + assert_true(replies.back()->Success); assert_false(replies.back()->Installed); + assert_int_equal(replies.back()->NextOffset, 8); assert_int_equal(app->Loads, 0); + raft->Process({}, Chunk(snapshot, bytes, 8, static_cast(bytes.size() - 8))); + assert_true(replies.back()->Success); assert_true(replies.back()->Installed); + assert_int_equal(replies.back()->NextOffset, bytes.size()); + assert_true(*state->GetSnapshot() == snapshot); + assert_int_equal(app->Loads, 1); assert_true(Read(*app, "a") == "stable snapshot"); + } + auto restarted = std::make_shared(state, 2, TNodeDict{{1, peer}}); + auto recovered = std::make_shared(); TRequestProcessor recovery(restarted, recovered, {}); + assert_int_equal(recovered->LastAppliedIndex, snapshot.Index); + assert_true(Read(*recovered, "a") == "stable snapshot"); + assert_true(restarted->GetState()->GetSnapshot()->StableConfiguration); +} void test_idle_partial_snapshot_expires_on_learner_despite_heartbeats(void**) { auto state = std::make_shared(); state->BindVoters({1}); @@ -848,6 +893,7 @@ int main() { cmocka_unit_test(test_snapshot_is_rechecked_when_commit_advances_mid_transfer), cmocka_unit_test(test_memory_snapshot_storage_rejects_term_regression), cmocka_unit_test(test_joint_snapshot_is_rejected_even_if_boundary_is_already_covered), + cmocka_unit_test(test_stable_configuration_snapshot_installs_and_recovers_with_fixed_membership), cmocka_unit_test(test_idle_partial_snapshot_expires_on_learner_despite_heartbeats), cmocka_unit_test(test_remote_snapshot_retains_suffix_without_applying_it_before_commit), cmocka_unit_test(test_snapshot_boundary_index_exhaustion_does_not_append_wrapped_noop), diff --git a/test/test_snapshots.cpp b/test/test_snapshots.cpp index a1e4533..129f67c 100644 --- a/test/test_snapshots.cpp +++ b/test/test_snapshots.cpp @@ -130,6 +130,19 @@ void test_stable_configuration_snapshot_codec_and_runtime_guards(void**) { memory->Snapshot = std::make_shared(snapshot); Reject([&] { TRaft unsupported(memory, 1, {{2, std::make_shared()}, {3, std::make_shared()}}); }); assert_true(memory->GetBoundVoters().empty()); + auto bound = std::make_shared(); + bound->BindVoters(snapshot.Voters); + bound->InstallSnapshot(snapshot); + TRaft recovered(bound, 1, {{2, std::make_shared()}, {3, std::make_shared()}}); + assert_int_equal(recovered.GetVolatileState()->CommitIndex, snapshot.Index); + auto changedBinding = std::make_shared(); + changedBinding->BindVoters({1, 2, 4}); + changedBinding->CurrentTerm = snapshot.Term; + changedBinding->Snapshot = std::make_shared(snapshot); + changedBinding->SnapshotIndex = changedBinding->LastLogIndex = snapshot.Index; + changedBinding->SnapshotTerm = changedBinding->LastLogTerm = snapshot.Term; + Reject([&] { TRaft invalid(changedBinding, 1, {{2, std::make_shared()}, {3, std::make_shared()}}); }); + assert_true(changedBinding->GetBoundVoters() == std::vector({1, 2, 4})); #ifdef __linux__ TFiles files; { TSnapshotStore store(files.Name, 1); store.Save(snapshot); assert_true(store.Load().value() == snapshot); } @@ -352,27 +365,46 @@ void test_startup_restores_stale_application_from_durable_snapshot_boundary(void assert_int_equal(raft->GetVolatileState()->CommitIndex, 2); } #ifdef __linux__ -void test_stable_configuration_snapshot_durable_roundtrip_but_runtime_rejects(void**) { - TFiles files; const auto path = files.Name + ".durable.1"; - auto snapshot = Sample(); snapshot.StableConfiguration = true; +void test_stable_configuration_snapshot_durable_runtime_recovery_and_compaction(void**) { + TFiles files; + TKv source; Apply(source, "a", "one", 1); Apply(source, "b", "two", 2); + TSnapshot snapshot{2, 1, {1, 2, 3}, source.SaveSnapshot()}; + snapshot.StableConfiguration = true; snapshot.ConfigurationIndex = 1; snapshot.ConfigurationTerm = 1; { TDurableState state(files.Name, 1); state.BindVoters(snapshot.Voters); state.InstallSnapshot(snapshot); } - std::string bytes; + TSnapshot compacted; { - TDurableState state(files.Name, 1); - assert_true(state.GetBoundVoters() == snapshot.Voters); - assert_true(*state.GetSnapshot() == snapshot); - std::ifstream input(path, std::ios::binary); - bytes.assign(std::istreambuf_iterator(input), {}); + auto state = std::make_shared(files.Name, 1); + assert_true(state->GetBoundVoters() == snapshot.Voters); + assert_true(*state->GetSnapshot() == snapshot); + auto raft = std::make_shared(state, 1, + TNodeDict{{2, std::make_shared()}, {3, std::make_shared()}}); + auto app = std::make_shared(); + TRequestProcessor processor(raft, app, {}); + assert_int_equal(app->LastAppliedIndex, 2); + assert_true(Read(*app, "a") == "one"); assert_true(Read(*app, "b") == "two"); + + auto entry = app->Prepare(Command("tail", "after recovery")); entry->Term = 1; + state->Append(entry); app->Write(entry, 3); + const_cast(raft->GetVolatileState())->CommitIndex = 3; + compacted = raft->Compact(); + assert_true(compacted.StableConfiguration); + assert_int_equal(compacted.ConfigurationIndex, snapshot.ConfigurationIndex); + assert_int_equal(compacted.ConfigurationTerm, snapshot.ConfigurationTerm); + assert_int_equal(compacted.Index, 3); } - Reject([&] { TRaft raft(std::make_shared(files.Name, 1), 1, - {{2, std::make_shared()}, {3, std::make_shared()}}); }); - std::ifstream after(path, std::ios::binary); - assert_true(std::string(std::istreambuf_iterator(after), {}) == bytes); + auto state = std::make_shared(files.Name, 1); + assert_true(*state->GetSnapshot() == compacted); + auto raft = std::make_shared(state, 1, + TNodeDict{{2, std::make_shared()}, {3, std::make_shared()}}); + auto app = std::make_shared(); TRequestProcessor processor(raft, app, {}); + assert_int_equal(app->LastAppliedIndex, compacted.Index); + assert_true(Read(*app, "a") == "one"); assert_true(Read(*app, "b") == "two"); + assert_true(Read(*app, "tail") == "after recovery"); } void test_unbound_durable_configuration_snapshot_recovery_fails_closed(void**) { TFiles files; const auto path = files.Name + ".durable.1"; @@ -555,7 +587,7 @@ int main() { cmocka_unit_test(test_restore_rejects_mismatched_log_configuration_and_used_application), cmocka_unit_test(test_startup_restores_stale_application_from_durable_snapshot_boundary), #ifdef __linux__ - cmocka_unit_test(test_stable_configuration_snapshot_durable_roundtrip_but_runtime_rejects), + cmocka_unit_test(test_stable_configuration_snapshot_durable_runtime_recovery_and_compaction), cmocka_unit_test(test_unbound_durable_configuration_snapshot_recovery_fails_closed), cmocka_unit_test(test_joint_snapshot_export_and_durable_roundtrip_but_runtime_rejects), cmocka_unit_test(test_snapshot_store_rejects_oversized_files_and_temp_symlinks), From 05b33435bb60c45551267e81e74ebe83df811a6c Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Sat, 14 Nov 2026 17:29:54 -0500 Subject: [PATCH 47/66] Preserve stable snapshot identity across refreshes (#47) --- README.md | 2 + docs/stable-configuration-snapshot-runtime.md | 14 ++++-- docs/stable-snapshot-metadata-continuity.md | 17 +++++++ miniraft/raft.cpp | 26 ++++++++-- test/test_compaction.cpp | 47 +++++++++++++++++-- 5 files changed, 97 insertions(+), 9 deletions(-) create mode 100644 docs/stable-snapshot-metadata-continuity.md diff --git a/README.md b/README.md index 20dc66f..82dd952 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,8 @@ The [durable configuration snapshot follow-up](docs/durable-configuration-snapsh documents its voter-binding requirement and recovery boundary. The [stable snapshot runtime follow-up](docs/stable-configuration-snapshot-runtime.md) carries that identity through restore, InstallSnapshot, and later compaction. +The [metadata continuity follow-up](docs/stable-snapshot-metadata-continuity.md) +preserves the known identity across legacy-format refreshes and rejects conflicts. The [snapshot term validation fix](docs/snapshot-committed-terms.md) rejects snapshot boundaries that contradict the follower's committed prefix. The [message buffer follow-up](docs/message-extents.md) checks allocated buffer diff --git a/docs/stable-configuration-snapshot-runtime.md b/docs/stable-configuration-snapshot-runtime.md index 5d0a200..7b64b34 100644 --- a/docs/stable-configuration-snapshot-runtime.md +++ b/docs/stable-configuration-snapshot-runtime.md @@ -11,6 +11,13 @@ configuration entry index and term into the new image. A follower whose committe prefix already covers a stable snapshot validates the RPC boundary, voter list, configuration identity and encoded data length before acknowledging it. +Once a node has a stable configuration identity, installing a newer MRFTS001 +snapshot carries that identity into the stored MRFTS003 image. An incoming +MRFTS003 image with a different identity is rejected, including on the +already-covered fast path. This prevents a legacy-format refresh from silently +erasing configuration history and prevents a peer from replacing it with a +conflicting baseline. + This does not implement membership changes. Configuration log entries remain disabled, the voter binding cannot change, and a stable snapshot claiming a different voter list is rejected. Joint consensus, transition rollback and @@ -19,6 +26,7 @@ voters before sending MRFTS003 images because older binaries may acknowledge an already-covered unknown snapshot format. Tests cover durable application restore, identity preservation through -compaction and restart, chunked InstallSnapshot, covered-prefix validation of -stable metadata, and rejection of joint snapshots. Self-review completed; no -independent reviewer approval is claimed. +compaction, restart, and legacy-format refresh, chunked InstallSnapshot, +covered-prefix validation of stable metadata, and rejection of conflicting +identities and joint snapshots. Self-review completed; no independent reviewer +approval is claimed. diff --git a/docs/stable-snapshot-metadata-continuity.md b/docs/stable-snapshot-metadata-continuity.md new file mode 100644 index 0000000..7030cd5 --- /dev/null +++ b/docs/stable-snapshot-metadata-continuity.md @@ -0,0 +1,17 @@ +# Stable snapshot metadata continuity: v0.1.0-alpha.47 + +A node that already stores an MRFTS003 snapshot has a stable configuration +identity tied to its persisted fixed voter set. A newer incoming MRFTS001 image +does not include that identity. The receiver now carries its trusted identity +forward before persisting the new image, so legacy-format transfer cannot erase +the baseline needed by recovery and later compaction. + +Incoming MRFTS003 snapshots must retain the known configuration index and term. +The receiver rejects conflicting identities both when installing a newer image +and when acknowledging an image whose boundary is already covered. These checks +do not enable membership changes: the node still requires the same bound voters, +and joint snapshots and configuration log entries remain unsupported. + +The focused compaction test covers a covered-boundary conflict, a legacy-format +refresh followed by restart, and a conflicting newer stable snapshot. The change +was self-reviewed; no independent reviewer approval is claimed. diff --git a/miniraft/raft.cpp b/miniraft/raft.cpp index 2183a3c..0f91332 100644 --- a/miniraft/raft.cpp +++ b/miniraft/raft.cpp @@ -38,7 +38,8 @@ bool ReadSnapshotInteger(std::string_view bytes, size_t offset, unsigned width, } TCoveredSnapshotProbe ProbeCoveredSnapshot(std::string_view prefix, uint64_t totalBytes, - uint64_t index, uint64_t term, const std::vector& voters) { + uint64_t index, uint64_t term, const std::vector& voters, + const TSnapshot* knownSnapshot) { if (prefix.size() < 8) return {false, false, 8}; const auto magic = prefix.substr(0, 8); const bool stable = magic == "MRFTS003"; @@ -63,7 +64,10 @@ TCoveredSnapshotProbe ProbeCoveredSnapshot(std::string_view prefix, uint64_t tot if (!ReadSnapshotInteger(prefix, 36, 8, configurationIndex) || !ReadSnapshotInteger(prefix, 44, 8, configurationTerm) || !configurationIndex || configurationIndex > embeddedIndex || !configurationTerm || configurationTerm > embeddedTerm || - (configurationIndex == embeddedIndex && configurationTerm != embeddedTerm)) + (configurationIndex == embeddedIndex && configurationTerm != embeddedTerm) || + (knownSnapshot && knownSnapshot->StableConfiguration && + (configurationIndex != knownSnapshot->ConfigurationIndex || + configurationTerm != knownSnapshot->ConfigurationTerm))) return {true, false, 0}; } @@ -972,8 +976,9 @@ void TRaft::OnInstallSnapshot(ITimeSource::Time now, TMessageHolderData, count); IncomingSnapshot->LastProgress = now; + const auto knownSnapshot = State->GetSnapshot(); const auto probe = ProbeCoveredSnapshot(prefix, request->TotalBytes, - request->SnapshotIndex, request->SnapshotTerm, GetVoterIds()); + request->SnapshotIndex, request->SnapshotTerm, GetVoterIds(), knownSnapshot.get()); if (!probe.Complete) { reply(true, false, prefix.size()); return; } IncomingSnapshot.reset(); reply(probe.Supported, probe.Supported, probe.Supported ? request->TotalBytes : 0); @@ -1010,6 +1015,21 @@ void TRaft::OnInstallSnapshot(ITimeSource::Time now, TMessageHolderSnapshotIndex || snapshot.Term != request->SnapshotTerm || snapshot.Voters != GetVoterIds()) throw TMessageError("Snapshot metadata mismatch"); + const auto knownSnapshot = State->GetSnapshot(); + if (knownSnapshot && knownSnapshot->StableConfiguration) { + if (snapshot.StableConfiguration && + (snapshot.ConfigurationIndex != knownSnapshot->ConfigurationIndex || + snapshot.ConfigurationTerm != knownSnapshot->ConfigurationTerm)) + throw TMessageError("Stable snapshot configuration identity changed"); + if (!snapshot.StableConfiguration) { + // A legacy-format refresh has no configuration identity. Keep + // the receiver's trusted identity when storing the new image. + snapshot.StableConfiguration = true; + snapshot.ConfigurationIndex = knownSnapshot->ConfigurationIndex; + snapshot.ConfigurationTerm = knownSnapshot->ConfigurationTerm; + } + } + ValidateSnapshot(snapshot); application->ValidateSnapshot(snapshot.Data, snapshot.Index); } catch (const std::exception&) { IncomingSnapshot.reset(); reply(false, false, 0); return; diff --git a/test/test_compaction.cpp b/test/test_compaction.cpp index 895411e..a9cf635 100644 --- a/test/test_compaction.cpp +++ b/test/test_compaction.cpp @@ -478,12 +478,53 @@ void test_stable_configuration_snapshot_installs_and_recovers_with_fixed_members assert_int_equal(replies.back()->NextOffset, bytes.size()); assert_true(*state->GetSnapshot() == snapshot); assert_int_equal(app->Loads, 1); assert_true(Read(*app, "a") == "stable snapshot"); + + // A covered stable image with a different configuration identity must + // not be acknowledged as progress. + auto conflictingCovered = snapshot; + conflictingCovered.ConfigurationIndex = 3; + const auto conflictingCoveredBytes = EncodeSnapshot(conflictingCovered); + raft->Process({}, Chunk(conflictingCovered, conflictingCoveredBytes, 0, + static_cast(conflictingCoveredBytes.size()))); + assert_false(replies.back()->Success); assert_false(replies.back()->Installed); + assert_true(*state->GetSnapshot() == snapshot); assert_int_equal(app->Loads, 1); + + // A newer legacy image can be installed, but it must retain the + // receiver's already trusted stable configuration identity. + source.Write(source.Prepare(Command("a", "legacy refresh")), 6); + TSnapshot legacy{6, 1, {1, 2}, source.SaveSnapshot()}; + const auto legacyBytes = EncodeSnapshot(legacy); + raft->Process({}, Chunk(legacy, legacyBytes, 0, static_cast(legacyBytes.size()))); + assert_true(replies.back()->Success); assert_true(replies.back()->Installed); + auto preserved = legacy; + preserved.StableConfiguration = true; + preserved.ConfigurationIndex = snapshot.ConfigurationIndex; + preserved.ConfigurationTerm = snapshot.ConfigurationTerm; + assert_true(*state->GetSnapshot() == preserved); + assert_int_equal(app->Loads, 2); assert_true(Read(*app, "a") == "legacy refresh"); + + // Conflicting metadata at a newer boundary is rejected before either + // durable Raft state or the application can be replaced. + source.Write(source.Prepare(Command("a", "conflicting refresh")), 7); + TSnapshot conflicting{7, 1, {1, 2}, source.SaveSnapshot()}; + conflicting.StableConfiguration = true; + conflicting.ConfigurationIndex = 3; conflicting.ConfigurationTerm = 1; + const auto conflictingBytes = EncodeSnapshot(conflicting); + raft->Process({}, Chunk(conflicting, conflictingBytes, 0, + static_cast(conflictingBytes.size()))); + assert_false(replies.back()->Success); assert_false(replies.back()->Installed); + assert_true(*state->GetSnapshot() == preserved); + assert_int_equal(app->Loads, 2); assert_int_equal(app->LastAppliedIndex, 6); + assert_true(Read(*app, "a") == "legacy refresh"); } auto restarted = std::make_shared(state, 2, TNodeDict{{1, peer}}); auto recovered = std::make_shared(); TRequestProcessor recovery(restarted, recovered, {}); - assert_int_equal(recovered->LastAppliedIndex, snapshot.Index); - assert_true(Read(*recovered, "a") == "stable snapshot"); - assert_true(restarted->GetState()->GetSnapshot()->StableConfiguration); + assert_int_equal(recovered->LastAppliedIndex, 6); + assert_true(Read(*recovered, "a") == "legacy refresh"); + const auto recoveredSnapshot = restarted->GetState()->GetSnapshot(); + assert_true(recoveredSnapshot->StableConfiguration); + assert_int_equal(recoveredSnapshot->ConfigurationIndex, 4); + assert_int_equal(recoveredSnapshot->ConfigurationTerm, 1); } void test_idle_partial_snapshot_expires_on_learner_despite_heartbeats(void**) { auto state = std::make_shared(); From 004591aadaacd69a7ac3602de20d8556ffbee604 Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Sat, 14 Nov 2026 17:43:00 -0500 Subject: [PATCH 48/66] Preflight unsupported snapshot formats before term update (#48) --- README.md | 2 ++ docs/joint-snapshots.md | 13 +++++++---- docs/unsupported-snapshot-term-preflight.md | 26 +++++++++++++++++++++ miniraft/raft.cpp | 14 ++++++++++- test/test_compaction.cpp | 21 +++++++++++++++++ 5 files changed, 70 insertions(+), 6 deletions(-) create mode 100644 docs/unsupported-snapshot-term-preflight.md diff --git a/README.md b/README.md index 82dd952..38394d9 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,8 @@ arithmetic for future membership changes; live voter reconfiguration remains dis The [joint snapshot metadata foundation](docs/joint-snapshots.md) adds a versioned representation for a joint configuration at a snapshot boundary; runtime adoption and membership changes remain disabled. +The [snapshot term preflight follow-up](docs/unsupported-snapshot-term-preflight.md) +rejects unsupported snapshot formats before a higher RPC term changes hard state. The [configuration entry foundation](docs/configuration-entries.md) defines an experimental entry codec and rejects that entry kind in the fixed-membership runtime. The [configuration history validator](docs/configuration-history.md) checks an diff --git a/docs/joint-snapshots.md b/docs/joint-snapshots.md index 02d7368..510074c 100644 --- a/docs/joint-snapshots.md +++ b/docs/joint-snapshots.md @@ -2,6 +2,8 @@ The [alpha.27 follow-up](configuration-history.md) requires the configuration term to equal the snapshot term when both refer to the same index. +The [alpha.48 follow-up](unsupported-snapshot-term-preflight.md) checks the +initial snapshot format before accepting a higher RPC term. This step adds an experimental snapshot representation for a joint configuration effective at the included snapshot boundary. MRFTS002 records the old and new @@ -26,11 +28,12 @@ matches the snapshot's old voter set. Raft startup and RPC paths still reject joint snapshots, and compaction only creates fixed-voter snapshots. A covered snapshot is acknowledged only after its prefix identifies supported fixed v1 or stable v3 metadata, including when the prefix arrives in several chunks. -Unsupported joint images are rejected without -replacing application data or publishing a joint snapshot. A valid higher-term -RPC header can still update durable hard state before its unsupported snapshot -contents are recognized. These rules avoid falsely acknowledging a joint image -merely because its index is already committed locally. +Unsupported joint images are rejected without replacing application data or +publishing a joint snapshot. Since alpha.48, a higher-term InstallSnapshot can +advance hard state only after its initial chunk identifies a supported MRFTS001 +or MRFTS003 format. A short, unknown, joint-format, or continuation chunk cannot +change the term before rejection. These rules avoid falsely acknowledging a +joint image merely because its index is already committed locally. Older binaries do not understand MRFTS002 and may acknowledge an already-covered image without checking its version. Do not send or activate joint snapshots in a diff --git a/docs/unsupported-snapshot-term-preflight.md b/docs/unsupported-snapshot-term-preflight.md new file mode 100644 index 0000000..d048d5b --- /dev/null +++ b/docs/unsupported-snapshot-term-preflight.md @@ -0,0 +1,26 @@ +# Unsupported snapshot term preflight: v0.1.0-alpha.48 + +Before this change, `TRaft::Process` persisted a higher RPC term before the +InstallSnapshot handler had read enough bytes to recognize its format. A joint +MRFTS002 image is intentionally unsupported, but its first chunk could still +advance the follower's durable term before the handler rejected it. + +For a higher-term InstallSnapshot request, the RPC validator now requires an +initial chunk with at least the eight-byte magic and accepts the term only when +that magic identifies a supported MRFTS001 or MRFTS003 image. Higher-term +continuation chunks are rejected because a term change discards partial +transfers. Current-term malformed or joint images continue through the ordinary +fail-closed rejection path. The outbound sender already places the complete +magic in its initial chunk. + +This closes the hard-state mutation gap for snapshots that can be recognized as +unsupported from their first chunk. It does not enable joint snapshot transfer, +joint snapshot recovery, or live membership changes. + +The compaction regression test sends a full joint image, a short first chunk, +and a continuation chunk at a higher term, then verifies that term, vote, log, +snapshot and responses remain unchanged. It also verifies the same-term rejection +path. Release and ASAN network-enabled CTest passed 21/21 each; hosted C++ build, +Debug, Release, sanitizer, and test-result checks passed. + +Self-review completed. No independent reviewer approval is claimed. diff --git a/miniraft/raft.cpp b/miniraft/raft.cpp index 0f91332..456bfae 100644 --- a/miniraft/raft.cpp +++ b/miniraft/raft.cpp @@ -615,7 +615,7 @@ bool TRaft::ValidateRpc(TMessageHolder message) const { // A later boundary cannot have an older term than the committed prefix. // For an already committed, retained boundary, the terms must match; // an older compacted boundary cannot have a newer term than the snapshot. - return request->LeaderId == header->Src && request->SnapshotIndex > 0 && request->SnapshotIndex <= maxIndex && + const bool valid = request->LeaderId == header->Src && request->SnapshotIndex > 0 && request->SnapshotIndex <= maxIndex && request->SnapshotTerm > 0 && request->SnapshotTerm <= header->Term && request->Done <= 1 && (request->SnapshotIndex <= VolatileState->CommitIndex || request->SnapshotTerm >= State->LogTerm(VolatileState->CommitIndex)) && @@ -626,6 +626,18 @@ bool TRaft::ValidateRpc(TMessageHolder message) const { request->Offset <= request->TotalBytes && length > 0 && length <= TInstallSnapshotRequest::MaxChunkBytes && length <= request->TotalBytes - request->Offset && bool(request->Done) == (request->Offset + length == request->TotalBytes); + if (!valid) return false; + if (header->Term > State->CurrentTerm) { + // Do not persist a higher term based on an InstallSnapshot request + // whose format cannot be accepted by this runtime. A continuation + // cannot be valid across a term change because term changes discard + // the partial transfer; an initial chunk must identify a supported + // fixed/stable format before it can establish leader contact. + if (request->Offset != 0 || length < 8) return false; + const std::string_view magic(request->Data, 8); + if (magic != "MRFTS001" && magic != "MRFTS003") return false; + } + return true; } case EMessageType::INSTALL_SNAPSHOT_RESPONSE: { auto response = message.Cast(); diff --git a/test/test_compaction.cpp b/test/test_compaction.cpp index a9cf635..4e6dc86 100644 --- a/test/test_compaction.cpp +++ b/test/test_compaction.cpp @@ -390,6 +390,27 @@ void test_joint_snapshot_is_rejected_even_if_boundary_is_already_covered(void**) auto joint = fixed; joint.JointVoters = {2, 3}; joint.ConfigurationIndex = 4; joint.ConfigurationTerm = 1; const auto bytes = EncodeSnapshot(joint); const auto before = static_cast(*state); + const auto repliesBeforeUnsupported = replies.size(); + + // An unsupported snapshot format or an unclassifiable short first chunk + // must not persist a higher term before the runtime can reject it. + auto higherTermJoint = Chunk(joint, bytes, 0, static_cast(bytes.size())); + higherTermJoint->Term = state->CurrentTerm + 1; + raft->Process({}, higherTermJoint); + assert_true(static_cast(*state) == before); + assert_int_equal(replies.size(), repliesBeforeUnsupported); + auto higherTermShort = Chunk(joint, bytes, 0, 4); + higherTermShort->Term = state->CurrentTerm + 1; + raft->Process({}, higherTermShort); + assert_true(static_cast(*state) == before); + assert_int_equal(replies.size(), repliesBeforeUnsupported); + auto higherTermContinuation = Chunk(joint, bytes, 4, 4); + higherTermContinuation->Term = state->CurrentTerm + 1; + raft->Process({}, higherTermContinuation); + assert_true(static_cast(*state) == before); + assert_int_equal(replies.size(), repliesBeforeUnsupported); + + // At the current term, normal fail-closed rejection still returns a reply. raft->Process({}, Chunk(joint, bytes, 0, bytes.size())); assert_false(replies.back()->Success); assert_false(replies.back()->Installed); assert_true(static_cast(*state) == before); assert_int_equal(app->Loads, 1); From 9b09e1da5c9771899b755af9b0e9b25120fd4e63 Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Sat, 14 Nov 2026 17:54:36 -0500 Subject: [PATCH 49/66] Validate snapshot metadata before higher-term update (#49) --- README.md | 2 + docs/joint-snapshots.md | 13 ++++-- docs/snapshot-metadata-term-preflight.md | 26 +++++++++++ docs/unsupported-snapshot-term-preflight.md | 3 ++ miniraft/raft.cpp | 26 ++++++----- test/test_compaction.cpp | 49 ++++++++++++++++----- 6 files changed, 93 insertions(+), 26 deletions(-) create mode 100644 docs/snapshot-metadata-term-preflight.md diff --git a/README.md b/README.md index 38394d9..5fc7485 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,8 @@ representation for a joint configuration at a snapshot boundary; runtime adoptio and membership changes remain disabled. The [snapshot term preflight follow-up](docs/unsupported-snapshot-term-preflight.md) rejects unsupported snapshot formats before a higher RPC term changes hard state. +The [snapshot metadata preflight follow-up](docs/snapshot-metadata-term-preflight.md) +checks the boundary, fixed voters, and stable identity before that term update. The [configuration entry foundation](docs/configuration-entries.md) defines an experimental entry codec and rejects that entry kind in the fixed-membership runtime. The [configuration history validator](docs/configuration-history.md) checks an diff --git a/docs/joint-snapshots.md b/docs/joint-snapshots.md index 510074c..3aeea62 100644 --- a/docs/joint-snapshots.md +++ b/docs/joint-snapshots.md @@ -3,7 +3,9 @@ The [alpha.27 follow-up](configuration-history.md) requires the configuration term to equal the snapshot term when both refer to the same index. The [alpha.48 follow-up](unsupported-snapshot-term-preflight.md) checks the -initial snapshot format before accepting a higher RPC term. +initial snapshot format before accepting a higher RPC term. The [alpha.49 follow-up](snapshot-metadata-term-preflight.md) +also checks the embedded boundary, fixed voters, and known stable configuration +identity. This step adds an experimental snapshot representation for a joint configuration effective at the included snapshot boundary. MRFTS002 records the old and new @@ -31,9 +33,12 @@ stable v3 metadata, including when the prefix arrives in several chunks. Unsupported joint images are rejected without replacing application data or publishing a joint snapshot. Since alpha.48, a higher-term InstallSnapshot can advance hard state only after its initial chunk identifies a supported MRFTS001 -or MRFTS003 format. A short, unknown, joint-format, or continuation chunk cannot -change the term before rejection. These rules avoid falsely acknowledging a -joint image merely because its index is already committed locally. +or MRFTS003 format. Since alpha.49, that chunk must also contain the complete +bounded metadata prefix and match the RPC boundary, fixed voter set, and any +known stable configuration identity. Short, unknown, joint-format, mismatched, +or continuation chunks cannot change the term before rejection. These rules +avoid falsely acknowledging a joint image merely because its index is already +committed locally. Older binaries do not understand MRFTS002 and may acknowledge an already-covered image without checking its version. Do not send or activate joint snapshots in a diff --git a/docs/snapshot-metadata-term-preflight.md b/docs/snapshot-metadata-term-preflight.md new file mode 100644 index 0000000..c9cde1a --- /dev/null +++ b/docs/snapshot-metadata-term-preflight.md @@ -0,0 +1,26 @@ +# Snapshot metadata term preflight: v0.1.0-alpha.49 + +Alpha.48 checked a higher-term InstallSnapshot's magic before accepting its +term. That left one metadata gap: MRFTS001 or MRFTS003 could have a boundary, +voter list, or stable configuration identity that the receiver would later +reject, after its hard state had already advanced. + +The higher-term preflight now uses the bounded metadata probe before term +processing. The initial chunk must contain the full format header and voter +list. The probe checks that the embedded index and term match the RPC, voter +count and IDs match the receiver's fixed membership, the encoded lengths fit the +declared total, and any existing stable configuration identity is preserved. +The largest metadata prefix is 16,436 bytes, within the sender's 64 KiB initial +chunk. Payload bytes and CRC remain the full install path's responsibility. + +Malformed or conflicting metadata is rejected before changing the term, vote, +or leader state. Same-term handling remains unchanged. This does not install +joint snapshots or enable voter membership changes; those still require +transition-aware recovery and quorum rules. + +Regression tests cover higher-term joint images, wrong voters, conflicting +stable identities, short or continued chunks, and same-term rejection. Release +and ASAN network-enabled CTest passed 21/21 each, and hosted build, Debug, +Release, sanitizer, and unit-test checks passed. + +Self-review completed. No independent reviewer approval is claimed. diff --git a/docs/unsupported-snapshot-term-preflight.md b/docs/unsupported-snapshot-term-preflight.md index d048d5b..20f0b61 100644 --- a/docs/unsupported-snapshot-term-preflight.md +++ b/docs/unsupported-snapshot-term-preflight.md @@ -1,5 +1,8 @@ # Unsupported snapshot term preflight: v0.1.0-alpha.48 +The [alpha.49 follow-up](snapshot-metadata-term-preflight.md) extends this +preflight through the complete snapshot metadata and voter list. + Before this change, `TRaft::Process` persisted a higher RPC term before the InstallSnapshot handler had read enough bytes to recognize its format. A joint MRFTS002 image is intentionally unsupported, but its first chunk could still diff --git a/miniraft/raft.cpp b/miniraft/raft.cpp index 456bfae..8e355e1 100644 --- a/miniraft/raft.cpp +++ b/miniraft/raft.cpp @@ -23,7 +23,7 @@ static uint32_t rand_(uint32_t* seed) { return *seed; } -struct TCoveredSnapshotProbe { +struct TSnapshotMetadataProbe { bool Complete = false; bool Supported = false; size_t Need = 8; @@ -37,7 +37,7 @@ bool ReadSnapshotInteger(std::string_view bytes, size_t offset, unsigned width, return true; } -TCoveredSnapshotProbe ProbeCoveredSnapshot(std::string_view prefix, uint64_t totalBytes, +TSnapshotMetadataProbe ProbeSnapshotMetadata(std::string_view prefix, uint64_t totalBytes, uint64_t index, uint64_t term, const std::vector& voters, const TSnapshot* knownSnapshot) { if (prefix.size() < 8) return {false, false, 8}; @@ -45,11 +45,9 @@ TCoveredSnapshotProbe ProbeCoveredSnapshot(std::string_view prefix, uint64_t tot const bool stable = magic == "MRFTS003"; if (!stable && magic != "MRFTS001") return {true, false, 0}; - // A covered snapshot is not installed, but its identity must still agree - // with the RPC and the receiver's fixed membership before it can count as - // replication progress. Stable metadata identity is checked as well. The - // application payload and CRC are unnecessary because the receiver already - // has this committed log prefix. + // Inspect only the bounded format header and voter list. Callers use this + // before granting replication progress or accepting a higher term; payload + // bytes and CRC are validated by the full install path when needed. const size_t fixedHeaderBytes = stable ? 52 : 36; if (prefix.size() < fixedHeaderBytes) return {false, false, fixedHeaderBytes}; uint64_t embeddedIndex = 0, embeddedTerm = 0, voterCount = 0, dataBytes = 0; @@ -632,10 +630,14 @@ bool TRaft::ValidateRpc(TMessageHolder message) const { // whose format cannot be accepted by this runtime. A continuation // cannot be valid across a term change because term changes discard // the partial transfer; an initial chunk must identify a supported - // fixed/stable format before it can establish leader contact. - if (request->Offset != 0 || length < 8) return false; - const std::string_view magic(request->Data, 8); - if (magic != "MRFTS001" && magic != "MRFTS003") return false; + // fixed/stable format and complete metadata prefix before it can + // establish leader contact. + if (request->Offset != 0) return false; + const auto knownSnapshot = State->GetSnapshot(); + const auto probe = ProbeSnapshotMetadata(std::string_view(request->Data, length), + request->TotalBytes, request->SnapshotIndex, request->SnapshotTerm, + GetVoterIds(), knownSnapshot.get()); + if (!probe.Complete || !probe.Supported) return false; } return true; } @@ -989,7 +991,7 @@ void TRaft::OnInstallSnapshot(ITimeSource::Time now, TMessageHolderData, count); IncomingSnapshot->LastProgress = now; const auto knownSnapshot = State->GetSnapshot(); - const auto probe = ProbeCoveredSnapshot(prefix, request->TotalBytes, + const auto probe = ProbeSnapshotMetadata(prefix, request->TotalBytes, request->SnapshotIndex, request->SnapshotTerm, GetVoterIds(), knownSnapshot.get()); if (!probe.Complete) { reply(true, false, prefix.size()); return; } IncomingSnapshot.reset(); diff --git a/test/test_compaction.cpp b/test/test_compaction.cpp index 4e6dc86..9be92d4 100644 --- a/test/test_compaction.cpp +++ b/test/test_compaction.cpp @@ -399,6 +399,22 @@ void test_joint_snapshot_is_rejected_even_if_boundary_is_already_covered(void**) raft->Process({}, higherTermJoint); assert_true(static_cast(*state) == before); assert_int_equal(replies.size(), repliesBeforeUnsupported); + auto wrongEmbeddedBoundary = fixed; wrongEmbeddedBoundary.Index = 4; + const auto wrongEmbeddedBoundaryBytes = EncodeSnapshot(wrongEmbeddedBoundary); + auto higherTermWrongBoundary = Chunk(fixed, wrongEmbeddedBoundaryBytes, + 0, static_cast(wrongEmbeddedBoundaryBytes.size())); + higherTermWrongBoundary->Term = state->CurrentTerm + 1; + raft->Process({}, higherTermWrongBoundary); + assert_true(static_cast(*state) == before); + assert_int_equal(replies.size(), repliesBeforeUnsupported); + auto wrongVotersBeforeTerm = fixed; wrongVotersBeforeTerm.Voters = {1, 3}; + const auto wrongVotersBeforeTermBytes = EncodeSnapshot(wrongVotersBeforeTerm); + auto higherTermWrongVoters = Chunk(wrongVotersBeforeTerm, wrongVotersBeforeTermBytes, + 0, static_cast(wrongVotersBeforeTermBytes.size())); + higherTermWrongVoters->Term = state->CurrentTerm + 1; + raft->Process({}, higherTermWrongVoters); + assert_true(static_cast(*state) == before); + assert_int_equal(replies.size(), repliesBeforeUnsupported); auto higherTermShort = Chunk(joint, bytes, 0, 4); higherTermShort->Term = state->CurrentTerm + 1; raft->Process({}, higherTermShort); @@ -491,10 +507,12 @@ void test_stable_configuration_snapshot_installs_and_recovers_with_fixed_members { auto raft = std::make_shared(state, 2, TNodeDict{{1, peer}}); auto app = std::make_shared(); TRequestProcessor processor(raft, app, {}); - raft->Process({}, Chunk(snapshot, bytes, 0, 8)); + const size_t metadataBytes = 52 + 4 * snapshot.Voters.size(); + raft->Process({}, Chunk(snapshot, bytes, 0, metadataBytes)); assert_true(replies.back()->Success); assert_false(replies.back()->Installed); - assert_int_equal(replies.back()->NextOffset, 8); assert_int_equal(app->Loads, 0); - raft->Process({}, Chunk(snapshot, bytes, 8, static_cast(bytes.size() - 8))); + assert_int_equal(replies.back()->NextOffset, metadataBytes); assert_int_equal(app->Loads, 0); + raft->Process({}, Chunk(snapshot, bytes, metadataBytes, + static_cast(bytes.size() - metadataBytes))); assert_true(replies.back()->Success); assert_true(replies.back()->Installed); assert_int_equal(replies.back()->NextOffset, bytes.size()); assert_true(*state->GetSnapshot() == snapshot); @@ -505,6 +523,16 @@ void test_stable_configuration_snapshot_installs_and_recovers_with_fixed_members auto conflictingCovered = snapshot; conflictingCovered.ConfigurationIndex = 3; const auto conflictingCoveredBytes = EncodeSnapshot(conflictingCovered); + const auto beforeConflictingCovered = static_cast(*state); + const auto repliesBeforeConflictingCovered = replies.size(); + auto higherTermConflictingCovered = Chunk(conflictingCovered, conflictingCoveredBytes, 0, + static_cast(conflictingCoveredBytes.size())); + higherTermConflictingCovered->Term = state->CurrentTerm + 1; + raft->Process({}, higherTermConflictingCovered); + assert_true(static_cast(*state) == beforeConflictingCovered); + assert_int_equal(replies.size(), repliesBeforeConflictingCovered); + assert_true(*state->GetSnapshot() == snapshot); assert_int_equal(app->Loads, 1); + raft->Process({}, Chunk(conflictingCovered, conflictingCoveredBytes, 0, static_cast(conflictingCoveredBytes.size()))); assert_false(replies.back()->Success); assert_false(replies.back()->Installed); @@ -561,10 +589,11 @@ void test_idle_partial_snapshot_expires_on_learner_despite_heartbeats(void**) { TKv source; source.Write(source.Prepare(Command("key", "value")), 1); TSnapshot snapshot{1, 1, {1}, source.SaveSnapshot()}; const auto bytes = EncodeSnapshot(snapshot); + const size_t metadataBytes = 36 + 4 * snapshot.Voters.size(); - raft->Process({}, Chunk(snapshot, bytes, 0, 8)); + raft->Process({}, Chunk(snapshot, bytes, 0, metadataBytes)); assert_true(replies.back()->Success); assert_false(replies.back()->Installed); - assert_int_equal(replies.back()->NextOffset, 8); assert_int_equal(app->Loads, 0); + assert_int_equal(replies.back()->NextOffset, metadataBytes); assert_int_equal(app->Loads, 0); // Ordinary leader contact keeps Raft's election timer fresh, but must not // keep an abandoned multi-chunk snapshot allocation alive. @@ -573,18 +602,18 @@ void test_idle_partial_snapshot_expires_on_learner_despite_heartbeats(void**) { TAppendEntriesRequest{.PrevLogIndex = 1, .PrevLogTerm = 1, .LeaderCommit = 0, .LeaderId = 1}); raft->Process(heartbeatAt, heartbeat); // A matching retry is valid transfer activity and refreshes the deadline. - raft->Process(heartbeatAt, Chunk(snapshot, bytes, 0, 8)); - assert_true(replies.back()->Success); assert_int_equal(replies.back()->NextOffset, 8); + raft->Process(heartbeatAt, Chunk(snapshot, bytes, 0, metadataBytes)); + assert_true(replies.back()->Success); assert_int_equal(replies.back()->NextOffset, metadataBytes); const auto nextChunkAt = heartbeatAt + std::chrono::seconds(10); raft->ProcessTimeout(nextChunkAt); - raft->Process(nextChunkAt, Chunk(snapshot, bytes, 8, 8)); - assert_true(replies.back()->Success); assert_int_equal(replies.back()->NextOffset, 16); + raft->Process(nextChunkAt, Chunk(snapshot, bytes, metadataBytes, 8)); + assert_true(replies.back()->Success); assert_int_equal(replies.back()->NextOffset, metadataBytes + 8); // A continuation after expiry is rejected with offset zero so the leader // restarts from the beginning instead of extending a discarded image. const auto expiredAt = nextChunkAt + TTimeout::SnapshotTransferIdle + std::chrono::milliseconds(1); raft->ProcessTimeout(expiredAt); - raft->Process(expiredAt, Chunk(snapshot, bytes, 16, 8)); + raft->Process(expiredAt, Chunk(snapshot, bytes, metadataBytes + 8, 8)); assert_false(replies.back()->Success); assert_false(replies.back()->Installed); assert_int_equal(replies.back()->NextOffset, 0); assert_int_equal(app->Loads, 0); assert_int_equal(app->LastAppliedIndex, 0); assert_int_equal(state->SnapshotIndex, 0); From b22a28752785a898d687090bc035e23214e8d703 Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Sat, 14 Nov 2026 18:08:25 -0500 Subject: [PATCH 50/66] Add trusted voter binding migration for legacy durable state (#50) --- README.md | 2 + docs/durable-configuration-snapshots.md | 7 +- docs/trusted-legacy-voter-binding.md | 27 ++++ miniraft/durable.cpp | 44 +++++++ miniraft/durable.h | 3 + test/test_learners.cpp | 163 +++++++++++++++++++++++- 6 files changed, 244 insertions(+), 2 deletions(-) create mode 100644 docs/trusted-legacy-voter-binding.md diff --git a/README.md b/README.md index 5fc7485..051ee70 100644 --- a/README.md +++ b/README.md @@ -86,6 +86,8 @@ snapshots when their voter list matches the node's existing fixed membership; joint snapshots and live voter changes remain disabled. The [durable configuration snapshot follow-up](docs/durable-configuration-snapshots.md) documents its voter-binding requirement and recovery boundary. +The [trusted legacy voter binding follow-up](docs/trusted-legacy-voter-binding.md) +adds an explicit migration path when an operator has an independent record of the original set. The [stable snapshot runtime follow-up](docs/stable-configuration-snapshot-runtime.md) carries that identity through restore, InstallSnapshot, and later compaction. The [metadata continuity follow-up](docs/stable-snapshot-metadata-continuity.md) diff --git a/docs/durable-configuration-snapshots.md b/docs/durable-configuration-snapshots.md index 20c9361..3ea27f7 100644 --- a/docs/durable-configuration-snapshots.md +++ b/docs/durable-configuration-snapshots.md @@ -5,7 +5,12 @@ joint and MRFTS003 stable snapshots so offline tools can validate configuration history after restart. A store must already have a voter binding, and `snapshot.Voters` must equal that binding. Durable recovery checks the same invariant before exposing the image. An unbound store cannot infer its original -voter set from configuration metadata and continues to reject the snapshot. +voter set from configuration metadata. The [alpha.50 follow-up](trusted-legacy-voter-binding.md) +adds an explicit durable migration API that accepts a caller-supplied trusted +original set, validates it against the recovered vote, snapshot, and retained +log, then persists the binding before Raft startup. The API does not establish +that the caller's assertion is true; absent an independent trusted baseline, the +store still cannot infer its voter set. The durable file layout is unchanged. The backend persists the snapshot together with its existing voter binding; it does not reinterpret that binding from the diff --git a/docs/trusted-legacy-voter-binding.md b/docs/trusted-legacy-voter-binding.md new file mode 100644 index 0000000..cd7bc2d --- /dev/null +++ b/docs/trusted-legacy-voter-binding.md @@ -0,0 +1,27 @@ +# Trusted legacy voter binding: v0.1.0-alpha.50 + +Older durable MRFTD001/MRFTD002 files do not persist the complete fixed voter +set. Raft cannot infer that set safely from a vote, a legacy snapshot voter +list, or the current process's peer list alone. + +`TDurableState::BindVotersFromTrustedBaseline(voters)` provides an explicit +migration path for an unbound, recovered durable store. The caller must supply +the complete original fixed voter set from an independently trusted source, +such as the cluster's verified deployment record. The operation checks that the +stored vote belongs to the set, any legacy snapshot names the same voters and +has no configuration metadata, retained terms are valid, and no reserved +configuration entries are present. It then atomically republishes the durable +state with its voter binding before the store is passed to `TRaft`. + +This API records the caller's assertion; it cannot prove the assertion is +correct, discover prior membership transitions hidden by a legacy snapshot, or +authorize a membership change. Existing `BindVoters` behavior is unchanged. +Do not call the migration API when the original complete voter set is unknown. +The migration applies to `TDurableState`; legacy `TDiskState` remains unable to +persist voter bindings. + +Tests cover MRFTD001 log-only and MRFTD002 snapshot-backed migration, rejection +of inconsistent vote/snapshot baselines and configuration log entries without +rewriting the source file, and recovered MRFTD003 binding after restart. + +Self-review completed. No independent reviewer approval is claimed. diff --git a/miniraft/durable.cpp b/miniraft/durable.cpp index 764fe79..a08c117 100644 --- a/miniraft/durable.cpp +++ b/miniraft/durable.cpp @@ -96,6 +96,7 @@ struct TDurableState::TImpl { std::vector> Log; std::shared_ptr Snapshot; std::vector Voters; + bool Recovered = false; THook Hook; std::string File, Temp; #ifdef __linux__ @@ -201,6 +202,7 @@ TDurableState::TDurableState(const std::string& name, uint32_t id, THook hook) // image. Stabilize the selected image and its name before serving requests. Sync(file.Value); Sync(Impl->Directory.Value); + Impl->Recovered = true; // A leftover temp is never a recovery candidate, even if its checksum is valid. #endif } @@ -338,3 +340,45 @@ void TDurableState::BindVoters(const std::vector& voters) { Impl->Voters = voters; Commit(); // Bind before acknowledging any vote/replication traffic. } + +void TDurableState::BindVotersFromTrustedBaseline(const std::vector& voters) { + ValidateFixedVoters(voters); + if (!Impl->Voters.empty()) { + if (Impl->Voters == voters) return; + throw std::invalid_argument("Durable voter binding already exists"); + } + if (!CurrentTerm || LastLogIndex > uint64_t(INT64_MAX) || SnapshotIndex > LastLogIndex || + LastLogIndex - SnapshotIndex != Impl->Log.size()) + throw std::invalid_argument("Invalid legacy state bounds"); + if (!Impl->Recovered) + throw std::invalid_argument("Trusted legacy baseline is only for recovered state"); + if (VotedFor && !std::binary_search(voters.begin(), voters.end(), VotedFor)) + throw std::invalid_argument("Stored vote is outside the trusted voter baseline"); + + if (Impl->Snapshot) { + const auto& snapshot = *Impl->Snapshot; + ValidateSnapshot(snapshot); + if (snapshot.HasConfigurationMetadata() || snapshot.Index != SnapshotIndex || + snapshot.Term != SnapshotTerm || snapshot.Term > CurrentTerm || snapshot.Voters != voters) + throw std::invalid_argument("Legacy snapshot does not match the trusted voter baseline"); + } else if (SnapshotIndex || SnapshotTerm) { + throw std::invalid_argument("Legacy compacted state is missing its snapshot"); + } + + auto previousTerm = SnapshotTerm; + for (uint64_t index = SnapshotIndex + 1; index <= LastLogIndex; ++index) { + if (RetainedLogFlags(index) & TLogEntry::EConfiguration) + throw std::invalid_argument("Configuration log migration requires an explicit history baseline"); + const auto term = RetainedLogTerm(index); + if (!term || term < previousTerm || term > CurrentTerm) + throw std::invalid_argument("Invalid legacy log term sequence"); + previousTerm = term; + } + if (LastLogTerm != previousTerm) + throw std::invalid_argument("Legacy log last term mismatch"); + + // Persist the operator-supplied baseline before Raft can serve votes or + // install configuration snapshots against it. + Impl->Voters = voters; + Commit(); +} diff --git a/miniraft/durable.h b/miniraft/durable.h index 8c4ea24..b596b7b 100644 --- a/miniraft/durable.h +++ b/miniraft/durable.h @@ -28,6 +28,9 @@ class TDurableState : public IState { bool SupportsVoterBinding() const override { return true; } std::vector GetBoundVoters() const override; void BindVoters(const std::vector& voters) override; + // Explicit legacy migration. The caller asserts that voters is the + // complete original fixed voter set, verified outside this store. + void BindVotersFromTrustedBaseline(const std::vector& voters); private: struct TImpl; std::unique_ptr Impl; diff --git a/test/test_learners.cpp b/test/test_learners.cpp index 7783d80..5006c0f 100644 --- a/test/test_learners.cpp +++ b/test/test_learners.cpp @@ -418,7 +418,10 @@ struct TFiles { std::filesystem::path Directory; std::string Name; TFiles() { static unsigned next = 0; - Directory = std::filesystem::temp_directory_path() / ("miniraft-learners-" + std::to_string(getpid()) + "-" + std::to_string(next++)); + do { + Directory = std::filesystem::temp_directory_path() / + ("miniraft-learners-" + std::to_string(getpid()) + "-" + std::to_string(next++)); + } while (std::filesystem::exists(Directory)); assert_true(std::filesystem::create_directory(Directory)); Name = (Directory / "raft").string(); } ~TFiles() { std::error_code ignored; std::filesystem::remove_all(Directory, ignored); } @@ -459,6 +462,161 @@ void test_durable_voter_binding_is_atomic_before_first_vote(void**) { assert_int_equal(state.CurrentTerm, 1); assert_int_equal(state.VotedFor, 0); assert_int_equal(state.LastLogIndex, 0); } } +void test_trusted_legacy_voter_baseline_migrates_durable_v1_and_v2(void**) { + { + TFiles files; + { + TDurableState fresh(files.Name, 1); + Reject([&] { fresh.BindVotersFromTrustedBaseline({1, 2, 3}); }); + } + auto recovered = std::make_shared(files.Name, 1); + assert_int_equal(recovered->CurrentTerm, 1); + assert_int_equal(recovered->LastLogIndex, 0); + recovered->BindVotersFromTrustedBaseline({1, 2, 3}); + assert_true(recovered->GetBoundVoters() == (std::vector{1, 2, 3})); + TRaft raft(recovered, 1, Peers(1, 3)); + assert_true(raft.GetVoterIds() == (std::vector{1, 2, 3})); + } + { + TFiles files; TStateFields before; + { + TDurableState legacy(files.Name, 1); + legacy.CurrentTerm = 3; legacy.VotedFor = 1; + auto entry = NewHoldedMessage(); entry->Term = 2; + legacy.Append(entry); before = legacy; + } + const auto path = files.Name + ".durable.1"; + std::ifstream originalInput(path, std::ios::binary); + const std::string original(std::istreambuf_iterator(originalInput), {}); + assert_true(original.substr(0, 8) == "MRFTD001"); + + { + auto state = std::make_shared(files.Name, 1); + assert_true(state->GetBoundVoters().empty()); + Reject([&] { state->BindVoters({1, 2, 3}); }); + Reject([&] { state->BindVotersFromTrustedBaseline({2, 3}); }); + assert_true(static_cast(*state) == before); + assert_true(state->GetBoundVoters().empty()); + std::ifstream unchanged(path, std::ios::binary); + assert_true(std::string(std::istreambuf_iterator(unchanged), {}) == original); + + state->BindVotersFromTrustedBaseline({1, 2, 3}); + assert_true(static_cast(*state) == before); + assert_true(state->GetBoundVoters() == (std::vector{1, 2, 3})); + assert_int_equal(state->RetainedLogTerm(1), 2); + } + { std::ifstream migrated(path, std::ios::binary); std::string magic(8, '\0'); + migrated.read(magic.data(), magic.size()); assert_true(magic == "MRFTD003"); } + auto recovered = std::make_shared(files.Name, 1); + assert_true(static_cast(*recovered) == before); + assert_true(recovered->GetBoundVoters() == (std::vector{1, 2, 3})); + TRaft raft(recovered, 1, Peers(1, 3)); + assert_true(raft.GetVoterIds() == (std::vector{1, 2, 3})); + } + { + TFiles files; TStateFields before; std::string application; + { + TDurableState legacy(files.Name, 1); TKv image; + legacy.CurrentTerm = 3; legacy.VotedFor = 1; + auto entry = image.Prepare(Command("migrated", "value")); entry->Term = 2; + legacy.Append(entry); image.Write(entry, 1); + application = image.SaveSnapshot(); + legacy.InstallSnapshot({1, 2, {1, 2, 3}, application}); + auto tail = NewHoldedMessage(); tail->Term = 3; legacy.Append(tail); + before = legacy; + } + const auto path = files.Name + ".durable.1"; + std::ifstream originalInput(path, std::ios::binary); + const std::string original(std::istreambuf_iterator(originalInput), {}); + assert_true(original.substr(0, 8) == "MRFTD002"); + + { + auto state = std::make_shared(files.Name, 1); + Reject([&] { state->BindVotersFromTrustedBaseline({1, 2}); }); + assert_true(static_cast(*state) == before); + assert_true(state->GetBoundVoters().empty()); + std::ifstream unchanged(path, std::ios::binary); + assert_true(std::string(std::istreambuf_iterator(unchanged), {}) == original); + state->BindVotersFromTrustedBaseline({1, 2, 3}); + assert_true(static_cast(*state) == before); + assert_true(state->GetSnapshot()->Data == application); + assert_true(state->GetBoundVoters() == (std::vector{1, 2, 3})); + } + { std::ifstream migrated(path, std::ios::binary); std::string magic(8, '\0'); + migrated.read(magic.data(), magic.size()); assert_true(magic == "MRFTD003"); } + auto recovered = std::make_shared(files.Name, 1); + assert_true(static_cast(*recovered) == before); + assert_true(recovered->GetBoundVoters() == (std::vector{1, 2, 3})); + auto raft = std::make_shared(recovered, 1, Peers(1, 3)); + auto app = std::make_shared(); TRequestProcessor processor(raft, app, {}); + assert_true(Read(*app, "migrated") == "value"); + assert_true(bool(recovered->Get(2))); + } +} +void test_trusted_legacy_voter_baseline_refuses_configuration_logs(void**) { + TFiles files; + { + TDurableState legacy(files.Name, 1); + legacy.CurrentTerm = 2; + auto entry = NewHoldedMessage(); entry->Term = 2; entry->Flags = TLogEntry::EConfiguration; + legacy.Append(entry); + } + const auto path = files.Name + ".durable.1"; + std::ifstream originalInput(path, std::ios::binary); + const std::string original(std::istreambuf_iterator(originalInput), {}); + auto state = std::make_shared(files.Name, 1); + Reject([&] { state->BindVotersFromTrustedBaseline({1, 2, 3}); }); + assert_true(state->GetBoundVoters().empty()); assert_int_equal(state->LastLogIndex, 1); + std::ifstream unchanged(path, std::ios::binary); + assert_true(std::string(std::istreambuf_iterator(unchanged), {}) == original); +} +void test_trusted_legacy_voter_binding_is_atomic_at_each_publish_stage(void**) { + using TStage = TDurableState::EStage; + const std::vector stages = {TStage::TempCreated, TStage::Written, TStage::FileSynced, + TStage::Renamed, TStage::DirectorySynced}; + for (bool withSnapshot : {false, true}) { + for (auto stage : stages) { + TFiles files; + { + TDurableState legacy(files.Name, 1); legacy.CurrentTerm = 3; legacy.VotedFor = 1; + if (withSnapshot) { + TKv image; auto entry = image.Prepare(Command("atomic", "snapshot")); + entry->Term = 2; legacy.Append(entry); image.Write(entry, 1); + legacy.InstallSnapshot({1, 2, {1, 2, 3}, image.SaveSnapshot()}); + auto tail = NewHoldedMessage(); tail->Term = 3; legacy.Append(tail); + } else { + auto entry = NewHoldedMessage(); entry->Term = 2; legacy.Append(entry); + } + } + auto child = fork(); assert_true(child >= 0); + if (!child) { + try { + TDurableState state(files.Name, 1, [&](auto point) { + if (point == stage) _exit(90); + }); + state.BindVotersFromTrustedBaseline({1, 2, 3}); + } catch (...) { _exit(91); } + _exit(92); + } + int status = 0; assert_int_equal(waitpid(child, &status, 0), child); + assert_true(WIFEXITED(status)); assert_int_equal(WEXITSTATUS(status), 90); + + TDurableState recovered(files.Name, 1); + if (stage < TStage::Renamed) { + assert_true(recovered.GetBoundVoters().empty()); + assert_int_equal(recovered.LastLogIndex, withSnapshot ? 2 : 1); + } else { + assert_true(recovered.GetBoundVoters() == (std::vector{1, 2, 3})); + assert_int_equal(recovered.LastLogIndex, withSnapshot ? 2 : 1); + } + if (withSnapshot) { + assert_true(bool(recovered.GetSnapshot())); + assert_int_equal(recovered.SnapshotIndex, 1); + assert_int_equal(recovered.SnapshotTerm, 2); + } + } + } +} void test_durable_learner_snapshot_suffix_and_role_survive_restart(void**) { TFiles files; { @@ -580,6 +738,9 @@ int main() { cmocka_unit_test(test_durable_learner_snapshot_suffix_and_role_survive_restart), cmocka_unit_test(test_durable_v3_corrupt_roles_fail_without_rewriting), cmocka_unit_test(test_durable_v2_snapshot_adoption_preserves_vote_log_and_application), + cmocka_unit_test(test_trusted_legacy_voter_baseline_migrates_durable_v1_and_v2), + cmocka_unit_test(test_trusted_legacy_voter_baseline_refuses_configuration_logs), + cmocka_unit_test(test_trusted_legacy_voter_binding_is_atomic_at_each_publish_stage), cmocka_unit_test(test_legacy_disk_backend_cannot_enable_learner_roles), cmocka_unit_test(test_legacy_leader_cannot_stage_online_learner), cmocka_unit_test(test_online_learner_staging_does_not_change_durable_binding), From b8e31ae296426e94c5e9636a3ec81347075e2d5e Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Sat, 14 Nov 2026 18:29:37 -0500 Subject: [PATCH 51/66] Restrict legacy voter migration to pre-startup (#51) --- README.md | 2 ++ docs/trusted-legacy-voter-lifecycle.md | 21 +++++++++++++++++++++ miniraft/durable.cpp | 5 +++++ miniraft/durable.h | 6 ++++-- miniraft/raft.cpp | 1 + miniraft/state.h | 3 +++ test/test_learners.cpp | 20 ++++++++++++++++++++ 7 files changed, 56 insertions(+), 2 deletions(-) create mode 100644 docs/trusted-legacy-voter-lifecycle.md diff --git a/README.md b/README.md index 051ee70..ad90e6f 100644 --- a/README.md +++ b/README.md @@ -88,6 +88,8 @@ The [durable configuration snapshot follow-up](docs/durable-configuration-snapsh documents its voter-binding requirement and recovery boundary. The [trusted legacy voter binding follow-up](docs/trusted-legacy-voter-binding.md) adds an explicit migration path when an operator has an independent record of the original set. +The [migration lifecycle follow-up](docs/trusted-legacy-voter-lifecycle.md) +enforces that migration happens before Raft attaches to an unbound legacy store. The [stable snapshot runtime follow-up](docs/stable-configuration-snapshot-runtime.md) carries that identity through restore, InstallSnapshot, and later compaction. The [metadata continuity follow-up](docs/stable-snapshot-metadata-continuity.md) diff --git a/docs/trusted-legacy-voter-lifecycle.md b/docs/trusted-legacy-voter-lifecycle.md new file mode 100644 index 0000000..9efe741 --- /dev/null +++ b/docs/trusted-legacy-voter-lifecycle.md @@ -0,0 +1,21 @@ +# Trusted voter migration startup boundary: v0.1.0-alpha.51 + +The alpha.50 `TDurableState::BindVotersFromTrustedBaseline` migration must run +before a `TRaft` instance uses a recovered legacy store. A nonfresh legacy store +without a persisted binding can still start under its configured fixed peers for +compatibility. If migration were allowed afterward, the durable state could be +bound to a different voter set while the live Raft object continued using the +peer set it received at construction. + +`IState::MarkRaftStarted` lets the runtime notify storage after all startup +validation succeeds. `TDurableState` records that attachment in memory and +rejects trusted legacy migration from then on, before modifying or publishing +the durable file. The marker is process-local: after restart, recovery creates a +new state object and the operator can migrate it before constructing Raft. + +The guard does not infer or prove the original voter set, and it does not add +live voter reconfiguration. Tests verify that a post-start migration to a +different set is rejected without changing the binding or file; existing tests +continue to cover migration before startup and recovered restart behavior. + +Self-review completed; no independent reviewer approval is claimed. diff --git a/miniraft/durable.cpp b/miniraft/durable.cpp index a08c117..4778e76 100644 --- a/miniraft/durable.cpp +++ b/miniraft/durable.cpp @@ -97,6 +97,7 @@ struct TDurableState::TImpl { std::shared_ptr Snapshot; std::vector Voters; bool Recovered = false; + bool RaftStarted = false; THook Hook; std::string File, Temp; #ifdef __linux__ @@ -334,6 +335,8 @@ void TDurableState::InstallSnapshot(const TSnapshot& snapshot) { std::vector TDurableState::GetBoundVoters() const { return Impl->Voters; } +void TDurableState::MarkRaftStarted() { Impl->RaftStarted = true; } + void TDurableState::BindVoters(const std::vector& voters) { ValidateVoterBinding(*this, Impl->Voters, voters); if (Impl->Voters == voters) return; @@ -343,6 +346,8 @@ void TDurableState::BindVoters(const std::vector& voters) { void TDurableState::BindVotersFromTrustedBaseline(const std::vector& voters) { ValidateFixedVoters(voters); + if (Impl->RaftStarted) + throw std::invalid_argument("Trusted legacy voter migration must happen before Raft startup"); if (!Impl->Voters.empty()) { if (Impl->Voters == voters) return; throw std::invalid_argument("Durable voter binding already exists"); diff --git a/miniraft/durable.h b/miniraft/durable.h index b596b7b..729491a 100644 --- a/miniraft/durable.h +++ b/miniraft/durable.h @@ -28,8 +28,10 @@ class TDurableState : public IState { bool SupportsVoterBinding() const override { return true; } std::vector GetBoundVoters() const override; void BindVoters(const std::vector& voters) override; - // Explicit legacy migration. The caller asserts that voters is the - // complete original fixed voter set, verified outside this store. + void MarkRaftStarted() override; + // Explicit legacy migration. Call before constructing TRaft. The caller + // asserts that voters is the complete original fixed set, verified outside + // this store. void BindVotersFromTrustedBaseline(const std::vector& voters); private: struct TImpl; diff --git a/miniraft/raft.cpp b/miniraft/raft.cpp index 8e355e1..af3c8d1 100644 --- a/miniraft/raft.cpp +++ b/miniraft/raft.cpp @@ -298,6 +298,7 @@ TRaft::TRaft(std::shared_ptr state, int node, const TNodeDict& nodes, st if (snapshot) { VolatileState->CommitIndex = State->SnapshotIndex; } + State->MarkRaftStarted(); } void TRaft::OnRequestVote(ITimeSource::Time now, TMessageHolder message) { diff --git a/miniraft/state.h b/miniraft/state.h index b2d90fc..2bc6b85 100644 --- a/miniraft/state.h +++ b/miniraft/state.h @@ -40,6 +40,9 @@ struct IState: TStateFields { virtual bool SupportsVoterBinding() const { return false; } virtual std::vector GetBoundVoters() const { return {}; } virtual void BindVoters(const std::vector&) { throw std::logic_error("Storage voter binding is unsupported"); } + // Storage backends can close startup-only migration paths once a Raft + // instance has finished validating and attached to this state. + virtual void MarkRaftStarted() {} virtual ~IState() = default; uint64_t LogTerm(int64_t index = -1) const { diff --git a/test/test_learners.cpp b/test/test_learners.cpp index 5006c0f..9726ad1 100644 --- a/test/test_learners.cpp +++ b/test/test_learners.cpp @@ -570,6 +570,25 @@ void test_trusted_legacy_voter_baseline_refuses_configuration_logs(void**) { std::ifstream unchanged(path, std::ios::binary); assert_true(std::string(std::istreambuf_iterator(unchanged), {}) == original); } +void test_trusted_legacy_voter_baseline_must_precede_raft_startup(void**) { + TFiles files; + { + TDurableState legacy(files.Name, 1); + legacy.CurrentTerm = 2; legacy.VotedFor = 1; + legacy.Commit(); + } + const auto path = files.Name + ".durable.1"; + std::ifstream originalInput(path, std::ios::binary); + const std::string original(std::istreambuf_iterator(originalInput), {}); + auto state = std::make_shared(files.Name, 1); + TRaft raft(state, 1, Peers(1, 3)); + assert_true(raft.GetVoterIds() == (std::vector{1, 2, 3})); + assert_true(state->GetBoundVoters().empty()); + Reject([&] { state->BindVotersFromTrustedBaseline({1, 2, 4}); }); + assert_true(state->GetBoundVoters().empty()); assert_int_equal(state->VotedFor, 1); + std::ifstream unchanged(path, std::ios::binary); + assert_true(std::string(std::istreambuf_iterator(unchanged), {}) == original); +} void test_trusted_legacy_voter_binding_is_atomic_at_each_publish_stage(void**) { using TStage = TDurableState::EStage; const std::vector stages = {TStage::TempCreated, TStage::Written, TStage::FileSynced, @@ -740,6 +759,7 @@ int main() { cmocka_unit_test(test_durable_v2_snapshot_adoption_preserves_vote_log_and_application), cmocka_unit_test(test_trusted_legacy_voter_baseline_migrates_durable_v1_and_v2), cmocka_unit_test(test_trusted_legacy_voter_baseline_refuses_configuration_logs), + cmocka_unit_test(test_trusted_legacy_voter_baseline_must_precede_raft_startup), cmocka_unit_test(test_trusted_legacy_voter_binding_is_atomic_at_each_publish_stage), cmocka_unit_test(test_legacy_disk_backend_cannot_enable_learner_roles), cmocka_unit_test(test_legacy_leader_cannot_stage_online_learner), From 0c43a3fc0a99dc89f1d038579a311a3eaefd5b38 Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Sat, 14 Nov 2026 18:42:10 -0500 Subject: [PATCH 52/66] Require voter proof for legacy durable startup (#52) --- README.md | 2 ++ docs/legacy-voter-proof-startup.md | 26 ++++++++++++++++++++++++++ docs/trusted-legacy-voter-binding.md | 3 ++- docs/trusted-legacy-voter-lifecycle.md | 4 ++++ miniraft/durable.cpp | 4 ++++ miniraft/durable.h | 1 + miniraft/raft.cpp | 2 ++ miniraft/state.h | 3 +++ test/test_learners.cpp | 23 +++++++++++++++++------ 9 files changed, 61 insertions(+), 7 deletions(-) create mode 100644 docs/legacy-voter-proof-startup.md diff --git a/README.md b/README.md index ad90e6f..3ff9921 100644 --- a/README.md +++ b/README.md @@ -90,6 +90,8 @@ The [trusted legacy voter binding follow-up](docs/trusted-legacy-voter-binding.m adds an explicit migration path when an operator has an independent record of the original set. The [migration lifecycle follow-up](docs/trusted-legacy-voter-lifecycle.md) enforces that migration happens before Raft attaches to an unbound legacy store. +The [legacy voter startup proof follow-up](docs/legacy-voter-proof-startup.md) +requires recovered durable stores to provide voter proof before Raft can start. The [stable snapshot runtime follow-up](docs/stable-configuration-snapshot-runtime.md) carries that identity through restore, InstallSnapshot, and later compaction. The [metadata continuity follow-up](docs/stable-snapshot-metadata-continuity.md) diff --git a/docs/legacy-voter-proof-startup.md b/docs/legacy-voter-proof-startup.md new file mode 100644 index 0000000..81876dc --- /dev/null +++ b/docs/legacy-voter-proof-startup.md @@ -0,0 +1,26 @@ +# Require voter proof before legacy durable startup: v0.1.0-alpha.52 + +The alpha.51 guard prevented a live Raft instance from being rebound after +startup, but it still allowed a recovered log-only MRFTD001/MRFTD002 store to +start unbound using only the process's peer list. That left the persisted node +identity unprotected across restarts. + +Raft now rejects a recovered durable store with no voter binding and no +ordinary snapshot before it can bind peers or serve traffic. The operator must +first call `TDurableState::BindVotersFromTrustedBaseline` using an independently +verified complete original voter set. A recovered unbound store with an +ordinary snapshot retains the existing adoption path, which validates that the +snapshot voter list matches the configured peers before persisting the +binding. Newly initialized stores still bind their initial peer set during +startup. + +The rejection does not rewrite the durable file. The alpha.51 startup marker +also keeps the explicit migration API startup-only. `TDiskState` and in-memory +state behavior are unchanged. This closes unbound durable startup; it does not +implement live membership changes or joint snapshot recovery. + +Tests cover recovered log-only and idle stores failing without file changes, +successful startup after explicit trusted migration, ordinary-snapshot adoption, +and rejection of migration after a successful startup. + +Self-review completed; no independent reviewer approval is claimed. diff --git a/docs/trusted-legacy-voter-binding.md b/docs/trusted-legacy-voter-binding.md index cd7bc2d..37dd2cf 100644 --- a/docs/trusted-legacy-voter-binding.md +++ b/docs/trusted-legacy-voter-binding.md @@ -18,7 +18,8 @@ correct, discover prior membership transitions hidden by a legacy snapshot, or authorize a membership change. Existing `BindVoters` behavior is unchanged. Do not call the migration API when the original complete voter set is unknown. The migration applies to `TDurableState`; legacy `TDiskState` remains unable to -persist voter bindings. +persist voter bindings. Since alpha.52, Raft rejects recovered log-only durable +stores with no binding, so this migration must be completed before startup. Tests cover MRFTD001 log-only and MRFTD002 snapshot-backed migration, rejection of inconsistent vote/snapshot baselines and configuration log entries without diff --git a/docs/trusted-legacy-voter-lifecycle.md b/docs/trusted-legacy-voter-lifecycle.md index 9efe741..2f9d5e1 100644 --- a/docs/trusted-legacy-voter-lifecycle.md +++ b/docs/trusted-legacy-voter-lifecycle.md @@ -18,4 +18,8 @@ live voter reconfiguration. Tests verify that a post-start migration to a different set is rejected without changing the binding or file; existing tests continue to cover migration before startup and recovered restart behavior. +The [alpha.52 follow-up](legacy-voter-proof-startup.md) closes the remaining +startup gap: recovered log-only durable stores without a voter binding now fail +before Raft starts, requiring this migration path or another valid voter proof. + Self-review completed; no independent reviewer approval is claimed. diff --git a/miniraft/durable.cpp b/miniraft/durable.cpp index 4778e76..ef68cb1 100644 --- a/miniraft/durable.cpp +++ b/miniraft/durable.cpp @@ -335,6 +335,10 @@ void TDurableState::InstallSnapshot(const TSnapshot& snapshot) { std::vector TDurableState::GetBoundVoters() const { return Impl->Voters; } +bool TDurableState::HasUnboundRecoveredState() const { + return Impl->Recovered && Impl->Voters.empty(); +} + void TDurableState::MarkRaftStarted() { Impl->RaftStarted = true; } void TDurableState::BindVoters(const std::vector& voters) { diff --git a/miniraft/durable.h b/miniraft/durable.h index 729491a..033a7be 100644 --- a/miniraft/durable.h +++ b/miniraft/durable.h @@ -26,6 +26,7 @@ class TDurableState : public IState { std::shared_ptr GetSnapshot() const override; void InstallSnapshot(const TSnapshot& snapshot) override; bool SupportsVoterBinding() const override { return true; } + bool HasUnboundRecoveredState() const override; std::vector GetBoundVoters() const override; void BindVoters(const std::vector& voters) override; void MarkRaftStarted() override; diff --git a/miniraft/raft.cpp b/miniraft/raft.cpp index af3c8d1..428c7e3 100644 --- a/miniraft/raft.cpp +++ b/miniraft/raft.cpp @@ -277,6 +277,8 @@ TRaft::TRaft(std::shared_ptr state, int node, const TNodeDict& nodes, st } else if (State->SnapshotIndex || State->SnapshotTerm) { throw std::invalid_argument("Compacted state is missing its snapshot"); } + if (State->HasUnboundRecoveredState() && !snapshot) + throw std::invalid_argument("Recovered durable state requires voter proof before Raft startup"); // Reject before binding voters or serving traffic. Current Raft cannot // interpret configuration entries, including uncommitted suffix entries. auto previousTerm = State->SnapshotTerm; diff --git a/miniraft/state.h b/miniraft/state.h index 2bc6b85..d8a2dc1 100644 --- a/miniraft/state.h +++ b/miniraft/state.h @@ -38,6 +38,9 @@ struct IState: TStateFields { virtual std::shared_ptr GetSnapshot() const { return {}; } virtual void InstallSnapshot(const TSnapshot&) { throw std::logic_error("Storage snapshots are unsupported"); } virtual bool SupportsVoterBinding() const { return false; } + // True when this backend recovered a persistent image without a stored + // voter identity; Raft must obtain proof before serving this state. + virtual bool HasUnboundRecoveredState() const { return false; } virtual std::vector GetBoundVoters() const { return {}; } virtual void BindVoters(const std::vector&) { throw std::logic_error("Storage voter binding is unsupported"); } // Storage backends can close startup-only migration paths once a Raft diff --git a/test/test_learners.cpp b/test/test_learners.cpp index 9726ad1..59e193f 100644 --- a/test/test_learners.cpp +++ b/test/test_learners.cpp @@ -447,10 +447,12 @@ void test_legacy_recovered_vote_outside_voters_preserves_storage(void**) { void test_durable_voter_binding_is_atomic_before_first_vote(void**) { using TStage = TDurableState::EStage; for (auto stage : {TStage::TempCreated, TStage::Written, TStage::FileSynced, TStage::Renamed, TStage::DirectorySynced}) { - TFiles files; { TDurableState initial(files.Name, 4); } + TFiles files; auto child = fork(); assert_true(child >= 0); if (!child) { - try { auto state = std::make_shared(files.Name, 4, [&](auto point) { if (point == stage) _exit(90); }); + bool armed = false; + try { auto state = std::make_shared(files.Name, 4, [&](auto point) { if (armed && point == stage) _exit(90); }); + armed = true; TRaft raft(state, 4, Peers(4), {4}); } catch (...) { _exit(91); } _exit(92); } @@ -472,6 +474,7 @@ void test_trusted_legacy_voter_baseline_migrates_durable_v1_and_v2(void**) { auto recovered = std::make_shared(files.Name, 1); assert_int_equal(recovered->CurrentTerm, 1); assert_int_equal(recovered->LastLogIndex, 0); + Reject([&] { TRaft missingBaseline(recovered, 1, Peers(1, 3)); }); recovered->BindVotersFromTrustedBaseline({1, 2, 3}); assert_true(recovered->GetBoundVoters() == (std::vector{1, 2, 3})); TRaft raft(recovered, 1, Peers(1, 3)); @@ -581,13 +584,21 @@ void test_trusted_legacy_voter_baseline_must_precede_raft_startup(void**) { std::ifstream originalInput(path, std::ios::binary); const std::string original(std::istreambuf_iterator(originalInput), {}); auto state = std::make_shared(files.Name, 1); - TRaft raft(state, 1, Peers(1, 3)); - assert_true(raft.GetVoterIds() == (std::vector{1, 2, 3})); + Reject([&] { TRaft missingBaseline(state, 1, Peers(1, 3)); }); assert_true(state->GetBoundVoters().empty()); - Reject([&] { state->BindVotersFromTrustedBaseline({1, 2, 4}); }); - assert_true(state->GetBoundVoters().empty()); assert_int_equal(state->VotedFor, 1); + assert_int_equal(state->VotedFor, 1); std::ifstream unchanged(path, std::ios::binary); assert_true(std::string(std::istreambuf_iterator(unchanged), {}) == original); + + state->BindVotersFromTrustedBaseline({1, 2, 3}); + auto raft = std::make_shared(state, 1, Peers(1, 3)); + assert_true(raft->GetVoterIds() == (std::vector{1, 2, 3})); + assert_true(state->GetBoundVoters() == (std::vector{1, 2, 3})); + std::ifstream migratedInput(path, std::ios::binary); + const std::string migrated(std::istreambuf_iterator(migratedInput), {}); + Reject([&] { state->BindVotersFromTrustedBaseline({1, 2, 3}); }); + std::ifstream after(path, std::ios::binary); + assert_true(std::string(std::istreambuf_iterator(after), {}) == migrated); } void test_trusted_legacy_voter_binding_is_atomic_at_each_publish_stage(void**) { using TStage = TDurableState::EStage; From 4b3861c367a3736a3c84034492b1c9f4c36a55f5 Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Sat, 14 Nov 2026 18:54:21 -0500 Subject: [PATCH 53/66] Compare complete configuration state in history scans (#53) --- README.md | 2 ++ docs/configuration-history.md | 4 ++++ docs/configuration-scan-identity.md | 17 +++++++++++++++++ miniraft/configuration.h | 5 ++++- test/test_configuration.cpp | 17 +++++++++++++++++ 5 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 docs/configuration-scan-identity.md diff --git a/README.md b/README.md index 3ff9921..5f295ee 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,8 @@ The [configuration history validator](docs/configuration-history.md) checks an offline retained-log sequence against a trusted baseline or joint snapshot. The [configuration log scanner](docs/configuration-scan.md) validates a retained suffix at a caller-supplied commit boundary and at its end. +The [configuration state comparison follow-up](docs/configuration-scan-identity.md) +includes phase and voter sets when reporting an uncommitted configuration. The [online learner staging API](docs/online-learner-staging.md) lets embedders add and remove leader-local, non-voting replication targets. The [network learner follow-up](docs/online-learner-network.md) connects that diff --git a/docs/configuration-history.md b/docs/configuration-history.md index 1c71a6a..619bf2a 100644 --- a/docs/configuration-history.md +++ b/docs/configuration-history.md @@ -41,6 +41,10 @@ appropriate before supplying it. The validator does not prove commitment, cluste identity, promotion readiness, log completeness outside the supplied sequence or the correctness of the caller's trusted baseline. +The [alpha.53 follow-up](configuration-scan-identity.md) ensures that the scan's +uncommitted-configuration result compares phase and voter sets as well as +configuration index and term. + Joint snapshot metadata now rejects a configuration term different from the snapshot term when both indices are equal: one log index cannot have two terms. Earlier alpha.25 tooling could encode such structurally inconsistent v2 images; diff --git a/docs/configuration-scan-identity.md b/docs/configuration-scan-identity.md new file mode 100644 index 0000000..792f37f --- /dev/null +++ b/docs/configuration-scan-identity.md @@ -0,0 +1,17 @@ +# Compare full configuration state in history scans: v0.1.0-alpha.53 + +`TConfigurationScan::HasUncommittedConfiguration()` previously compared only +the configuration index and term. Two independently supplied histories with the +same identity but different voter sets or joint/stable phase could therefore be +reported as having no configuration difference. + +The result now compares the full effective configuration: phase, old and new +voter sets, configuration index, and configuration term. Different scan +boundaries alone do not count as a configuration change. Callers still supply +and substantiate the baseline and commit boundary; this helper does not prove +commitment or activate membership in Raft. + +Tests cover conflicting stable voter sets and a stable-versus-joint mismatch +with equal configuration index/term, plus the unchanged case. + +Self-review completed; no independent reviewer approval is claimed. diff --git a/miniraft/configuration.h b/miniraft/configuration.h index fb17eda..9fd3c3b 100644 --- a/miniraft/configuration.h +++ b/miniraft/configuration.h @@ -59,7 +59,10 @@ struct TConfigurationScan { TConfigurationHistory Latest; bool HasUncommittedConfiguration() const { return Committed.ConfigurationIndex() != Latest.ConfigurationIndex() || - Committed.ConfigurationTerm() != Latest.ConfigurationTerm(); + Committed.ConfigurationTerm() != Latest.ConfigurationTerm() || + Committed.IsJoint() != Latest.IsJoint() || + Committed.OldVoters() != Latest.OldVoters() || + Committed.NewVoters() != Latest.NewVoters(); } }; diff --git a/test/test_configuration.cpp b/test/test_configuration.cpp index 4d49baa..9e9acc5 100644 --- a/test/test_configuration.cpp +++ b/test/test_configuration.cpp @@ -103,6 +103,22 @@ void test_history_validates_joint_stable_chain_and_independent_rollback(void**) assert_true(pending.NewVoters() == joint.NewVoters); assert_false(prefix.IsJoint()); assert_int_equal(prefix.ThroughIndex(), 1); } +void test_configuration_scan_compares_full_configuration_state(void**) { + auto committed = TConfigurationHistory::Stable({1, 2, 3}, 7, 3, 9, 4); + auto differentVoters = TConfigurationHistory::Stable({1, 2, 4}, 7, 3, 9, 4); + TConfigurationScan voterMismatch{committed, differentVoters}; + assert_true(voterMismatch.HasUncommittedConfiguration()); + + TSnapshot jointSnapshot{9, 4, {1, 2, 3}, "opaque"}; + jointSnapshot.JointVoters = {3, 4, 5}; + jointSnapshot.ConfigurationIndex = 7; jointSnapshot.ConfigurationTerm = 3; + auto joint = TConfigurationHistory::FromJointSnapshot(jointSnapshot); + TConfigurationScan phaseMismatch{committed, joint}; + assert_true(phaseMismatch.HasUncommittedConfiguration()); + + TConfigurationScan unchanged{committed, committed}; + assert_false(unchanged.HasUncommittedConfiguration()); +} void test_history_from_joint_snapshot_requires_a_consistent_identity(void**) { TSnapshot snapshot{8, 4, {1, 2, 3}, "opaque"}; snapshot.JointVoters = {3, 4, 5}; snapshot.ConfigurationIndex = 5; snapshot.ConfigurationTerm = 3; @@ -445,6 +461,7 @@ int main() { const CMUnitTest tests[] = { cmocka_unit_test(test_configuration_snapshot_projection_and_history_resume), cmocka_unit_test(test_history_validates_joint_stable_chain_and_independent_rollback), + cmocka_unit_test(test_configuration_scan_compares_full_configuration_state), cmocka_unit_test(test_history_from_joint_snapshot_requires_a_consistent_identity), cmocka_unit_test(test_history_rejects_skips_wrong_predecessors_and_term_regression), cmocka_unit_test(test_scan_separates_commit_and_replaced_suffix), From 4ea7fc01fa04068ed3e05252c5b0580472abb2b8 Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Sat, 14 Nov 2026 19:18:54 -0500 Subject: [PATCH 54/66] Preserve stable snapshot identity in storage (#54) Protect stable configuration metadata across direct memory and durable snapshot installs. --- README.md | 2 + docs/stable-snapshot-storage-continuity.md | 25 ++++++++++++ miniraft/durable.cpp | 23 ++++++----- miniraft/raft.cpp | 14 +------ miniraft/snapshot.cpp | 17 ++++++++ miniraft/snapshot.h | 3 ++ miniraft/state.h | 23 ++++++----- test/test_snapshots.cpp | 46 ++++++++++++++++++++++ 8 files changed, 120 insertions(+), 33 deletions(-) create mode 100644 docs/stable-snapshot-storage-continuity.md diff --git a/README.md b/README.md index 5f295ee..be029b0 100644 --- a/README.md +++ b/README.md @@ -98,6 +98,8 @@ The [stable snapshot runtime follow-up](docs/stable-configuration-snapshot-runti carries that identity through restore, InstallSnapshot, and later compaction. The [metadata continuity follow-up](docs/stable-snapshot-metadata-continuity.md) preserves the known identity across legacy-format refreshes and rejects conflicts. +The [storage continuity follow-up](docs/stable-snapshot-storage-continuity.md) +applies the same identity rule to direct memory and durable snapshot installs. The [snapshot term validation fix](docs/snapshot-committed-terms.md) rejects snapshot boundaries that contradict the follower's committed prefix. The [message buffer follow-up](docs/message-extents.md) checks allocated buffer diff --git a/docs/stable-snapshot-storage-continuity.md b/docs/stable-snapshot-storage-continuity.md new file mode 100644 index 0000000..9122f83 --- /dev/null +++ b/docs/stable-snapshot-storage-continuity.md @@ -0,0 +1,25 @@ +# Stable snapshot storage continuity: v0.1.0-alpha.54 + +The stable snapshot identity checks previously lived in Raft's +`InstallSnapshot` receiver. Callers that used `TState::InstallSnapshot` or +`TDurableState::InstallSnapshot` directly could replace a known MRFTS003 +configuration identity with another stable identity, or refresh it with an +MRFTS001 image that erased the identity. + +The storage backends now apply the same continuity rule as the receiver. A +legacy refresh inherits the previously stored stable configuration index and +term before it is published. A stable image with a different identity is +rejected before the in-memory state or durable file changes. The receiver and +both backends share one helper, so future callers cannot bypass the rule by +installing snapshots below Raft. + +This preserves only the fixed-membership identity already trusted by storage. It +does not enable joint snapshot recovery, live membership changes, or membership +activation from an untrusted snapshot. Joint images remain explicit metadata and +are still available to the existing offline durable format tests. + +Tests install a legacy refresh and attempt a conflicting stable replacement +through both memory and durable storage. They check metadata preservation, +rejection atomicity, and durable recovery after restart. Release and ASan/UBSan +network-enabled CTest passed 21/21 each. This change was self-reviewed; no +independent reviewer approval is claimed. diff --git a/miniraft/durable.cpp b/miniraft/durable.cpp index ef68cb1..ef7c965 100644 --- a/miniraft/durable.cpp +++ b/miniraft/durable.cpp @@ -313,21 +313,24 @@ uint64_t TDurableState::RetainedLogFlags(int64_t index) const { std::shared_ptr TDurableState::GetSnapshot() const { return Impl->Snapshot; } void TDurableState::InstallSnapshot(const TSnapshot& snapshot) { - EncodeSnapshot(snapshot); - if (snapshot.HasConfigurationMetadata() && Impl->Voters.empty()) + ValidateSnapshot(snapshot); + TSnapshot image = snapshot; + PreserveStableSnapshotIdentity(Impl->Snapshot.get(), image); + ValidateSnapshot(image); + if (image.HasConfigurationMetadata() && Impl->Voters.empty()) throw std::invalid_argument("Configuration snapshots require a bound voter set"); - if (!Impl->Voters.empty() && snapshot.Voters != Impl->Voters) + if (!Impl->Voters.empty() && image.Voters != Impl->Voters) throw std::invalid_argument("Snapshot would change bound voters"); - if (snapshot.Index <= SnapshotIndex || snapshot.Term < SnapshotTerm || snapshot.Term > CurrentTerm) + if (image.Index <= SnapshotIndex || image.Term < SnapshotTerm || image.Term > CurrentTerm) throw std::invalid_argument("Obsolete or future durable snapshot"); - auto image = std::make_shared(snapshot); + auto saved = std::make_shared(std::move(image)); std::vector> suffix; - if (snapshot.Index <= LastLogIndex && LogTerm(snapshot.Index) == snapshot.Term) - suffix.assign(Impl->Log.begin() + (snapshot.Index - SnapshotIndex), Impl->Log.end()); + if (saved->Index <= LastLogIndex && LogTerm(saved->Index) == saved->Term) + suffix.assign(Impl->Log.begin() + (saved->Index - SnapshotIndex), Impl->Log.end()); Impl->Log.swap(suffix); - Impl->Snapshot = std::move(image); - SnapshotIndex = snapshot.Index; - SnapshotTerm = snapshot.Term; + Impl->Snapshot = std::move(saved); + SnapshotIndex = Impl->Snapshot->Index; + SnapshotTerm = Impl->Snapshot->Term; LastLogIndex = SnapshotIndex + Impl->Log.size(); LastLogTerm = Impl->Log.empty() ? SnapshotTerm : Impl->Log.back()->Term; Commit(); // Atomically publishes hard state, snapshot and the selected suffix. diff --git a/miniraft/raft.cpp b/miniraft/raft.cpp index 428c7e3..6db1017 100644 --- a/miniraft/raft.cpp +++ b/miniraft/raft.cpp @@ -1033,19 +1033,7 @@ void TRaft::OnInstallSnapshot(ITimeSource::Time now, TMessageHolderSnapshotIndex || snapshot.Term != request->SnapshotTerm || snapshot.Voters != GetVoterIds()) throw TMessageError("Snapshot metadata mismatch"); const auto knownSnapshot = State->GetSnapshot(); - if (knownSnapshot && knownSnapshot->StableConfiguration) { - if (snapshot.StableConfiguration && - (snapshot.ConfigurationIndex != knownSnapshot->ConfigurationIndex || - snapshot.ConfigurationTerm != knownSnapshot->ConfigurationTerm)) - throw TMessageError("Stable snapshot configuration identity changed"); - if (!snapshot.StableConfiguration) { - // A legacy-format refresh has no configuration identity. Keep - // the receiver's trusted identity when storing the new image. - snapshot.StableConfiguration = true; - snapshot.ConfigurationIndex = knownSnapshot->ConfigurationIndex; - snapshot.ConfigurationTerm = knownSnapshot->ConfigurationTerm; - } - } + PreserveStableSnapshotIdentity(knownSnapshot.get(), snapshot); ValidateSnapshot(snapshot); application->ValidateSnapshot(snapshot.Data, snapshot.Index); } catch (const std::exception&) { diff --git a/miniraft/snapshot.cpp b/miniraft/snapshot.cpp index f58b4b0..bc778f0 100644 --- a/miniraft/snapshot.cpp +++ b/miniraft/snapshot.cpp @@ -69,6 +69,23 @@ void Sync(int fd) { void ValidateSnapshot(const TSnapshot& snapshot) { Validate(snapshot); } +void PreserveStableSnapshotIdentity(const TSnapshot* current, TSnapshot& incoming) { + if (!current || !current->StableConfiguration) return; + if (incoming.StableConfiguration) { + if (incoming.ConfigurationIndex != current->ConfigurationIndex || + incoming.ConfigurationTerm != current->ConfigurationTerm) + throw std::invalid_argument("Stable snapshot configuration identity changed"); + return; + } + // A joint image carries an explicit new phase. A legacy image carries no + // configuration claim, so retain the already trusted stable identity. + if (!incoming.HasConfigurationMetadata()) { + incoming.StableConfiguration = true; + incoming.ConfigurationIndex = current->ConfigurationIndex; + incoming.ConfigurationTerm = current->ConfigurationTerm; + } +} + std::string EncodeSnapshot(const TSnapshot& snapshot) { Validate(snapshot); std::string out(snapshot.StableConfiguration ? StableMagic : snapshot.IsJoint() ? JointMagic : Magic); diff --git a/miniraft/snapshot.h b/miniraft/snapshot.h index b64e239..3d1429c 100644 --- a/miniraft/snapshot.h +++ b/miniraft/snapshot.h @@ -33,6 +33,9 @@ std::string EncodeSnapshot(const TSnapshot& snapshot); TSnapshot DecodeSnapshot(std::string_view bytes); // Validate metadata and size without copying the application image. void ValidateSnapshot(const TSnapshot& snapshot); +// Preserve a trusted stable configuration identity when a legacy image is +// refreshed, and reject a stable image that changes that identity. +void PreserveStableSnapshotIdentity(const TSnapshot* current, TSnapshot& incoming); TSnapshot CaptureSnapshot(const TRaft& raft, const IRsm& rsm); // Startup only, with an application behind the snapshot and // a matching local boundary/log. This can recover an app left behind by a crash diff --git a/miniraft/state.h b/miniraft/state.h index d8a2dc1..1765c88 100644 --- a/miniraft/state.h +++ b/miniraft/state.h @@ -156,21 +156,24 @@ struct TState: IState { } std::shared_ptr GetSnapshot() const override { return Snapshot; } void InstallSnapshot(const TSnapshot& snapshot) override { - EncodeSnapshot(snapshot); // Validate before changing any state. - if (snapshot.HasConfigurationMetadata() && BoundVoters.empty()) + ValidateSnapshot(snapshot); + TSnapshot image = snapshot; + PreserveStableSnapshotIdentity(Snapshot.get(), image); + ValidateSnapshot(image); // Validate before changing any state. + if (image.HasConfigurationMetadata() && BoundVoters.empty()) throw std::invalid_argument("Configuration snapshots require a bound voter set"); - if (!BoundVoters.empty() && snapshot.Voters != BoundVoters) + if (!BoundVoters.empty() && image.Voters != BoundVoters) throw std::invalid_argument("Snapshot would change bound voters"); - if (snapshot.Index <= SnapshotIndex || snapshot.Term < SnapshotTerm || snapshot.Term > CurrentTerm) + if (image.Index <= SnapshotIndex || image.Term < SnapshotTerm || image.Term > CurrentTerm) throw std::invalid_argument("Obsolete or future snapshot"); - auto image = std::make_shared(snapshot); + auto saved = std::make_shared(std::move(image)); std::vector> suffix; - if (snapshot.Index <= LastLogIndex && LogTerm(snapshot.Index) == snapshot.Term) - suffix.assign(Log.begin() + (snapshot.Index - SnapshotIndex), Log.end()); + if (saved->Index <= LastLogIndex && LogTerm(saved->Index) == saved->Term) + suffix.assign(Log.begin() + (saved->Index - SnapshotIndex), Log.end()); Log.swap(suffix); - Snapshot = std::move(image); - SnapshotIndex = snapshot.Index; - SnapshotTerm = snapshot.Term; + Snapshot = std::move(saved); + SnapshotIndex = Snapshot->Index; + SnapshotTerm = Snapshot->Term; LastLogIndex = SnapshotIndex + Log.size(); LastLogTerm = Log.empty() ? SnapshotTerm : Log.back()->Term; } diff --git a/test/test_snapshots.cpp b/test/test_snapshots.cpp index 129f67c..00a5af4 100644 --- a/test/test_snapshots.cpp +++ b/test/test_snapshots.cpp @@ -364,7 +364,51 @@ void test_startup_restores_stale_application_from_durable_snapshot_boundary(void assert_true(Read(*stale, "stale").empty()); assert_int_equal(raft->GetVolatileState()->CommitIndex, 2); } +void CheckStableSnapshotIdentityContinuity(IState& state) { + const std::vector voters{1, 2, 3}; + state.BindVoters(voters); + TSnapshot stable{2, 1, voters, "stable image"}; + stable.StableConfiguration = true; + stable.ConfigurationIndex = 1; + stable.ConfigurationTerm = 1; + state.InstallSnapshot(stable); + + TSnapshot legacy{3, 1, voters, "legacy refresh"}; + state.InstallSnapshot(legacy); + auto preserved = legacy; + preserved.StableConfiguration = true; + preserved.ConfigurationIndex = stable.ConfigurationIndex; + preserved.ConfigurationTerm = stable.ConfigurationTerm; + assert_true(*state.GetSnapshot() == preserved); + + auto conflicting = TSnapshot{4, 1, voters, "conflicting stable image"}; + conflicting.StableConfiguration = true; + conflicting.ConfigurationIndex = 2; + conflicting.ConfigurationTerm = 1; + const auto before = static_cast(state); + Reject([&] { state.InstallSnapshot(conflicting); }); + assert_true(static_cast(state) == before); + assert_true(*state.GetSnapshot() == preserved); +} +void test_memory_snapshot_storage_preserves_stable_configuration_identity(void**) { + TState state; + CheckStableSnapshotIdentityContinuity(state); +} #ifdef __linux__ +void test_durable_snapshot_storage_preserves_stable_configuration_identity(void**) { + TFiles files; + { + TDurableState state(files.Name, 1); + CheckStableSnapshotIdentityContinuity(state); + } + TDurableState recovered(files.Name, 1); + auto snapshot = recovered.GetSnapshot(); + assert_true(snapshot && snapshot->StableConfiguration); + assert_int_equal(snapshot->Index, 3); + assert_int_equal(snapshot->ConfigurationIndex, 1); + assert_int_equal(snapshot->ConfigurationTerm, 1); + assert_true(snapshot->Data == "legacy refresh"); +} void test_stable_configuration_snapshot_durable_runtime_recovery_and_compaction(void**) { TFiles files; TKv source; Apply(source, "a", "one", 1); Apply(source, "b", "two", 2); @@ -586,7 +630,9 @@ int main() { cmocka_unit_test(test_capture_requires_committed_applied_progress), cmocka_unit_test(test_restore_rejects_mismatched_log_configuration_and_used_application), cmocka_unit_test(test_startup_restores_stale_application_from_durable_snapshot_boundary), + cmocka_unit_test(test_memory_snapshot_storage_preserves_stable_configuration_identity), #ifdef __linux__ + cmocka_unit_test(test_durable_snapshot_storage_preserves_stable_configuration_identity), cmocka_unit_test(test_stable_configuration_snapshot_durable_runtime_recovery_and_compaction), cmocka_unit_test(test_unbound_durable_configuration_snapshot_recovery_fails_closed), cmocka_unit_test(test_joint_snapshot_export_and_durable_roundtrip_but_runtime_rejects), From 83dff780db4841e18400d7d8d12985790811a240 Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Sat, 14 Nov 2026 19:43:31 -0500 Subject: [PATCH 55/66] Check stable snapshot identity during checkpoint restore (#55) Reject standalone application checkpoints that conflict with authoritative stable snapshot identity. --- README.md | 2 + docs/stable-snapshot-checkpoint-continuity.md | 21 ++++++++++ miniraft/snapshot.cpp | 12 ++++-- test/test_snapshots.cpp | 38 +++++++++++++++++++ 4 files changed, 70 insertions(+), 3 deletions(-) create mode 100644 docs/stable-snapshot-checkpoint-continuity.md diff --git a/README.md b/README.md index be029b0..614276e 100644 --- a/README.md +++ b/README.md @@ -100,6 +100,8 @@ The [metadata continuity follow-up](docs/stable-snapshot-metadata-continuity.md) preserves the known identity across legacy-format refreshes and rejects conflicts. The [storage continuity follow-up](docs/stable-snapshot-storage-continuity.md) applies the same identity rule to direct memory and durable snapshot installs. +The [checkpoint restore follow-up](docs/stable-snapshot-checkpoint-continuity.md) +rejects standalone application checkpoints with a conflicting stable identity. The [snapshot term validation fix](docs/snapshot-committed-terms.md) rejects snapshot boundaries that contradict the follower's committed prefix. The [message buffer follow-up](docs/message-extents.md) checks allocated buffer diff --git a/docs/stable-snapshot-checkpoint-continuity.md b/docs/stable-snapshot-checkpoint-continuity.md new file mode 100644 index 0000000..35df207 --- /dev/null +++ b/docs/stable-snapshot-checkpoint-continuity.md @@ -0,0 +1,21 @@ +# Stable identity checks for checkpoint restore: v0.1.0-alpha.55 + +`RestoreSnapshot` can load a standalone checkpoint into the application when its +index and term match the local retained log. It previously checked the fixed +voter list but did not compare an MRFTS003 checkpoint's stable configuration +identity with the authoritative snapshot already stored by Raft. A checkpoint +with a conflicting configuration index or term could therefore replace +application state while Raft retained a different configuration baseline. + +Restore now rejects an explicit stable identity that conflicts with the local +authoritative snapshot before validating or loading application data. Legacy +checkpoints remain usable because they make no stable identity claim. Joint +snapshots remain rejected by the fixed-membership runtime. + +The regression test uses a checkpoint whose index, term and voter set match the +retained state but whose stable identity conflicts. It verifies rejection leaves +the application unchanged, then verifies a legacy checkpoint at the same +boundary can still restore. + +Release and ASan/UBSan network-enabled CTest each passed 21/21. This change is +self-reviewed; no independent reviewer approval is claimed. diff --git a/miniraft/snapshot.cpp b/miniraft/snapshot.cpp index bc778f0..da96de3 100644 --- a/miniraft/snapshot.cpp +++ b/miniraft/snapshot.cpp @@ -41,6 +41,12 @@ void Validate(const TSnapshot& snapshot) { validateVoters(snapshot.JointVoters); } } +void CheckStableSnapshotIdentity(const TSnapshot* current, const TSnapshot& incoming) { + if (current && current->StableConfiguration && incoming.StableConfiguration && + (incoming.ConfigurationIndex != current->ConfigurationIndex || + incoming.ConfigurationTerm != current->ConfigurationTerm)) + throw std::invalid_argument("Stable snapshot configuration identity changed"); +} void CheckLocalLog(const TSnapshot& snapshot, const TRaft& raft) { Validate(snapshot); if (snapshot.IsJoint() || snapshot.Voters != raft.GetVoterIds() || snapshot.Index > raft.GetLastIndex() || @@ -71,10 +77,8 @@ void ValidateSnapshot(const TSnapshot& snapshot) { Validate(snapshot); } void PreserveStableSnapshotIdentity(const TSnapshot* current, TSnapshot& incoming) { if (!current || !current->StableConfiguration) return; + CheckStableSnapshotIdentity(current, incoming); if (incoming.StableConfiguration) { - if (incoming.ConfigurationIndex != current->ConfigurationIndex || - incoming.ConfigurationTerm != current->ConfigurationTerm) - throw std::invalid_argument("Stable snapshot configuration identity changed"); return; } // A joint image carries an explicit new phase. A legacy image carries no @@ -158,6 +162,8 @@ TSnapshot CaptureSnapshot(const TRaft& raft, const IRsm& rsm) { } void RestoreSnapshot(const TSnapshot& snapshot, const TRaft& raft, IRsm& rsm) { + const auto knownSnapshot = raft.GetState()->GetSnapshot(); + CheckStableSnapshotIdentity(knownSnapshot.get(), snapshot); CheckLocalLog(snapshot, raft); if (rsm.LastAppliedIndex >= snapshot.Index) throw std::runtime_error("Application is not behind the snapshot"); diff --git a/test/test_snapshots.cpp b/test/test_snapshots.cpp index 00a5af4..0307cd3 100644 --- a/test/test_snapshots.cpp +++ b/test/test_snapshots.cpp @@ -349,6 +349,43 @@ void test_restore_rejects_mismatched_log_configuration_and_used_application(void Reject([&] { RestoreSnapshot(snapshot, raft, restored); }); assert_true(Read(restored, "a") == "one"); } +void test_restore_rejects_checkpoint_with_conflicting_stable_identity(void**) { + const std::vector voters{1, 2, 3}; + auto state = std::make_shared(); + state->BindVoters(voters); + TSnapshot authoritative{2, 1, voters, "authoritative"}; + authoritative.StableConfiguration = true; + authoritative.ConfigurationIndex = 1; + authoritative.ConfigurationTerm = 1; + state->InstallSnapshot(authoritative); + state->CurrentTerm = 2; + auto entry = NewHoldedMessage(); entry->Term = 2; + state->Append(entry); + TRaft raft(state, 1, {{2, std::make_shared()}, {3, std::make_shared()}}); + const_cast(raft.GetVolatileState())->CommitIndex = 3; + + TKv source; Apply(source, "a", "checkpoint value", 3); + TSnapshot checkpoint{3, 2, voters, source.SaveSnapshot()}; + checkpoint.StableConfiguration = true; + for (int mode = 0; mode < 2; ++mode) { + auto conflicting = checkpoint; + if (mode == 0) conflicting.ConfigurationIndex = 2; + else conflicting.ConfigurationTerm = 2; + TKv rejected; + Reject([&] { RestoreSnapshot(conflicting, raft, rejected); }); + assert_int_equal(rejected.LastAppliedIndex, 0); + assert_true(Read(rejected, "a").empty()); + } + + // Legacy checkpoints contain no identity; the local trusted identity is + // still enough to accept their application image at a matching boundary. + checkpoint.StableConfiguration = false; + checkpoint.ConfigurationIndex = checkpoint.ConfigurationTerm = 0; + TKv legacy; + RestoreSnapshot(checkpoint, raft, legacy); + assert_int_equal(legacy.LastAppliedIndex, 3); + assert_true(Read(legacy, "a") == "checkpoint value"); +} void test_startup_restores_stale_application_from_durable_snapshot_boundary(void**) { auto state = std::make_shared(); TKv source; Apply(source, "snapshot", "committed", 2); @@ -629,6 +666,7 @@ int main() { cmocka_unit_test(test_kv_invalid_snapshot_preserves_existing_state_and_progress), cmocka_unit_test(test_capture_requires_committed_applied_progress), cmocka_unit_test(test_restore_rejects_mismatched_log_configuration_and_used_application), + cmocka_unit_test(test_restore_rejects_checkpoint_with_conflicting_stable_identity), cmocka_unit_test(test_startup_restores_stale_application_from_durable_snapshot_boundary), cmocka_unit_test(test_memory_snapshot_storage_preserves_stable_configuration_identity), #ifdef __linux__ From 55eeecaa3e0d0e2423b00afe8c2df932309de3a8 Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Sat, 14 Nov 2026 20:46:15 -0500 Subject: [PATCH 56/66] Persist Raft commit index in durable state (#56) --- docs/compaction.md | 31 +++--- docs/durable-configuration-snapshots.md | 9 +- docs/durable-state.md | 37 ++++++-- docs/learners.md | 24 ++--- docs/trusted-legacy-voter-binding.md | 5 +- miniraft/durable.cpp | 57 ++++++++++-- miniraft/durable.h | 1 + miniraft/persist.cpp | 2 + miniraft/raft.cpp | 47 ++++++++-- miniraft/raft.h | 2 + miniraft/state.h | 12 +++ test/smoke_kv_checkpoints.py | 4 + test/test_compaction.cpp | 40 +++++++- test/test_durable.cpp | 119 +++++++++++++++++++++++- test/test_learners.cpp | 30 +++--- test/test_snapshots.cpp | 9 +- 16 files changed, 346 insertions(+), 83 deletions(-) diff --git a/docs/compaction.md b/docs/compaction.md index 7ebdf41..f2a73ad 100644 --- a/docs/compaction.md +++ b/docs/compaction.md @@ -1,7 +1,8 @@ # Compaction and InstallSnapshot: v0.1.0-alpha.16 -The [learner follow-up](learners.md) adds non-voting replication and v3 durable -voter binding while retaining backward decoding of the v1/v2 formats below. +The [learner follow-up](learners.md) adds non-voting replication and durable +voter binding. Current MRFTD004 images also persist commit progress; the reader +continues to accept the earlier v1/v2/v3 images below. This milestone adds prefix deletion and follower state transfer to the alpha.15 KV checkpoint foundation. TState and Linux TDurableState support compacted logs; @@ -22,7 +23,7 @@ Old binaries cannot read compacted durable images or process InstallSnapshot RPC The meaning of `--snapshots` changes from checkpoint-only to actual compaction. Mixed-version operation is unsupported. Existing v1 durable files and alpha.15 checkpoints can be read without destructive migration. The first compaction or -remote installation writes v2; downgrading that store is unsupported. Fixed voter +remote installation writes the current durable format; downgrading that store is unsupported. Fixed voter IDs must match the embedded snapshot. Static `--node` edits remain unsafe as a membership-change mechanism. @@ -63,16 +64,24 @@ does not install an image at or below known commitment, so stale snapshots canno rewind application state. Retained suffix entries are applied only after a leader establishes their commitment. -TDurableState v2 (`MRFTD002`) publishes hard state, the complete encoded snapshot -and suffix in one atomic image, using the existing temp-write/file-fsync/rename/ -directory-fsync protocol. Its header remains magic[8], payload length[8], CRC[4]. -The payload is CurrentTerm[8], VotedFor[4], retained count[8], snapshot length[8], -the existing MRFTS001 image, then the existing term/flags/length/data records. -The v2 CRC covers the version magic plus payload; v1's checksum rules are retained -for backward reading. Recovery bounds the embedded length, validates both CRCs, -the absolute count range and nondecreasing terms beginning at SnapshotTerm. +MRFTD004 publishes hard state, the persisted commit boundary, voter binding, +complete encoded snapshot and suffix in one atomic image, using the existing +temp-write/file-fsync/rename/directory-fsync protocol. Its header remains +magic[8], payload length[8], CRC[4]. The payload is CurrentTerm[8], VotedFor[4], +retained count[8], CommitIndex[8], voter count[4], voter IDs, snapshot length[8], +the encoded snapshot, then term/flags/length/data records. The CRC covers the v4 +magic plus payload. Recovery checks `SnapshotIndex <= CommitIndex <= LastLogIndex`, +bounds the embedded image and retained count, and validates nondecreasing terms +beginning at SnapshotTerm. Older v1/v2/v3 images remain readable and recover +their commit floor as SnapshotIndex; a successful later publication writes v4. Malformed recovery does not rewrite the image. Live publication failure is fatal. +Followers publish a newly learned commit index before acknowledging +AppendEntries. Leaders publish quorum commit advancement before application +processing. Snapshot installation commits at least the included boundary, and +storage refuses to truncate a committed entry. The legacy TDiskState backend +does not persist CommitIndex. + ## Transfer protocol The previously reserved message types 6/7 now carry InstallSnapshot requests and diff --git a/docs/durable-configuration-snapshots.md b/docs/durable-configuration-snapshots.md index 3ea27f7..780d221 100644 --- a/docs/durable-configuration-snapshots.md +++ b/docs/durable-configuration-snapshots.md @@ -12,10 +12,11 @@ log, then persists the binding before Raft startup. The API does not establish that the caller's assertion is true; absent an independent trusted baseline, the store still cannot infer its voter set. -The durable file layout is unchanged. The backend persists the snapshot together -with its existing voter binding; it does not reinterpret that binding from the -metadata. The history scanner also verifies that a joint baseline's old voters -match the bound set. +The current MRFTD004 durable format persists the snapshot, commit boundary and +existing voter binding in the same atomic image; it does not reinterpret that +binding from snapshot metadata. Older durable formats remain readable and use +their snapshot boundary as the conservative commit floor. The history scanner +also verifies that a joint baseline's old voters match the bound set. In alpha.45 these images were persistence for offline validation only. The alpha.46 follow-up enables runtime recovery and transfer of MRFTS003 stable diff --git a/docs/durable-state.md b/docs/durable-state.md index 73d270a..7a01151 100644 --- a/docs/durable-state.md +++ b/docs/durable-state.md @@ -1,5 +1,28 @@ # Linux durable state: v0.1.0-alpha.3 +## Commit recovery: MRFTD004 + +The current durable image stores `CommitIndex` alongside the current term, vote, +voter binding, snapshot and retained log. Recovery requires +`SnapshotIndex <= CommitIndex <= LastLogIndex`. Raft publishes a commit advance +before it acknowledges the corresponding AppendEntries request or exposes the +new boundary to application processing. Restart therefore replays committed +entries even when the application checkpoint trails them. Installed snapshots +advance the persisted commit floor to at least their included index, and storage +rejects truncation of committed entries. + +MRFTD004 keeps the 20-byte file header: magic[8], payload length[8] and CRC[4]. +Its payload is CurrentTerm[8], VotedFor[4], retained-log count[8], +CommitIndex[8], voter count[4], sorted voter IDs[4 each], snapshot length[8], +optional encoded snapshot, then retained log records. The CRC covers the v4 magic +and payload. MRFTD001, MRFTD002 and MRFTD003 remain readable; their commit floor +is conservatively initialized to SnapshotIndex, since those formats did not +record commit progress. Their next successful publication writes MRFTD004. + +This closes commit-boundary recovery for `TDurableState`; it does not implement +live membership changes or joint consensus. The legacy `TDiskState` format does +not persist this field and cannot provide the same recovery guarantee. + TDurableState is an opt-in storage backend with synchronous durable publication. It closes the missing file-sync and atomic-publication gap for new stores, under the filesystem assumptions below. The legacy TDiskState backend is unchanged @@ -31,10 +54,10 @@ Startup errors are exceptions and do not fall back to older/temp images. ## Format and ownership -The image has version magic, payload length and CRC32 followed by term, vote, -entry count and length-prefixed log entries. Recovery checks the checksum before -decoding entries and validates bounds and term ordering. CRC32 detects accidental -corruption but does not protect against malicious modification or all collisions. +The image has version magic, payload length and CRC32. Recovery checks the +checksum before decoding entries and validates bounds, commit metadata, snapshot +identity and term ordering. CRC32 detects accidental corruption but does not +protect against malicious modification or all collisions. The files are NAME.durable.ID, NAME.durable.ID.tmp and NAME.durable.ID.lock. A nonblocking advisory flock holds the lock for the object's lifetime. Use one @@ -68,9 +91,9 @@ TDurableState construction throws an explicit unsupported-platform exception. ## Validation and self-review -Twelve new tests cover publication ordering, 20 process-crash boundaries across -append/truncation/vote/replacement operations, four immediate exits after returned -updates, five interrupted initializations, a live injected failure, an actual +Tests cover publication ordering, process-crash boundaries across +append/truncation/vote/replacement/commit updates, immediate exits after returned +updates, interrupted initializations, a live injected failure, an actual write error using a child-only file-size limit, corrupted/truncated images, single-writer exclusion, missing images, legacy-store refusal, entry ownership and Raft election/application replay across a storage restart. diff --git a/docs/learners.md b/docs/learners.md index 222932a..1090d4b 100644 --- a/docs/learners.md +++ b/docs/learners.md @@ -90,19 +90,21 @@ all-voter operation retains its previous startup contract until that proof exist A standalone alpha.15 checkpoint alone is not an embedded configuration witness. Never delete voter state or edit votes/terms to work around this validation. -The new durable format `MRFTD003` retains the 20-byte header and stores: -CurrentTerm[8], VotedFor[4], retained-log count[8], voter count[4], sorted voter -IDs[4 each], snapshot length[8], optional MRFTS001, MRFTS002, or MRFTS003 image, -then retained log records. -An absent snapshot uses length zero. The CRC covers version magic plus payload. -Recovery bounds and validates the voter list and requires any snapshot to name -the same voters. Both v1 and v2 remain readable; the first binding writes v3 using -the existing atomic write/fsync/rename/directory-fsync protocol. A crash before -first binding leaves either a fresh unbound store with no votes/log, or a complete -bound store. Publication failure is fatal before the node can serve traffic. +Current durable images use `MRFTD004` and retain the 20-byte header. The payload +stores CurrentTerm[8], VotedFor[4], retained-log count[8], CommitIndex[8], voter +count[4], sorted voter IDs[4 each], snapshot length[8], optional MRFTS001, +MRFTS002, or MRFTS003 image, then retained log records. The CRC covers version +magic plus payload. Recovery bounds and validates the voter list, commit +boundary and requires any snapshot to name the same voters. MRFTD001 through +MRFTD003 remain readable; formats without commit metadata conservatively use +SnapshotIndex as their commit floor. The next successful publication writes v4 +using the existing atomic write/fsync/rename/directory-fsync protocol. A crash +before first binding leaves either a fresh unbound store with no votes/log, or a +complete bound store. Publication failure is fatal before the node can serve +traffic. Upgrade all voters before configuring any learners. Older binaries treat every -peer as a voter and cannot read v3 stores; mixed-role/mixed-version operation is +peer as a voter and cannot read v4 stores; mixed-role/mixed-version operation is unsupported. All fresh bootstrap configurations must agree. Persistent binding does not authenticate initial bootstrap choices or wrong-store copies. No cluster UUID, role negotiation or automatic migration from unknown membership is added. diff --git a/docs/trusted-legacy-voter-binding.md b/docs/trusted-legacy-voter-binding.md index 37dd2cf..b5db0f0 100644 --- a/docs/trusted-legacy-voter-binding.md +++ b/docs/trusted-legacy-voter-binding.md @@ -21,8 +21,9 @@ The migration applies to `TDurableState`; legacy `TDiskState` remains unable to persist voter bindings. Since alpha.52, Raft rejects recovered log-only durable stores with no binding, so this migration must be completed before startup. -Tests cover MRFTD001 log-only and MRFTD002 snapshot-backed migration, rejection +Tests cover MRFTD001, MRFTD002 and MRFTD003 recovery, rejection of inconsistent vote/snapshot baselines and configuration log entries without -rewriting the source file, and recovered MRFTD003 binding after restart. +rewriting the source file, and the current MRFTD004 binding after publication +and restart. Legacy formats recover their commit floor at the snapshot boundary. Self-review completed. No independent reviewer approval is claimed. diff --git a/miniraft/durable.cpp b/miniraft/durable.cpp index ef7c965..c4f5a48 100644 --- a/miniraft/durable.cpp +++ b/miniraft/durable.cpp @@ -20,6 +20,7 @@ using TBytes = std::vector; constexpr std::array Magic{'M','R','F','T','D','0','0','1'}; constexpr std::array SnapshotMagic{'M','R','F','T','D','0','0','2'}; constexpr std::array ConfigurationMagic{'M','R','F','T','D','0','0','3'}; +constexpr std::array CommitMagic{'M','R','F','T','D','0','0','4'}; void Put(TBytes& bytes, uint64_t value, unsigned width) { for (unsigned i = 0; i < width; ++i) bytes.push_back(uint8_t(value >> (8 * i))); } @@ -141,9 +142,10 @@ TDurableState::TDurableState(const std::string& name, uint32_t id, THook hook) return; } const auto bytes = ReadAll(file.Value); + const bool commitAware = std::equal(CommitMagic.begin(), CommitMagic.end(), bytes.begin()); const bool configured = std::equal(ConfigurationMagic.begin(), ConfigurationMagic.end(), bytes.begin()); const bool compacted = std::equal(SnapshotMagic.begin(), SnapshotMagic.end(), bytes.begin()); - const auto* version = configured ? &ConfigurationMagic : compacted ? &SnapshotMagic : nullptr; + const auto* version = commitAware ? &CommitMagic : configured ? &ConfigurationMagic : compacted ? &SnapshotMagic : nullptr; if (!version && !std::equal(Magic.begin(), Magic.end(), bytes.begin())) throw std::runtime_error("Unknown durable state format"); size_t position = Magic.size(); @@ -154,6 +156,7 @@ TDurableState::TDurableState(const std::string& name, uint32_t id, THook hook) CurrentTerm = Take(bytes, position, 8); VotedFor = Take(bytes, position, 4); const auto count = Take(bytes, position, 8); + if (commitAware) CommitIndex = Take(bytes, position, 8); if (configured) { const auto voters = Take(bytes, position, 4); if (!voters || voters > TSnapshot::MaxVoters || voters > (bytes.size() - position) / 4) @@ -162,7 +165,32 @@ TDurableState::TDurableState(const std::string& name, uint32_t id, THook hook) try { ValidateFixedVoters(Impl->Voters); } catch (const std::invalid_argument&) { throw std::runtime_error("Invalid durable voter configuration"); } } - if (compacted || configured) { + if (commitAware) { + const auto voters = Take(bytes, position, 4); + if (voters > TSnapshot::MaxVoters || voters > (bytes.size() - position) / 4) + throw std::runtime_error("Invalid durable voter count"); + for (uint32_t i = 0; i < voters; ++i) Impl->Voters.push_back(Take(bytes, position, 4)); + if (voters) { + try { ValidateFixedVoters(Impl->Voters); } + catch (const std::invalid_argument&) { throw std::runtime_error("Invalid durable voter configuration"); } + } + const auto length = Take(bytes, position, 8); + if (length > TSnapshot::MaxBytes || length > bytes.size() - position) + throw std::runtime_error("Invalid embedded snapshot size"); + if (length) { + auto snapshot = DecodeSnapshot(std::string_view(reinterpret_cast(bytes.data() + position), size_t(length))); + if (snapshot.HasConfigurationMetadata() && Impl->Voters.empty()) + throw std::runtime_error("Configuration snapshot recovery requires a bound voter set"); + if (snapshot.Term > CurrentTerm) throw std::runtime_error("Future durable snapshot term"); + if (!Impl->Voters.empty() && snapshot.Voters != Impl->Voters) + throw std::runtime_error("Durable snapshot/voter mismatch"); + position += size_t(length); + SnapshotIndex = snapshot.Index; + SnapshotTerm = snapshot.Term; + LastLogTerm = SnapshotTerm; + Impl->Snapshot = std::make_shared(std::move(snapshot)); + } + } else if (compacted || configured) { const auto length = Take(bytes, position, 8); if (length > TSnapshot::MaxBytes || length > bytes.size() - position) throw std::runtime_error("Invalid embedded snapshot size"); @@ -180,6 +208,7 @@ TDurableState::TDurableState(const std::string& name, uint32_t id, THook hook) Impl->Snapshot = std::make_shared(std::move(snapshot)); } } + if (!commitAware) CommitIndex = SnapshotIndex; if (!CurrentTerm || count > uint64_t(INT64_MAX) - SnapshotIndex || count > (bytes.size() - position) / 20) throw std::runtime_error("Invalid durable state metadata"); for (uint64_t i = 0; i < count; ++i) { @@ -199,6 +228,8 @@ TDurableState::TDurableState(const std::string& name, uint32_t id, THook hook) } if (position != bytes.size()) throw std::runtime_error("Trailing durable state data"); LastLogIndex = SnapshotIndex + count; + if (CommitIndex < SnapshotIndex || CommitIndex > LastLogIndex) + throw std::runtime_error("Invalid durable commit index"); // A crash after rename but before directory fsync may leave a visible new // image. Stabilize the selected image and its name before serving requests. Sync(file.Value); @@ -216,6 +247,7 @@ void TDurableState::Publish() throw std::runtime_error("TDurableState currently requires Linux"); #else if (!CurrentTerm || LastLogIndex > uint64_t(INT64_MAX) || LastLogIndex != SnapshotIndex + Impl->Log.size() || + CommitIndex < SnapshotIndex || CommitIndex > LastLogIndex || LastLogTerm != (Impl->Log.empty() ? SnapshotTerm : Impl->Log.back()->Term) || (Impl->Snapshot ? (Impl->Snapshot->Index != SnapshotIndex || Impl->Snapshot->Term != SnapshotTerm || SnapshotTerm > CurrentTerm) : (SnapshotIndex != 0 || SnapshotTerm != 0))) @@ -224,18 +256,17 @@ void TDurableState::Publish() Put(payload, CurrentTerm, 8); Put(payload, VotedFor, 4); Put(payload, Impl->Log.size(), 8); + Put(payload, CommitIndex, 8); + Put(payload, Impl->Voters.size(), 4); if (!Impl->Voters.empty()) { ValidateFixedVoters(Impl->Voters); if (Impl->Snapshot && Impl->Snapshot->Voters != Impl->Voters) throw std::runtime_error("Snapshot would change bound voters"); - Put(payload, Impl->Voters.size(), 4); for (auto id : Impl->Voters) Put(payload, id, 4); } - if (Impl->Snapshot || !Impl->Voters.empty()) { - const auto image = Impl->Snapshot ? EncodeSnapshot(*Impl->Snapshot) : std::string{}; - Put(payload, image.size(), 8); - payload.insert(payload.end(), image.begin(), image.end()); - } + const auto image = Impl->Snapshot ? EncodeSnapshot(*Impl->Snapshot) : std::string{}; + Put(payload, image.size(), 8); + payload.insert(payload.end(), image.begin(), image.end()); uint64_t previousTerm = SnapshotTerm; for (auto entry : Impl->Log) { if (!entry->Term || entry->Term < previousTerm || entry->Term > CurrentTerm) @@ -246,10 +277,10 @@ void TDurableState::Publish() Put(payload, entry->Len - sizeof(TLogEntry), 4); payload.insert(payload.end(), entry->Data, entry->Data + entry->Len - sizeof(TLogEntry)); } - const auto& magic = !Impl->Voters.empty() ? ConfigurationMagic : Impl->Snapshot ? SnapshotMagic : Magic; + const auto& magic = CommitMagic; TBytes bytes(magic.begin(), magic.end()); Put(bytes, payload.size(), 8); - Put(bytes, Checksum(payload, 0, !Impl->Voters.empty() ? &ConfigurationMagic : Impl->Snapshot ? &SnapshotMagic : nullptr), 4); + Put(bytes, Checksum(payload, 0, &CommitMagic), 4); bytes.insert(bytes.end(), payload.begin(), payload.end()); TFd temp(openat(Impl->Directory.Value, Impl->Temp.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC | O_NOFOLLOW, 0600)); @@ -288,6 +319,8 @@ void TDurableState::Append(TMessageHolder entry) void TDurableState::RemoveLast() { if (Impl->Log.empty()) return; + if (LastLogIndex <= CommitIndex) + throw std::logic_error("Cannot remove a committed log entry"); Impl->Log.pop_back(); LastLogIndex = SnapshotIndex + Impl->Log.size(); LastLogTerm = Impl->Log.empty() ? SnapshotTerm : Impl->Log.back()->Term; @@ -323,6 +356,9 @@ void TDurableState::InstallSnapshot(const TSnapshot& snapshot) { throw std::invalid_argument("Snapshot would change bound voters"); if (image.Index <= SnapshotIndex || image.Term < SnapshotTerm || image.Term > CurrentTerm) throw std::invalid_argument("Obsolete or future durable snapshot"); + if (image.Index <= CommitIndex && + (image.Index > LastLogIndex || LogTerm(image.Index) != image.Term)) + throw std::invalid_argument("Snapshot would discard committed log entries"); auto saved = std::make_shared(std::move(image)); std::vector> suffix; if (saved->Index <= LastLogIndex && LogTerm(saved->Index) == saved->Term) @@ -333,6 +369,7 @@ void TDurableState::InstallSnapshot(const TSnapshot& snapshot) { SnapshotTerm = Impl->Snapshot->Term; LastLogIndex = SnapshotIndex + Impl->Log.size(); LastLogTerm = Impl->Log.empty() ? SnapshotTerm : Impl->Log.back()->Term; + CommitIndex = std::max(CommitIndex, SnapshotIndex); Commit(); // Atomically publishes hard state, snapshot and the selected suffix. } diff --git a/miniraft/durable.h b/miniraft/durable.h index 033a7be..3c5891c 100644 --- a/miniraft/durable.h +++ b/miniraft/durable.h @@ -22,6 +22,7 @@ class TDurableState : public IState { uint64_t RetainedLogTerm(int64_t index) const override; uint64_t RetainedLogFlags(int64_t index) const override; void Commit() override; + bool SupportsCommitIndexPersistence() const override { return true; } bool SupportsSnapshots() const override { return true; } std::shared_ptr GetSnapshot() const override; void InstallSnapshot(const TSnapshot& snapshot) override; diff --git a/miniraft/persist.cpp b/miniraft/persist.cpp index 4c44cbd..e225190 100644 --- a/miniraft/persist.cpp +++ b/miniraft/persist.cpp @@ -123,6 +123,8 @@ void TDiskState::Write(TMessageHolder message) void TDiskState::RemoveLast() { if (LastLogIndex > 0) { + if (LastLogIndex <= CommitIndex) + throw std::logic_error("Cannot remove a committed log entry"); const auto previousTerm = LastLogIndex == 1 ? 0 : Get(LastLogIndex - 1)->Term; --LastLogIndex; LastLogTerm = previousTerm; diff --git a/miniraft/raft.cpp b/miniraft/raft.cpp index 6db1017..a21c325 100644 --- a/miniraft/raft.cpp +++ b/miniraft/raft.cpp @@ -260,8 +260,10 @@ TRaft::TRaft(std::shared_ptr state, int node, const TNodeDict& nodes, st if (!Learners.empty() && !State->SupportsVoterBinding()) throw std::invalid_argument("Learners require voter-binding-capable storage"); const auto bound = State->GetBoundVoters(); - if (State->SnapshotIndex > State->LastLogIndex || State->LastLogIndex > uint64_t(INT64_MAX)) + if (State->SnapshotIndex > State->LastLogIndex || State->LastLogIndex > uint64_t(INT64_MAX) || + State->CommitIndex > State->LastLogIndex) throw std::invalid_argument("Invalid retained log bounds"); + const bool commitFloorAdded = State->CommitIndex < State->SnapshotIndex; const auto snapshot = State->GetSnapshot(); if (snapshot) { ValidateSnapshot(*snapshot); @@ -297,9 +299,11 @@ TRaft::TRaft(std::shared_ptr state, int node, const TNodeDict& nodes, st const bool fresh = State->CurrentTerm == 1 && !State->VotedFor && !State->LastLogIndex; if (State->SupportsVoterBinding() && (!bound.empty() || fresh || snapshot || !Learners.empty())) State->BindVoters(Voters); - if (snapshot) { - VolatileState->CommitIndex = State->SnapshotIndex; + if (commitFloorAdded) { + State->CommitIndex = State->SnapshotIndex; + State->Commit(); } + VolatileState->CommitIndex = State->CommitIndex; State->MarkRaftStarted(); } @@ -396,9 +400,9 @@ void TRaft::OnAppendEntries(ITimeSource::Time now, TMessageHolderSrc, .Term = State->CurrentTerm, .Seqno = message->Seqno}, TAppendEntriesResponse {.MatchIndex = matchIndex, .Success = success}); + AdvanceCommitIndex(commitIndex); (*VolatileState) .SetLeaderId(message->LeaderId) - .SetCommitIndex(commitIndex) .SetElectionDue(MakeElection(now)); Become(EState::FOLLOWER); @@ -434,8 +438,8 @@ void TRaft::OnAppendEntries(TMessageHolder message) { .SetNextIndex(nodeId, matchIndex+1) .SetRpcDue(nodeId, ITimeSource::Time{}) .SetBatchSize(nodeId, 1024) - .SetBackOff(nodeId, 1) - .CommitAdvance(*VotingQuorum, Id, *State); + .SetBackOff(nodeId, 1); + AdvanceLeaderCommit(); } else if (message->MatchIndex > 0) { // A compacted-prefix hint is not replication evidence. Probe its boundary // with AppendEntries before advancing MatchIndex or confirming reads. @@ -701,7 +705,7 @@ void TRaft::LeaderTimeout(ITimeSource::Time now) { } if (Nservers == 1) { - VolatileState->CommitAdvance(*VotingQuorum, Id, *State); + AdvanceLeaderCommit(); } } @@ -890,6 +894,7 @@ TSnapshot TRaft::Compact() { throw std::runtime_error("Compaction deferred while snapshot transfers pin multiple generations"); auto snapshot = CaptureSnapshot(*this, *application); application->ValidateSnapshot(snapshot.Data, snapshot.Index); + AdvanceCommitIndex(VolatileState->CommitIndex); State->InstallSnapshot(snapshot); // Outstanding transfers own their immutable encoded bytes; discard any // otherwise-retained encoding of the old local snapshot. @@ -944,7 +949,8 @@ void TRaft::OnInstallSnapshot(TMessageHolder response) if (response->NextOffset != rpc.Bytes->size()) return; const auto index = std::max(VolatileState->MatchIndex[response->Src], response->SnapshotIndex); (*VolatileState).SetMatchIndex(response->Src, index).SetNextIndex(response->Src, index + 1) - .SetBatchSize(response->Src, 1024).SetBackOff(response->Src, 1).CommitAdvance(*VotingQuorum, Id, *State); + .SetBatchSize(response->Src, 1024).SetBackOff(response->Src, 1); + AdvanceLeaderCommit(); VolatileState->SnapshotRpcs.erase(found); // Snapshot completion is not a fresh-read acknowledgement. AppendEntries // must still establish the next read barrier and replicate the suffix. @@ -1044,7 +1050,7 @@ void TRaft::OnInstallSnapshot(ITimeSource::Time now, TMessageHolderLoadSnapshot(snapshot.Data, snapshot.Index); if (application->LastAppliedIndex != snapshot.Index) std::terminate(); } catch (...) { std::terminate(); } - VolatileState->CommitIndex = snapshot.Index; + VolatileState->CommitIndex = State->CommitIndex; IncomingSnapshot.reset(); reply(true, true, request->TotalBytes); } @@ -1054,7 +1060,28 @@ void TRaft::RestoreAppliedIndex(uint64_t index) { throw std::invalid_argument("Application progress exceeds recovered Raft log"); // A correctly recovered application marker is evidence of commitment. // Protect its prefix immediately, before any AppendEntries can be handled. - VolatileState->CommitIndex = std::max(VolatileState->CommitIndex, index); + AdvanceCommitIndex(std::max(VolatileState->CommitIndex, index)); +} + +void TRaft::AdvanceCommitIndex(uint64_t index) { + if (index > State->LastLogIndex) + throw std::logic_error("Invalid Raft commit index advance"); + const auto target = std::max({index, VolatileState->CommitIndex, State->CommitIndex}); + if (target > State->LastLogIndex) + throw std::logic_error("Invalid persisted Raft commit index"); + if (State->CommitIndex < target) { + State->CommitIndex = target; + State->Commit(); + } + VolatileState->CommitIndex = target; +} + +void TRaft::AdvanceLeaderCommit() { + const auto previous = VolatileState->CommitIndex; + VolatileState->CommitAdvance(*VotingQuorum, Id, *State); + const auto updated = VolatileState->CommitIndex; + VolatileState->CommitIndex = previous; + AdvanceCommitIndex(updated); } TRequestProcessor::TRequestProcessor(std::shared_ptr raft, std::shared_ptr rsm, const TNodeDict& nodes) diff --git a/miniraft/raft.h b/miniraft/raft.h index b69c57d..929e838 100644 --- a/miniraft/raft.h +++ b/miniraft/raft.h @@ -182,6 +182,8 @@ class TRaft { private: friend class TRequestProcessor; void RestoreAppliedIndex(uint64_t index); + void AdvanceCommitIndex(uint64_t index); + void AdvanceLeaderCommit(); void OnInstallSnapshot(ITimeSource::Time now, TMessageHolder message); void OnInstallSnapshot(TMessageHolder message); TMessageHolder CreateInstallSnapshot(uint32_t nodeId); diff --git a/miniraft/state.h b/miniraft/state.h index 1765c88..264ef74 100644 --- a/miniraft/state.h +++ b/miniraft/state.h @@ -11,6 +11,9 @@ struct TStateFields { uint64_t LastLogTerm = 0; uint64_t SnapshotIndex = 0; uint64_t SnapshotTerm = 0; + // Persisted commit boundary where the storage backend supports it. Older + // formats conservatively recover this as SnapshotIndex. + uint64_t CommitIndex = 0; bool operator==(const TStateFields& other) const = default; }; @@ -34,6 +37,9 @@ struct IState: TStateFields { return entry->Flags; } virtual void Commit() = 0; + // True only when Commit() durably records CommitIndex and recovery restores + // it. Membership protocols can require this before trusting recovered history. + virtual bool SupportsCommitIndexPersistence() const { return false; } virtual bool SupportsSnapshots() const { return false; } virtual std::shared_ptr GetSnapshot() const { return {}; } virtual void InstallSnapshot(const TSnapshot&) { throw std::logic_error("Storage snapshots are unsupported"); } @@ -121,6 +127,8 @@ struct TState: IState { void RemoveLast() override { if (Log.empty()) return; + if (LastLogIndex <= CommitIndex) + throw std::logic_error("Cannot remove a committed log entry"); Log.pop_back(); LastLogIndex = SnapshotIndex + Log.size(); LastLogTerm = Log.empty() ? SnapshotTerm : Log.back()->Term; @@ -166,6 +174,9 @@ struct TState: IState { throw std::invalid_argument("Snapshot would change bound voters"); if (image.Index <= SnapshotIndex || image.Term < SnapshotTerm || image.Term > CurrentTerm) throw std::invalid_argument("Obsolete or future snapshot"); + if (image.Index <= CommitIndex && + (image.Index > LastLogIndex || LogTerm(image.Index) != image.Term)) + throw std::invalid_argument("Snapshot would discard committed log entries"); auto saved = std::make_shared(std::move(image)); std::vector> suffix; if (saved->Index <= LastLogIndex && LogTerm(saved->Index) == saved->Term) @@ -176,6 +187,7 @@ struct TState: IState { SnapshotTerm = Snapshot->Term; LastLogIndex = SnapshotIndex + Log.size(); LastLogTerm = Log.empty() ? SnapshotTerm : Log.back()->Term; + CommitIndex = std::max(CommitIndex, SnapshotIndex); } void Commit() override { } diff --git a/test/smoke_kv_checkpoints.py b/test/smoke_kv_checkpoints.py index 08e261a..2265a20 100644 --- a/test/smoke_kv_checkpoints.py +++ b/test/smoke_kv_checkpoints.py @@ -50,6 +50,10 @@ def snapshot_boundary(path): voters = struct.unpack(" image = [] { return std::string{}; }) { + state.CurrentTerm = 2; + auto entry = NewHoldedMessage(); entry->Term = 2; state.Append(entry); + state.CommitIndex = 1; state.Commit(); + const auto before = static_cast(state); + const auto beforeImage = image(); + Reject([&] { state.InstallSnapshot({1, 1, {1}, "conflicting committed boundary"}); }); + assert_true(static_cast(state) == before); + assert_true(image() == beforeImage); +} +void test_memory_snapshot_cannot_replace_committed_index_with_conflicting_term(void**) { + TState state; CheckSnapshotCannotConflictAtCommit(state); +} +#ifdef __linux__ +void test_durable_snapshot_cannot_replace_committed_index_with_conflicting_term(void**) { + auto dir = std::filesystem::temp_directory_path() / ("miniraft-committed-snapshot-" + + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count())); + assert_true(std::filesystem::create_directory(dir)); + auto name = (dir / "raft").string(); + { TDurableState state(name, 1); CheckSnapshotCannotConflictAtCommit(state, [&] { + std::ifstream input(name + ".durable.1", std::ios::binary); + return std::string(std::istreambuf_iterator(input), {}); + }); } + { TDurableState state(name, 1); assert_int_equal(state.CommitIndex, 1); } + std::error_code ignored; std::filesystem::remove_all(dir, ignored); +} +#endif + void test_remote_snapshot_retains_suffix_without_applying_it_before_commit(void**) { auto state = std::make_shared(); TKv prefix; @@ -854,7 +882,7 @@ void ExerciseBoundary(IState& state) { assert_int_equal(state.LastLogIndex, 3); assert_int_equal(state.SnapshotIndex, 2); assert_int_equal(state.LogTerm(1), 0); assert_int_equal(state.LogTerm(2), 2); assert_int_equal(state.Get(3)->Term, 3); assert_false(state.Get(1)); assert_false(state.Get(2)); - state.RemoveLast(); state.RemoveLast(); + state.RemoveLast(); assert_int_equal(state.LastLogIndex, 2); assert_int_equal(state.LastLogTerm, 2); auto entry = NewHoldedMessage(); entry->Term = 3; state.Append(entry); assert_int_equal(state.LastLogIndex, 3); assert_int_equal(state.Get(3)->Term, 3); @@ -943,16 +971,16 @@ void test_compaction_reclaims_durable_prefix_bytes(void**) { assert_true(after < before / 10); assert_int_equal(state.LastLogIndex, 30); assert_true(bool(state.Get(30))); assert_false(state.Get(29)); } -void test_durable_v2_semantic_corruption_rejects_without_rewriting(void**) { +void test_durable_v4_semantic_corruption_rejects_without_rewriting(void**) { for (int invalid = 0; invalid < 3; ++invalid) { TFiles files; { TDurableState state(files.Name, 1); ExerciseBoundary(state); } const auto path = files.Name + ".durable.1"; std::ifstream input(path, std::ios::binary); std::string bytes(std::istreambuf_iterator(input), {}); input.close(); - assert_true(bytes.substr(0, 8) == "MRFTD002"); + assert_true(bytes.substr(0, 8) == "MRFTD004"); if (invalid < 2) { - const auto offset = invalid == 0 ? 40 : 32; // snapshot length or retained entry count + const auto offset = invalid == 0 ? 52 : 32; // snapshot length or retained entry count for (size_t i = offset; i < size_t(offset + 8); ++i) bytes[i] = char(0xff); } else bytes.back() ^= 1; // inner snapshot checksum with a valid outer checksum const auto crc = NSnapshotCodec::Checksum(bytes.substr(0, 8) + bytes.substr(20)); @@ -983,6 +1011,7 @@ int main() { cmocka_unit_test(test_older_snapshot_cannot_exceed_compacted_boundary_term), cmocka_unit_test(test_snapshot_is_rechecked_when_commit_advances_mid_transfer), cmocka_unit_test(test_memory_snapshot_storage_rejects_term_regression), + cmocka_unit_test(test_memory_snapshot_cannot_replace_committed_index_with_conflicting_term), cmocka_unit_test(test_joint_snapshot_is_rejected_even_if_boundary_is_already_covered), cmocka_unit_test(test_stable_configuration_snapshot_installs_and_recovers_with_fixed_membership), cmocka_unit_test(test_idle_partial_snapshot_expires_on_learner_despite_heartbeats), @@ -992,12 +1021,13 @@ int main() { cmocka_unit_test(test_memory_snapshot_boundary_retains_matching_suffix_and_replaces_conflict), #ifdef __linux__ cmocka_unit_test(test_durable_snapshot_storage_rejects_term_regression), + cmocka_unit_test(test_durable_snapshot_cannot_replace_committed_index_with_conflicting_term), cmocka_unit_test(test_durable_storage_rejects_index_exhaustion), cmocka_unit_test(test_durable_term_lookup_avoids_payload_copies_across_compaction), cmocka_unit_test(test_durable_compacted_boundary_survives_restart), cmocka_unit_test(test_snapshot_and_suffix_are_atomic_at_every_publication_stage), cmocka_unit_test(test_compaction_reclaims_durable_prefix_bytes), - cmocka_unit_test(test_durable_v2_semantic_corruption_rejects_without_rewriting), + cmocka_unit_test(test_durable_v4_semantic_corruption_rejects_without_rewriting), #endif }; return cmocka_run_group_tests(tests, nullptr, nullptr); diff --git a/test/test_durable.cpp b/test/test_durable.cpp index 1be7c11..86e6477 100644 --- a/test/test_durable.cpp +++ b/test/test_durable.cpp @@ -1,10 +1,12 @@ #include #include #include +#include #include #include #include #include +#include #include #include #include @@ -33,12 +35,72 @@ struct TFiles { fs::path Image() const { return Name + ".durable.1"; } void Seed(); }; +void Put(std::string& bytes, uint64_t value, unsigned width) { + for (unsigned i = 0; i < width; ++i) bytes.push_back(char(value >> (8 * i))); +} +uint32_t Crc32(std::string_view bytes) { + uint32_t crc = UINT32_MAX; + for (unsigned char byte : bytes) { + crc ^= byte; + for (int bit = 0; bit < 8; ++bit) crc = (crc >> 1) ^ (0xedb88320u & (0u - (crc & 1u))); + } + return ~crc; +} +std::string LegacyImage(std::string_view magic, bool hasSnapshot, bool configured) { + std::string payload; + Put(payload, 3, 8); // current term + Put(payload, 1, 4); // vote + Put(payload, 1, 8); // retained entry count + if (configured) { + Put(payload, 2, 4); Put(payload, 1, 4); Put(payload, 2, 4); + } + if (hasSnapshot) { + const auto image = EncodeSnapshot(TSnapshot{1, 2, {1, 2}, "legacy snapshot"}); + Put(payload, image.size(), 8); payload.append(image); + } + Put(payload, 3, 8); Put(payload, 0, 8); Put(payload, 4, 4); payload.append("tail"); + std::string checksumInput; + if (magic != "MRFTD001") checksumInput.append(magic); + checksumInput.append(payload); + std::string bytes(magic); + Put(bytes, payload.size(), 8); Put(bytes, Crc32(checksumInput), 4); bytes.append(payload); + return bytes; +} TMessageHolder Entry(uint64_t term, const std::string& data) { auto entry = NewHoldedMessage(sizeof(TLogEntry) + data.size()); entry->Term = term; memcpy(entry->Data, data.data(), data.size()); return entry; } +std::string Data(TDurableState& state, int64_t index); + +void test_v1_v2_v3_images_recover_conservatively_and_upgrade_on_publish(void**) { + for (const auto [magic, hasSnapshot, configured] : { + std::tuple{"MRFTD001", false, false}, + {"MRFTD002", true, false}, + {"MRFTD003", true, true}, + }) { + TFiles files; + const auto legacy = LegacyImage(magic, hasSnapshot, configured); + { std::ofstream output(files.Image(), std::ios::binary); output.write(legacy.data(), legacy.size()); } + { + TDurableState state(files.Name, 1); + assert_int_equal(state.CurrentTerm, 3); + assert_int_equal(state.SnapshotIndex, hasSnapshot ? 1 : 0); + assert_int_equal(state.CommitIndex, state.SnapshotIndex); + assert_int_equal(state.LastLogIndex, hasSnapshot ? 2 : 1); + if (!configured) state.BindVotersFromTrustedBaseline({1, 2}); + assert_true((state.GetBoundVoters() == std::vector{1, 2})); + assert_true(state.SupportsCommitIndexPersistence()); + if (hasSnapshot) assert_true(state.GetSnapshot()->Data == "legacy snapshot"); + assert_string_equal(Data(state, state.LastLogIndex).c_str(), "tail"); + state.Commit(); + } + std::ifstream upgraded(files.Image(), std::ios::binary); + std::string published(8, '\0'); upgraded.read(published.data(), published.size()); + assert_true(published == "MRFTD004"); + } +} void TFiles::Seed() { TDurableState state(Name, 1); state.CurrentTerm = 3; @@ -50,9 +112,13 @@ std::string Data(TDurableState& state, int64_t index) { auto entry = state.Get(index); return std::string(entry->Data, entry->Len - sizeof(TLogEntry)); } -enum class EOperation { Append, Truncate, Vote, Replace }; +enum class EOperation { Append, Truncate, Vote, Replace, CommitIndex }; void Operate(TDurableState& state, EOperation operation) { if (operation == EOperation::Truncate) state.RemoveLast(); + else if (operation == EOperation::CommitIndex) { + state.CommitIndex = 1; + state.Commit(); + } else if (operation == EOperation::Vote) { state.CurrentTerm = 4; state.VotedFor = 3; @@ -64,8 +130,9 @@ void Check(TDurableState& state, EOperation operation, bool updated) { assert_int_equal(state.VotedFor, updated && operation == EOperation::Vote ? 3 : 2); const auto oldCount = operation == EOperation::Replace ? 1 : 2; const auto count = !updated ? oldCount : operation == EOperation::Truncate ? 1 : - operation == EOperation::Vote ? 2 : oldCount + 1; + operation == EOperation::Vote || operation == EOperation::CommitIndex ? 2 : oldCount + 1; assert_int_equal(state.LastLogIndex, count); + assert_int_equal(state.CommitIndex, updated && operation == EOperation::CommitIndex ? 1 : 0); assert_string_equal(Data(state, 1).c_str(), "first"); if (count >= 2) { const bool replaced = updated && operation == EOperation::Replace; @@ -83,7 +150,7 @@ void Wait(pid_t child, int expected) { } void test_crash_at_every_publication_boundary(void**) { - for (auto operation : {EOperation::Append, EOperation::Truncate, EOperation::Vote, EOperation::Replace}) { + for (auto operation : {EOperation::Append, EOperation::Truncate, EOperation::Vote, EOperation::Replace, EOperation::CommitIndex}) { for (auto stage : {TStage::TempCreated, TStage::Written, TStage::FileSynced, TStage::Renamed, TStage::DirectorySynced}) { TFiles files; @@ -110,7 +177,7 @@ void test_crash_at_every_publication_boundary(void**) { } void test_acknowledged_updates_survive_immediate_exit(void**) { - for (auto operation : {EOperation::Append, EOperation::Truncate, EOperation::Vote, EOperation::Replace}) { + for (auto operation : {EOperation::Append, EOperation::Truncate, EOperation::Vote, EOperation::Replace, EOperation::CommitIndex}) { TFiles files; files.Seed(); const auto child = fork(); @@ -129,6 +196,48 @@ void test_acknowledged_updates_survive_immediate_exit(void**) { } } +struct TEndpoint : INode { + std::vector> Sent; + std::vector* Events = nullptr; + void Send(TMessageHolder message) override { + Sent.push_back(std::move(message)); + if (Events) Events->push_back("ack"); + } + void Drain() override {} +}; + +void test_follower_persists_commit_before_ack_and_recovers_it(void**) { + TFiles files; + std::vector events; + auto peer = std::make_shared(); + peer->Events = &events; + { + auto state = std::make_shared(files.Name, 1, [&](auto stage) { + if (stage == TStage::DirectorySynced) events.push_back("durable"); + }); + TRaft raft(state, 1, {{2, peer}}); + events.clear(); // Ignore initial image and voter-binding publications. + auto request = NewHoldedMessage(TMessageEx{.Src = 2, .Dst = 1, .Term = 1}, + TAppendEntriesRequest{.PrevLogIndex = 0, .PrevLogTerm = 0, .LeaderCommit = 1, + .LeaderId = 2, .Nentries = 1}); + request.InitPayload(1); + request.Payload[0] = Entry(1, "committed"); + raft.Process(ITimeSource::Time{}, request); + assert_int_equal(state->CommitIndex, 1); + assert_int_equal(peer->Sent.size(), 1); + assert_true(peer->Sent.back().Cast()->Success); + assert_true((events == std::vector{"durable", "durable", "ack"})); + } + TDurableState recovered(files.Name, 1); + assert_int_equal(recovered.CommitIndex, 1); + assert_int_equal(recovered.LastLogIndex, 1); + bool protectedEntry = false; + try { recovered.RemoveLast(); } + catch (const std::logic_error&) { protectedEntry = true; } + assert_true(protectedEntry); + assert_int_equal(recovered.LastLogIndex, 1); +} + void test_publication_order(void**) { TFiles files; files.Seed(); @@ -306,7 +415,9 @@ int main() { const CMUnitTest tests[] = { cmocka_unit_test(test_crash_at_every_publication_boundary), cmocka_unit_test(test_acknowledged_updates_survive_immediate_exit), + cmocka_unit_test(test_follower_persists_commit_before_ack_and_recovers_it), cmocka_unit_test(test_publication_order), + cmocka_unit_test(test_v1_v2_v3_images_recover_conservatively_and_upgrade_on_publish), cmocka_unit_test(test_live_publication_failure_is_fatal), cmocka_unit_test(test_write_error_is_fatal_and_preserves_image), cmocka_unit_test(test_interrupted_initialization_is_not_reset), diff --git a/test/test_learners.cpp b/test/test_learners.cpp index 59e193f..1c41e45 100644 --- a/test/test_learners.cpp +++ b/test/test_learners.cpp @@ -464,7 +464,7 @@ void test_durable_voter_binding_is_atomic_before_first_vote(void**) { assert_int_equal(state.CurrentTerm, 1); assert_int_equal(state.VotedFor, 0); assert_int_equal(state.LastLogIndex, 0); } } -void test_trusted_legacy_voter_baseline_migrates_durable_v1_and_v2(void**) { +void test_trusted_legacy_voter_baseline_migrates_durable_state(void**) { { TFiles files; { @@ -491,7 +491,7 @@ void test_trusted_legacy_voter_baseline_migrates_durable_v1_and_v2(void**) { const auto path = files.Name + ".durable.1"; std::ifstream originalInput(path, std::ios::binary); const std::string original(std::istreambuf_iterator(originalInput), {}); - assert_true(original.substr(0, 8) == "MRFTD001"); + assert_true(original.substr(0, 8) == "MRFTD004"); { auto state = std::make_shared(files.Name, 1); @@ -509,7 +509,7 @@ void test_trusted_legacy_voter_baseline_migrates_durable_v1_and_v2(void**) { assert_int_equal(state->RetainedLogTerm(1), 2); } { std::ifstream migrated(path, std::ios::binary); std::string magic(8, '\0'); - migrated.read(magic.data(), magic.size()); assert_true(magic == "MRFTD003"); } + migrated.read(magic.data(), magic.size()); assert_true(magic == "MRFTD004"); } auto recovered = std::make_shared(files.Name, 1); assert_true(static_cast(*recovered) == before); assert_true(recovered->GetBoundVoters() == (std::vector{1, 2, 3})); @@ -531,7 +531,7 @@ void test_trusted_legacy_voter_baseline_migrates_durable_v1_and_v2(void**) { const auto path = files.Name + ".durable.1"; std::ifstream originalInput(path, std::ios::binary); const std::string original(std::istreambuf_iterator(originalInput), {}); - assert_true(original.substr(0, 8) == "MRFTD002"); + assert_true(original.substr(0, 8) == "MRFTD004"); { auto state = std::make_shared(files.Name, 1); @@ -546,7 +546,7 @@ void test_trusted_legacy_voter_baseline_migrates_durable_v1_and_v2(void**) { assert_true(state->GetBoundVoters() == (std::vector{1, 2, 3})); } { std::ifstream migrated(path, std::ios::binary); std::string magic(8, '\0'); - migrated.read(magic.data(), magic.size()); assert_true(magic == "MRFTD003"); } + migrated.read(magic.data(), magic.size()); assert_true(magic == "MRFTD004"); } auto recovered = std::make_shared(files.Name, 1); assert_true(static_cast(*recovered) == before); assert_true(recovered->GetBoundVoters() == (std::vector{1, 2, 3})); @@ -666,15 +666,15 @@ void test_durable_learner_snapshot_suffix_and_role_survive_restart(void**) { assert_true(Read(*app, "b") == "tail"); assert_int_equal(state->VotedFor, 0); assert_true(state->GetBoundVoters() == (std::vector{1, 2, 3})); } -void test_durable_v3_corrupt_roles_fail_without_rewriting(void**) { - for (size_t offset : {size_t(40), size_t(44), size_t(48), size_t(56)}) { +void test_durable_v4_corrupt_commit_or_roles_fail_without_rewriting(void**) { + for (size_t offset : {size_t(40), size_t(52), size_t(56), size_t(64)}) { TFiles files; { auto state = std::make_shared(files.Name, 4); TRaft raft(state, 4, Peers(4), {4}); } const auto path = files.Name + ".durable.4"; std::ifstream input(path, std::ios::binary); std::string bytes(std::istreambuf_iterator(input), {}); input.close(); - assert_true(bytes.substr(0, 8) == "MRFTD003"); - const auto width = offset == 56 ? 8 : 4; - for (size_t i = offset; i < offset + width; ++i) bytes[i] = offset == 44 || offset == 48 ? 0 : char(0xff); + assert_true(bytes.substr(0, 8) == "MRFTD004"); + const auto width = offset == 40 || offset == 64 ? 8 : 4; + for (size_t i = offset; i < offset + width; ++i) bytes[i] = offset == 52 || offset == 56 ? 0 : char(0xff); const auto crc = NSnapshotCodec::Checksum(bytes.substr(0, 8) + bytes.substr(20)); for (int i = 0; i < 4; ++i) bytes[16 + i] = char(crc >> (8 * i)); { std::ofstream out(path, std::ios::binary); out.write(bytes.data(), bytes.size()); } @@ -682,7 +682,7 @@ void test_durable_v3_corrupt_roles_fail_without_rewriting(void**) { std::ifstream after(path, std::ios::binary); assert_true(std::string(std::istreambuf_iterator(after), {}) == bytes); } } -void test_durable_v2_snapshot_adoption_preserves_vote_log_and_application(void**) { +void test_durable_snapshot_adoption_preserves_vote_log_and_application(void**) { TFiles files; TStateFields before; std::string application; { TDurableState state(files.Name, 1); TKv app; @@ -692,7 +692,7 @@ void test_durable_v2_snapshot_adoption_preserves_vote_log_and_application(void** state.Append(app.Prepare(Command("b", "tail"))); before = state; } const auto path = files.Name + ".durable.1"; - { std::ifstream input(path, std::ios::binary); std::string magic(8, '\0'); input.read(magic.data(), 8); assert_true(magic == "MRFTD002"); } + { std::ifstream input(path, std::ios::binary); std::string magic(8, '\0'); input.read(magic.data(), 8); assert_true(magic == "MRFTD004"); } { auto state = std::make_shared(files.Name, 1); auto raft = std::make_shared(state, 1, Peers(1), std::vector{4}); @@ -766,9 +766,9 @@ int main() { cmocka_unit_test(test_legacy_recovered_vote_outside_voters_preserves_storage), cmocka_unit_test(test_durable_voter_binding_is_atomic_before_first_vote), cmocka_unit_test(test_durable_learner_snapshot_suffix_and_role_survive_restart), - cmocka_unit_test(test_durable_v3_corrupt_roles_fail_without_rewriting), - cmocka_unit_test(test_durable_v2_snapshot_adoption_preserves_vote_log_and_application), - cmocka_unit_test(test_trusted_legacy_voter_baseline_migrates_durable_v1_and_v2), + cmocka_unit_test(test_durable_v4_corrupt_commit_or_roles_fail_without_rewriting), + cmocka_unit_test(test_durable_snapshot_adoption_preserves_vote_log_and_application), + cmocka_unit_test(test_trusted_legacy_voter_baseline_migrates_durable_state), cmocka_unit_test(test_trusted_legacy_voter_baseline_refuses_configuration_logs), cmocka_unit_test(test_trusted_legacy_voter_baseline_must_precede_raft_startup), cmocka_unit_test(test_trusted_legacy_voter_binding_is_atomic_at_each_publish_stage), diff --git a/test/test_snapshots.cpp b/test/test_snapshots.cpp index 0307cd3..aa5b5fc 100644 --- a/test/test_snapshots.cpp +++ b/test/test_snapshots.cpp @@ -495,12 +495,12 @@ void test_unbound_durable_configuration_snapshot_recovery_fails_closed(void**) { } std::ifstream input(path, std::ios::binary); std::string bytes(std::istreambuf_iterator(input), {}); input.close(); - assert_true(bytes.substr(0, 8) == "MRFTD002"); + assert_true(bytes.substr(0, 8) == "MRFTD004"); auto snapshot = Sample(); snapshot.StableConfiguration = true; snapshot.ConfigurationIndex = 1; snapshot.ConfigurationTerm = 1; const auto image = EncodeSnapshot(snapshot); - constexpr size_t imageLengthOffset = 40; - constexpr size_t imageOffset = 48; + constexpr size_t imageLengthOffset = 52; + constexpr size_t imageOffset = 60; uint64_t oldLength = 0; for (int i = 0; i < 8; ++i) oldLength |= uint64_t(uint8_t(bytes[imageLengthOffset + i])) << (8 * i); assert_true(oldLength == bytes.size() - imageOffset); @@ -643,7 +643,8 @@ void test_durable_kv_restart_restores_checkpoint_and_replays_only_tail(void**) { RestoreSnapshot(*snapshot, *raft, *kv); assert_true(Read(*kv, "a") == "one"); assert_true(Read(*kv, "b").empty()); TRequestProcessor processor(raft, kv, {}); - processor.ProcessCommitted(); assert_int_equal(kv->Applied, 0); + processor.ProcessCommitted(); assert_int_equal(kv->Applied, 1); + assert_true(Read(*kv, "b") == "tail"); raft->ProcessTimeout(ITimeSource::Time{}); processor.ProcessCommitted(); assert_int_equal(kv->Applied, 1); assert_true(Read(*kv, "a") == "one"); assert_true(Read(*kv, "b") == "tail"); From 86599d70275a522f2c78aa0e96ea60d6f53365a3 Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Sat, 14 Nov 2026 21:01:59 -0500 Subject: [PATCH 57/66] Use persisted commit index for configuration scans (#57) --- docs/durable-configuration-snapshots.md | 6 ++++ miniraft/configuration.cpp | 8 +++++ miniraft/configuration.h | 6 ++++ test/test_configuration.cpp | 42 +++++++++++++++++++++++++ 4 files changed, 62 insertions(+) diff --git a/docs/durable-configuration-snapshots.md b/docs/durable-configuration-snapshots.md index 780d221..c47f954 100644 --- a/docs/durable-configuration-snapshots.md +++ b/docs/durable-configuration-snapshots.md @@ -18,6 +18,12 @@ binding from snapshot metadata. Older durable formats remain readable and use their snapshot boundary as the conservative commit floor. The history scanner also verifies that a joint baseline's old voters match the bound set. +`ScanCommittedConfigurationHistory` uses the recovered commit index directly +and requires both durable commit-index support and a persisted voter binding. +The caller still supplies a trusted history baseline through the snapshot +boundary. This remains an offline validation API; it does not activate joint +quorums or permit live configuration changes. + In alpha.45 these images were persistence for offline validation only. The alpha.46 follow-up enables runtime recovery and transfer of MRFTS003 stable snapshots when the voter list equals the node's existing fixed membership. diff --git a/miniraft/configuration.cpp b/miniraft/configuration.cpp index f243562..151aa02 100644 --- a/miniraft/configuration.cpp +++ b/miniraft/configuration.cpp @@ -202,3 +202,11 @@ TConfigurationScan ScanConfigurationHistory(const IState& state, throw std::invalid_argument("Configuration scan last term mismatch"); return {committed, latest}; } + +TConfigurationScan ScanCommittedConfigurationHistory(const IState& state, + TConfigurationHistory baseline) { + if (!state.SupportsCommitIndexPersistence() || !state.SupportsVoterBinding() || + state.GetBoundVoters().empty()) + throw std::invalid_argument("Committed configuration scan requires durable commit and voter metadata"); + return ScanConfigurationHistory(state, std::move(baseline), state.CommitIndex); +} diff --git a/miniraft/configuration.h b/miniraft/configuration.h index 9fd3c3b..9a02688 100644 --- a/miniraft/configuration.h +++ b/miniraft/configuration.h @@ -70,3 +70,9 @@ struct TConfigurationScan { // the baseline and the commit boundary; neither is proven by this scan. TConfigurationScan ScanConfigurationHistory(const IState& state, TConfigurationHistory baseline, uint64_t commitIndex); + +// Read the committed boundary from storage instead of accepting a caller's +// guess. Requires voter binding and a backend that persists CommitIndex; the +// caller must still supply a trusted baseline through SnapshotIndex. +TConfigurationScan ScanCommittedConfigurationHistory(const IState& state, + TConfigurationHistory baseline); diff --git a/test/test_configuration.cpp b/test/test_configuration.cpp index 9e9acc5..6108c18 100644 --- a/test/test_configuration.cpp +++ b/test/test_configuration.cpp @@ -455,6 +455,47 @@ void test_offline_history_scan_after_durable_snapshot_and_restart(void**) { assert_true(std::string(std::istreambuf_iterator(after), {}) == before); std::error_code ignored; std::filesystem::remove_all(dir, ignored); } + +void test_persisted_commit_boundary_drives_committed_configuration_scan(void**) { + auto dir = std::filesystem::temp_directory_path() / ("miniraft-committed-history-" + + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count())); + assert_true(std::filesystem::create_directory(dir)); + const auto name = (dir / "raft").string(); + TConfigurationRecord joint{EConfigurationPhase::Joint, 2, 0, 0, {1}, {1, 2}}; + auto final = joint; final.Phase = EConfigurationPhase::Stable; final.Term = 3; + final.PreviousIndex = 2; final.PreviousTerm = 2; + { + auto state = std::make_shared(name, 1); + { TRaft bootstrap(state, 1, {}); } + state->CurrentTerm = 3; state->Commit(); + state->Append(Ordinary(1)); + state->Append(EncodeConfigurationEntry(joint)); + state->Append(Ordinary(2)); + state->Append(EncodeConfigurationEntry(final)); + state->CommitIndex = 3; state->Commit(); + } + { + TDurableState state(name, 1); + auto baseline = TConfigurationHistory::Stable({1}, 0, 0, 0, 0); + const auto scan = ScanCommittedConfigurationHistory(state, baseline); + assert_int_equal(state.CommitIndex, 3); + assert_true(scan.Committed.IsJoint()); + assert_int_equal(scan.Committed.ConfigurationIndex(), 2); + assert_false(scan.Latest.IsJoint()); + assert_true(scan.Latest.OldVoters() == final.NewVoters); + assert_true(scan.HasUncommittedConfiguration()); + Reject([&] { + auto wrong = TConfigurationHistory::Stable({2}, 0, 0, 0, 0); + ScanCommittedConfigurationHistory(state, wrong); + }); + } + TState volatileState; volatileState.BindVoters({1}); + Reject([&] { + auto baseline = TConfigurationHistory::Stable({1}, 0, 0, 0, 0); + ScanCommittedConfigurationHistory(volatileState, baseline); + }); + std::error_code ignored; std::filesystem::remove_all(dir, ignored); +} #endif } int main() { @@ -476,6 +517,7 @@ int main() { cmocka_unit_test(test_durable_restart_rejects_reserved_entry_without_rewriting), cmocka_unit_test(test_legacy_restart_uses_safe_flag_lookup_and_preserves_files), cmocka_unit_test(test_offline_history_scan_after_durable_snapshot_and_restart), + cmocka_unit_test(test_persisted_commit_boundary_drives_committed_configuration_scan), #endif }; return cmocka_run_group_tests(tests, nullptr, nullptr); From e1d4c8be13ceeed7ab1bbcd018f920071941c66f Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Sat, 14 Nov 2026 21:28:57 -0500 Subject: [PATCH 58/66] Preserve original voter binding across offline config snapshots (#58) --- docs/configuration-history.md | 10 +-- docs/configuration-scan.md | 21 +++-- docs/durable-configuration-snapshots.md | 40 +++++---- docs/joint-snapshots.md | 9 +- docs/learners.md | 9 +- docs/stable-configuration-snapshot-runtime.md | 9 +- docs/stable-configuration-snapshots.md | 19 +++-- miniraft/configuration.cpp | 9 +- miniraft/durable.cpp | 21 +++-- miniraft/snapshot.h | 5 +- miniraft/state.h | 19 ++++- test/test_configuration.cpp | 84 +++++++++++++++++-- test/test_snapshots.cpp | 3 +- 13 files changed, 185 insertions(+), 73 deletions(-) diff --git a/docs/configuration-history.md b/docs/configuration-history.md index 619bf2a..4e1dd90 100644 --- a/docs/configuration-history.md +++ b/docs/configuration-history.md @@ -10,11 +10,11 @@ and that the supplied application image corresponds exactly to that boundary. The method validates snapshot metadata and size; it does not capture application state, compact or mutate storage, establish commitment, or activate voters. Joint configuration snapshots remain rejected by the live runtime. Stable -metadata snapshots can be recovered and installed when they match the node's -existing fixed voter binding; this does not enable voter changes. -Bound storage can retain and recover stable or joint metadata snapshots for -offline validation, provided the snapshot voter list matches the store's existing -voter binding. +metadata snapshots are runtime-compatible only when they match the node's +existing fixed voter set. Bound storage can retain stable or joint metadata +snapshots for offline validation while preserving the original voter binding, +even when the active set in the snapshot differs. This does not enable voter +changes. TConfigurationHistory checks a sequence of retained log entries against a trusted configuration baseline. It joins the experimental configuration-entry codec with diff --git a/docs/configuration-scan.md b/docs/configuration-scan.md index 99a11e0..26295f6 100644 --- a/docs/configuration-scan.md +++ b/docs/configuration-scan.md @@ -12,9 +12,13 @@ The caller must independently establish the baseline and commit index: the scan does not prove that any entry is committed, recover a commit index, authenticate the snapshot, or establish cluster identity. A fixed v1 snapshot does not identify its latest configuration entry, so its configuration identity must come from -other trusted information. Experimental stable and joint snapshots contain that identity and the scan -checks that the supplied baseline matches it. The scan checks snapshot and log -boundaries, bound fixed voters, terms, configuration predecessors, and transitions. +other trusted information. Experimental stable and joint snapshots contain that +identity and the scan checks that the supplied active configuration matches it. +The persisted voter binding identifies the original trusted set; configuration +metadata can describe an active set reached by later transitions. Legacy images +without configuration metadata still require the original and snapshot voter +sets to agree. The scan checks snapshot and log boundaries, voter history, terms, +configuration predecessors, and transitions. It requires a consistent, externally serialized view of the state; it does not mutate storage, apply entries, activate voters, or change Raft roles. @@ -24,9 +28,8 @@ copying ordinary payloads; external backends using default metadata accessors ma read full entries. Configuration records are read and validated in full. Configuration entries and joint snapshots remain rejected by the live runtime. -Stable metadata snapshots can be recovered and installed if their voter list -matches the persisted fixed membership. Bound in-memory and durable stores can -also retain joint metadata for offline validation; for joint snapshots, the -scanner requires the baseline's old voter set to match that binding. Live -membership activation, commit rules, and snapshot installation across membership -transitions remain future work. +Bound in-memory and durable stores can retain stable or joint metadata snapshots +for offline validation while preserving the original voter binding. The scanner +still requires a trusted configuration baseline; metadata presence alone does +not authenticate a transition. Live membership activation, commit rules, and +snapshot installation across membership transitions remain future work. diff --git a/docs/durable-configuration-snapshots.md b/docs/durable-configuration-snapshots.md index c47f954..f3f1836 100644 --- a/docs/durable-configuration-snapshots.md +++ b/docs/durable-configuration-snapshots.md @@ -2,10 +2,13 @@ Bound `TState` and `TDurableState` instances can install and recover MRFTS002 joint and MRFTS003 stable snapshots so offline tools can validate configuration -history after restart. A store must already have a voter binding, and -`snapshot.Voters` must equal that binding. Durable recovery checks the same -invariant before exposing the image. An unbound store cannot infer its original -voter set from configuration metadata. The [alpha.50 follow-up](trusted-legacy-voter-binding.md) +history after restart. A store must already have a voter binding. For a +configuration-bearing snapshot, `snapshot.Voters` describes the active set at +the snapshot boundary and may differ from that original binding. Durable +recovery preserves the original binding and exposes the active snapshot set +separately. A legacy snapshot without configuration metadata must still match +the binding. An unbound store cannot infer its original voter set from +configuration metadata. The [alpha.50 follow-up](trusted-legacy-voter-binding.md) adds an explicit durable migration API that accepts a caller-supplied trusted original set, validates it against the recovered vote, snapshot, and retained log, then persists the binding before Raft startup. The API does not establish @@ -13,26 +16,33 @@ that the caller's assertion is true; absent an independent trusted baseline, the store still cannot infer its voter set. The current MRFTD004 durable format persists the snapshot, commit boundary and -existing voter binding in the same atomic image; it does not reinterpret that +original voter binding in the same atomic image; it does not reinterpret that binding from snapshot metadata. Older durable formats remain readable and use their snapshot boundary as the conservative commit floor. The history scanner -also verifies that a joint baseline's old voters match the bound set. +checks configuration-bearing snapshot metadata against the active baseline; +the original binding may refer to an earlier stable set. `ScanCommittedConfigurationHistory` uses the recovered commit index directly -and requires both durable commit-index support and a persisted voter binding. -The caller still supplies a trusted history baseline through the snapshot -boundary. This remains an offline validation API; it does not activate joint -quorums or permit live configuration changes. +and requires both durable commit-index support and a persisted original voter +binding. For configuration-bearing snapshots, the caller supplies the active +history baseline through the snapshot boundary; the scanner verifies it against +the snapshot metadata. Legacy snapshots still require the baseline voters to +match the original binding. This remains an offline validation API; the snapshot +metadata is not proof of commitment or cluster identity, and this does not +activate joint quorums or permit live configuration changes. In alpha.45 these images were persistence for offline validation only. The alpha.46 follow-up enables runtime recovery and transfer of MRFTS003 stable snapshots when the voter list equals the node's existing fixed membership. Compaction carries forward that stable configuration identity. MRFTS002 joint snapshots and configuration log entries remain disabled in the live runtime. -Stable snapshots that claim another voter set are still rejected because -membership transition semantics are not implemented. +Since alpha.58, bound storage can preserve its original voter binding while +installing configuration-bearing snapshots whose active set changed. Raft +startup and network snapshot handling still reject those transition snapshots; +the storage support is only for offline history validation and trusted inputs. -Tests cover in-memory installation, durable joint and stable round-trips, bound -voter mismatch rejection without rewriting storage, offline scan baseline checks, -stable runtime restore/transfer/compaction, and continued joint snapshot rejection. +Tests cover in-memory installation, durable joint and stable round-trips, +rejection of unbound configuration snapshots and mismatched legacy snapshots, +offline scan baselines across multiple committed transitions, stable runtime +restore/transfer/compaction, and continued joint snapshot rejection. No independent reviewer approval is claimed. diff --git a/docs/joint-snapshots.md b/docs/joint-snapshots.md index 3aeea62..2844a83 100644 --- a/docs/joint-snapshots.md +++ b/docs/joint-snapshots.md @@ -25,9 +25,12 @@ snapshot store can export and load v2 images for testing or offline tooling. This is a format and offline persistence foundation, not live joint consensus or complete configuration history. A bound memory or durable store can install and -recover a joint snapshot for offline validation when its persisted voter list -matches the snapshot's old voter set. Raft startup and RPC paths still reject -joint snapshots, and compaction only creates fixed-voter snapshots. A covered +recover a joint snapshot for offline validation while preserving its original +voter binding, which may differ from the snapshot's active old voter set after +earlier transitions. The caller must supply a trusted history baseline; the +snapshot codec and store do not prove that the transition was committed. Raft +startup and RPC paths still reject joint snapshots, and compaction only creates +fixed-voter snapshots. A covered snapshot is acknowledged only after its prefix identifies supported fixed v1 or stable v3 metadata, including when the prefix arrives in several chunks. Unsupported joint images are rejected without replacing application data or diff --git a/docs/learners.md b/docs/learners.md index 1090d4b..d9eff1e 100644 --- a/docs/learners.md +++ b/docs/learners.md @@ -95,9 +95,12 @@ stores CurrentTerm[8], VotedFor[4], retained-log count[8], CommitIndex[8], voter count[4], sorted voter IDs[4 each], snapshot length[8], optional MRFTS001, MRFTS002, or MRFTS003 image, then retained log records. The CRC covers version magic plus payload. Recovery bounds and validates the voter list, commit -boundary and requires any snapshot to name the same voters. MRFTD001 through -MRFTD003 remain readable; formats without commit metadata conservatively use -SnapshotIndex as their commit floor. The next successful publication writes v4 +boundary. Legacy snapshots must name the original bound voters; configuration- +bearing snapshots can describe a later active set while retaining that original +binding for offline history scans. Raft startup still requires the runtime voter +set and binding to match. MRFTD001 through MRFTD003 remain readable; formats +without commit metadata conservatively use SnapshotIndex as their commit floor. +The next successful publication writes v4 using the existing atomic write/fsync/rename/directory-fsync protocol. A crash before first binding leaves either a fresh unbound store with no votes/log, or a complete bound store. Publication failure is fatal before the node can serve diff --git a/docs/stable-configuration-snapshot-runtime.md b/docs/stable-configuration-snapshot-runtime.md index 7b64b34..b653525 100644 --- a/docs/stable-configuration-snapshot-runtime.md +++ b/docs/stable-configuration-snapshot-runtime.md @@ -19,9 +19,12 @@ erasing configuration history and prevents a peer from replacing it with a conflicting baseline. This does not implement membership changes. Configuration log entries remain -disabled, the voter binding cannot change, and a stable snapshot claiming a -different voter list is rejected. Joint consensus, transition rollback and -snapshot installation across a membership transition remain open. Upgrade all +disabled, and the original voter binding cannot change. Raft startup and network +installation still reject a stable snapshot whose active set differs from the +configured fixed set. Since alpha.58, offline storage can retain such +configuration-bearing snapshots without changing the original binding. Joint +consensus, transition rollback and snapshot installation across a membership +transition remain open. Upgrade all voters before sending MRFTS003 images because older binaries may acknowledge an already-covered unknown snapshot format. diff --git a/docs/stable-configuration-snapshots.md b/docs/stable-configuration-snapshots.md index c44af17..8d61382 100644 --- a/docs/stable-configuration-snapshots.md +++ b/docs/stable-configuration-snapshots.md @@ -18,12 +18,19 @@ trusted snapshot and commit boundary independently. MRFTS003 cannot assert that there was no earlier configuration entry; an explicit trusted baseline remains necessary for such histories. -The runtime accepts MRFTS003 only when its voter list matches the node's existing -fixed voter binding. It can recover the image at startup, restore the application, -install it from a peer, and preserve its configuration identity through later -compaction. The covered-snapshot path checks the stable configuration identity -and voter list before acknowledging an already-covered boundary. These paths do -not activate a changed voter set or enable joint consensus. +The live runtime accepts MRFTS003 only when its voter list matches the node's +configured fixed voter set and persisted binding. It can recover the image at +startup, restore the application, install it from a peer, and preserve its +configuration identity through later compaction. The covered-snapshot path +checks the stable configuration identity and voter list before acknowledging an +already-covered boundary. These paths do not activate a changed voter set or +enable joint consensus. + +Since alpha.58, bound in-memory and durable storage can also retain an +MRFTS003/MRFTS002 image whose active set differs from the original persisted +binding, for offline history validation. That storage support does not make the +image acceptable to live Raft or authenticate its membership claim; callers must +provide a trusted history baseline. Older binaries may falsely acknowledge covered unknown formats; do not send these images to a mixed-version cluster. Joint snapshot installation, live diff --git a/miniraft/configuration.cpp b/miniraft/configuration.cpp index 151aa02..33e2ff2 100644 --- a/miniraft/configuration.cpp +++ b/miniraft/configuration.cpp @@ -178,8 +178,13 @@ TConfigurationScan ScanConfigurationHistory(const IState& state, throw std::invalid_argument("Invalid empty snapshot boundary"); } const auto bound = state.GetBoundVoters(); - if (!bound.empty() && bound != baseline.OldVoters()) - throw std::invalid_argument("Configuration baseline differs from bound voters"); + // The binding is the trusted original set. Configuration-bearing snapshots + // carry the active set and its identity after a transition, so those voters + // may legitimately differ. Legacy snapshots still require the original + // set because they cannot describe or validate that transition boundary. + if (!bound.empty() && bound != baseline.OldVoters() && + (!snapshot || !snapshot->HasConfigurationMetadata())) + throw std::invalid_argument("Configuration baseline differs from the trusted voter history"); auto latest = baseline; auto committed = baseline; diff --git a/miniraft/durable.cpp b/miniraft/durable.cpp index c4f5a48..fcc7ae1 100644 --- a/miniraft/durable.cpp +++ b/miniraft/durable.cpp @@ -96,6 +96,8 @@ TBytes ReadAll(int fd) { struct TDurableState::TImpl { std::vector> Log; std::shared_ptr Snapshot; + // Original trusted voter set; configuration snapshots can carry a later + // active set without rewriting this binding. std::vector Voters; bool Recovered = false; bool RaftStarted = false; @@ -179,11 +181,9 @@ TDurableState::TDurableState(const std::string& name, uint32_t id, THook hook) throw std::runtime_error("Invalid embedded snapshot size"); if (length) { auto snapshot = DecodeSnapshot(std::string_view(reinterpret_cast(bytes.data() + position), size_t(length))); - if (snapshot.HasConfigurationMetadata() && Impl->Voters.empty()) - throw std::runtime_error("Configuration snapshot recovery requires a bound voter set"); + if (!SnapshotCompatibleWithVoterBinding(snapshot, Impl->Voters)) + throw std::runtime_error("Durable snapshot is incompatible with the persisted voter binding"); if (snapshot.Term > CurrentTerm) throw std::runtime_error("Future durable snapshot term"); - if (!Impl->Voters.empty() && snapshot.Voters != Impl->Voters) - throw std::runtime_error("Durable snapshot/voter mismatch"); position += size_t(length); SnapshotIndex = snapshot.Index; SnapshotTerm = snapshot.Term; @@ -200,7 +200,8 @@ TDurableState::TDurableState(const std::string& name, uint32_t id, THook hook) if (snapshot.HasConfigurationMetadata() && !configured) throw std::runtime_error("Configuration snapshot recovery requires a bound voter set"); if (snapshot.Term > CurrentTerm) throw std::runtime_error("Future durable snapshot term"); - if (configured && snapshot.Voters != Impl->Voters) throw std::runtime_error("Durable snapshot/voter mismatch"); + if (configured && !SnapshotCompatibleWithVoterBinding(snapshot, Impl->Voters)) + throw std::runtime_error("Durable snapshot is incompatible with the persisted voter binding"); position += size_t(length); SnapshotIndex = snapshot.Index; SnapshotTerm = snapshot.Term; @@ -260,10 +261,10 @@ void TDurableState::Publish() Put(payload, Impl->Voters.size(), 4); if (!Impl->Voters.empty()) { ValidateFixedVoters(Impl->Voters); - if (Impl->Snapshot && Impl->Snapshot->Voters != Impl->Voters) - throw std::runtime_error("Snapshot would change bound voters"); for (auto id : Impl->Voters) Put(payload, id, 4); } + if (Impl->Snapshot && !SnapshotCompatibleWithVoterBinding(*Impl->Snapshot, Impl->Voters)) + throw std::runtime_error("Durable snapshot is incompatible with the persisted voter binding"); const auto image = Impl->Snapshot ? EncodeSnapshot(*Impl->Snapshot) : std::string{}; Put(payload, image.size(), 8); payload.insert(payload.end(), image.begin(), image.end()); @@ -350,10 +351,8 @@ void TDurableState::InstallSnapshot(const TSnapshot& snapshot) { TSnapshot image = snapshot; PreserveStableSnapshotIdentity(Impl->Snapshot.get(), image); ValidateSnapshot(image); - if (image.HasConfigurationMetadata() && Impl->Voters.empty()) - throw std::invalid_argument("Configuration snapshots require a bound voter set"); - if (!Impl->Voters.empty() && image.Voters != Impl->Voters) - throw std::invalid_argument("Snapshot would change bound voters"); + if (!SnapshotCompatibleWithVoterBinding(image, Impl->Voters)) + throw std::invalid_argument("Snapshot is incompatible with the persisted voter binding"); if (image.Index <= SnapshotIndex || image.Term < SnapshotTerm || image.Term > CurrentTerm) throw std::invalid_argument("Obsolete or future durable snapshot"); if (image.Index <= CommitIndex && diff --git a/miniraft/snapshot.h b/miniraft/snapshot.h index 3d1429c..02a5576 100644 --- a/miniraft/snapshot.h +++ b/miniraft/snapshot.h @@ -22,8 +22,9 @@ struct TSnapshot { // Runtime installation is disabled until membership recovery is implemented. std::vector JointVoters; uint64_t ConfigurationIndex = 0, ConfigurationTerm = 0; - // Experimental stable-configuration identity; runtime requires the same - // persisted fixed voter binding. + // Experimental stable-configuration identity. Offline storage may retain + // an active voter set that differs from its original persisted binding; + // live Raft still rejects that configuration until transitions are enabled. bool StableConfiguration = false; bool IsJoint() const { return !JointVoters.empty() || (!StableConfiguration && (ConfigurationIndex || ConfigurationTerm)); } bool HasConfigurationMetadata() const { return StableConfiguration || IsJoint(); } diff --git a/miniraft/state.h b/miniraft/state.h index 264ef74..10cdd31 100644 --- a/miniraft/state.h +++ b/miniraft/state.h @@ -47,6 +47,8 @@ struct IState: TStateFields { // True when this backend recovered a persistent image without a stored // voter identity; Raft must obtain proof before serving this state. virtual bool HasUnboundRecoveredState() const { return false; } + // Original trusted voter set persisted with this store. It may differ from + // the active set in a configuration-bearing offline snapshot. virtual std::vector GetBoundVoters() const { return {}; } virtual void BindVoters(const std::vector&) { throw std::logic_error("Storage voter binding is unsupported"); } // Storage backends can close startup-only migration paths once a Raft @@ -90,6 +92,17 @@ inline void ValidateVoterBinding(const IState& state, const std::vector& voters) { + if (snapshot.HasConfigurationMetadata()) return !voters.empty(); + return voters.empty() || snapshot.Voters == voters; +} + struct TState: IState { static TMessageHolder CloneEntry(const TMessageHolder& entry) { if (!entry.HasValidExtent() || entry->Type != uint32_t(EMessageType::LOG_ENTRY) || entry->Len < sizeof(TLogEntry)) @@ -168,10 +181,8 @@ struct TState: IState { TSnapshot image = snapshot; PreserveStableSnapshotIdentity(Snapshot.get(), image); ValidateSnapshot(image); // Validate before changing any state. - if (image.HasConfigurationMetadata() && BoundVoters.empty()) - throw std::invalid_argument("Configuration snapshots require a bound voter set"); - if (!BoundVoters.empty() && image.Voters != BoundVoters) - throw std::invalid_argument("Snapshot would change bound voters"); + if (!SnapshotCompatibleWithVoterBinding(image, BoundVoters)) + throw std::invalid_argument("Snapshot is incompatible with the persisted voter binding"); if (image.Index <= SnapshotIndex || image.Term < SnapshotTerm || image.Term > CurrentTerm) throw std::invalid_argument("Obsolete or future snapshot"); if (image.Index <= CommitIndex && diff --git a/test/test_configuration.cpp b/test/test_configuration.cpp index 6108c18..89ee8f3 100644 --- a/test/test_configuration.cpp +++ b/test/test_configuration.cpp @@ -203,11 +203,10 @@ void test_scan_separates_commit_and_replaced_suffix(void**) { void test_scan_from_joint_snapshot_and_rejects_mismatched_baselines(void**) { TSnapshot snapshot{2, 2, {1, 2}, "image"}; snapshot.JointVoters = {2, 3}; snapshot.ConfigurationIndex = 2; snapshot.ConfigurationTerm = 2; - // Synthetic offline state with the original voter set durably bound. - TState state; state.Snapshot = std::make_shared(snapshot); - state.SnapshotIndex = state.LastLogIndex = 2; state.SnapshotTerm = state.LastLogTerm = 2; - state.BoundVoters = snapshot.Voters; - state.CurrentTerm = 3; state.Append(Ordinary(2)); + // The original binding is retained while the snapshot describes an active + // configuration reached by an earlier committed transition. + TState state; state.BindVoters({7, 8}); state.CurrentTerm = 2; + state.InstallSnapshot(snapshot); state.CurrentTerm = 3; state.Append(Ordinary(2)); TConfigurationRecord final{EConfigurationPhase::Stable, 3, 2, 2, {1, 2}, {2, 3}}; state.Append(EncodeConfigurationEntry(final)); auto base = TConfigurationHistory::FromJointSnapshot(snapshot); @@ -221,8 +220,8 @@ void test_scan_from_joint_snapshot_and_rejects_mismatched_baselines(void**) { auto wrong = snapshot; wrong.JointVoters = {2, 4}; Reject([&] { ScanConfigurationHistory(state, TConfigurationHistory::FromJointSnapshot(wrong), 2); }); state.BoundVoters = {1, 2, 4}; - Reject([&] { ScanConfigurationHistory(state, base, 2); }); - state.BoundVoters = snapshot.Voters; + assert_true(ScanConfigurationHistory(state, base, 2).Committed.IsJoint()); + state.BoundVoters = {7, 8}; state.LastLogTerm = 2; Reject([&] { ScanConfigurationHistory(state, base, 2); }); } @@ -232,8 +231,8 @@ void test_scan_from_stable_configuration_snapshot(void**) { auto decoded = DecodeSnapshot(EncodeSnapshot(snapshot)); auto baseline = TConfigurationHistory::FromStableSnapshot(decoded); assert_false(baseline.IsJoint()); assert_int_equal(baseline.ConfigurationIndex(), 4); - TState state; state.Snapshot = std::make_shared(decoded); - state.SnapshotIndex = state.LastLogIndex = 5; state.SnapshotTerm = state.LastLogTerm = 2; + TState state; state.BindVoters({7, 8}); + state.CurrentTerm = 2; state.InstallSnapshot(decoded); state.CurrentTerm = 3; TConfigurationRecord joint{EConfigurationPhase::Joint, 3, 4, 2, {1, 2}, {2, 3}}; state.Append(EncodeConfigurationEntry(joint)); @@ -496,6 +495,72 @@ void test_persisted_commit_boundary_drives_committed_configuration_scan(void**) }); std::error_code ignored; std::filesystem::remove_all(dir, ignored); } + +void test_configuration_snapshots_preserve_original_binding_across_transitions(void**) { + auto dir = std::filesystem::temp_directory_path() / ("miniraft-config-snapshot-binding-" + + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count())); + assert_true(std::filesystem::create_directory(dir)); + const auto name = (dir / "raft").string(); + const std::vector original{1, 2, 3}; + const std::vector firstActive{2, 3, 4}; + const std::vector secondActive{2, 4, 5}; + const std::vector thirdActive{4, 5, 6}; + + TSnapshot jointSnapshot{5, 4, firstActive, "joint image"}; + jointSnapshot.JointVoters = secondActive; + jointSnapshot.ConfigurationIndex = 4; jointSnapshot.ConfigurationTerm = 3; + TConfigurationRecord stable{EConfigurationPhase::Stable, 5, 4, 3, firstActive, secondActive}; + + { + TDurableState state(name, 2); + state.BindVoters(original); + state.CurrentTerm = 4; state.Commit(); + state.InstallSnapshot(jointSnapshot); + assert_true(state.GetBoundVoters() == original); + assert_true(*state.GetSnapshot() == jointSnapshot); + } + + TSnapshot stableSnapshot{6, 5, secondActive, "stable image"}; + stableSnapshot.StableConfiguration = true; + stableSnapshot.ConfigurationIndex = 6; stableSnapshot.ConfigurationTerm = 5; + TConfigurationRecord nextJoint{EConfigurationPhase::Joint, 6, 6, 5, secondActive, thirdActive}; + { + TDurableState state(name, 2); + assert_true(state.GetBoundVoters() == original); + assert_true(*state.GetSnapshot() == jointSnapshot); + auto history = TConfigurationHistory::FromJointSnapshot(*state.GetSnapshot()); + state.CurrentTerm = 5; state.Commit(); + state.Append(EncodeConfigurationEntry(stable)); + state.CommitIndex = state.LastLogIndex; state.Commit(); + auto scan = ScanCommittedConfigurationHistory(state, history); + assert_false(scan.Committed.IsJoint()); + assert_true(scan.Committed.OldVoters() == secondActive); + + state.InstallSnapshot(stableSnapshot); + state.CurrentTerm = 6; state.Commit(); + state.Append(EncodeConfigurationEntry(nextJoint)); + state.CommitIndex = state.LastLogIndex; state.Commit(); + } + + { + auto state = std::make_shared(name, 2); + assert_true(state->GetBoundVoters() == original); + assert_true(*state->GetSnapshot() == stableSnapshot); + auto baseline = TConfigurationHistory::FromStableSnapshot(*state->GetSnapshot()); + const auto scan = ScanCommittedConfigurationHistory(*state, baseline); + assert_true(scan.Committed.IsJoint()); + assert_true(scan.Committed.OldVoters() == secondActive); + assert_true(scan.Committed.NewVoters() == thirdActive); + assert_false(scan.HasUncommittedConfiguration()); + + const auto fields = static_cast(*state); + Reject([&] { TRaft disabled(state, 2, {{4, std::make_shared()}, + {5, std::make_shared()}}); }); + assert_true(static_cast(*state) == fields); + assert_true(state->GetBoundVoters() == original); + } + std::error_code ignored; std::filesystem::remove_all(dir, ignored); +} #endif } int main() { @@ -518,6 +583,7 @@ int main() { cmocka_unit_test(test_legacy_restart_uses_safe_flag_lookup_and_preserves_files), cmocka_unit_test(test_offline_history_scan_after_durable_snapshot_and_restart), cmocka_unit_test(test_persisted_commit_boundary_drives_committed_configuration_scan), + cmocka_unit_test(test_configuration_snapshots_preserve_original_binding_across_transitions), #endif }; return cmocka_run_group_tests(tests, nullptr, nullptr); diff --git a/test/test_snapshots.cpp b/test/test_snapshots.cpp index aa5b5fc..c87c64f 100644 --- a/test/test_snapshots.cpp +++ b/test/test_snapshots.cpp @@ -530,7 +530,8 @@ void test_joint_snapshot_export_and_durable_roundtrip_but_runtime_rejects(void** assert_true(state.GetBoundVoters() == JointSample().Voters); assert_true(*state.GetSnapshot() == JointSample()); const auto before = static_cast(state); - auto mismatched = JointSample(); mismatched.Voters = {1, 2, 4}; + // Legacy snapshots still cannot change the original voter binding. + TSnapshot mismatched{3, 1, {1, 2, 4}, "legacy changed voters"}; std::ifstream input(path, std::ios::binary); bytes.assign(std::istreambuf_iterator(input), {}); Reject([&] { state.InstallSnapshot(mismatched); }); From e3922bed16ab32daf16e9efe8e1d07ef5f166fce Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Sat, 14 Nov 2026 21:44:42 -0500 Subject: [PATCH 59/66] Validate configuration continuity across snapshots (#59) --- docs/configuration-snapshot-continuity.md | 34 +++++++++ docs/durable-configuration-snapshots.md | 3 + docs/joint-snapshots.md | 5 ++ docs/stable-snapshot-storage-continuity.md | 4 + miniraft/snapshot.cpp | 58 +++++++++++---- miniraft/snapshot.h | 4 +- test/test_snapshots.cpp | 85 ++++++++++++++++++++++ 7 files changed, 177 insertions(+), 16 deletions(-) create mode 100644 docs/configuration-snapshot-continuity.md diff --git a/docs/configuration-snapshot-continuity.md b/docs/configuration-snapshot-continuity.md new file mode 100644 index 0000000..386178b --- /dev/null +++ b/docs/configuration-snapshot-continuity.md @@ -0,0 +1,34 @@ +# Configuration snapshot continuity: v0.1.0-alpha.59 + +Alpha.58 separated a store's original voter binding from the active voter set +recorded in configuration-bearing snapshots. That exposed a continuity check +that previously relied on the original binding to catch conflicting snapshot +memberships. + +When storage already has configuration metadata, a replacement snapshot must +extend that known state: + +- Stable to stable snapshots keep both the configuration entry index/term and + the active voter set unchanged. +- Stable to joint snapshots use the current active set as their old voters and + place the new configuration entry after the current snapshot boundary without + regressing its log term. +- Joint snapshots may advance their included log boundary only while keeping the + same joint identity and both voter sets. Replacing joint metadata with a + different joint record requires a stable successor first. +- Joint to stable snapshots select the joint snapshot's new voter set and use a + stable entry after the joint snapshot boundary without regressing its log + term. +- A legacy refresh of a stable snapshot must retain its active voter set; storage + carries the stable identity forward. A legacy image cannot erase joint state. + +These checks run in both storage backends and application snapshot recovery. They +check continuity with locally stored metadata, but they do not prove that a +configuration entry was committed or authenticate the sender. Callers that use +the offline install APIs must still provide a trusted snapshot and history +baseline. Live configuration entries, joint quorum activation and +membership-aware network installation remain disabled. + +Deterministic memory and durable tests cover conflicting same-identity voter +sets, stale legacy membership, stable-to-joint and joint-to-stable transitions, +rejected phase downgrades and durable recovery of the original binding. diff --git a/docs/durable-configuration-snapshots.md b/docs/durable-configuration-snapshots.md index f3f1836..90db787 100644 --- a/docs/durable-configuration-snapshots.md +++ b/docs/durable-configuration-snapshots.md @@ -40,6 +40,9 @@ Since alpha.58, bound storage can preserve its original voter binding while installing configuration-bearing snapshots whose active set changed. Raft startup and network snapshot handling still reject those transition snapshots; the storage support is only for offline history validation and trusted inputs. +The [alpha.59 follow-up](configuration-snapshot-continuity.md) checks that a +replacement configuration snapshot extends the previously stored phase, +identity, and active voter sets. Tests cover in-memory installation, durable joint and stable round-trips, rejection of unbound configuration snapshots and mismatched legacy snapshots, diff --git a/docs/joint-snapshots.md b/docs/joint-snapshots.md index 2844a83..0e6f216 100644 --- a/docs/joint-snapshots.md +++ b/docs/joint-snapshots.md @@ -23,6 +23,11 @@ bounded before allocation. Fixed snapshots continue to encode as MRFTS001 with exactly the previous bytes; a golden v1 image tests compatibility. The standalone snapshot store can export and load v2 images for testing or offline tooling. +The [alpha.59 continuity follow-up](configuration-snapshot-continuity.md) checks +that a stored stable configuration advances to a joint image using the same old +voters, and that a joint image can only continue with the same joint identity or +move to a stable successor selecting its new voters. + This is a format and offline persistence foundation, not live joint consensus or complete configuration history. A bound memory or durable store can install and recover a joint snapshot for offline validation while preserving its original diff --git a/docs/stable-snapshot-storage-continuity.md b/docs/stable-snapshot-storage-continuity.md index 9122f83..4c06da9 100644 --- a/docs/stable-snapshot-storage-continuity.md +++ b/docs/stable-snapshot-storage-continuity.md @@ -18,6 +18,10 @@ does not enable joint snapshot recovery, live membership changes, or membership activation from an untrusted snapshot. Joint images remain explicit metadata and are still available to the existing offline durable format tests. +The [alpha.59 follow-up](configuration-snapshot-continuity.md) extends this +continuity check to active voter sets, stable-to-joint transitions and joint +snapshot recovery, while retaining the same offline-only trust boundary. + Tests install a legacy refresh and attempt a conflicting stable replacement through both memory and durable storage. They check metadata preservation, rejection atomicity, and durable recovery after restart. Release and ASan/UBSan diff --git a/miniraft/snapshot.cpp b/miniraft/snapshot.cpp index da96de3..d34c409 100644 --- a/miniraft/snapshot.cpp +++ b/miniraft/snapshot.cpp @@ -41,11 +41,44 @@ void Validate(const TSnapshot& snapshot) { validateVoters(snapshot.JointVoters); } } -void CheckStableSnapshotIdentity(const TSnapshot* current, const TSnapshot& incoming) { - if (current && current->StableConfiguration && incoming.StableConfiguration && - (incoming.ConfigurationIndex != current->ConfigurationIndex || - incoming.ConfigurationTerm != current->ConfigurationTerm)) - throw std::invalid_argument("Stable snapshot configuration identity changed"); +void CheckSnapshotConfigurationContinuity(const TSnapshot* current, const TSnapshot& incoming) { + if (!current || !current->HasConfigurationMetadata()) return; + if (current->StableConfiguration) { + if (incoming.StableConfiguration) { + if (incoming.ConfigurationIndex != current->ConfigurationIndex || + incoming.ConfigurationTerm != current->ConfigurationTerm) + throw std::invalid_argument("Stable snapshot configuration identity changed"); + if (incoming.Voters != current->Voters) + throw std::invalid_argument("Stable snapshot voters changed without a new configuration identity"); + return; + } + if (incoming.IsJoint()) { + if (incoming.Voters != current->Voters || + incoming.ConfigurationIndex <= current->Index || + incoming.ConfigurationTerm < current->Term) + throw std::invalid_argument("Joint snapshot does not extend the stable configuration"); + return; + } + if (incoming.Voters != current->Voters) + throw std::invalid_argument("Legacy snapshot would change the active voter set"); + return; + } + + if (incoming.IsJoint()) { + if (incoming.Voters != current->Voters || incoming.JointVoters != current->JointVoters || + incoming.ConfigurationIndex != current->ConfigurationIndex || + incoming.ConfigurationTerm != current->ConfigurationTerm) + throw std::invalid_argument("Joint snapshot configuration changed without a stable successor"); + return; + } + if (incoming.StableConfiguration) { + if (incoming.Voters != current->JointVoters || + incoming.ConfigurationIndex <= current->Index || + incoming.ConfigurationTerm < current->Term) + throw std::invalid_argument("Stable snapshot does not extend the joint configuration"); + return; + } + throw std::invalid_argument("Legacy snapshot cannot discard joint configuration metadata"); } void CheckLocalLog(const TSnapshot& snapshot, const TRaft& raft) { Validate(snapshot); @@ -76,14 +109,10 @@ void Sync(int fd) { void ValidateSnapshot(const TSnapshot& snapshot) { Validate(snapshot); } void PreserveStableSnapshotIdentity(const TSnapshot* current, TSnapshot& incoming) { - if (!current || !current->StableConfiguration) return; - CheckStableSnapshotIdentity(current, incoming); - if (incoming.StableConfiguration) { - return; - } - // A joint image carries an explicit new phase. A legacy image carries no - // configuration claim, so retain the already trusted stable identity. - if (!incoming.HasConfigurationMetadata()) { + CheckSnapshotConfigurationContinuity(current, incoming); + if (current && current->StableConfiguration && !incoming.HasConfigurationMetadata()) { + // Legacy images carry no membership claim, so retain the known stable + // identity after confirming their active voters are unchanged. incoming.StableConfiguration = true; incoming.ConfigurationIndex = current->ConfigurationIndex; incoming.ConfigurationTerm = current->ConfigurationTerm; @@ -163,7 +192,8 @@ TSnapshot CaptureSnapshot(const TRaft& raft, const IRsm& rsm) { void RestoreSnapshot(const TSnapshot& snapshot, const TRaft& raft, IRsm& rsm) { const auto knownSnapshot = raft.GetState()->GetSnapshot(); - CheckStableSnapshotIdentity(knownSnapshot.get(), snapshot); + ValidateSnapshot(snapshot); + CheckSnapshotConfigurationContinuity(knownSnapshot.get(), snapshot); CheckLocalLog(snapshot, raft); if (rsm.LastAppliedIndex >= snapshot.Index) throw std::runtime_error("Application is not behind the snapshot"); diff --git a/miniraft/snapshot.h b/miniraft/snapshot.h index 02a5576..ece53c2 100644 --- a/miniraft/snapshot.h +++ b/miniraft/snapshot.h @@ -34,8 +34,8 @@ std::string EncodeSnapshot(const TSnapshot& snapshot); TSnapshot DecodeSnapshot(std::string_view bytes); // Validate metadata and size without copying the application image. void ValidateSnapshot(const TSnapshot& snapshot); -// Preserve a trusted stable configuration identity when a legacy image is -// refreshed, and reject a stable image that changes that identity. +// Preserve a trusted stable identity on legacy refresh and validate snapshot +// configuration continuity: stable -> joint -> stable, with voter-set checks. void PreserveStableSnapshotIdentity(const TSnapshot* current, TSnapshot& incoming); TSnapshot CaptureSnapshot(const TRaft& raft, const IRsm& rsm); // Startup only, with an application behind the snapshot and diff --git a/test/test_snapshots.cpp b/test/test_snapshots.cpp index c87c64f..9dd993b 100644 --- a/test/test_snapshots.cpp +++ b/test/test_snapshots.cpp @@ -427,10 +427,81 @@ void CheckStableSnapshotIdentityContinuity(IState& state) { assert_true(static_cast(state) == before); assert_true(*state.GetSnapshot() == preserved); } +TSnapshot CheckConfigurationSnapshotTransitionContinuity(IState& state) { + const std::vector original{1, 2, 3}; + const std::vector active{2, 3, 4}; + const std::vector next{3, 4, 5}; + state.BindVoters(original); + + TSnapshot stable{2, 1, active, "stable before transition"}; + stable.StableConfiguration = true; + stable.ConfigurationIndex = 1; stable.ConfigurationTerm = 1; + state.InstallSnapshot(stable); + + // Neither a legacy refresh nor a stable image with the same identity may + // silently restore the original or any other voter list. + TSnapshot legacyWithOriginal{3, 1, original, "legacy stale membership"}; + Reject([&] { state.InstallSnapshot(legacyWithOriginal); }); + auto stableWithConflict = stable; stableWithConflict.Index = 3; + stableWithConflict.Voters = original; + Reject([&] { state.InstallSnapshot(stableWithConflict); }); + assert_true(*state.GetSnapshot() == stable); + + TSnapshot legacy{3, 1, active, "legacy refresh"}; + state.InstallSnapshot(legacy); + stable.Index = legacy.Index; stable.Data = legacy.Data; + assert_true(*state.GetSnapshot() == stable); + + state.CurrentTerm = 2; state.Commit(); + TSnapshot wrongJoint{4, 2, original, "wrong old set"}; + wrongJoint.JointVoters = next; + wrongJoint.ConfigurationIndex = 4; wrongJoint.ConfigurationTerm = 2; + Reject([&] { state.InstallSnapshot(wrongJoint); }); + TSnapshot staleJoint{4, 2, active, "configuration predates stable snapshot"}; + staleJoint.JointVoters = next; + staleJoint.ConfigurationIndex = 3; staleJoint.ConfigurationTerm = 1; + Reject([&] { state.InstallSnapshot(staleJoint); }); + assert_true(*state.GetSnapshot() == stable); + + TSnapshot joint{4, 2, active, "joint transition"}; + joint.JointVoters = next; + joint.ConfigurationIndex = 4; joint.ConfigurationTerm = 2; + state.InstallSnapshot(joint); + auto conflictingJoint = joint; conflictingJoint.Index = 5; + conflictingJoint.JointVoters = {3, 5, 6}; + Reject([&] { state.InstallSnapshot(conflictingJoint); }); + + TSnapshot legacyDowngrade{5, 2, active, "cannot erase joint phase"}; + Reject([&] { state.InstallSnapshot(legacyDowngrade); }); + TSnapshot wrongStable{5, 2, active, "wrong stable set"}; + wrongStable.StableConfiguration = true; + wrongStable.ConfigurationIndex = 5; wrongStable.ConfigurationTerm = 2; + Reject([&] { state.InstallSnapshot(wrongStable); }); + TSnapshot regressedStable{5, 2, next, "configuration term predates joint snapshot"}; + regressedStable.StableConfiguration = true; + regressedStable.ConfigurationIndex = 5; regressedStable.ConfigurationTerm = 1; + Reject([&] { state.InstallSnapshot(regressedStable); }); + + TSnapshot final{5, 2, next, "stable after transition"}; + final.StableConfiguration = true; + final.ConfigurationIndex = 5; final.ConfigurationTerm = 2; + state.InstallSnapshot(final); + + auto conflictingFinal = final; conflictingFinal.Index = 6; + conflictingFinal.Voters = {3, 4, 6}; + Reject([&] { state.InstallSnapshot(conflictingFinal); }); + assert_true(*state.GetSnapshot() == final); + assert_true(state.GetBoundVoters() == original); + return final; +} void test_memory_snapshot_storage_preserves_stable_configuration_identity(void**) { TState state; CheckStableSnapshotIdentityContinuity(state); } +void test_memory_snapshot_storage_checks_configuration_transition_continuity(void**) { + TState state; + CheckConfigurationSnapshotTransitionContinuity(state); +} #ifdef __linux__ void test_durable_snapshot_storage_preserves_stable_configuration_identity(void**) { TFiles files; @@ -446,6 +517,18 @@ void test_durable_snapshot_storage_preserves_stable_configuration_identity(void* assert_int_equal(snapshot->ConfigurationTerm, 1); assert_true(snapshot->Data == "legacy refresh"); } +void test_durable_snapshot_storage_checks_configuration_transition_continuity(void**) { + TFiles files; + const std::vector original{1, 2, 3}; + TSnapshot final; + { + TDurableState state(files.Name, 1); + final = CheckConfigurationSnapshotTransitionContinuity(state); + } + TDurableState recovered(files.Name, 1); + assert_true(recovered.GetBoundVoters() == original); + assert_true(recovered.GetSnapshot() && *recovered.GetSnapshot() == final); +} void test_stable_configuration_snapshot_durable_runtime_recovery_and_compaction(void**) { TFiles files; TKv source; Apply(source, "a", "one", 1); Apply(source, "b", "two", 2); @@ -671,8 +754,10 @@ int main() { cmocka_unit_test(test_restore_rejects_checkpoint_with_conflicting_stable_identity), cmocka_unit_test(test_startup_restores_stale_application_from_durable_snapshot_boundary), cmocka_unit_test(test_memory_snapshot_storage_preserves_stable_configuration_identity), + cmocka_unit_test(test_memory_snapshot_storage_checks_configuration_transition_continuity), #ifdef __linux__ cmocka_unit_test(test_durable_snapshot_storage_preserves_stable_configuration_identity), + cmocka_unit_test(test_durable_snapshot_storage_checks_configuration_transition_continuity), cmocka_unit_test(test_stable_configuration_snapshot_durable_runtime_recovery_and_compaction), cmocka_unit_test(test_unbound_durable_configuration_snapshot_recovery_fails_closed), cmocka_unit_test(test_joint_snapshot_export_and_durable_roundtrip_but_runtime_rejects), From 5b1f56d3704f78548bbb66dbe420d4444e5486da Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Sat, 14 Nov 2026 22:22:12 -0500 Subject: [PATCH 60/66] Validate snapshot metadata before accepting leader contact (#60) --- README.md | 2 ++ docs/snapshot-contact-validation.md | 28 +++++++++++++++++++++++++ miniraft/raft.cpp | 32 +++++++++++++++++++++++------ test/test_compaction.cpp | 12 +++++++++++ 4 files changed, 68 insertions(+), 6 deletions(-) create mode 100644 docs/snapshot-contact-validation.md diff --git a/README.md b/README.md index 614276e..e3659e5 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,8 @@ The [network learner follow-up](docs/online-learner-network.md) connects that API to the built-in server's runtime drain loop. The [covered-snapshot follow-up](docs/covered-snapshot-identity.md) validates snapshot boundary and voter metadata before acknowledging an already-covered image. +The [snapshot contact follow-up](docs/snapshot-contact-validation.md) waits for +a supported metadata prefix before a same-term image can refresh leader contact. The [snapshot transfer expiry follow-up](docs/snapshot-transfer-expiry.md) releases incomplete incoming images after 30 seconds without a valid chunk, including on learners. diff --git a/docs/snapshot-contact-validation.md b/docs/snapshot-contact-validation.md new file mode 100644 index 0000000..12f18a9 --- /dev/null +++ b/docs/snapshot-contact-validation.md @@ -0,0 +1,28 @@ +# InstallSnapshot leader-contact validation + +InstallSnapshot previously demoted a candidate and refreshed its election timer +as soon as a structurally valid RPC arrived. The runtime only discovered that a +same-term image used an unsupported format or conflicting metadata after those +state changes. A peer sending an unusable snapshot could therefore delay a new +election without providing a usable recovery image. + +The receiver now accumulates the bounded metadata prefix before treating a +same-term InstallSnapshot as leader contact. Partial prefixes may be retained +and acknowledged so chunked transfer can continue, but they do not change the +Raft role, leader ID or election deadline. Unknown formats and invalid metadata +are rejected without those changes. Once the prefix matches a supported fixed +or stable image, normal transfer handling refreshes leader contact. Higher-term +requests retain the existing preflight rule: their first chunk must contain a +complete supported metadata prefix before hard state advances. + +Deterministic regression coverage checks an unsupported joint image split across +chunks while the receiver is a candidate. The first partial prefix does not +demote it or move its deadline; the chunk that identifies the unsupported +format is rejected with the same state preserved. Existing valid split-prefix, +snapshot installation, stale-term and higher-term preflight tests continue to +cover the accepted paths. No wire or storage format changed. + +This validates format and boundary metadata before recognizing leader contact; +the complete image CRC and application payload are still checked only when an +image is installed. It does not authenticate peers or enable joint snapshot +installation or live membership changes. diff --git a/miniraft/raft.cpp b/miniraft/raft.cpp index a21c325..a5a7774 100644 --- a/miniraft/raft.cpp +++ b/miniraft/raft.cpp @@ -29,6 +29,13 @@ struct TSnapshotMetadataProbe { size_t Need = 8; }; +bool IsSupportedSnapshotMagicPrefix(std::string_view prefix) { + constexpr std::string_view legacy = "MRFTS001"; + constexpr std::string_view stable = "MRFTS003"; + const auto bytes = prefix.substr(0, std::min(prefix.size(), legacy.size())); + return legacy.substr(0, bytes.size()) == bytes || stable.substr(0, bytes.size()) == bytes; +} + bool ReadSnapshotInteger(std::string_view bytes, size_t offset, unsigned width, uint64_t& value) { if (offset > bytes.size() || width > bytes.size() - offset) return false; value = 0; @@ -970,9 +977,14 @@ void TRaft::OnInstallSnapshot(ITimeSource::Time now, TMessageHolderTerm < State->CurrentTerm) { reply(false, false, 0); return; } - Become(EState::FOLLOWER); - VolatileState->LeaderId = request->LeaderId; - VolatileState->ElectionDue = MakeElection(now); + // Partial or unsupported image headers are not evidence of leader contact. + // In particular, do not demote a candidate or refresh its election timer + // until the complete bounded metadata prefix is supported. + auto acceptLeaderContact = [&] { + Become(EState::FOLLOWER); + VolatileState->LeaderId = request->LeaderId; + VolatileState->ElectionDue = MakeElection(now); + }; if (request->SnapshotIndex <= VolatileState->CommitIndex) { const auto term = State->LogTerm(request->SnapshotIndex); const bool covered = request->SnapshotIndex < State->SnapshotIndex || term == request->SnapshotTerm; @@ -999,11 +1011,15 @@ void TRaft::OnInstallSnapshot(ITimeSource::Time now, TMessageHolderData, count); IncomingSnapshot->LastProgress = now; + if (!IsSupportedSnapshotMagicPrefix(prefix)) { + IncomingSnapshot.reset(); reply(false, false, 0); return; + } const auto knownSnapshot = State->GetSnapshot(); const auto probe = ProbeSnapshotMetadata(prefix, request->TotalBytes, request->SnapshotIndex, request->SnapshotTerm, GetVoterIds(), knownSnapshot.get()); if (!probe.Complete) { reply(true, false, prefix.size()); return; } IncomingSnapshot.reset(); + if (probe.Supported) acceptLeaderContact(); reply(probe.Supported, probe.Supported, probe.Supported ? request->TotalBytes : 0); return; } @@ -1028,17 +1044,21 @@ void TRaft::OnInstallSnapshot(ITimeSource::Time now, TMessageHolderData, count); IncomingSnapshot->LastProgress = now; - if (bytes.size() >= 8 && std::string_view(bytes).substr(0, 8) != "MRFTS001" && - std::string_view(bytes).substr(0, 8) != "MRFTS003") { + if (!IsSupportedSnapshotMagicPrefix(bytes)) { IncomingSnapshot.reset(); reply(false, false, 0); return; } + const auto knownSnapshot = State->GetSnapshot(); + const auto probe = ProbeSnapshotMetadata(bytes, request->TotalBytes, + request->SnapshotIndex, request->SnapshotTerm, GetVoterIds(), knownSnapshot.get()); + if (!probe.Complete) { reply(true, false, bytes.size()); return; } + if (!probe.Supported) { IncomingSnapshot.reset(); reply(false, false, 0); return; } + acceptLeaderContact(); if (bytes.size() < request->TotalBytes) { reply(true, false, bytes.size()); return; } TSnapshot snapshot; try { snapshot = DecodeSnapshot(bytes); if (snapshot.IsJoint() || snapshot.Index != request->SnapshotIndex || snapshot.Term != request->SnapshotTerm || snapshot.Voters != GetVoterIds()) throw TMessageError("Snapshot metadata mismatch"); - const auto knownSnapshot = State->GetSnapshot(); PreserveStableSnapshotIdentity(knownSnapshot.get(), snapshot); ValidateSnapshot(snapshot); application->ValidateSnapshot(snapshot.Data, snapshot.Index); diff --git a/test/test_compaction.cpp b/test/test_compaction.cpp index cb16ad3..f300355 100644 --- a/test/test_compaction.cpp +++ b/test/test_compaction.cpp @@ -427,8 +427,20 @@ void test_joint_snapshot_is_rejected_even_if_boundary_is_already_covered(void**) assert_int_equal(replies.size(), repliesBeforeUnsupported); // At the current term, normal fail-closed rejection still returns a reply. + raft->Become(EState::CANDIDATE); + const auto electionDueBeforeUnsupported = raft->GetVolatileState()->ElectionDue; + raft->Process({}, Chunk(joint, bytes, 0, 4)); + assert_true(replies.back()->Success); assert_false(replies.back()->Installed); + assert_int_equal(raft->CurrentStateName(), EState::CANDIDATE); + assert_true(raft->GetVolatileState()->ElectionDue == electionDueBeforeUnsupported); + raft->Process({}, Chunk(joint, bytes, 4, 4)); + assert_false(replies.back()->Success); assert_false(replies.back()->Installed); + assert_int_equal(raft->CurrentStateName(), EState::CANDIDATE); + assert_true(raft->GetVolatileState()->ElectionDue == electionDueBeforeUnsupported); raft->Process({}, Chunk(joint, bytes, 0, bytes.size())); assert_false(replies.back()->Success); assert_false(replies.back()->Installed); + assert_int_equal(raft->CurrentStateName(), EState::CANDIDATE); + assert_true(raft->GetVolatileState()->ElectionDue == electionDueBeforeUnsupported); assert_true(static_cast(*state) == before); assert_int_equal(app->Loads, 1); // Covered fast-path ACKs still require the embedded boundary and voter set From 92c8b7e28eaaff379da1d0b0f0ccfc82095c2ec5 Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Sat, 14 Nov 2026 22:34:39 -0500 Subject: [PATCH 61/66] Cover unsupported snapshots beyond the commit boundary (#61) --- docs/snapshot-contact-validation.md | 5 +++-- test/test_compaction.cpp | 33 +++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/docs/snapshot-contact-validation.md b/docs/snapshot-contact-validation.md index 12f18a9..866b198 100644 --- a/docs/snapshot-contact-validation.md +++ b/docs/snapshot-contact-validation.md @@ -16,8 +16,9 @@ requests retain the existing preflight rule: their first chunk must contain a complete supported metadata prefix before hard state advances. Deterministic regression coverage checks an unsupported joint image split across -chunks while the receiver is a candidate. The first partial prefix does not -demote it or move its deadline; the chunk that identifies the unsupported +chunks while the receiver is a candidate, both when its boundary is already +covered and when it would require full installation. The first partial prefix +does not demote it or move its deadline; the chunk that identifies the unsupported format is rejected with the same state preserved. Existing valid split-prefix, snapshot installation, stale-term and higher-term preflight tests continue to cover the accepted paths. No wire or storage format changed. diff --git a/test/test_compaction.cpp b/test/test_compaction.cpp index f300355..aabb72c 100644 --- a/test/test_compaction.cpp +++ b/test/test_compaction.cpp @@ -507,6 +507,38 @@ void test_joint_snapshot_is_rejected_even_if_boundary_is_already_covered(void**) assert_true(static_cast(*state) == before); assert_int_equal(app->Loads, 1); assert_true(Read(*app, "a") == "old"); } +void test_unsupported_new_snapshot_does_not_accept_leader_contact(void**) { + TKv source; source.Write(source.Prepare(Command("a", "joint")), 5); + TSnapshot joint{5, 2, {1, 2}, source.SaveSnapshot()}; + joint.JointVoters = {2, 3}; joint.ConfigurationIndex = 4; joint.ConfigurationTerm = 2; + const auto bytes = EncodeSnapshot(joint); + std::vector> replies; + auto peer = std::make_shared([&](auto message) { + replies.push_back(message.template Cast()); + }); + auto state = std::make_shared(); + auto raft = std::make_shared(state, 2, TNodeDict{{1, peer}}); + auto app = std::make_shared(); TRequestProcessor processor(raft, app, {}); + state->CurrentTerm = 3; + raft->Become(EState::CANDIDATE); + const auto before = static_cast(*state); + const auto electionDue = raft->GetVolatileState()->ElectionDue; + + // This image is newer than the local commit boundary, so it exercises the + // full installation path rather than the covered-prefix fast path. + raft->Process({}, Chunk(joint, bytes, 0, 4)); + assert_true(replies.back()->Success); assert_false(replies.back()->Installed); + assert_int_equal(raft->CurrentStateName(), EState::CANDIDATE); + assert_true(raft->GetVolatileState()->ElectionDue == electionDue); + raft->Process({}, Chunk(joint, bytes, 4, 4)); + assert_false(replies.back()->Success); assert_false(replies.back()->Installed); + raft->Process({}, Chunk(joint, bytes, 0, static_cast(bytes.size()))); + assert_false(replies.back()->Success); assert_false(replies.back()->Installed); + assert_int_equal(raft->CurrentStateName(), EState::CANDIDATE); + assert_true(raft->GetVolatileState()->ElectionDue == electionDue); + assert_true(static_cast(*state) == before); + assert_int_equal(app->Loads, 0); assert_int_equal(app->LastAppliedIndex, 0); +} void test_stable_configuration_snapshot_installs_and_recovers_with_fixed_membership(void**) { TKv source; source.Write(source.Prepare(Command("a", "stable snapshot")), 5); TSnapshot snapshot{5, 1, {1, 2}, source.SaveSnapshot()}; @@ -1025,6 +1057,7 @@ int main() { cmocka_unit_test(test_memory_snapshot_storage_rejects_term_regression), cmocka_unit_test(test_memory_snapshot_cannot_replace_committed_index_with_conflicting_term), cmocka_unit_test(test_joint_snapshot_is_rejected_even_if_boundary_is_already_covered), + cmocka_unit_test(test_unsupported_new_snapshot_does_not_accept_leader_contact), cmocka_unit_test(test_stable_configuration_snapshot_installs_and_recovers_with_fixed_membership), cmocka_unit_test(test_idle_partial_snapshot_expires_on_learner_despite_heartbeats), cmocka_unit_test(test_remote_snapshot_retains_suffix_without_applying_it_before_commit), From 94ed732b6ea3d277b35d11b2111acea44c8825bd Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Sat, 14 Nov 2026 23:14:42 -0500 Subject: [PATCH 62/66] Track learner commit acknowledgements (#62) * Track learner commit acknowledgements * Cover learner snapshot commit acknowledgement --- docs/online-learner-staging.md | 4 ++++ miniraft/raft.cpp | 23 +++++++++++++++++++++-- miniraft/raft.h | 8 +++++++- test/test_learners.cpp | 27 +++++++++++++++++++++++++++ 4 files changed, 59 insertions(+), 3 deletions(-) diff --git a/docs/online-learner-staging.md b/docs/online-learner-staging.md index 9e7702b..f596e58 100644 --- a/docs/online-learner-staging.md +++ b/docs/online-learner-staging.md @@ -7,6 +7,10 @@ binding do not change. Staged learners receive the ordinary log or a snapshot when needed, but never count toward election, commit, or strong-read quorums. `IsLearnerCaughtUp` remains a point-in-time log watermark, not an authorization to promote or a guarantee of application health. +`IsLearnerCommitAcknowledged` separately reports whether a correlated successful +replication reply confirms the leader's current Raft commit boundary while the +learner also matches the leader's log. It does not report application-applied +progress and does not authorize promotion. These are serialized embedding APIs, not a replicated cluster configuration or a promotion API. The caller must ensure the destination is configured as a learner diff --git a/miniraft/raft.cpp b/miniraft/raft.cpp index a5a7774..53c2449 100644 --- a/miniraft/raft.cpp +++ b/miniraft/raft.cpp @@ -431,6 +431,7 @@ void TRaft::OnAppendEntries(TMessageHolder message) { }); if (request == requests.end() || (message->Success && message->MatchIndex != request->LastIndex)) return; + const auto acknowledgedLeaderCommit = request->LeaderCommit; // A newer response supersedes older RPCs. Reordered/duplicate failures // must not undo progress or repeatedly increase backoff. requests.erase(requests.begin(), std::next(request)); @@ -440,12 +441,15 @@ void TRaft::OnAppendEntries(TMessageHolder message) { acknowledged = std::max(acknowledged, message->Seqno); } auto matchIndex = std::max(VolatileState->MatchIndex[nodeId], message->MatchIndex); + auto commitAckIndex = std::max(VolatileState->CommitAckIndex[nodeId], + std::min(acknowledgedLeaderCommit, message->MatchIndex)); (*VolatileState) .SetMatchIndex(nodeId, matchIndex) .SetNextIndex(nodeId, matchIndex+1) .SetRpcDue(nodeId, ITimeSource::Time{}) .SetBatchSize(nodeId, 1024) .SetBackOff(nodeId, 1); + VolatileState->CommitAckIndex[nodeId] = commitAckIndex; AdvanceLeaderCommit(); } else if (message->MatchIndex > 0) { // A compacted-prefix hint is not replication evidence. Probe its boundary @@ -496,12 +500,13 @@ TMessageHolder TRaft::CreateAppendEntries(uint32_t nodeId } lastIndex = prevIndex + entries.size(); + const auto leaderCommit = std::min(VolatileState->CommitIndex, lastIndex); auto mes = NewHoldedMessage( TMessageEx {.Src = Id, .Dst = nodeId, .Term = State->CurrentTerm, .Seqno = NextSeqno()}, TAppendEntriesRequest { .PrevLogIndex = prevIndex, .PrevLogTerm = State->LogTerm(prevIndex), - .LeaderCommit = std::min(VolatileState->CommitIndex, lastIndex), + .LeaderCommit = leaderCommit, .LeaderId = Id, .Nentries = static_cast(lastIndex - prevIndex), }); @@ -512,7 +517,7 @@ TMessageHolder TRaft::CreateAppendEntries(uint32_t nodeId } VolatileState->LastSentSeqno[nodeId] = mes->Seqno; auto& pending = VolatileState->PendingAppendRpcs[nodeId]; - pending.push_back({mes->Seqno, lastIndex}); + pending.push_back({mes->Seqno, lastIndex, leaderCommit}); if (pending.size() > TVolatileState::MaxPendingAppendRpcs) pending.pop_front(); return mes; } @@ -873,6 +878,7 @@ std::shared_ptr TRaft::RemoveLearnerTarget(uint32_t id) { auto node = peer->second; Nodes.erase(peer); Learners.erase(learner); VolatileState->NextIndex.erase(id); VolatileState->MatchIndex.erase(id); + VolatileState->CommitAckIndex.erase(id); VolatileState->ReadAckSeqno.erase(id); VolatileState->LastSentSeqno.erase(id); VolatileState->PendingAppendRpcs.erase(id); VolatileState->SnapshotRpcs.erase(id); VolatileState->HeartbeatDue.erase(id); VolatileState->RpcDue.erase(id); @@ -889,6 +895,16 @@ bool TRaft::IsLearnerCaughtUp(uint32_t id) const { return match != VolatileState->MatchIndex.end() && match->second >= State->LastLogIndex; } +bool TRaft::IsLearnerCommitAcknowledged(uint32_t id) const { + if (StateName != EState::LEADER || !std::binary_search(Learners.begin(), Learners.end(), id) || + VolatileState->CommitIndex == 0 || State->LogTerm(VolatileState->CommitIndex) != State->CurrentTerm) + return false; + const auto match = VolatileState->MatchIndex.find(id); + const auto commitAck = VolatileState->CommitAckIndex.find(id); + return match != VolatileState->MatchIndex.end() && match->second >= State->LastLogIndex && + commitAck != VolatileState->CommitAckIndex.end() && commitAck->second >= VolatileState->CommitIndex; +} + TSnapshot TRaft::Compact() { if (!State->SupportsSnapshots()) throw std::logic_error("Storage cannot compact snapshots"); auto application = SnapshotApplication.lock(); @@ -957,6 +973,9 @@ void TRaft::OnInstallSnapshot(TMessageHolder response) const auto index = std::max(VolatileState->MatchIndex[response->Src], response->SnapshotIndex); (*VolatileState).SetMatchIndex(response->Src, index).SetNextIndex(response->Src, index + 1) .SetBatchSize(response->Src, 1024).SetBackOff(response->Src, 1); + VolatileState->CommitAckIndex[response->Src] = std::max( + VolatileState->CommitAckIndex[response->Src], + std::min(VolatileState->CommitIndex, response->SnapshotIndex)); AdvanceLeaderCommit(); VolatileState->SnapshotRpcs.erase(found); // Snapshot completion is not a fresh-read acknowledgement. AppendEntries diff --git a/miniraft/raft.h b/miniraft/raft.h index 929e838..3b7e7af 100644 --- a/miniraft/raft.h +++ b/miniraft/raft.h @@ -54,7 +54,7 @@ struct TDummyRsm: public IRsm { using TNodeDict = std::unordered_map>; struct TVolatileState { - struct TAppendRpc { uint64_t Seqno, LastIndex; }; + struct TAppendRpc { uint64_t Seqno, LastIndex, LeaderCommit; }; static constexpr size_t MaxPendingAppendRpcs = 256; // Distinct snapshot images retained by active peer transfers. New // compaction is deferred when this many generations are already pinned. @@ -76,6 +76,8 @@ struct TVolatileState { uint32_t LeaderId = 0; std::unordered_map NextIndex; std::unordered_map MatchIndex; + // Highest leader commit boundary confirmed by a correlated replication ACK. + std::unordered_map CommitAckIndex; std::unordered_set Votes; std::unordered_map HeartbeatDue; std::unordered_map RpcDue; @@ -133,6 +135,10 @@ class TRaft { bool IsVoter(uint32_t id) const; // Leader-only, point-in-time log catch-up; not permission to promote. bool IsLearnerCaughtUp(uint32_t id) const; + // True when a correlated ACK confirms the learner's Raft commit boundary + // through this leader's current commit index, and its log matches the leader. + // This does not attest to application progress or authorize promotion. + bool IsLearnerCommitAcknowledged(uint32_t id) const; // Leader-local replication staging; not a membership change. Caller owns // endpoint lifecycle, reconciles targets after failover/restart, and must // serialize these calls with all Raft processing. diff --git a/test/test_learners.cpp b/test/test_learners.cpp index 1c41e45..260ee88 100644 --- a/test/test_learners.cpp +++ b/test/test_learners.cpp @@ -203,6 +203,31 @@ void test_learner_replication_cannot_commit_or_satisfy_strong_read(void**) { cluster.Isolated = {3}; cluster.Pump(1, 80); assert_int_equal(replies, 1); assert_true(Read(*cluster.Apps[3], "x") == "new"); } +void test_learner_commit_ack_waits_for_commit_propagation(void**) { + TCluster cluster; cluster.Elect(1); cluster.Pump(); + auto leader = cluster.Nodes[0]; + const auto index = leader->Append(cluster.Apps[0]->Prepare(Command("pending", "value"))); + + // The learner can have the complete log while the voters cannot yet commit + // this entry. Its ACK only proves the previous commit boundary. + cluster.Isolated = {2, 3}; cluster.Pump(1, 40); + assert_int_equal(leader->GetVolatileState()->CommitIndex, index - 1); + assert_true(leader->GetVolatileState()->MatchIndex.at(4) >= index); + assert_true(leader->IsLearnerCaughtUp(4)); + assert_true(leader->IsLearnerCommitAcknowledged(4)); + + // Now let the voter majority commit while the learner is partitioned. The + // log watermark remains true, but the correlated commit acknowledgement is + // stale until the leader sends the new boundary and receives a reply. + cluster.Isolated = {4}; cluster.Pump(1, 40); + assert_int_equal(leader->GetVolatileState()->CommitIndex, index); + assert_true(leader->IsLearnerCaughtUp(4)); + assert_false(leader->IsLearnerCommitAcknowledged(4)); + + cluster.Isolated.clear(); cluster.Pump(1, 40); + assert_true(leader->IsLearnerCommitAcknowledged(4)); + assert_int_equal(cluster.Apps[3]->LastAppliedIndex, index); +} void test_learner_snapshot_catchup_status_and_restart_preserve_role(void**) { TCluster cluster; cluster.Isolated = {4}; cluster.Elect(1); for (const std::string key : {"a", "b", "c"}) cluster.Write(key, std::string(60000, key[0])); @@ -210,6 +235,7 @@ void test_learner_snapshot_catchup_status_and_restart_preserve_role(void**) { assert_true(snapshot.Voters == (std::vector{1, 2, 3})); assert_false(cluster.Nodes[0]->IsLearnerCaughtUp(4)); cluster.Isolated.clear(); cluster.Pump(1, 100, true); assert_int_equal(cluster.Apps[3]->Loads, 1); assert_true(cluster.Nodes[0]->IsLearnerCaughtUp(4)); + assert_true(cluster.Nodes[0]->IsLearnerCommitAcknowledged(4)); assert_false(cluster.Nodes[0]->IsLearnerCaughtUp(2)); assert_false(cluster.Nodes[0]->IsLearnerCaughtUp(99)); assert_true(Read(*cluster.Apps[3], "c") == std::string(60000, 'c')); cluster.Nodes[0]->Append(cluster.Apps[0]->Prepare(Command("tail", "later"))); @@ -753,6 +779,7 @@ int main() { cmocka_unit_test(test_single_voter_commits_and_reads_without_learner_acknowledgements), cmocka_unit_test(test_learner_grants_cannot_elect_candidate_and_no_votes_are_solicited), cmocka_unit_test(test_learner_replication_cannot_commit_or_satisfy_strong_read), + cmocka_unit_test(test_learner_commit_ack_waits_for_commit_propagation), cmocka_unit_test(test_learner_snapshot_catchup_status_and_restart_preserve_role), cmocka_unit_test(test_restart_cannot_promote_learner_or_change_voters_but_can_add_transport), cmocka_unit_test(test_nonfresh_unbound_store_requires_snapshot_proof_for_learners), From 1e71d0145ca3ac9aec22e938a127a3e268c71cdb Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Sat, 14 Nov 2026 23:26:45 -0500 Subject: [PATCH 63/66] Require committed configuration predecessors (#63) --- docs/configuration-scan.md | 3 +++ miniraft/configuration.cpp | 6 ++++++ test/test_configuration.cpp | 27 +++++++++++++++++++++++++-- 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/docs/configuration-scan.md b/docs/configuration-scan.md index 26295f6..da6a8c1 100644 --- a/docs/configuration-scan.md +++ b/docs/configuration-scan.md @@ -21,6 +21,9 @@ sets to agree. The scan checks snapshot and log boundaries, voter history, terms configuration predecessors, and transitions. It requires a consistent, externally serialized view of the state; it does not mutate storage, apply entries, activate voters, or change Raft roles. +It also rejects a configuration entry that follows an earlier configuration +entry outside the supplied commit boundary. In particular, a stable successor +cannot make an uncommitted joint configuration disappear from the latest view. For ordinary entries, the scan trusts term and flag metadata from the storage backend, which must validate its own records. Memory and durable backends avoid diff --git a/miniraft/configuration.cpp b/miniraft/configuration.cpp index 33e2ff2..3318eb9 100644 --- a/miniraft/configuration.cpp +++ b/miniraft/configuration.cpp @@ -193,6 +193,12 @@ TConfigurationScan ScanConfigurationHistory(const IState& state, const auto flags = state.RetainedLogFlags(index); if (!term || term > state.CurrentTerm) throw std::invalid_argument("Invalid retained log term"); if (flags & TLogEntry::EConfiguration) { + // Do not accept a second transition until the preceding config has + // committed. Otherwise an uncommitted joint entry could be followed + // by a stable entry and make Latest appear to leave joint mode + // without ever establishing the required old/new quorum. + if (latest.ConfigurationIndex() > commitIndex) + throw std::invalid_argument("Configuration transition precedes predecessor commitment"); auto entry = state.Get(index); if (!entry.HasValidExtent() || entry->Len < sizeof(TLogEntry) || entry->Term != term || entry->Flags != flags) diff --git a/test/test_configuration.cpp b/test/test_configuration.cpp index 89ee8f3..a947d10 100644 --- a/test/test_configuration.cpp +++ b/test/test_configuration.cpp @@ -166,6 +166,29 @@ void test_history_rejects_skips_wrong_predecessors_and_term_regression(void**) { assert_false(prefix.IsJoint()); assert_int_equal(prefix.ThroughIndex(), 1); assert_true(pending.IsJoint()); assert_int_equal(pending.ThroughIndex(), 2); } +void test_scan_requires_each_configuration_predecessor_to_be_committed(void**) { + TState state; state.CurrentTerm = 4; + state.Append(Ordinary(1)); + const TConfigurationRecord joint{EConfigurationPhase::Joint, 2, 0, 0, {1}, {1, 2}}; + state.Append(EncodeConfigurationEntry(joint)); + const TConfigurationRecord stable{EConfigurationPhase::Stable, 3, 2, 2, {1}, {1, 2}}; + state.Append(EncodeConfigurationEntry(stable)); + const TConfigurationRecord nextJoint{EConfigurationPhase::Joint, 4, 3, 3, {1, 2}, {2, 3}}; + state.Append(EncodeConfigurationEntry(nextJoint)); + + const auto baseline = TConfigurationHistory::Stable({1}, 0, 0, 0, 0); + // The first joint entry is committed, but its stable successor is not. + // The next joint transition must wait for that stable entry to commit. + Reject([&] { ScanConfigurationHistory(state, baseline, 2); }); + + const auto scan = ScanConfigurationHistory(state, baseline, 3); + assert_false(scan.Committed.IsJoint()); + assert_true(scan.Committed.OldVoters() == stable.NewVoters); + assert_true(scan.Latest.IsJoint()); + assert_true(scan.Latest.OldVoters() == nextJoint.OldVoters); + assert_true(scan.Latest.NewVoters() == nextJoint.NewVoters); + assert_true(scan.HasUncommittedConfiguration()); +} struct TCountingState : TState { mutable unsigned Gets = 0; TMessageHolder Get(int64_t index) const override { ++Gets; return TState::Get(index); } @@ -195,8 +218,7 @@ void test_scan_separates_commit_and_replaced_suffix(void**) { scan = ScanConfigurationHistory(state, base, 2); assert_true(scan.Latest.IsJoint()); assert_false(scan.HasUncommittedConfiguration()); state.Append(EncodeConfigurationEntry(final)); - scan = ScanConfigurationHistory(state, base, 1); - assert_false(scan.Committed.IsJoint()); assert_true(scan.HasUncommittedConfiguration()); + Reject([&] { ScanConfigurationHistory(state, base, 1); }); Reject([&] { base.ObserveMetadata(1, 1, TLogEntry::EConfiguration); }); Reject([&] { base.ObserveMetadata(2, 1, 0); }); } @@ -570,6 +592,7 @@ int main() { cmocka_unit_test(test_configuration_scan_compares_full_configuration_state), cmocka_unit_test(test_history_from_joint_snapshot_requires_a_consistent_identity), cmocka_unit_test(test_history_rejects_skips_wrong_predecessors_and_term_regression), + cmocka_unit_test(test_scan_requires_each_configuration_predecessor_to_be_committed), cmocka_unit_test(test_scan_separates_commit_and_replaced_suffix), cmocka_unit_test(test_scan_from_joint_snapshot_and_rejects_mismatched_baselines), cmocka_unit_test(test_scan_from_stable_configuration_snapshot), From dfae33e0ba69bec2c0ada8dd7c482fbc81b86129 Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Sat, 14 Nov 2026 23:40:37 -0500 Subject: [PATCH 64/66] Apply configuration entries as Raft metadata (#64) --- docs/configuration-entries.md | 7 +++++-- miniraft/raft.cpp | 10 ++++++++-- test/test_configuration.cpp | 24 ++++++++++++++++++------ 3 files changed, 31 insertions(+), 10 deletions(-) diff --git a/docs/configuration-entries.md b/docs/configuration-entries.md index 7d1d3ec..4587b6e 100644 --- a/docs/configuration-entries.md +++ b/docs/configuration-entries.md @@ -19,8 +19,11 @@ authentication. CRC is an accidental-corruption check, not authentication. Current Raft rejects entries carrying the reserved bit in local proposals and incoming AppendEntries, before changing term or log. Startup scans the retained -suffix and rejects such entries before voter binding or serving traffic; the -application path also refuses them. Joint snapshots remain rejected as described +suffix and rejects such entries before voter binding or serving traffic. If a +configuration record reaches committed replay in a controlled test or future +runtime, the request processor validates it, advances its applied index, and +does not pass it to the application state machine. This no-op application rule +does not activate the configuration. Joint snapshots remain rejected as described in [joint snapshot metadata](joint-snapshots.md). The codec and storage backends can represent records for deterministic testing and offline tools, but they are not an active replication or recovery protocol. diff --git a/miniraft/raft.cpp b/miniraft/raft.cpp index 53c2449..f8ca8d3 100644 --- a/miniraft/raft.cpp +++ b/miniraft/raft.cpp @@ -13,6 +13,7 @@ #include "raft.h" #include "messages.h" #include "timesource.h" +#include "configuration.h" namespace { @@ -1339,8 +1340,13 @@ void TRequestProcessor::ProcessCommitted() { auto& state = Raft->GetState(); for (auto i = Rsm->LastAppliedIndex+1; i <= commitIndex; i++) { auto entry = state->Get(i); - if (entry->Flags & TLogEntry::EConfiguration) - throw std::logic_error("Configuration application is unsupported"); + if (entry->Flags & TLogEntry::EConfiguration) { + // Configuration is Raft metadata, never an application command. + // Validate its control record and advance the application boundary + // without exposing it to the state machine. + DecodeConfigurationEntry(entry); + continue; + } if (entry->Flags == TLogEntry::EStub) { continue; } diff --git a/test/test_configuration.cpp b/test/test_configuration.cpp index a947d10..94e692c 100644 --- a/test/test_configuration.cpp +++ b/test/test_configuration.cpp @@ -37,6 +37,13 @@ struct TEndpoint : INode { void Send(TMessageHolder) override { ++Sent; } void Drain() override {} }; +struct TCountingRsm : TDummyRsm { + unsigned Writes = 0; + TMessageHolder Write(TMessageHolder entry, uint64_t index) override { + ++Writes; + return TDummyRsm::Write(std::move(entry), index); + } +}; TMessageHolder Ordinary(uint64_t term) { auto entry = NewHoldedMessage(); entry->Term = term; return entry; } @@ -374,18 +381,23 @@ void test_configuration_rpc_and_proposal_are_rejected_before_mutation(void**) { Reject([&] { leader.Append(proposed); }); assert_true(static_cast(*leaderState) == leaderBefore); assert_int_equal(proposed->Term, 2); } -void test_unsupported_configuration_never_reaches_application_or_binds_voters(void**) { +void test_configuration_application_advances_without_calling_state_machine(void**) { auto state = std::make_shared(); auto record = Joint(); record.Term = 1; record.PreviousIndex = record.PreviousTerm = 0; state->Append(EncodeConfigurationEntry(record)); Reject([&] { TRaft invalid(state, 1, {}); }); assert_true(state->GetBoundVoters().empty()); auto live = std::make_shared(); auto raft = std::make_shared(live, 1, TNodeDict{}); - auto app = std::make_shared(); TRequestProcessor processor(raft, app, {}); + auto app = std::make_shared(); TRequestProcessor processor(raft, app, {}); + raft->ProcessTimeout({}); + processor.ProcessCommitted(); + assert_int_equal(app->LastAppliedIndex, 1); // Leader's initial stub. + record.Term = live->CurrentTerm; live->Append(EncodeConfigurationEntry(record)); - const_cast(raft->GetVolatileState())->CommitIndex = 1; - Reject([&] { processor.ProcessCommitted(); }); - assert_int_equal(app->LastAppliedIndex, 0); + raft->ProcessTimeout(ITimeSource::Time{} + std::chrono::seconds(1)); + processor.ProcessCommitted(); + assert_int_equal(app->LastAppliedIndex, 2); + assert_int_equal(app->Writes, 0); auto compacted = std::make_shared(); { TRaft bootstrap(compacted, 1, {}); } compacted->Append(NewHoldedMessage()); @@ -600,7 +612,7 @@ int main() { cmocka_unit_test(test_configuration_roundtrips_both_phases_and_bounded_maximum), cmocka_unit_test(test_configuration_rejects_invalid_fields_and_valid_crc_corruption), cmocka_unit_test(test_configuration_rpc_and_proposal_are_rejected_before_mutation), - cmocka_unit_test(test_unsupported_configuration_never_reaches_application_or_binds_voters), + cmocka_unit_test(test_configuration_application_advances_without_calling_state_machine), #ifdef __linux__ cmocka_unit_test(test_durable_restart_rejects_reserved_entry_without_rewriting), cmocka_unit_test(test_legacy_restart_uses_safe_flag_lookup_and_preserves_files), From 8fa4f752ed4fd985ffd27e5037a4a05230ecad41 Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Sun, 15 Nov 2026 09:44:11 -0500 Subject: [PATCH 65/66] Recover runtime joint configuration quorum (#65) --- docs/configuration-entries.md | 48 +++++++------ docs/joint-quorum.md | 30 +++++--- miniraft/configuration.h | 8 ++- miniraft/raft.cpp | 125 +++++++++++++++++++++++++++----- miniraft/raft.h | 19 ++++- test/test_configuration.cpp | 132 +++++++++++++++++++++++++++++++++- 6 files changed, 304 insertions(+), 58 deletions(-) diff --git a/docs/configuration-entries.md b/docs/configuration-entries.md index 4587b6e..17fa5b8 100644 --- a/docs/configuration-entries.md +++ b/docs/configuration-entries.md @@ -17,25 +17,28 @@ and term; a stable entry requires a nonzero predecessor. The codec checks shape, not log position, predecessor identity, commitment, role eligibility or cluster authentication. CRC is an accidental-corruption check, not authentication. -Current Raft rejects entries carrying the reserved bit in local proposals and -incoming AppendEntries, before changing term or log. Startup scans the retained -suffix and rejects such entries before voter binding or serving traffic. If a -configuration record reaches committed replay in a controlled test or future -runtime, the request processor validates it, advances its applied index, and -does not pass it to the application state machine. This no-op application rule -does not activate the configuration. Joint snapshots remain rejected as described -in [joint snapshot metadata](joint-snapshots.md). The codec and storage backends -can represent records for deterministic testing and offline tools, but they are -not an active replication or recovery protocol. +Local proposals and incoming AppendEntries still reject entries carrying the +reserved bit, before changing term or log. Startup now scans and reconstructs +valid retained configuration history, then selects the stable or joint quorum for +elections, commit advancement and strong-read confirmation. Retained transitions +require a bound original voter set, a persisted commit index and peer endpoints +for every active voter. If a configuration record reaches committed replay, the +request processor validates it, advances its applied index, and does not pass it +to the application state machine. Joint snapshot recovery and snapshot transfer +across a membership transition remain unsupported, so compaction is blocked while +the retained suffix carries a membership change. This is recovery support, not a +live reconfiguration API. Joint snapshots remain rejected as described in +[joint snapshot metadata](joint-snapshots.md). Compatibility matters: previous versions allowed custom entry flags. Before -upgrading, audit applications and retained logs for use of the high bit. A node -with that bit in its retained suffix now refuses startup; this release does not -rewrite or migrate it. A fixed snapshot may already contain effects of older +upgrading, audit applications and retained logs for use of the high bit. Raft now +reserves it for configuration records and refuses startup unless every retained +record forms a valid, trusted transition chain; this release does not rewrite or +migrate custom uses. A fixed snapshot may already contain effects of older entries, which this suffix scan cannot detect. Older binaries may apply an unknown -configuration entry as application data, so no future version should replicate -these entries until every voter runs a compatible implementation. Do not delete -or edit committed state to bypass rejection. +configuration entry as application data, so these entries must not be replicated +until every voter runs a compatible implementation. Do not delete or edit +committed state to bypass rejection. The optimized memory and durable backends scan flags without copying payloads. External and legacy backends use the default Get-based fallback, so startup can @@ -45,9 +48,10 @@ RetainedLogFlags consistent. No existing durable file or wire frame layout chang Tests cover both phases, bootstrap and maximum-size records, every-byte corruption and truncation, semantic corruption with valid CRC, malformed frames, rejected RPCs -and proposals, application isolation, compacted suffixes and restart rejection -without rewriting durable or legacy files. Self-review checked fail-closed ordering, -the flag scan, predecessor shape and compatibility limits. No independent reviewer -approval is claimed. Dynamic promotion/removal, configuration activation and -rollback, joint snapshot recovery, discovery and full configuration history remain -open; this experimental encoding may change when those semantics are implemented. +and proposals, application isolation, compacted suffixes and restart recovery of a +joint quorum without rewriting durable files. Self-review checked fail-closed +ordering, predecessor shape, runtime quorum selection and compatibility limits. No +independent reviewer approval is claimed. Dynamic promotion/removal APIs, live +configuration RPCs, suffix rollback after replacement, joint snapshot recovery, +discovery and full transition testing remain open; this experimental encoding may +change when those semantics are implemented. diff --git a/docs/joint-quorum.md b/docs/joint-quorum.md index 3bf88bc..b8adc70 100644 --- a/docs/joint-quorum.md +++ b/docs/joint-quorum.md @@ -10,8 +10,13 @@ TQuorum validates positive, sorted, unique, nonempty voter sets and computes stable or joint election results and watermarks. Its value callback must return the same value for a voter throughout a call. TVolatileState can apply this arithmetic to commit advancement, retaining the current-term commit restriction -and monotonic commit index. Existing fixed-membership election, commit and strong -read paths now use the same stable-quorum arithmetic; they do not enter joint mode. +and monotonic commit index. Validated retained configuration history now selects +the runtime quorum used by elections, commit advancement and strong-read +confirmation. An uncommitted joint entry uses both voter sets; after that entry +commits, the joint quorum remains active until its stable successor commits. +The fixed-membership snapshot runtime still requires stable snapshot voters to +match the node's configured and bound set; joint snapshot recovery remains +disabled. Tests exhaust every pair of nonempty old/new subsets of five IDs and every vote subset, including disjoint and overlapping sets, even-sized groups, absent self @@ -20,16 +25,19 @@ watermarks and check current-term gating. Separate stable-set tests compare the calculation with strict-majority order statistics for sizes 1–8. Existing Raft, partition, learner, snapshot and read tests exercise the fixed-membership paths. -This is arithmetic foundation, not a reconfiguration API. It does not replicate -configuration entries, change roles, elect/remove leaders, bind joint membership -durably or encode configuration history in snapshots. A safe next step must define -configuration entry ordering and recovery, activation and rollback of uncommitted -configurations, promotion readiness, old/new quorum use at every protocol phase, -and snapshot installation across transitions. Until those pieces are implemented -and tested, stored voter bindings remain immutable and promotion stays disabled. +This adds runtime recovery and quorum selection, not a reconfiguration API. Local +configuration proposals and incoming configuration entries remain disabled. +Recovered suffix transitions require a bound original voter set, a persisted +commit index and endpoints for every active voter. Joint snapshot recovery and +snapshot transfer across a membership transition remain unsupported; compaction +is rejected while the retained log contains a membership change beyond its +snapshot baseline. Promotion readiness, live configuration RPCs, rollback on +suffix replacement, and removed-leader behavior still need end-to-end coverage. The change adds a public type and changes TRaft's in-memory layout; rebuild the library and dependents together. Wire and durable formats are unchanged. Self-review checked overlap, strict majorities, current-term commitment, stable -path equivalence and learner exclusion. No independent reviewer approval is -claimed. There is no live membership change to exercise or benchmark yet. +path equivalence and learner exclusion. Runtime tests verify that elections, +commit advancement and strong-read confirmation each wait for both majorities +under an in-progress joint configuration. No independent reviewer approval is +claimed. The live transition API remains disabled. diff --git a/miniraft/configuration.h b/miniraft/configuration.h index 9a02688..679c781 100644 --- a/miniraft/configuration.h +++ b/miniraft/configuration.h @@ -5,7 +5,9 @@ #include #include -// Experimental record encoding only. Current Raft rejects these entries. +// Experimental configuration record encoding. The runtime can recover a +// validated retained history, but proposals and incoming config entries remain +// disabled until the complete transition/install protocol is implemented. enum class EConfigurationPhase : uint32_t { Joint = 1, Stable = 2 }; struct TConfigurationRecord { EConfigurationPhase Phase = EConfigurationPhase::Joint; @@ -66,8 +68,8 @@ struct TConfigurationScan { } }; -// Offline, read-only validation. The caller supplies and must substantiate both -// the baseline and the commit boundary; neither is proven by this scan. +// Read-only validation. The caller supplies and must substantiate both the +// baseline and commit boundary; neither is proven by this scan. TConfigurationScan ScanConfigurationHistory(const IState& state, TConfigurationHistory baseline, uint64_t commitIndex); diff --git a/miniraft/raft.cpp b/miniraft/raft.cpp index f8ca8d3..144d313 100644 --- a/miniraft/raft.cpp +++ b/miniraft/raft.cpp @@ -253,17 +253,14 @@ TRaft::TRaft(std::shared_ptr state, int node, const TNodeDict& nodes, st throw std::invalid_argument("Invalid or duplicate learner ID"); previous = id; } - if (!std::binary_search(Learners.begin(), Learners.end(), Id)) Voters.push_back(Id); + if (!std::binary_search(Learners.begin(), Learners.end(), Id)) InitialVoters.push_back(Id); for (const auto& [id, _] : Nodes) - if (!std::binary_search(Learners.begin(), Learners.end(), id)) Voters.push_back(id); - std::sort(Voters.begin(), Voters.end()); - ValidateFixedVoters(Voters); + if (!std::binary_search(Learners.begin(), Learners.end(), id)) InitialVoters.push_back(id); + std::sort(InitialVoters.begin(), InitialVoters.end()); + ValidateFixedVoters(InitialVoters); + Voters = InitialVoters; VotingQuorum.emplace(Voters); if (!State->CurrentTerm) throw std::invalid_argument("Stored Raft term must be positive"); - if (!IsVoter(Id) && State->VotedFor) - throw std::invalid_argument("A learner store must not contain a vote"); - if (State->VotedFor && !IsVoter(State->VotedFor)) - throw std::invalid_argument("Stored vote does not name a configured voter"); Nservers = Voters.size(); MinVotes = Nservers / 2 + 1; if (!Learners.empty() && !State->SupportsVoterBinding()) throw std::invalid_argument("Learners require voter-binding-capable storage"); @@ -277,10 +274,11 @@ TRaft::TRaft(std::shared_ptr state, int node, const TNodeDict& nodes, st ValidateSnapshot(*snapshot); if (snapshot->IsJoint()) throw std::invalid_argument("Joint snapshot recovery is unsupported"); - if (snapshot->StableConfiguration && bound != Voters) - throw std::invalid_argument("Stable configuration snapshot requires an existing voter binding"); + if (snapshot->StableConfiguration && bound != InitialVoters) + throw std::invalid_argument("Stable configuration snapshot requires the existing voter binding"); if (snapshot->Index != State->SnapshotIndex || snapshot->Term != State->SnapshotTerm || - snapshot->Term > State->CurrentTerm || snapshot->Voters != Voters || + snapshot->Term > State->CurrentTerm || + snapshot->Voters != InitialVoters || State->LastLogTerm < State->SnapshotTerm || State->LastLogTerm > State->CurrentTerm || (State->LastLogIndex == State->SnapshotIndex && State->LastLogTerm != State->SnapshotTerm)) throw std::invalid_argument("Compacted state snapshot boundary mismatch"); @@ -289,12 +287,8 @@ TRaft::TRaft(std::shared_ptr state, int node, const TNodeDict& nodes, st } if (State->HasUnboundRecoveredState() && !snapshot) throw std::invalid_argument("Recovered durable state requires voter proof before Raft startup"); - // Reject before binding voters or serving traffic. Current Raft cannot - // interpret configuration entries, including uncommitted suffix entries. auto previousTerm = State->SnapshotTerm; for (uint64_t index = State->SnapshotIndex + 1; index <= State->LastLogIndex; ++index) { - if (State->RetainedLogFlags(index) & TLogEntry::EConfiguration) - throw std::invalid_argument("Configuration log recovery is unsupported"); if (snapshot) { const auto term = State->RetainedLogTerm(index); if (!term || term < previousTerm || term > State->CurrentTerm) @@ -305,8 +299,26 @@ TRaft::TRaft(std::shared_ptr state, int node, const TNodeDict& nodes, st if (snapshot && previousTerm != State->LastLogTerm) throw std::invalid_argument("Compacted log last term mismatch"); const bool fresh = State->CurrentTerm == 1 && !State->VotedFor && !State->LastLogIndex; - if (State->SupportsVoterBinding() && (!bound.empty() || fresh || snapshot || !Learners.empty())) - State->BindVoters(Voters); + if (!bound.empty()) { + if (bound != InitialVoters) + throw std::invalid_argument("Persisted voter configuration differs from the configured peers"); + } + const bool bindInitialVoters = bound.empty() && State->SupportsVoterBinding() && + (fresh || snapshot || !Learners.empty()); + const auto baseline = snapshot && snapshot->StableConfiguration + ? TConfigurationHistory::FromStableSnapshot(*snapshot) + : TConfigurationHistory::Stable(InitialVoters, 0, 0, State->SnapshotIndex, State->SnapshotTerm); + ConfigurationBaseline = baseline; + RefreshConfiguration(std::max(State->CommitIndex, State->SnapshotIndex)); + const bool hasRetainedConfiguration = Configuration->Latest.ConfigurationIndex() > baseline.ConfigurationIndex(); + if (hasRetainedConfiguration && (!State->SupportsVoterBinding() || bound.empty() || + !State->SupportsCommitIndexPersistence())) + throw std::invalid_argument("Retained membership changes require bound voters and a persisted commit index"); + if (State->VotedFor && !IsVoter(State->VotedFor) && !hasRetainedConfiguration) + throw std::invalid_argument("Stored vote does not name a configured voter"); + if (!IsVoter(Id) && State->VotedFor && !hasRetainedConfiguration) + throw std::invalid_argument("A learner store must not contain a vote"); + if (bindInitialVoters) State->BindVoters(InitialVoters); if (commitFloorAdded) { State->CommitIndex = State->SnapshotIndex; State->Commit(); @@ -315,6 +327,76 @@ TRaft::TRaft(std::shared_ptr state, int node, const TNodeDict& nodes, st State->MarkRaftStarted(); } +void TRaft::RefreshConfiguration(uint64_t commitIndex) { + if (!ConfigurationBaseline) + throw std::logic_error("Missing trusted configuration baseline"); + auto scan = ScanConfigurationHistory(*State, *ConfigurationBaseline, commitIndex); + + // An uncommitted joint entry already requires both majorities to commit. + // Once the joint entry commits, keep joint consensus until its stable + // successor commits as well. This makes truncating either suffix restore + // the quorum implied by the surviving log. + const TConfigurationHistory* quorumConfiguration = nullptr; + if (scan.Committed.IsJoint()) quorumConfiguration = &scan.Committed; + else if (scan.Latest.IsJoint()) quorumConfiguration = &scan.Latest; + else quorumConfiguration = &scan.Committed; + + std::vector voters = quorumConfiguration->OldVoters(); + std::optional quorum; + int minVotes = 0; + if (quorumConfiguration->IsJoint()) { + voters.insert(voters.end(), quorumConfiguration->NewVoters().begin(), + quorumConfiguration->NewVoters().end()); + std::sort(voters.begin(), voters.end()); + voters.erase(std::unique(voters.begin(), voters.end()), voters.end()); + quorum.emplace(quorumConfiguration->OldVoters(), quorumConfiguration->NewVoters()); + minVotes = std::max(int(quorumConfiguration->OldVoters().size() / 2 + 1), + int(quorumConfiguration->NewVoters().size() / 2 + 1)); + } else { + quorum.emplace(quorumConfiguration->OldVoters()); + minVotes = int(quorumConfiguration->OldVoters().size() / 2 + 1); + } + for (auto id : voters) + if (id != Id && !Nodes.contains(id)) + throw std::invalid_argument("Active configuration has no replication target for a voter"); + + std::vector learners; + if (!std::binary_search(voters.begin(), voters.end(), Id)) learners.push_back(Id); + for (const auto& [id, _] : Nodes) + if (!std::binary_search(voters.begin(), voters.end(), id)) learners.push_back(id); + std::sort(learners.begin(), learners.end()); + + Voters.swap(voters); + Learners.swap(learners); + VotingQuorum.swap(quorum); + MinVotes = minVotes; + Nservers = int(Voters.size()); + Configuration = std::move(scan); + const auto& baseline = *ConfigurationBaseline; + const auto& latest = Configuration->Latest; + ConfigurationScanRequired = baseline.ConfigurationIndex() != latest.ConfigurationIndex() || + baseline.ConfigurationTerm() != latest.ConfigurationTerm() || baseline.IsJoint() != latest.IsJoint() || + baseline.OldVoters() != latest.OldVoters() || baseline.NewVoters() != latest.NewVoters(); + + if (!IsVoter(Id)) { + VolatileState->Votes.clear(); + if (StateName == EState::CANDIDATE || StateName == EState::LEADER) { + StateName = EState::FOLLOWER; + VolatileState->LeaderId = 0; + } + } +} + +bool TRaft::HasMembershipHistory() const { + if (!ConfigurationBaseline || !Configuration) return false; + const auto& baseline = *ConfigurationBaseline; + const auto& latest = Configuration->Latest; + return baseline.ConfigurationIndex() != latest.ConfigurationIndex() || + baseline.ConfigurationTerm() != latest.ConfigurationTerm() || + baseline.IsJoint() != latest.IsJoint() || baseline.OldVoters() != latest.OldVoters() || + baseline.NewVoters() != latest.NewVoters(); +} + void TRaft::OnRequestVote(ITimeSource::Time now, TMessageHolder message) { if (!IsVoter(Id)) { Nodes.at(message->Src)->Send(NewHoldedMessage( @@ -856,7 +938,7 @@ void TRaft::AddLearnerTarget(uint32_t id, std::shared_ptr node) { if (!id || id == Id || !node || Nodes.contains(id) || IsVoter(id) || std::binary_search(Learners.begin(), Learners.end(), id)) throw std::invalid_argument("Invalid or duplicate learner target"); - if (!State->SupportsVoterBinding() || State->GetBoundVoters() != Voters) + if (!State->SupportsVoterBinding() || State->GetBoundVoters() != InitialVoters) throw std::logic_error("Learner staging requires a bound voter configuration"); if (Nodes.size() >= size_t(INT_MAX)) throw std::overflow_error("Too many replication targets"); // Build all allocating changes before publishing any part of the target. @@ -908,6 +990,8 @@ bool TRaft::IsLearnerCommitAcknowledged(uint32_t id) const { TSnapshot TRaft::Compact() { if (!State->SupportsSnapshots()) throw std::logic_error("Storage cannot compact snapshots"); + if (HasMembershipHistory()) + throw std::logic_error("Compaction after a log membership change requires runtime configuration snapshots"); auto application = SnapshotApplication.lock(); if (!application) throw std::logic_error("Compaction requires a bound application"); if (application->LastAppliedIndex == State->SnapshotIndex && State->GetSnapshot()) return *State->GetSnapshot(); @@ -920,6 +1004,10 @@ TSnapshot TRaft::Compact() { application->ValidateSnapshot(snapshot.Data, snapshot.Index); AdvanceCommitIndex(VolatileState->CommitIndex); State->InstallSnapshot(snapshot); + ConfigurationBaseline = snapshot.StableConfiguration + ? TConfigurationHistory::FromStableSnapshot(snapshot) + : TConfigurationHistory::Stable(Voters, 0, 0, snapshot.Index, snapshot.Term); + RefreshConfiguration(VolatileState->CommitIndex); // Outstanding transfers own their immutable encoded bytes; discard any // otherwise-retained encoding of the old local snapshot. EncodedSnapshot.reset(); @@ -1114,6 +1202,7 @@ void TRaft::AdvanceCommitIndex(uint64_t index) { State->Commit(); } VolatileState->CommitIndex = target; + if (ConfigurationScanRequired) RefreshConfiguration(target); } void TRaft::AdvanceLeaderCommit() { diff --git a/miniraft/raft.h b/miniraft/raft.h index 3b7e7af..972cad1 100644 --- a/miniraft/raft.h +++ b/miniraft/raft.h @@ -13,6 +13,7 @@ #include "timesource.h" #include "state.h" #include "quorum.h" +#include "configuration.h" struct INode { virtual ~INode() = default; @@ -118,8 +119,9 @@ enum class EState: int { class TRaft { public: - // Nodes contains every replication peer; learners may include the local ID. - // Voter roles remain fixed; learner replication targets can be staged locally. + // Nodes contains every configured replication peer; learners may include + // the local ID. Retained validated configuration history determines active + // voter roles and joint quorum arithmetic. TRaft(std::shared_ptr state, int node, const TNodeDict& nodes, std::vector learners = {}); void Process(ITimeSource::Time now, TMessageHolder message, const std::shared_ptr& replyTo = {}); @@ -173,6 +175,9 @@ class TRaft { } int GetMinVotes() const { + // Stable configurations return their strict-majority threshold. Joint + // configurations return the larger of the two thresholds; callers + // must use protocol outcomes, not this scalar, to decide a quorum. return MinVotes; } @@ -211,10 +216,18 @@ class TRaft { ITimeSource::Time MakeElection(ITimeSource::Time now); uint64_t NextSeqno(); bool ValidateRpc(TMessageHolder message) const; + void RefreshConfiguration(uint64_t commitIndex); + bool HasMembershipHistory() const; uint32_t Id; TNodeDict Nodes; - std::vector Voters, Learners; + // InitialVoters is the immutable storage binding. Voters is the active + // voting union (the union of both sets while a joint configuration is in + // force); Learners is derived from that active membership. + std::vector InitialVoters, Voters, Learners; + std::optional ConfigurationBaseline; + std::optional Configuration; + bool ConfigurationScanRequired = false; std::optional VotingQuorum; int MinVotes; int Npeers; diff --git a/test/test_configuration.cpp b/test/test_configuration.cpp index 94e692c..2812aab 100644 --- a/test/test_configuration.cpp +++ b/test/test_configuration.cpp @@ -34,7 +34,8 @@ void FixChecksum(TMessageHolder entry) { } struct TEndpoint : INode { unsigned Sent = 0; - void Send(TMessageHolder) override { ++Sent; } + std::vector> Messages; + void Send(TMessageHolder message) override { ++Sent; Messages.push_back(std::move(message)); } void Drain() override {} }; struct TCountingRsm : TDummyRsm { @@ -488,6 +489,132 @@ void test_offline_history_scan_after_durable_snapshot_and_restart(void**) { assert_true(std::string(std::istreambuf_iterator(after), {}) == before); std::error_code ignored; std::filesystem::remove_all(dir, ignored); } +void test_raft_recovers_joint_quorum_for_candidate_election(void**) { + auto dir = std::filesystem::temp_directory_path() / ("miniraft-joint-quorum-" + + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count())); + assert_true(std::filesystem::create_directory(dir)); + const auto name = (dir / "raft").string(); + { + auto state = std::make_shared(name, 4); + state->BindVoters({1, 2, 3}); + state->CurrentTerm = 2; + state->Commit(); + TConfigurationRecord joint{EConfigurationPhase::Joint, 2, 0, 0, {1, 2, 3}, {3, 4, 5}}; + state->Append(EncodeConfigurationEntry(joint)); + state->Commit(); + + TNodeDict peers; + std::unordered_map> endpoints; + for (uint32_t id : {1, 2, 3, 5}) { + auto endpoint = std::make_shared(); + endpoints.emplace(id, endpoint); + peers.emplace(id, endpoint); + } + auto raft = std::make_shared(state, 4, peers, std::vector{4, 5}); + assert_true(raft->IsVoter(4)); + assert_true(raft->IsVoter(5)); + assert_int_equal(raft->GetNservers(), 5); + + auto now = ITimeSource::Time{}; + raft->ProcessTimeout(now); // self vote alone cannot satisfy either joint majority + assert_true(raft->CurrentStateName() == EState::CANDIDATE); + auto vote = [&](uint32_t source) { + raft->Process(now, NewHoldedMessage(TMessageEx{.Src = source, .Dst = 4, .Term = 3}, + TRequestVoteResponse{.VoteGranted = 1})); + raft->ProcessTimeout(now + std::chrono::milliseconds(1)); + }; + + vote(1); // This leaves both voter sets below their strict majority. + assert_true(raft->CurrentStateName() == EState::CANDIDATE); + vote(3); // 3 plus self satisfies new, while 1 plus 3 satisfies old. + assert_true(raft->CurrentStateName() == EState::LEADER); + + raft->ProcessTimeout(now + std::chrono::milliseconds(2)); + auto ack = [&](uint32_t source, uint64_t expectedIndex, int responseMillis) { + auto request = endpoints.at(source)->Messages.back().Cast(); + const auto matchIndex = request->PrevLogIndex + request->Nentries; + assert_int_equal(matchIndex, expectedIndex); + raft->Process(now + std::chrono::milliseconds(responseMillis), + NewHoldedMessage(TMessageEx{.Src = source, .Dst = 4, .Term = 3, .Seqno = request->Seqno}, + TAppendEntriesResponse{.MatchIndex = matchIndex, .Success = 1})); + }; + ack(3, 1, 3); + ack(5, 1, 3); + ack(1, 1, 3); + assert_int_equal(raft->GetVolatileState()->CommitIndex, 0); + + raft->ProcessTimeout(now + std::chrono::milliseconds(4)); + ack(3, 2, 5); + assert_int_equal(raft->GetVolatileState()->CommitIndex, 0); + ack(5, 2, 5); + assert_int_equal(raft->GetVolatileState()->CommitIndex, 0); + assert_int_equal(raft->CommitSeqno(), 0); + ack(1, 2, 5); + assert_int_equal(raft->GetVolatileState()->CommitIndex, 2); + assert_true(raft->CommitSeqno() > 0); + } + std::error_code ignored; std::filesystem::remove_all(dir, ignored); +} +void test_raft_keeps_joint_quorum_until_stable_successor_commits(void**) { + auto dir = std::filesystem::temp_directory_path() / ("miniraft-joint-finalize-" + + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count())); + assert_true(std::filesystem::create_directory(dir)); + auto state = std::make_shared((dir / "raft").string(), 4); + state->BindVoters({1, 2, 3}); + state->CurrentTerm = 3; + state->Commit(); + TConfigurationRecord joint{EConfigurationPhase::Joint, 2, 0, 0, {1, 2, 3}, {3, 4, 5}}; + state->Append(EncodeConfigurationEntry(joint)); + auto stable = joint; + stable.Phase = EConfigurationPhase::Stable; + stable.Term = 3; + stable.PreviousIndex = 1; + stable.PreviousTerm = 2; + state->Append(EncodeConfigurationEntry(stable)); + state->CommitIndex = 1; + state->Commit(); + + TNodeDict peers; + for (uint32_t id : {1, 2, 3, 5}) peers.emplace(id, std::make_shared()); + { + TRaft inJoint(state, 4, peers, std::vector{4, 5}); + assert_true(inJoint.IsVoter(1)); + assert_true(inJoint.IsVoter(5)); + assert_int_equal(inJoint.GetNservers(), 5); + } + + state->CommitIndex = 2; + state->Commit(); + { + TRaft afterStableCommit(state, 4, peers, std::vector{4, 5}); + assert_false(afterStableCommit.IsVoter(1)); + assert_false(afterStableCommit.IsVoter(2)); + assert_true(afterStableCommit.IsVoter(3)); + assert_true(afterStableCommit.IsVoter(5)); + assert_int_equal(afterStableCommit.GetNservers(), 3); + } + state.reset(); + std::error_code ignored; std::filesystem::remove_all(dir, ignored); +} +void test_raft_rejects_stable_snapshot_that_changes_active_voters(void**) { + auto state = std::make_shared(); + state->BindVoters({1, 2, 3}); + state->CurrentTerm = 5; + TSnapshot snapshot{5, 4, {2, 3, 4}, "stable image"}; + snapshot.StableConfiguration = true; + snapshot.ConfigurationIndex = 4; + snapshot.ConfigurationTerm = 3; + state->InstallSnapshot(snapshot); + + TNodeDict peers{ + {3, std::make_shared()}, + {4, std::make_shared()}, + }; + const auto fields = static_cast(*state); + Reject([&] { TRaft raft(state, 2, peers); }); + assert_true(static_cast(*state) == fields); + assert_true(state->GetBoundVoters() == std::vector({1, 2, 3})); +} void test_persisted_commit_boundary_drives_committed_configuration_scan(void**) { auto dir = std::filesystem::temp_directory_path() / ("miniraft-committed-history-" + @@ -617,6 +744,9 @@ int main() { cmocka_unit_test(test_durable_restart_rejects_reserved_entry_without_rewriting), cmocka_unit_test(test_legacy_restart_uses_safe_flag_lookup_and_preserves_files), cmocka_unit_test(test_offline_history_scan_after_durable_snapshot_and_restart), + cmocka_unit_test(test_raft_recovers_joint_quorum_for_candidate_election), + cmocka_unit_test(test_raft_keeps_joint_quorum_until_stable_successor_commits), + cmocka_unit_test(test_raft_rejects_stable_snapshot_that_changes_active_voters), cmocka_unit_test(test_persisted_commit_boundary_drives_committed_configuration_scan), cmocka_unit_test(test_configuration_snapshots_preserve_original_binding_across_transitions), #endif From 6bf346a08e76f449cee1d076d4eab8e6afbfeebb Mon Sep 17 00:00:00 2001 From: Michael Rayva Date: Sun, 15 Nov 2026 10:33:36 -0500 Subject: [PATCH 66/66] Enable guarded live joint membership --- README.md | 15 +- docs/configuration-entries.md | 108 +++++------ docs/configuration-scan.md | 13 +- docs/covered-snapshot-identity.md | 4 +- docs/durable-configuration-snapshots.md | 10 +- docs/joint-quorum.md | 15 +- docs/learners.md | 4 +- docs/online-learner-network.md | 7 +- docs/online-learner-staging.md | 7 +- docs/proposal-admission.md | 5 +- docs/snapshot-pinning.md | 4 +- docs/snapshot-retries.md | 7 +- docs/stable-configuration-snapshot-runtime.md | 16 +- docs/stable-snapshot-metadata-continuity.md | 5 +- miniraft/configuration.cpp | 2 +- miniraft/configuration.h | 10 +- miniraft/raft.cpp | 150 +++++++++++++++- miniraft/raft.h | 9 + test/test_configuration.cpp | 167 +++++++++++++++++- 19 files changed, 449 insertions(+), 109 deletions(-) diff --git a/README.md b/README.md index e3659e5..9f7e6f3 100644 --- a/README.md +++ b/README.md @@ -38,8 +38,11 @@ fully retained in that baseline. The [compaction follow-up](docs/compaction.md) enables prefix deletion and chunked InstallSnapshot catch-up; fixed membership remains required. Upgrade all voters before enabling the new compaction behavior. The [learner follow-up](docs/learners.md) adds non-voting catch-up and durable -voter binding. KV uses `--learner id` on every replica's configuration. Promotion -and joint-consensus voter changes remain disabled. +voter binding. KV uses `--learner id` on every replica's configuration. The +[live joint-membership step](docs/configuration-entries.md) adds a durable-store +`TRaft` API for adding and removing nonleader voters through joint consensus; +leader removal, hot server role reconciliation and membership-aware snapshot +transfer remain unsupported. The [snapshot retry follow-up](docs/snapshot-retries.md) prevents timeout retries and strong-read traffic from starving delayed snapshot acknowledgements, and prevents stale reader failures from destroying newer reconnect attempts. @@ -55,7 +58,7 @@ caller-owned entry buffers from changing entries already stored in TState. The [term lookup follow-up](docs/term-lookups.md) avoids copying full payloads when Raft needs only a retained entry's term. The [joint quorum foundation](docs/joint-quorum.md) adds and tests dual-majority -arithmetic for future membership changes; live voter reconfiguration remains disabled. +arithmetic used by the live membership transition path. The [joint snapshot metadata foundation](docs/joint-snapshots.md) adds a versioned representation for a joint configuration at a snapshot boundary; runtime adoption and membership changes remain disabled. @@ -63,8 +66,8 @@ The [snapshot term preflight follow-up](docs/unsupported-snapshot-term-preflight rejects unsupported snapshot formats before a higher RPC term changes hard state. The [snapshot metadata preflight follow-up](docs/snapshot-metadata-term-preflight.md) checks the boundary, fixed voters, and stable identity before that term update. -The [configuration entry foundation](docs/configuration-entries.md) defines an -experimental entry codec and rejects that entry kind in the fixed-membership runtime. +The [configuration entry support](docs/configuration-entries.md) describes the +live joint-to-stable transition API and its durable-storage and snapshot limits. The [configuration history validator](docs/configuration-history.md) checks an offline retained-log sequence against a trusted baseline or joint snapshot. The [configuration log scanner](docs/configuration-scan.md) validates a retained @@ -87,7 +90,7 @@ transfer retention to two image generations and defers local compaction at the c The [stable configuration snapshot foundation](docs/stable-configuration-snapshots.md) preserves configuration identity. Raft can recover and install stable metadata snapshots when their voter list matches the node's existing fixed membership; -joint snapshots and live voter changes remain disabled. +joint snapshots and membership-aware snapshot transfer remain disabled. The [durable configuration snapshot follow-up](docs/durable-configuration-snapshots.md) documents its voter-binding requirement and recovery boundary. The [trusted legacy voter binding follow-up](docs/trusted-legacy-voter-binding.md) diff --git a/docs/configuration-entries.md b/docs/configuration-entries.md index 17fa5b8..40ed685 100644 --- a/docs/configuration-entries.md +++ b/docs/configuration-entries.md @@ -1,57 +1,63 @@ -# Configuration entry foundation: v0.1.0-alpha.26 +# Live joint membership -This milestone defines an experimental representation for a joint or stable -configuration log entry. It does not enable live membership changes. Each record -contains its phase, entry term, previous configuration entry index and term, and -the old and new sorted voter sets. The stable record carries both sets as context; -future transition logic must decide how to validate and apply it. +The runtime accepts membership changes through a joint-consensus log sequence. +`TRaft::ProposeJointConfiguration(newVoters)` appends the joint entry, which +activates both the old and new voter majorities. After the joint entry commits, +`TRaft::FinalizeConfiguration()` appends its stable successor. The stable set +becomes active only after that successor commits. Elections, commit advancement +and strong-read confirmation use the active stable or joint quorum. -The MRFTC001 payload is magic[8], phase[4] (joint=1, stable=2), entry term[8], -previous configuration index[8], previous configuration term[8], old count[4], -new count[4], old voter IDs[4 each], new voter IDs[4 each], and CRC32[4] over -the preceding payload. The entry's outer term must equal the encoded term, and -its flags must equal EConfiguration, the high bit of the 64-bit flags field. -Counts, lengths, ordering and bounds are checked before allocation. A joint -entry may refer to an implicit bootstrap configuration with zero previous index -and term; a stable entry requires a nonzero predecessor. The codec checks shape, -not log position, predecessor identity, commitment, role eligibility or cluster -authentication. CRC is an accidental-corruption check, not authentication. +Both APIs require storage that persists the commit index and original voter +binding. `TDurableState` provides those guarantees. In-memory `TState` and legacy +stores cannot start a live transition. The leader must have a committed entry in +its current term. Every newly added voter must first be staged as a learner and +have a correlated replication acknowledgement through the leader's committed +boundary. The leader must be a member of the new set. The currently supported +API cannot remove its active leader; this library does not yet provide a +leadership-transfer API. Every node must have replication endpoints for all +voters before it accepts the transition. Endpoint discovery and configuration +distribution remain responsibilities of the embedding application. +On restart, construct each Raft object with the persisted original voter set and +configure later members as learners in that bootstrap view; the retained log +then recovers their active roles. Supplying the latest voter set as if it were +the original binding is rejected. -Local proposals and incoming AppendEntries still reject entries carrying the -reserved bit, before changing term or log. Startup now scans and reconstructs -valid retained configuration history, then selects the stable or joint quorum for -elections, commit advancement and strong-read confirmation. Retained transitions -require a bound original voter set, a persisted commit index and peer endpoints -for every active voter. If a configuration record reaches committed replay, the -request processor validates it, advances its applied index, and does not pass it -to the application state machine. Joint snapshot recovery and snapshot transfer -across a membership transition remain unsupported, so compaction is blocked while -the retained suffix carries a membership change. This is recovery support, not a -live reconfiguration API. Joint snapshots remain rejected as described in -[joint snapshot metadata](joint-snapshots.md). +These are `TRaft` APIs. `TRaftServer` currently starts outbound client-response +readers from the roles present when `Serve()` starts and does not reconcile that +reader set after a live promotion. Raft replication and quorum processing are +updated live, but a promoted follower that forwards client requests may need a +server restart before its forwarded responses are handled. Hot server role +reconciliation remains a follow-up. -Compatibility matters: previous versions allowed custom entry flags. Before -upgrading, audit applications and retained logs for use of the high bit. Raft now -reserves it for configuration records and refuses startup unless every retained -record forms a valid, trusted transition chain; this release does not rewrite or -migrate custom uses. A fixed snapshot may already contain effects of older -entries, which this suffix scan cannot detect. Older binaries may apply an unknown -configuration entry as application data, so these entries must not be replicated -until every voter runs a compatible implementation. Do not delete or edit -committed state to bypass rejection. +Incoming AppendEntries configuration records are decoded and the complete +candidate suffix is scanned before a higher RPC term is persisted. The scan +checks predecessor identity, transition order, committed-prefix constraints, +voter binding and endpoint availability. Same-term configuration entries must +match byte for byte. Replacing an uncommitted configuration suffix recomputes +the active roles and quorum from the surviving log; committed entries cannot be +truncated. A malformed or semantically invalid candidate suffix is rejected +without changing the term, commit index, log or response state. -The optimized memory and durable backends scan flags without copying payloads. -External and legacy backends use the default Get-based fallback, so startup can -read the full retained log; the interface's vtable changed and dependent binaries -must be rebuilt. Backends that override Get with different semantics must make -RetainedLogFlags consistent. No existing durable file or wire frame layout changed. +The record payload remains MRFTC001: magic[8], phase[4] (joint=1, stable=2), +entry term[8], previous configuration index[8], previous configuration +term[8], old count[4], new count[4], old voter IDs[4 each], new voter IDs[4 +each], and CRC32[4] over the preceding payload. The outer entry term must match +the encoded term and its flags must equal `EConfiguration`. CRC checks accidental +corruption; it is not authentication. All voters must run a compatible binary +before a transition because older versions reject this entry type. -Tests cover both phases, bootstrap and maximum-size records, every-byte corruption -and truncation, semantic corruption with valid CRC, malformed frames, rejected RPCs -and proposals, application isolation, compacted suffixes and restart recovery of a -joint quorum without rewriting durable files. Self-review checked fail-closed -ordering, predecessor shape, runtime quorum selection and compatibility limits. No -independent reviewer approval is claimed. Dynamic promotion/removal APIs, live -configuration RPCs, suffix rollback after replacement, joint snapshot recovery, -discovery and full transition testing remain open; this experimental encoding may -change when those semantics are implemented. +Membership-aware compaction and joint snapshot installation are still open. +Compaction remains blocked after retained membership history, and a lagging +peer that requires snapshot transfer across a transition is not yet supported. +Existing fixed-membership snapshots and legacy stores do not prove the active +voter set after a transition. Do not remove or edit committed state to bypass +these limits. + +Deterministic durable-store tests cover learner catch-up, voter addition and +nonleader removal, per-node activation, joint commit propagation, stable +finalization, rejection of a bad predecessor before term mutation, and rollback +of an uncommitted joint suffix. The test suite also checks recovery of retained +joint and stable histories, including post-restart role recovery after adding +and removing a voter. This is a first live transition slice; it does not +claim snapshot continuity across membership changes or explicit leadership +transfer. diff --git a/docs/configuration-scan.md b/docs/configuration-scan.md index da6a8c1..477b8eb 100644 --- a/docs/configuration-scan.md +++ b/docs/configuration-scan.md @@ -30,9 +30,10 @@ backend, which must validate its own records. Memory and durable backends avoid copying ordinary payloads; external backends using default metadata accessors may read full entries. Configuration records are read and validated in full. -Configuration entries and joint snapshots remain rejected by the live runtime. -Bound in-memory and durable stores can retain stable or joint metadata snapshots -for offline validation while preserving the original voter binding. The scanner -still requires a trusted configuration baseline; metadata presence alone does -not authenticate a transition. Live membership activation, commit rules, and -snapshot installation across membership transitions remain future work. +The standalone scanner remains read-only. The Raft runtime uses it to validate +a simulated AppendEntries suffix before accepting configuration entries and to +recompute roles after commit or suffix replacement. Live changes additionally +require durable commit-index persistence, the original voter binding and local +endpoints for every active voter. Joint snapshot installation and snapshot +transfer across a transition remain unsupported. Snapshot metadata alone does +not authenticate a transition. diff --git a/docs/covered-snapshot-identity.md b/docs/covered-snapshot-identity.md index 9e838a7..faa9df9 100644 --- a/docs/covered-snapshot-identity.md +++ b/docs/covered-snapshot-identity.md @@ -18,5 +18,7 @@ application payload before replacing state. Deterministic regression cases cover mismatched embedded voters, a snapshot boundary that disagrees with the request, invalid stable metadata, and valid metadata prefixes split across chunks. Rejected images do not change durable Raft -or application state. Joint snapshots and live voter changes remain disabled. +or application state. Joint snapshots remain unsupported; live voter changes use +the configuration log protocol described in +[live membership](configuration-entries.md). Self-review completed; no independent reviewer approval is claimed. diff --git a/docs/durable-configuration-snapshots.md b/docs/durable-configuration-snapshots.md index 90db787..4fda75c 100644 --- a/docs/durable-configuration-snapshots.md +++ b/docs/durable-configuration-snapshots.md @@ -27,15 +27,17 @@ and requires both durable commit-index support and a persisted original voter binding. For configuration-bearing snapshots, the caller supplies the active history baseline through the snapshot boundary; the scanner verifies it against the snapshot metadata. Legacy snapshots still require the baseline voters to -match the original binding. This remains an offline validation API; the snapshot -metadata is not proof of commitment or cluster identity, and this does not -activate joint quorums or permit live configuration changes. +match the original binding. This scanner is an offline validation API; snapshot +metadata is not proof of commitment or cluster identity. The separate live log +transition API uses the durable commit boundary and original binding, while +joint snapshot installation remains unsupported. In alpha.45 these images were persistence for offline validation only. The alpha.46 follow-up enables runtime recovery and transfer of MRFTS003 stable snapshots when the voter list equals the node's existing fixed membership. Compaction carries forward that stable configuration identity. MRFTS002 joint -snapshots and configuration log entries remain disabled in the live runtime. +snapshots remain disabled in the live runtime; configuration log entries are +now accepted through the guarded joint-consensus path. Since alpha.58, bound storage can preserve its original voter binding while installing configuration-bearing snapshots whose active set changed. Raft startup and network snapshot handling still reject those transition snapshots; diff --git a/docs/joint-quorum.md b/docs/joint-quorum.md index b8adc70..e64a3ac 100644 --- a/docs/joint-quorum.md +++ b/docs/joint-quorum.md @@ -25,14 +25,13 @@ watermarks and check current-term gating. Separate stable-set tests compare the calculation with strict-majority order statistics for sizes 1–8. Existing Raft, partition, learner, snapshot and read tests exercise the fixed-membership paths. -This adds runtime recovery and quorum selection, not a reconfiguration API. Local -configuration proposals and incoming configuration entries remain disabled. -Recovered suffix transitions require a bound original voter set, a persisted -commit index and endpoints for every active voter. Joint snapshot recovery and -snapshot transfer across a membership transition remain unsupported; compaction -is rejected while the retained log contains a membership change beyond its -snapshot baseline. Promotion readiness, live configuration RPCs, rollback on -suffix replacement, and removed-leader behavior still need end-to-end coverage. +This arithmetic now backs the bounded live transition API described in +[configuration entries](configuration-entries.md). Recovered suffix transitions +and live changes require a bound original voter set, a persisted commit index and +endpoints for every active voter. Joint snapshot recovery and snapshot transfer +across a membership transition remain unsupported; compaction is rejected while +the retained log contains a membership change beyond its snapshot baseline. +Leader removal and explicit leadership transfer are not implemented. The change adds a public type and changes TRaft's in-memory layout; rebuild the library and dependents together. Wire and durable formats are unchanged. diff --git a/docs/learners.md b/docs/learners.md index d9eff1e..0f6808f 100644 --- a/docs/learners.md +++ b/docs/learners.md @@ -46,7 +46,9 @@ The KV CLI uses static transport discovery. Library embedders can use [network learner staging](online-learner-network.md). That local, non-voting replication target is not persisted or cluster membership. Deleting `--learner 4` to turn it into a voter is rejected for bound stores; it is -not a membership-change procedure. Voter-set changes remain disabled. +not a membership-change procedure. Durable Raft embedders can use the separate +[live joint-membership API](configuration-entries.md) after catch-up and commit +acknowledgement. ## Quorum and term isolation diff --git a/docs/online-learner-network.md b/docs/online-learner-network.md index dd00ea0..9ccaffc 100644 --- a/docs/online-learner-network.md +++ b/docs/online-learner-network.md @@ -25,9 +25,10 @@ lifecycle. The KV CLI still discovers peers at startup. Staging is leader-local and is lost on restart or failover; the new leader and the application must reconcile -it. This adds network transport support for non-voting catch-up only. Voter -promotion/removal, replicated joint consensus and snapshot installation across -membership transitions remain unsupported. +it. This adds network transport support for non-voting catch-up. Voter promotion +and replicated joint consensus are now available through the bounded durable +transition path in [live membership](configuration-entries.md). Membership-aware +snapshot installation and leader removal remain unsupported. The deterministic server test checks target registration, voter/quorum invariance, removal and rollback after a nonleader rejection. The existing diff --git a/docs/online-learner-staging.md b/docs/online-learner-staging.md index f596e58..76b02f9 100644 --- a/docs/online-learner-staging.md +++ b/docs/online-learner-staging.md @@ -38,6 +38,7 @@ multi-chunk snapshot with delayed acknowledgements, invalid calls, and refusal on unbound or legacy stores. Durable-store tests verify that staging does not rewrite the binding and is absent on restart. A leader-change test verifies that an embedding application must re-register the target on the new leader. -Dynamic voter promotion/removal, replicated joint consensus, persistent discovery, -and membership-aware snapshot installation are still required before live voting -membership changes are supported. +A bounded durable-store promotion/finalization path now uses these catch-up +acknowledgements; see [live joint membership](configuration-entries.md). +Persistent endpoint discovery, leader removal and membership-aware snapshot +installation remain open. diff --git a/docs/proposal-admission.md b/docs/proposal-admission.md index 5976053..a6c5ba5 100644 --- a/docs/proposal-admission.md +++ b/docs/proposal-admission.md @@ -39,5 +39,6 @@ The role check is not a quorum lease: an isolated leader may still accept entrie that remain uncommitted, and a successful Append return is only a local log index. Callers must serialize Raft access and wait for committed application/client completion as appropriate. Directly editing IState or forcing states through the -test-facing Become API is outside proposal admission. Joint consensus and voter -promotion/removal remain open. +test-facing Become API is outside proposal admission. The current bounded +joint-consensus path is described in [live membership](configuration-entries.md); +leader removal and membership-aware snapshots remain open. diff --git a/docs/snapshot-pinning.md b/docs/snapshot-pinning.md index c30abf3..6d33c59 100644 --- a/docs/snapshot-pinning.md +++ b/docs/snapshot-pinning.md @@ -59,4 +59,6 @@ durable formats are unchanged; no additional migration is required. This guarantees progress of the selected finite image under eventual chunk delivery and ACKs. Reaching the live log end still requires enough replication capacity relative to writes/compaction and sufficiently stable leadership. -Dynamic promotion/removal and joint consensus remain open. +The bounded promotion and joint-consensus path is described in +[live membership](configuration-entries.md). Membership-aware snapshots and +leader removal remain open. diff --git a/docs/snapshot-retries.md b/docs/snapshot-retries.md index 27b6d72..878cbce 100644 --- a/docs/snapshot-retries.md +++ b/docs/snapshot-retries.md @@ -58,6 +58,7 @@ isolation. Self-review checked connection generation ownership and outstanding connector lifetime as well as chunk identity lifetime. No independent approval is claimed. -Dynamic promotion/removal and replicated joint-consensus configurations remain -open. Snapshot catch-up is a prerequisite for promotion, not authorization to -change the fixed voter set. +Snapshot catch-up is a prerequisite for promotion, not authorization by itself. +The bounded durable joint-consensus API is described in +[live membership](configuration-entries.md); leader removal and +membership-aware snapshot transfer remain open. diff --git a/docs/stable-configuration-snapshot-runtime.md b/docs/stable-configuration-snapshot-runtime.md index b653525..86e1cf7 100644 --- a/docs/stable-configuration-snapshot-runtime.md +++ b/docs/stable-configuration-snapshot-runtime.md @@ -18,15 +18,13 @@ already-covered fast path. This prevents a legacy-format refresh from silently erasing configuration history and prevents a peer from replacing it with a conflicting baseline. -This does not implement membership changes. Configuration log entries remain -disabled, and the original voter binding cannot change. Raft startup and network -installation still reject a stable snapshot whose active set differs from the -configured fixed set. Since alpha.58, offline storage can retain such -configuration-bearing snapshots without changing the original binding. Joint -consensus, transition rollback and snapshot installation across a membership -transition remain open. Upgrade all -voters before sending MRFTS003 images because older binaries may acknowledge an -already-covered unknown snapshot format. +The original voter binding still identifies the trusted initial set. The live +joint-consensus log protocol can move the active set without rewriting that +binding, but the snapshot runtime here still requires snapshot voters to match +the configured set. Joint snapshot installation and snapshot transfer across a +membership transition remain open. Upgrade all voters before sending MRFTS003 +images because older binaries may acknowledge an already-covered unknown +snapshot format. Tests cover durable application restore, identity preservation through compaction, restart, and legacy-format refresh, chunked InstallSnapshot, diff --git a/docs/stable-snapshot-metadata-continuity.md b/docs/stable-snapshot-metadata-continuity.md index 7030cd5..76a1f29 100644 --- a/docs/stable-snapshot-metadata-continuity.md +++ b/docs/stable-snapshot-metadata-continuity.md @@ -9,8 +9,9 @@ the baseline needed by recovery and later compaction. Incoming MRFTS003 snapshots must retain the known configuration index and term. The receiver rejects conflicting identities both when installing a newer image and when acknowledging an image whose boundary is already covered. These checks -do not enable membership changes: the node still requires the same bound voters, -and joint snapshots and configuration log entries remain unsupported. +do not enable membership changes through snapshots: the node still requires the +same configured voters, and joint snapshots remain unsupported. Membership now +changes through the separate configuration log protocol. The focused compaction test covers a covered-boundary conflict, a legacy-format refresh followed by restart, and a conflicting newer stable snapshot. The change diff --git a/miniraft/configuration.cpp b/miniraft/configuration.cpp index 3318eb9..fa311ae 100644 --- a/miniraft/configuration.cpp +++ b/miniraft/configuration.cpp @@ -118,7 +118,7 @@ TConfigurationHistory TConfigurationHistory::Advance(uint64_t index, TMessageHol throw std::invalid_argument("Configuration entry does not extend the trusted history"); TConfigurationHistory next = *this; if (!IsJoint()) { - if (record.Phase != EConfigurationPhase::Joint || record.OldVoters != Old) + if (record.Phase != EConfigurationPhase::Joint || record.OldVoters != Old || record.NewVoters == Old) throw std::invalid_argument("Expected a joint configuration from the stable voter set"); next.New = record.NewVoters; } else { diff --git a/miniraft/configuration.h b/miniraft/configuration.h index 679c781..8405220 100644 --- a/miniraft/configuration.h +++ b/miniraft/configuration.h @@ -5,9 +5,8 @@ #include #include -// Experimental configuration record encoding. The runtime can recover a -// validated retained history, but proposals and incoming config entries remain -// disabled until the complete transition/install protocol is implemented. +// Configuration record encoding for the live joint-consensus log protocol. +// Snapshot installation across membership transitions remains unsupported. enum class EConfigurationPhase : uint32_t { Joint = 1, Stable = 2 }; struct TConfigurationRecord { EConfigurationPhase Phase = EConfigurationPhase::Joint; @@ -19,8 +18,9 @@ struct TConfigurationRecord { TMessageHolder EncodeConfigurationEntry(const TConfigurationRecord& record); TConfigurationRecord DecodeConfigurationEntry(TMessageHolder entry); -// Offline transition validation only; does not establish commitment, activate -// roles, or make a fixed-membership Raft node accept configuration entries. +// Validates transition ordering and derives configuration identity from a +// trusted baseline and commit boundary. Runtime callers use this before +// activating roles or accepting a candidate AppendEntries suffix. class TConfigurationHistory { public: TConfigurationHistory(const TConfigurationHistory&) = default; diff --git a/miniraft/raft.cpp b/miniraft/raft.cpp index 144d313..f218777 100644 --- a/miniraft/raft.cpp +++ b/miniraft/raft.cpp @@ -462,6 +462,7 @@ void TRaft::OnAppendEntries(ITimeSource::Time now, TMessageHolderCommitIndex; bool success = false; + bool configurationTouched = false; if (message->PrevLogIndex >= State->SnapshotIndex && (message->PrevLogIndex == 0 || (message->PrevLogIndex <= State->LastLogIndex && State->LogTerm(message->PrevLogIndex) == message->PrevLogTerm))) @@ -474,10 +475,13 @@ void TRaft::OnAppendEntries(ITimeSource::Time now, TMessageHolderLogTerm(index) != entry->Term) { + if (Configuration && index <= Configuration->Latest.ConfigurationIndex()) + configurationTouched = true; while (State->LastLogIndex > index-1) { State->RemoveLast(); } State->Append(entry); + if (entry->Flags & TLogEntry::EConfiguration) configurationTouched = true; } } @@ -491,6 +495,7 @@ void TRaft::OnAppendEntries(ITimeSource::Time now, TMessageHolderCommitIndex); (*VolatileState) .SetLeaderId(message->LeaderId) .SetElectionDue(MakeElection(now)); @@ -758,18 +763,105 @@ bool TRaft::ValidateRpc(TMessageHolder message) const { auto previousTerm = append->PrevLogTerm; for (uint32_t i = 0; i < message.PayloadSize; ++i) { auto entry = message.Payload[i].Cast(); - if (entry->Flags & TLogEntry::EConfiguration) return false; if (!entry->Term || entry->Term < previousTerm || entry->Term > header->Term) return false; previousTerm = entry->Term; const auto index = append->PrevLogIndex + i + 1; if (index >= State->SnapshotIndex && index <= VolatileState->CommitIndex && State->LogTerm(index) != entry->Term) return false; } - return true; + return ValidateAppendConfiguration(append); } default: return false; } } +bool TRaft::ValidateAppendConfiguration(TMessageHolder message) const { + if (message->Nentries != message.PayloadSize) return false; + bool hasConfigurationEntry = false; + for (uint32_t i = 0; i < message.PayloadSize; ++i) { + auto entry = message.Payload[i].Cast(); + if (!(entry->Flags & TLogEntry::EConfiguration)) continue; + try { + const auto record = DecodeConfigurationEntry(entry); + // A transition leader must be able to remain a voter through the + // stable successor. Leader removal needs an explicit transfer and + // commit-propagation protocol, which this runtime does not yet have. + if (!std::binary_search(record.NewVoters.begin(), record.NewVoters.end(), message->LeaderId)) + return false; + } catch (const std::exception&) { + return false; + } + hasConfigurationEntry = true; + } + if (hasConfigurationEntry && (!ConfigurationBaseline || !State->SupportsVoterBinding() || + !State->SupportsCommitIndexPersistence() || State->GetBoundVoters() != InitialVoters)) + return false; + + // A mismatched previous boundary cannot change this log, so after checking + // the entry frames we can safely return the normal AppendEntries conflict. + if (message->Term < State->CurrentTerm || message->PrevLogIndex < State->SnapshotIndex || + (message->PrevLogIndex && (message->PrevLogIndex > State->LastLogIndex || + State->LogTerm(message->PrevLogIndex) != message->PrevLogTerm))) + return true; + + bool scanRequired = hasConfigurationEntry; + for (uint32_t i = 0; i < message.PayloadSize; ++i) { + const auto index = message->PrevLogIndex + i + 1; + auto incoming = message.Payload[i].Cast(); + const auto localTerm = State->LogTerm(index); + const bool incomingConfiguration = incoming->Flags & TLogEntry::EConfiguration; + const bool localConfiguration = index <= State->LastLogIndex && + (State->RetainedLogFlags(index) & TLogEntry::EConfiguration); + if (localTerm == incoming->Term && (incomingConfiguration || localConfiguration)) { + auto local = State->Get(index); + if (!local || local->Flags != incoming->Flags || local->Len != incoming->Len || + memcmp(local.Mes, incoming.Mes, incoming->Len) != 0) + return false; + } + if (localTerm != incoming->Term && Configuration && + index <= Configuration->Latest.ConfigurationIndex()) + scanRequired = true; + } + if (!scanRequired) return true; + if (!ConfigurationBaseline || !State->SupportsVoterBinding() || + !State->SupportsCommitIndexPersistence() || State->GetBoundVoters() != InitialVoters) + return false; + + try { + TState shadow; + static_cast(shadow) = static_cast(*State); + shadow.Snapshot = State->GetSnapshot(); + shadow.BoundVoters = State->GetBoundVoters(); + shadow.LastLogIndex = State->SnapshotIndex; + shadow.LastLogTerm = State->SnapshotTerm; + for (uint64_t index = State->SnapshotIndex + 1; index <= State->LastLogIndex; ++index) { + auto entry = State->Get(index); + if (!entry) return false; + shadow.Append(std::move(entry)); + } + shadow.CurrentTerm = message->Term; + auto index = message->PrevLogIndex; + for (uint32_t i = 0; i < message.PayloadSize; ++i) { + auto entry = message.Payload[i].Cast(); + ++index; + if (shadow.LogTerm(index) == entry->Term) continue; + while (shadow.LastLogIndex > index - 1) shadow.RemoveLast(); + shadow.Append(std::move(entry)); + } + const auto commitIndex = std::max(VolatileState->CommitIndex, + std::min(message->LeaderCommit, index)); + const auto scan = ScanConfigurationHistory(shadow, *ConfigurationBaseline, commitIndex); + const auto* active = scan.Committed.IsJoint() ? &scan.Committed : + scan.Latest.IsJoint() ? &scan.Latest : &scan.Committed; + for (auto id : active->OldVoters()) + if (id != Id && !Nodes.contains(id)) return false; + for (auto id : active->NewVoters()) + if (id != Id && !Nodes.contains(id)) return false; + } catch (const std::exception&) { + return false; + } + return true; +} + void TRaft::FollowerTimeout(ITimeSource::Time now) { if (VolatileState->ElectionDue <= now) { Become(EState::CANDIDATE); @@ -921,6 +1013,60 @@ uint64_t TRaft::Append(TMessageHolder entry) { return State->LastLogIndex; } +void TRaft::RequireLiveConfigurationStorage() const { + if (!State->SupportsVoterBinding() || !State->SupportsCommitIndexPersistence() || + InitialVoters.empty() || State->GetBoundVoters() != InitialVoters) + throw std::logic_error("Live membership changes require persisted commit and voter metadata"); +} + +uint64_t TRaft::ProposeJointConfiguration(std::vector newVoters) { + if (StateName != EState::LEADER) throw std::logic_error("Only a leader can change membership"); + RequireLiveConfigurationStorage(); + ValidateFixedVoters(newVoters); + if (!Configuration || Configuration->HasUncommittedConfiguration() || Configuration->Latest.IsJoint()) + throw std::logic_error("A stable committed configuration is required to start a transition"); + if (State->LastLogIndex >= uint64_t(INT64_MAX)) throw std::overflow_error("Raft log index exhausted"); + if (!VolatileState->CommitIndex || State->LogTerm(VolatileState->CommitIndex) != State->CurrentTerm) + throw std::logic_error("A current-term entry must be committed before changing membership"); + + const auto& oldVoters = Configuration->Committed.OldVoters(); + if (newVoters == oldVoters || !std::binary_search(newVoters.begin(), newVoters.end(), Id)) + throw std::invalid_argument("The new voter set must change membership and retain the current leader"); + for (auto id : newVoters) { + if (id != Id && !Nodes.contains(id)) + throw std::invalid_argument("Every new voter must have a configured replication target"); + if (!std::binary_search(oldVoters.begin(), oldVoters.end(), id) && + !IsLearnerCommitAcknowledged(id)) + throw std::logic_error("Every added voter must be a staged learner with a committed catch-up ACK"); + } + + const auto& latest = Configuration->Latest; + TConfigurationRecord record{EConfigurationPhase::Joint, State->CurrentTerm, + latest.ConfigurationIndex(), latest.ConfigurationTerm(), oldVoters, std::move(newVoters)}; + State->Append(EncodeConfigurationEntry(record)); + RefreshConfiguration(VolatileState->CommitIndex); + return State->LastLogIndex; +} + +uint64_t TRaft::FinalizeConfiguration() { + if (StateName != EState::LEADER) throw std::logic_error("Only a leader can change membership"); + RequireLiveConfigurationStorage(); + if (!Configuration || Configuration->HasUncommittedConfiguration() || !Configuration->Committed.IsJoint()) + throw std::logic_error("The joint configuration must commit before finalization"); + if (State->LastLogIndex >= uint64_t(INT64_MAX)) throw std::overflow_error("Raft log index exhausted"); + if (!VolatileState->CommitIndex || State->LogTerm(VolatileState->CommitIndex) != State->CurrentTerm) + throw std::logic_error("A current-term entry must be committed before finalizing membership"); + + const auto& joint = Configuration->Committed; + if (!std::binary_search(joint.NewVoters().begin(), joint.NewVoters().end(), Id)) + throw std::logic_error("Leadership must transfer to a voter in the new set before finalization"); + TConfigurationRecord record{EConfigurationPhase::Stable, State->CurrentTerm, + joint.ConfigurationIndex(), joint.ConfigurationTerm(), joint.OldVoters(), joint.NewVoters()}; + State->Append(EncodeConfigurationEntry(record)); + RefreshConfiguration(VolatileState->CommitIndex); + return State->LastLogIndex; +} + uint32_t TRaft::GetLeaderId() const { return VolatileState->LeaderId; } diff --git a/miniraft/raft.h b/miniraft/raft.h index 972cad1..bc8b1f2 100644 --- a/miniraft/raft.h +++ b/miniraft/raft.h @@ -130,6 +130,13 @@ class TRaft { // Local proposals require leader state. Role rejection leaves storage and // the supplied entry unchanged; followers receive entries through Process. uint64_t Append(TMessageHolder entry); + // Start a joint-consensus transition. Every newly added voter must already + // be staged as a learner and have acknowledged this leader's committed + // boundary. This first runtime path requires the current leader to remain + // in the new voter set; leadership transfer must precede leader removal. + uint64_t ProposeJointConfiguration(std::vector newVoters); + // Append the stable successor after the joint entry has committed. + uint64_t FinalizeConfiguration(); uint32_t GetLeaderId() const; uint64_t GetLastIndex() const; std::vector GetVoterIds() const; @@ -216,6 +223,8 @@ class TRaft { ITimeSource::Time MakeElection(ITimeSource::Time now); uint64_t NextSeqno(); bool ValidateRpc(TMessageHolder message) const; + bool ValidateAppendConfiguration(TMessageHolder message) const; + void RequireLiveConfigurationStorage() const; void RefreshConfiguration(uint64_t commitIndex); bool HasMembershipHistory() const; diff --git a/test/test_configuration.cpp b/test/test_configuration.cpp index 2812aab..fae2fc6 100644 --- a/test/test_configuration.cpp +++ b/test/test_configuration.cpp @@ -35,7 +35,12 @@ void FixChecksum(TMessageHolder entry) { struct TEndpoint : INode { unsigned Sent = 0; std::vector> Messages; - void Send(TMessageHolder message) override { ++Sent; Messages.push_back(std::move(message)); } + std::function)> Route; + void Send(TMessageHolder message) override { + ++Sent; + if (Route) Route(std::move(message)); + else Messages.push_back(std::move(message)); + } void Drain() override {} }; struct TCountingRsm : TDummyRsm { @@ -156,6 +161,8 @@ void test_history_rejects_skips_wrong_predecessors_and_term_regression(void**) { Reject([&] { base.Advance(1, Ordinary(1)); }); auto wrong = joint; wrong.OldVoters = {1, 2, 4}; Reject([&] { base.Advance(1, EncodeConfigurationEntry(wrong)); }); + auto noOp = joint; noOp.NewVoters = noOp.OldVoters; + Reject([&] { base.Advance(1, EncodeConfigurationEntry(noOp)); }); auto prefix = base.Observe(1, Ordinary(3)); Reject([&] { prefix.Advance(2, entry); }); Reject([&] { prefix.Observe(2, Ordinary(2)); }); @@ -381,6 +388,161 @@ void test_configuration_rpc_and_proposal_are_rejected_before_mutation(void**) { auto proposed = EncodeConfigurationEntry(record); Reject([&] { leader.Append(proposed); }); assert_true(static_cast(*leaderState) == leaderBefore); assert_int_equal(proposed->Term, 2); + Reject([&] { leader.ProposeJointConfiguration({1, 2}); }); // Volatile stores cannot authorize a live change. + assert_true(static_cast(*leaderState) == leaderBefore); +} +void test_joint_membership_addition_and_finalization_replicate_deterministically(void**) { +#ifdef __linux__ + auto dir = std::filesystem::temp_directory_path() / ("miniraft-live-config-" + + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count())); + assert_true(std::filesystem::create_directory(dir)); + const auto name = (dir / "raft").string(); + { + auto toLearner = std::make_shared(); + auto toLeader = std::make_shared(); + auto leaderState = std::make_shared(name, 1); + auto learnerState = std::make_shared(name, 2); + TRaft leader(leaderState, 1, {{2, toLearner}}, {2}); + TRaft learner(learnerState, 2, {{1, toLeader}}, {2}); + const auto now = ITimeSource::Time{}; + toLearner->Route = [&](auto message) { learner.Process(now, std::move(message)); }; + toLeader->Route = [&](auto message) { leader.Process(now, std::move(message)); }; + + // A well-formed but wrong predecessor must not make the higher term + // durable or earn an AppendEntries response. + TConfigurationRecord invalid{EConfigurationPhase::Joint, 2, 0, 0, {2}, {1, 2}}; + auto invalidRequest = NewHoldedMessage(TMessageEx{.Src = 1, .Dst = 2, .Term = 2}, + TAppendEntriesRequest{.LeaderId = 1, .Nentries = 1}); + invalidRequest.InitPayload(1); invalidRequest.Payload[0] = EncodeConfigurationEntry(invalid); + const auto beforeInvalidRequest = static_cast(*learnerState); + learner.Process(now, invalidRequest); + assert_true(static_cast(*learnerState) == beforeInvalidRequest); + assert_int_equal(toLeader->Sent, 0); + + leader.ProcessTimeout(now); + assert_true(leader.CurrentStateName() == EState::LEADER); + leader.ProcessTimeout(now + std::chrono::seconds(10)); + assert_true(leader.IsLearnerCommitAcknowledged(2)); + + const auto beforeRejectedProposal = static_cast(*leaderState); + Reject([&] { leader.ProposeJointConfiguration({2}); }); // Cannot remove the active leader. + assert_true(static_cast(*leaderState) == beforeRejectedProposal); + const auto jointIndex = leader.ProposeJointConfiguration({1, 2}); + assert_int_equal(jointIndex, 2); + assert_true(leader.IsVoter(2)); + assert_true(leader.GetVoterIds() == (std::vector{1, 2})); + + leader.ProcessTimeout(now + std::chrono::seconds(20)); + assert_true(learner.IsVoter(2)); + assert_int_equal(leader.GetVolatileState()->CommitIndex, jointIndex); + assert_int_equal(learnerState->CommitIndex, 1); // The commit watermark needs another RPC. + leader.ProcessTimeout(now + std::chrono::seconds(30)); + assert_int_equal(learnerState->CommitIndex, jointIndex); + + const auto stableIndex = leader.FinalizeConfiguration(); + assert_int_equal(stableIndex, 3); + leader.ProcessTimeout(now + std::chrono::seconds(40)); + assert_int_equal(leader.GetVolatileState()->CommitIndex, stableIndex); + assert_true(leader.GetVoterIds() == (std::vector{1, 2})); + leader.ProcessTimeout(now + std::chrono::seconds(50)); + assert_int_equal(learnerState->CommitIndex, stableIndex); + assert_true(leader.GetLearnerIds().empty()); + assert_true(learner.IsVoter(2)); + assert_true(learner.GetVoterIds() == (std::vector{1, 2})); + + // Removing a different voter still requires both old and new + // majorities until the stable successor commits. + const auto removalJoint = leader.ProposeJointConfiguration({1}); + assert_int_equal(removalJoint, 4); + assert_true(leader.IsVoter(2)); + leader.ProcessTimeout(now + std::chrono::seconds(60)); + assert_int_equal(leader.GetVolatileState()->CommitIndex, removalJoint); + leader.ProcessTimeout(now + std::chrono::seconds(70)); + assert_int_equal(learnerState->CommitIndex, removalJoint); + const auto removalStable = leader.FinalizeConfiguration(); + assert_int_equal(removalStable, 5); + leader.ProcessTimeout(now + std::chrono::seconds(80)); + assert_int_equal(leader.GetVolatileState()->CommitIndex, removalStable); + assert_false(leader.IsVoter(2)); + leader.ProcessTimeout(now + std::chrono::seconds(90)); + assert_int_equal(learnerState->CommitIndex, removalStable); + assert_false(leader.IsVoter(2)); + assert_false(learner.IsVoter(2)); + assert_true(leader.GetVoterIds() == (std::vector{1})); + assert_true(learner.GetVoterIds() == (std::vector{1})); + } + std::error_code ignored; std::filesystem::remove_all(dir, ignored); +#endif +} +void test_uncommitted_joint_suffix_replacement_restores_old_voters(void**) { +#ifdef __linux__ + auto dir = std::filesystem::temp_directory_path() / ("miniraft-config-rollback-" + + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count())); + assert_true(std::filesystem::create_directory(dir)); + const auto name = (dir / "raft").string(); + { + auto peer = std::make_shared(); + auto state = std::make_shared(name, 2); + state->BindVoters({1}); + state->CurrentTerm = 2; + auto joint = TConfigurationRecord{EConfigurationPhase::Joint, 2, 0, 0, {1}, {1, 2}}; + state->Append(EncodeConfigurationEntry(joint)); + state->Commit(); + TRaft follower(state, 2, {{1, peer}}, {2}); + assert_true(follower.IsVoter(2)); + + auto ordinary = Ordinary(3); + auto replacement = NewHoldedMessage(TMessageEx{.Src = 1, .Dst = 2, .Term = 3}, + TAppendEntriesRequest{.LeaderId = 1, .Nentries = 1}); + replacement.InitPayload(1); replacement.Payload[0] = ordinary; + follower.Process(ITimeSource::Time{}, replacement); + assert_int_equal(state->CurrentTerm, 3); + assert_int_equal(state->LastLogIndex, 1); + assert_int_equal(state->RetainedLogFlags(1), 0); + assert_false(follower.IsVoter(2)); + assert_true(follower.GetVoterIds() == (std::vector{1})); + assert_int_equal(peer->Sent, 1); + } + std::error_code ignored; std::filesystem::remove_all(dir, ignored); +#endif +} +void test_durable_restart_recovers_stable_membership_roles(void**) { +#ifdef __linux__ + auto dir = std::filesystem::temp_directory_path() / ("miniraft-config-restart-" + + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count())); + assert_true(std::filesystem::create_directory(dir)); + const auto name = (dir / "raft").string(); + { + auto state = std::make_shared(name, 2); + state->BindVoters({1}); // Original binding stays fixed through both transitions. + state->CurrentTerm = 3; + auto addJoint = TConfigurationRecord{EConfigurationPhase::Joint, 2, 0, 0, {1}, {1, 2}}; + state->Append(EncodeConfigurationEntry(addJoint)); + state->CommitIndex = 1; state->Commit(); + auto addStable = addJoint; addStable.Phase = EConfigurationPhase::Stable; + addStable.PreviousIndex = 1; addStable.PreviousTerm = 2; + state->Append(EncodeConfigurationEntry(addStable)); + state->CommitIndex = 2; state->Commit(); + auto removeJoint = TConfigurationRecord{EConfigurationPhase::Joint, 3, 2, 2, {1, 2}, {1}}; + state->Append(EncodeConfigurationEntry(removeJoint)); + state->CommitIndex = 3; state->Commit(); + auto removeStable = removeJoint; removeStable.Phase = EConfigurationPhase::Stable; + removeStable.PreviousIndex = 3; removeStable.PreviousTerm = 3; + state->Append(EncodeConfigurationEntry(removeStable)); + state->CommitIndex = 4; state->Commit(); + } + { + auto state = std::make_shared(name, 2); + auto peer = std::make_shared(); + TRaft raft(state, 2, {{1, peer}}, {2}); // Reconstruct from the original binding. + assert_true(state->GetBoundVoters() == (std::vector{1})); + assert_false(raft.IsVoter(2)); + assert_true(raft.GetVoterIds() == (std::vector{1})); + assert_true(raft.GetLearnerIds() == (std::vector{2})); + assert_int_equal(raft.GetVolatileState()->CommitIndex, 4); + } + std::error_code ignored; std::filesystem::remove_all(dir, ignored); +#endif } void test_configuration_application_advances_without_calling_state_machine(void**) { auto state = std::make_shared(); auto record = Joint(); record.Term = 1; @@ -739,8 +901,11 @@ int main() { cmocka_unit_test(test_configuration_roundtrips_both_phases_and_bounded_maximum), cmocka_unit_test(test_configuration_rejects_invalid_fields_and_valid_crc_corruption), cmocka_unit_test(test_configuration_rpc_and_proposal_are_rejected_before_mutation), + cmocka_unit_test(test_joint_membership_addition_and_finalization_replicate_deterministically), + cmocka_unit_test(test_uncommitted_joint_suffix_replacement_restores_old_voters), cmocka_unit_test(test_configuration_application_advances_without_calling_state_machine), #ifdef __linux__ + cmocka_unit_test(test_durable_restart_recovers_stable_membership_roles), cmocka_unit_test(test_durable_restart_rejects_reserved_entry_without_rewriting), cmocka_unit_test(test_legacy_restart_uses_safe_flag_lookup_and_preserves_files), cmocka_unit_test(test_offline_history_scan_after_durable_snapshot_and_restart),