diff --git a/.github/workflows/correctness.yml b/.github/workflows/correctness.yml new file mode 100644 index 0000000..a9f2c6c --- /dev/null +++ b/.github/workflows/correctness.yml @@ -0,0 +1,34 @@ +name: Consensus correctness +on: [push, pull_request] +permissions: + contents: read +jobs: + core: + runs-on: ubuntu-latest + strategy: + matrix: + build_type: [Debug, Release] + steps: + - uses: actions/checkout@v4 + - run: sudo apt-get update && sudo apt-get install -y libcmocka-dev libsqlite3-dev + - name: Configure core without networking dependencies + run: >- + cmake -S . -B build/core + -DMINIRAFT_BUILD_NETWORK=OFF + -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} + - run: cmake --build build/core --parallel 2 + - run: ctest --test-dir build/core --output-on-failure + sanitizers: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - run: sudo apt-get update && sudo apt-get install -y libcmocka-dev liburing-dev libssl-dev libsqlite3-dev + - name: Configure full build with ASan and UBSan + run: >- + cmake -S . -B build/full -DCMAKE_BUILD_TYPE=Debug + -DCMAKE_CXX_FLAGS="-fsanitize=address,undefined -fno-sanitize-recover=all" + -DCMAKE_EXE_LINKER_FLAGS="-fsanitize=address,undefined" + - run: cmake --build build/full --parallel 2 + - run: ctest --test-dir build/full --output-on-failure diff --git a/.gitmodules b/.gitmodules index bc42759..0e292cc 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "coroio"] path = coroio - url = ../coroio.git + url = https://github.com/resetius/coroio.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 3e108ad..a421d79 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,10 +5,15 @@ if (EXISTS ${CMAKE_BINARY_DIR}/local.cmake) include(${CMAKE_BINARY_DIR}/local.cmake) endif () -enable_testing() +include(CTest) +option(MINIRAFT_BUILD_NETWORK "Build coroio networking and examples" ON) -add_subdirectory(coroio/coroio) add_subdirectory(miniraft) -add_subdirectory(miniraft/net) -add_subdirectory(examples) -add_subdirectory(test) \ No newline at end of file +if(MINIRAFT_BUILD_NETWORK) + add_subdirectory(coroio/coroio) + add_subdirectory(miniraft/net) + add_subdirectory(examples) +endif() +if(BUILD_TESTING) + add_subdirectory(test) +endif() diff --git a/README.md b/README.md index 8daf485..9f7e6f3 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,120 @@ # MiniRaft-CPP +This fork's first milestone fixes strict-majority commit selection and FIFO +request completion, and adds deterministic consensus failure tests. +See the [correctness baseline and review](docs/correctness-baseline.md) for +validation, build instructions and remaining production release gates. +The milestone is experimental; durable crash recovery is not established. +The [persistence recovery follow-up](docs/persistence-recovery.md) rejects +incomplete or structurally corrupt storage and fixes truncation/restart behavior. +For new Linux stores, the opt-in [durable state backend](docs/durable-state.md) +adds checksummed atomic publication with file and directory synchronization. +The KV example selects it with `--durable`; legacy `--persist` is unchanged. +The [strong-read follow-up](docs/strong-reads.md) requires a fresh majority for +`--consistent` reads and rejects pending reads when their leadership term ends. +The [pending request follow-up](docs/pending-requests.md) prevents replacement +write acknowledgements and cleans up obsolete or disconnected client requests. +The [forwarding follow-up](docs/forwarding.md) preserves client cookies and binds +forwarded replies to their original leader, term and outbound peer. +The [network framing follow-up](docs/network-frames.md) handles fragmented input +and enforces type, frame, batch and AppendEntries payload bounds. +The [consensus validation follow-up](docs/consensus-validation.md) checks RPC +identities and indices, protects committed prefixes and correlates acknowledgements. +The [application payload follow-up](docs/application-payloads.md) validates KV, +SQL and dummy-RSM data before access and provides pre-admission validation hooks. +The [SQL read policy](docs/sql-readonly.md) prevents requests marked as reads +from executing unreplicated mutations or connection controls. +The [SQL application follow-up](docs/sql-application.md) protects managed write +transactions and commits application progress together with successful changes. +The [SQL read budget follow-up](docs/sql-read-budgets.md) interrupts expensive +reads and bounds serialized output before appending each field or separator. +The [application recovery follow-up](docs/application-recovery.md) preserves +recovered progress and prevents committed commands from being applied again. +The [legacy log-term follow-up](docs/legacy-log-terms.md) rejects impossible +stored term sequences before they influence elections or log matching. +The [snapshot foundation](docs/snapshots.md) adds opt-in durable KV checkpoints +and tail-only application recovery with `--durable --snapshots N`. Logs remain +fully retained in that baseline. The [compaction follow-up](docs/compaction.md) +enables prefix deletion and chunked InstallSnapshot catch-up; fixed membership +remains required. Upgrade all voters before enabling the new compaction behavior. +The [learner follow-up](docs/learners.md) adds non-voting catch-up and durable +voter binding. KV uses `--learner id` on every replica's configuration. The +[live joint-membership step](docs/configuration-entries.md) adds a durable-store +`TRaft` API for adding and removing nonleader voters through joint consensus; +leader removal, hot server role reconciliation and membership-aware snapshot +transfer remain unsupported. +The [snapshot retry follow-up](docs/snapshot-retries.md) prevents timeout retries +and strong-read traffic from starving delayed snapshot acknowledgements, and +prevents stale reader failures from destroying newer reconnect attempts. +The [snapshot pinning follow-up](docs/snapshot-pinning.md) lets an active transfer +finish while the leader continues compacting newer application state, and avoids +the dependency's unsafe connect-deadline path. TCP connect failure timing follows +the OS; a safe bounded application deadline remains open. +The [proposal admission follow-up](docs/proposal-admission.md) restricts direct +Append calls to leaders, preventing nonleaders from creating conflicting commands +at the same log index and term. +The [memory log ownership follow-up](docs/memory-log-ownership.md) prevents +caller-owned entry buffers from changing entries already stored in TState. +The [term lookup follow-up](docs/term-lookups.md) avoids copying full payloads +when Raft needs only a retained entry's term. +The [joint quorum foundation](docs/joint-quorum.md) adds and tests dual-majority +arithmetic used by the live membership transition path. +The [joint snapshot metadata foundation](docs/joint-snapshots.md) adds a versioned +representation for a joint configuration at a snapshot boundary; runtime adoption +and membership changes remain disabled. +The [snapshot term preflight follow-up](docs/unsupported-snapshot-term-preflight.md) +rejects unsupported snapshot formats before a higher RPC term changes hard state. +The [snapshot metadata preflight follow-up](docs/snapshot-metadata-term-preflight.md) +checks the boundary, fixed voters, and stable identity before that term update. +The [configuration entry support](docs/configuration-entries.md) describes the +live joint-to-stable transition API and its durable-storage and snapshot limits. +The [configuration history validator](docs/configuration-history.md) checks an +offline retained-log sequence against a trusted baseline or joint snapshot. +The [configuration log scanner](docs/configuration-scan.md) validates a retained +suffix at a caller-supplied commit boundary and at its end. +The [configuration state comparison follow-up](docs/configuration-scan-identity.md) +includes phase and voter sets when reporting an uncommitted configuration. +The [online learner staging API](docs/online-learner-staging.md) lets embedders +add and remove leader-local, non-voting replication targets. +The [network learner follow-up](docs/online-learner-network.md) connects that +API to the built-in server's runtime drain loop. +The [covered-snapshot follow-up](docs/covered-snapshot-identity.md) validates +snapshot boundary and voter metadata before acknowledging an already-covered image. +The [snapshot contact follow-up](docs/snapshot-contact-validation.md) waits for +a supported metadata prefix before a same-term image can refresh leader contact. +The [snapshot transfer expiry follow-up](docs/snapshot-transfer-expiry.md) +releases incomplete incoming images after 30 seconds without a valid chunk, +including on learners. +The [snapshot pin budget follow-up](docs/snapshot-pin-budget.md) bounds outgoing +transfer retention to two image generations and defers local compaction at the cap. +The [stable configuration snapshot foundation](docs/stable-configuration-snapshots.md) +preserves configuration identity. Raft can recover and install stable metadata +snapshots when their voter list matches the node's existing fixed membership; +joint snapshots and membership-aware snapshot transfer remain disabled. +The [durable configuration snapshot follow-up](docs/durable-configuration-snapshots.md) +documents its voter-binding requirement and recovery boundary. +The [trusted legacy voter binding follow-up](docs/trusted-legacy-voter-binding.md) +adds an explicit migration path when an operator has an independent record of the original set. +The [migration lifecycle follow-up](docs/trusted-legacy-voter-lifecycle.md) +enforces that migration happens before Raft attaches to an unbound legacy store. +The [legacy voter startup proof follow-up](docs/legacy-voter-proof-startup.md) +requires recovered durable stores to provide voter proof before Raft can start. +The [stable snapshot runtime follow-up](docs/stable-configuration-snapshot-runtime.md) +carries that identity through restore, InstallSnapshot, and later compaction. +The [metadata continuity follow-up](docs/stable-snapshot-metadata-continuity.md) +preserves the known identity across legacy-format refreshes and rejects conflicts. +The [storage continuity follow-up](docs/stable-snapshot-storage-continuity.md) +applies the same identity rule to direct memory and durable snapshot installs. +The [checkpoint restore follow-up](docs/stable-snapshot-checkpoint-continuity.md) +rejects standalone application checkpoints with a conflicting stable identity. +The [snapshot term validation fix](docs/snapshot-committed-terms.md) rejects +snapshot boundaries that contradict the follower's committed prefix. +The [message buffer follow-up](docs/message-extents.md) checks allocated buffer +and batch extents before accepting locally constructed frames. +The [SQL snapshot follow-up](docs/sql-snapshots.md) adds bounded SQLite snapshot +capture, validated restore and recovery from an application checkpoint left +behind a durably installed Raft snapshot. + [![Number of GitHub stars](https://img.shields.io/github/stars/resetius/miniraft-cpp)](https://github.com/resetius/miniraft-cpp/stargazers) ![GitHub commit activity](https://img.shields.io/github/commit-activity/m/resetius/miniraft-cpp) [![GitHub license which is BSD-2-Clause license](https://img.shields.io/github/license/resetius/miniraft-cpp)](https://github.com/resetius/miniraft-cpp) @@ -31,8 +146,8 @@ log replication, and fault tolerance in a distributed system. - **Leader Election** (supported) - **Log Replication** (supported) - **Persistence** (supported) -- **Snapshots** (not supported yet) -- **Membership Change** (not supported yet) +- **Snapshots** (opt-in KV compaction and InstallSnapshot; Linux durable backend) +- **Membership Change** (non-voting learners; voting changes not supported yet) The project focuses on a clear, modular design that leverages C++20 coroutines to simplify asynchronous flows. diff --git a/docs/application-payloads.md b/docs/application-payloads.md new file mode 100644 index 0000000..08743ff --- /dev/null +++ b/docs/application-payloads.md @@ -0,0 +1,49 @@ +# Application payload validation: v0.1.0-alpha.9 + +KV key/value lengths and SQL query lengths are checked against the actual +remaining bytes before preparing, reading or applying a command. Checks use the +payload's offset rather than sizeof(derived structure), preserving legacy native +layout/tail padding. Invalid direct application writes do not advance application +progress or mutate KV state. + +IRsm gains a default no-op const Validate hook. TRequestProcessor calls it before +queueing, preparing or forwarding requests. KV, SQL and the dummy RSM implement +their shape checks. Custom applications must supply their own validator: the +default preserves source compatibility but cannot infer custom payload formats. +Admission validation exceptions propagate to the inbound handler, closing that +client connection before the invalid operation is admitted. + +Read-time standard exceptions are returned as ErrorCode=2 with the original +cookie, allowing other requests and timer processing to continue. Exceptions +while applying a committed entry terminate the process: continuing after a +partial or invalid committed application is unsafe. This is an intentional +behavioral change for embedders whose Write callback throws. + +The dummy RSM requires a complete read index and copies only the log entry's +data into its reply, fixing a header-size error that overread by 16 bytes. +Message factories take size_t allocation sizes and reject values above UINT32_MAX +before narrowing/allocation. KV list and SQL reply construction enforce the +existing frame cap. Blank/malformed KV CLI commands no longer dereference missing +tokens or attempt to send an empty request. + +KV/SQL state machines are split from their CLI drivers so the real application +methods are directly testable without networking. SQL tests use SQLite's genuine +:memory: database; file-backed naming remains unchanged. SQLite development files +are installed in the core CI job. SQL tests are optional when SQLite is absent. + +Fifteen new tests cover malformed lengths, valid KV/SQL round trips, rejection +before admission/forwarding, no mutation/progress on rejection, oversized factory +arguments, dummy read bounds, recoverable read failures and fatal committed +application failures. All 145 tests pass locally on Linux with SQLite under +ASan/UBSan; core Release checks pass. Hosted CI executes the same targets. +The fatal-application subprocess test is excluded on Windows. + +Self-review checked validation before data access and admission, widening before +length addition, native-layout offsets, response allocation bounds, CLI tokens +and exception routing. No independent reviewer approval is claimed. + +This is payload memory-safety validation, not SQL authorization or complete state +machine correctness. SQL read-only enforcement, nondeterministic SQL, transaction +and metadata semantics, result-set memory quotas, authentication, deadlines, +admission limits and retry deduplication remain open. Callback access remains +serialized and non-reentrant. Write outcomes after interruption can be unknown. diff --git a/docs/application-recovery.md b/docs/application-recovery.md new file mode 100644 index 0000000..3b87004 --- /dev/null +++ b/docs/application-recovery.md @@ -0,0 +1,67 @@ +# Recovered application progress: v0.1.0-alpha.13 + +The [compaction follow-up](compaction.md) extends this recovery contract to +embedded snapshots and absolute log boundaries. + +After restarting a durable application, Raft's volatile commit index began at +zero. ProcessCommitted unconditionally copied that zero into LastAppliedIndex, +discarding the application's recovered marker. Re-establishing commitment then +reapplied old commands. A file-backed SQL/TDiskState regression reproduced an +INSERT being applied twice after a clean restart. A separate restart scenario +also exposed the loss of committed-prefix protection before leader contact. + +TRequestProcessor construction now initializes a commit floor from the recovered +application marker. The marker must not exceed the recovered local log or the +signed log-index range; invalid pairings throw before changing the commit index. +The floor preserves any higher commitment already known to Raft. The existing +AppendEntries validation then protects this applied prefix immediately, while +allowing replacement of an uncommitted suffix. The restoration helper is private +to Raft and the request processor. + +ProcessCommitted returns when commitment is at or below application progress. +It never rewinds the marker or computes LastAppliedIndex+1 in that case. New +commitment applies only the remaining entries. Framework progress still advances +over internal no-op entries; an application need only persist progress alongside +its own mutations, so harmless no-ops can be revisited after restart. + +Recovery contract for embedders: + +* Restore application state and LastAppliedIndex atomically before constructing + TRequestProcessor, and construct the processor before accepting network traffic. +* A nonzero marker must represent genuinely committed progress in the same local + Raft log. Do not derive it from the log's last index: a suffix may be uncommitted. +* A volatile application starts at zero and rebuilds after commitment is known. +* Keep application and Raft storage paired. An application ahead of the local log + now fails startup instead of silently forgetting or skipping progress. + +An applied marker is trusted evidence of commitment; a range check cannot detect +a forged marker, mismatched data at the same index or application state persisted +without its marker. Snapshot/compaction recovery is not supported. Strong reads +still require a current-term commit and a fresh majority; the recovered floor +does not replace those checks. This follows the monotonic application-progress +and state-machine safety rules in [Raft's Figure 2 and Figure 3](https://raft.github.io/raft.pdf). + +Six new tests failed against the prior implementation and now pass: + +1. Restarted application progress survives an early timer and is not replayed. +2. Lower commitment input cannot rewind application progress. +3. A recovered applied prefix resists conflicting RPCs while its suffix can change. +4. Application progress ahead of the recovered log is rejected without mutation. +5. A three-node restart preserves durable application histories, rebuilds a + volatile application, replaces an uncommitted old-leader suffix and tolerates + delayed, reversed and duplicate traffic after a new election. It also checks + that recovered commitment cannot authorize new-term reads by itself. +6. Real SQL and legacy Raft files reopen without repeating a committed INSERT. + +All 175 tests across 11 CTest targets pass locally under ASan/UBSan with SQLite; +the Release core suite also passes. Self-review covered initialization ordering, +range checks before mutation, monotonicity, no-op handling, conflict protection, +current-term read gating and repeat application calls. No independent reviewer +approval is claimed. + +The simulator retains stable term/vote/log objects and reconstructs Raft volatile +state; it is a deterministic process-restart model, not torn-write simulation. +The SQL integration test performs clean file close/reopen. These additions do not +establish power-loss durability for TDiskState or SQLite, authenticated pairing +of stores, exhaustive fault schedules, snapshots or retry deduplication. Those +and independent review remain production gates. diff --git a/docs/compaction.md b/docs/compaction.md new file mode 100644 index 0000000..f2a73ad --- /dev/null +++ b/docs/compaction.md @@ -0,0 +1,160 @@ +# Compaction and InstallSnapshot: v0.1.0-alpha.16 + +The [learner follow-up](learners.md) adds non-voting replication and durable +voter binding. Current MRFTD004 images also persist commit progress; the reader +continues to accept the earlier v1/v2/v3 images below. + +This milestone adds prefix deletion and follower state transfer to the alpha.15 +KV checkpoint foundation. TState and Linux TDurableState support compacted logs; +legacy TDiskState does not. KV implements snapshot validation/restoration. SQL +support is described in [the SQL snapshot follow-up](sql-snapshots.md); the dummy +state machine remains unsupported for compaction and installation. + +## Operation and compatibility + +The existing Linux KV option `--durable --snapshots N` now compacts after N more +applied indices. Raft::Compact captures committed applied state, validates its +application image and atomically stores the snapshot plus retained suffix. The +separate `state.snapshot.` remains a checkpoint export for compatibility; the +embedded snapshot in `state.durable.` is authoritative after compaction. + +Stop and upgrade every voter before enabling this release in an existing cluster. +Old binaries cannot read compacted durable images or process InstallSnapshot RPCs. +The meaning of `--snapshots` changes from checkpoint-only to actual compaction. +Mixed-version operation is unsupported. Existing v1 durable files and alpha.15 +checkpoints can be read without destructive migration. The first compaction or +remote installation writes the current durable format; downgrading that store is unsupported. Fixed voter +IDs must match the embedded snapshot. Static `--node` edits remain unsafe as a +membership-change mechanism. + +Startup restores an application behind the embedded snapshot before request +processing, including after a crash between durable publication and application +replacement. A newer standalone +checkpoint can restore progress within the retained suffix; an older one never +replaces embedded state. With periodic snapshots enabled, an existing corrupt +standalone file is still rejected conservatively. Snapshot recovery never resets +the persisted current term or vote. + +## Boundaries and atomic persistence + +Raft startup validates a recovered snapshot before binding voters: its image, +index and term must agree with the storage boundary, its voter set must match +the fixed configuration, and the last term must be consistent with the +boundary and current term. It scans retained terms for nonzero, nondecreasing +values no newer than the current term and verifies the recorded last term. A +nonzero boundary without an image is rejected. Custom backends must report +accurate entry metadata, validate payloads, and supply a consistent, serialized +view; these checks are not a complete storage audit or proof of snapshot contents. +The scan uses `RetainedLogTerm`; backends relying on its default implementation +may read full entry payloads at startup. + +SnapshotIndex/SnapshotTerm identify the last included entry. Get only returns +entries after SnapshotIndex, while LogTerm at the boundary returns SnapshotTerm. +LastLogIndex and all RPC/application indices remain absolute. Appends, truncation, +votes, current-term commitment and strong reads use these absolute boundaries. +Truncation cannot remove a snapshot. Election no-op creation also checks signed +index exhaustion, now reachable without allocating an enormous prefix. Memory +and durable storage reject a direct append once the last index is INT64_MAX, +before mutating the log or publishing an image. Appending the entry at INT64_MAX +itself remains valid when the previous index is INT64_MAX - 1. + +An installation retains the suffix only when the local entry at the included +index has the same term. Otherwise its uncommitted log is discarded. A follower +does not install an image at or below known commitment, so stale snapshots cannot +rewind application state. Retained suffix entries are applied only after a leader +establishes their commitment. + +MRFTD004 publishes hard state, the persisted commit boundary, voter binding, +complete encoded snapshot and suffix in one atomic image, using the existing +temp-write/file-fsync/rename/directory-fsync protocol. Its header remains +magic[8], payload length[8], CRC[4]. The payload is CurrentTerm[8], VotedFor[4], +retained count[8], CommitIndex[8], voter count[4], voter IDs, snapshot length[8], +the encoded snapshot, then term/flags/length/data records. The CRC covers the v4 +magic plus payload. Recovery checks `SnapshotIndex <= CommitIndex <= LastLogIndex`, +bounds the embedded image and retained count, and validates nondecreasing terms +beginning at SnapshotTerm. Older v1/v2/v3 images remain readable and recover +their commit floor as SnapshotIndex; a successful later publication writes v4. +Malformed recovery does not rewrite the image. Live publication failure is fatal. + +Followers publish a newly learned commit index before acknowledging +AppendEntries. Leaders publish quorum commit advancement before application +processing. Snapshot installation commits at least the included boundary, and +storage refuses to truncate a committed entry. The legacy TDiskState backend +does not persist CommitIndex. + +## Transfer protocol + +The previously reserved message types 6/7 now carry InstallSnapshot requests and +responses. A request identifies leader/term, boundary index/term, total image size, +offset and final-chunk flag. Responses carry the boundary, next offset, success +and an explicit Installed flag. Seqno correlates acknowledgements to the active +per-peer RPC. Frames are capped at a 64 KiB body before allocation; complete +images remain bounded at 64 MiB with at most 4,096 fixed voters. + +Leaders use stop-and-wait chunks when NextIndex is at or below their compacted +boundary. Offset and RPC context survive retries within the leadership term; +delayed/duplicate responses cannot confirm another image. Followers hold one +bounded partial image, validate duplicate bytes, reject gaps and bind assembly +to leader, term, index, boundary term and total length. A restarted follower asks +for offset zero; partial state is not persisted. Transfers restart on newer terms +or newer images, and valid chunks refresh the election timer. + +Partial chunks never change application progress or MatchIndex. Completion first +checks CRC, metadata, fixed voters and the application payload without changing +live application state. The follower durably installs snapshot plus suffix, then +replaces application state, then acknowledges Installed. Failure after durable +installation terminates the process; startup restores the selected image. A lost +completion ACK can be retried without applying the image again. + +Only Installed completion advances leader MatchIndex. Neither partial nor final +snapshot acknowledgements count as fresh-read confirmations: subsequent +AppendEntries must establish read barriers and replicate any suffix. A compacted +follower can return its boundary in a failed AppendEntries MatchIndex field as a +probe hint. That hint advances only NextIndex; a matching AppendEntries probe is +required before counting the index as replicated or acknowledging reads. + +This implements the boundary/suffix rules of [Raft section 7 and Figure 13](https://raft.github.io/raft.pdf). +Transfer is deliberately serialized and bounded; it is not disk-streaming or a +parallel multi-image protocol. + +## Validation and review + +Fourteen new deterministic core/storage tests cover lagging follower catch-up and +tail replication; restart mid-transfer; delayed, duplicate and impossible ACKs; +leadership changes; strong-read gating; compacted-prefix probing; corrupt images +and invalid headers; voter and application mismatch; retained but uncommitted +suffixes; memory/durable boundary behavior; index exhaustion; v2 semantic +corruption; disk-space reclamation; and first/replacement image selection at all +five publication boundaries for retained/discarded suffixes. + +One new transport test checks fragmented request/response round trips and rejects +empty/oversized chunks before body allocation. The existing CLI smoke now checks +v2 publication and restoration with periodic snapshotting disabled. A new real +three-process test compacts a two-voter majority, restarts both to clear queued +old log traffic, starts an empty third voter, verifies local reads after snapshot +installation, replicates a tail, then restarts that voter and checks recovery. +It uses bounded waits, separately from deterministic virtual-time fault schedules. + +That integration test exposed a pre-existing election tie: all nodes used the +same RNG seed and campaigned in lockstep. Node-specific nonzero deterministic +seeds now provide distinct election schedules; a simultaneous-startup regression +verifies election progress after the initial split vote. This is scheduling +diversification, not a security/entropy source. + +Local validation: 207 unit tests and two CLI tests pass under ASan/UBSan across +15 CTest targets; Release core checks pass. Self-review covered boundary arithmetic, +suffix selection, persistence/application/ACK ordering, parser allocation limits, +transfer context and retries, stale snapshots, fresh-read separation, recovery +ordering and backward v1 decoding. No independent reviewer approval is claimed. +Publication-stage process crashes do not replace hardware power-cut testing. + +## Remaining work + +Membership remains fixed. Learners, catch-up gates, replicated joint-consensus +configuration entries, leader removal and corresponding quorum/read changes are +next. All participating applications/backends must support snapshots before +compaction is enabled; no capability negotiation is implemented. Asynchronous/COW +capture, disk-streamed transfer, bounded transport queues, total heap quotas, +cluster/store authentication and independent review remain open. +The conservative backend still rewrites the entire snapshot and retained suffix +on each update, and capture/publication runs synchronously on the event loop. diff --git a/docs/configuration-entries.md b/docs/configuration-entries.md new file mode 100644 index 0000000..40ed685 --- /dev/null +++ b/docs/configuration-entries.md @@ -0,0 +1,63 @@ +# Live joint membership + +The runtime accepts membership changes through a joint-consensus log sequence. +`TRaft::ProposeJointConfiguration(newVoters)` appends the joint entry, which +activates both the old and new voter majorities. After the joint entry commits, +`TRaft::FinalizeConfiguration()` appends its stable successor. The stable set +becomes active only after that successor commits. Elections, commit advancement +and strong-read confirmation use the active stable or joint quorum. + +Both APIs require storage that persists the commit index and original voter +binding. `TDurableState` provides those guarantees. In-memory `TState` and legacy +stores cannot start a live transition. The leader must have a committed entry in +its current term. Every newly added voter must first be staged as a learner and +have a correlated replication acknowledgement through the leader's committed +boundary. The leader must be a member of the new set. The currently supported +API cannot remove its active leader; this library does not yet provide a +leadership-transfer API. Every node must have replication endpoints for all +voters before it accepts the transition. Endpoint discovery and configuration +distribution remain responsibilities of the embedding application. +On restart, construct each Raft object with the persisted original voter set and +configure later members as learners in that bootstrap view; the retained log +then recovers their active roles. Supplying the latest voter set as if it were +the original binding is rejected. + +These are `TRaft` APIs. `TRaftServer` currently starts outbound client-response +readers from the roles present when `Serve()` starts and does not reconcile that +reader set after a live promotion. Raft replication and quorum processing are +updated live, but a promoted follower that forwards client requests may need a +server restart before its forwarded responses are handled. Hot server role +reconciliation remains a follow-up. + +Incoming AppendEntries configuration records are decoded and the complete +candidate suffix is scanned before a higher RPC term is persisted. The scan +checks predecessor identity, transition order, committed-prefix constraints, +voter binding and endpoint availability. Same-term configuration entries must +match byte for byte. Replacing an uncommitted configuration suffix recomputes +the active roles and quorum from the surviving log; committed entries cannot be +truncated. A malformed or semantically invalid candidate suffix is rejected +without changing the term, commit index, log or response state. + +The record payload remains MRFTC001: magic[8], phase[4] (joint=1, stable=2), +entry term[8], previous configuration index[8], previous configuration +term[8], old count[4], new count[4], old voter IDs[4 each], new voter IDs[4 +each], and CRC32[4] over the preceding payload. The outer entry term must match +the encoded term and its flags must equal `EConfiguration`. CRC checks accidental +corruption; it is not authentication. All voters must run a compatible binary +before a transition because older versions reject this entry type. + +Membership-aware compaction and joint snapshot installation are still open. +Compaction remains blocked after retained membership history, and a lagging +peer that requires snapshot transfer across a transition is not yet supported. +Existing fixed-membership snapshots and legacy stores do not prove the active +voter set after a transition. Do not remove or edit committed state to bypass +these limits. + +Deterministic durable-store tests cover learner catch-up, voter addition and +nonleader removal, per-node activation, joint commit propagation, stable +finalization, rejection of a bad predecessor before term mutation, and rollback +of an uncommitted joint suffix. The test suite also checks recovery of retained +joint and stable histories, including post-restart role recovery after adding +and removing a voter. This is a first live transition slice; it does not +claim snapshot continuity across membership changes or explicit leadership +transfer. diff --git a/docs/configuration-history.md b/docs/configuration-history.md new file mode 100644 index 0000000..4e1dd90 --- /dev/null +++ b/docs/configuration-history.md @@ -0,0 +1,69 @@ +# Offline configuration history validation: v0.1.0-alpha.27 + +`TConfigurationHistory::MakeSnapshot` projects the history at its through-index +and term into an offline snapshot. It selects MRFTS002 for a joint configuration, +MRFTS003 for a stable configuration with a known configuration entry, and the +legacy MRFTS001 format when the trusted baseline asserts no earlier configuration +entry. That legacy format does not encode or prove the zero-identity assertion. +The caller must independently establish that the history boundary is committed +and that the supplied application image corresponds exactly to that boundary. +The method validates snapshot metadata and size; it does not capture application +state, compact or mutate storage, establish commitment, or activate voters. +Joint configuration snapshots remain rejected by the live runtime. Stable +metadata snapshots are runtime-compatible only when they match the node's +existing fixed voter set. Bound storage can retain stable or joint metadata +snapshots for offline validation while preserving the original voter binding, +even when the active set in the snapshot differs. This does not enable voter +changes. + +TConfigurationHistory checks a sequence of retained log entries against a trusted +configuration baseline. It joins the experimental configuration-entry codec with +the joint-snapshot metadata and checks that joint and stable records have the +expected predecessor index/term and voter sets. A stable configuration can advance +to a joint one; that joint configuration can advance to a stable record naming the +same old and new sets. A successful stable transition selects the new voters. + +The caller visits every retained entry in order: Observe handles ordinary entries, +and Advance handles configuration entries. Both return a new history value and +leave the original unchanged on success or failure. They reject index gaps, +nonpositive or decreasing terms, malformed records, wrong predecessors, wrong voter +sets and invalid phase transitions. A caller can retain a previous value and scan +a replacement branch from that point. This models validation of rollback; it does +not truncate a live log or decide when a configuration becomes active. + +FromJointSnapshot validates the snapshot metadata and starts at its included +index and term, using its joint configuration identity. Stable accepts an explicit +configuration identity and scanned-prefix boundary. The caller must establish that +baseline independently: a v1 fixed snapshot contains voter IDs but does not prove +the latest configuration entry's identity. Passing a zero identity asserts that no +earlier configuration entry exists. Decode and authenticate any persisted input as +appropriate before supplying it. The validator does not prove commitment, cluster +identity, promotion readiness, log completeness outside the supplied sequence or +the correctness of the caller's trusted baseline. + +The [alpha.53 follow-up](configuration-scan-identity.md) ensures that the scan's +uncommitted-configuration result compares phase and voter sets as well as +configuration index and term. + +Joint snapshot metadata now rejects a configuration term different from the +snapshot term when both indices are equal: one log index cannot have two terms. +Earlier alpha.25 tooling could encode such structurally inconsistent v2 images; +they are now rejected and are not rewritten automatically. The live runtime never +accepted joint snapshots. Fixed v1 snapshot encoding and existing durable formats +are unchanged. + +Tests cover joint/stable chains, alternate branches, joint snapshot boundaries, +invalid baseline identities, skipped entries, stale terms, wrong sets and +predecessors, malformed entries, index exhaustion and CRC-valid snapshot metadata +corruption. A durable test scans configuration entries after a fixed snapshot and +restart, confirms the fixture's known bootstrap baseline, and verifies that Raft +still rejects the unsupported entries without rewriting storage. Full tests also +cover existing snapshot installation and learner catch-up. + +This is configuration history validation for fixed membership, not live +reconfiguration. Joint entry activation and rollback, snapshot installation +across membership transitions, promotion gates and transport discovery remain +open. Self-review +checked contiguous scanning, term and predecessor relationships, immutable branch +results, snapshot boundary consistency and the unknown-v1-identity case. No +independent reviewer approval is claimed. diff --git a/docs/configuration-scan-identity.md b/docs/configuration-scan-identity.md new file mode 100644 index 0000000..792f37f --- /dev/null +++ b/docs/configuration-scan-identity.md @@ -0,0 +1,17 @@ +# Compare full configuration state in history scans: v0.1.0-alpha.53 + +`TConfigurationScan::HasUncommittedConfiguration()` previously compared only +the configuration index and term. Two independently supplied histories with the +same identity but different voter sets or joint/stable phase could therefore be +reported as having no configuration difference. + +The result now compares the full effective configuration: phase, old and new +voter sets, configuration index, and configuration term. Different scan +boundaries alone do not count as a configuration change. Callers still supply +and substantiate the baseline and commit boundary; this helper does not prove +commitment or activate membership in Raft. + +Tests cover conflicting stable voter sets and a stable-versus-joint mismatch +with equal configuration index/term, plus the unchanged case. + +Self-review completed; no independent reviewer approval is claimed. diff --git a/docs/configuration-scan.md b/docs/configuration-scan.md new file mode 100644 index 0000000..477b8eb --- /dev/null +++ b/docs/configuration-scan.md @@ -0,0 +1,39 @@ +# Offline configuration log scan + +`ScanConfigurationHistory` validates a retained suffix starting at a trusted +configuration history at the snapshot boundary. It returns the history at the +caller-supplied commit index and at the end of the suffix. The two results make +it possible to distinguish the configuration in that committed prefix from a +later configuration record, including after a suffix is replaced. A malformed +entry anywhere in the suffix fails the entire scan; the scanner does not decide +whether a damaged tail may be truncated. + +The caller must independently establish the baseline and commit index: the scan +does not prove that any entry is committed, recover a commit index, authenticate +the snapshot, or establish cluster identity. A fixed v1 snapshot does not identify +its latest configuration entry, so its configuration identity must come from +other trusted information. Experimental stable and joint snapshots contain that +identity and the scan checks that the supplied active configuration matches it. +The persisted voter binding identifies the original trusted set; configuration +metadata can describe an active set reached by later transitions. Legacy images +without configuration metadata still require the original and snapshot voter +sets to agree. The scan checks snapshot and log boundaries, voter history, terms, +configuration predecessors, and transitions. +It requires a consistent, externally serialized view of the state; it does not +mutate storage, apply entries, activate voters, or change Raft roles. +It also rejects a configuration entry that follows an earlier configuration +entry outside the supplied commit boundary. In particular, a stable successor +cannot make an uncommitted joint configuration disappear from the latest view. + +For ordinary entries, the scan trusts term and flag metadata from the storage +backend, which must validate its own records. Memory and durable backends avoid +copying ordinary payloads; external backends using default metadata accessors may +read full entries. Configuration records are read and validated in full. + +The standalone scanner remains read-only. The Raft runtime uses it to validate +a simulated AppendEntries suffix before accepting configuration entries and to +recompute roles after commit or suffix replacement. Live changes additionally +require durable commit-index persistence, the original voter binding and local +endpoints for every active voter. Joint snapshot installation and snapshot +transfer across a transition remain unsupported. Snapshot metadata alone does +not authenticate a transition. diff --git a/docs/configuration-snapshot-continuity.md b/docs/configuration-snapshot-continuity.md new file mode 100644 index 0000000..386178b --- /dev/null +++ b/docs/configuration-snapshot-continuity.md @@ -0,0 +1,34 @@ +# Configuration snapshot continuity: v0.1.0-alpha.59 + +Alpha.58 separated a store's original voter binding from the active voter set +recorded in configuration-bearing snapshots. That exposed a continuity check +that previously relied on the original binding to catch conflicting snapshot +memberships. + +When storage already has configuration metadata, a replacement snapshot must +extend that known state: + +- Stable to stable snapshots keep both the configuration entry index/term and + the active voter set unchanged. +- Stable to joint snapshots use the current active set as their old voters and + place the new configuration entry after the current snapshot boundary without + regressing its log term. +- Joint snapshots may advance their included log boundary only while keeping the + same joint identity and both voter sets. Replacing joint metadata with a + different joint record requires a stable successor first. +- Joint to stable snapshots select the joint snapshot's new voter set and use a + stable entry after the joint snapshot boundary without regressing its log + term. +- A legacy refresh of a stable snapshot must retain its active voter set; storage + carries the stable identity forward. A legacy image cannot erase joint state. + +These checks run in both storage backends and application snapshot recovery. They +check continuity with locally stored metadata, but they do not prove that a +configuration entry was committed or authenticate the sender. Callers that use +the offline install APIs must still provide a trusted snapshot and history +baseline. Live configuration entries, joint quorum activation and +membership-aware network installation remain disabled. + +Deterministic memory and durable tests cover conflicting same-identity voter +sets, stale legacy membership, stable-to-joint and joint-to-stable transitions, +rejected phase downgrades and durable recovery of the original binding. diff --git a/docs/consensus-validation.md b/docs/consensus-validation.md new file mode 100644 index 0000000..7a77536 --- /dev/null +++ b/docs/consensus-validation.md @@ -0,0 +1,41 @@ +# Consensus RPC validation: v0.1.0-alpha.8 + +Validate complete RPCs before changing terms, votes or logs. Unknown senders, +wrong destinations, inconsistent leader/candidate IDs, invalid booleans, +impossible log metadata and malformed payloads are ignored. IDs must be positive; +peer configuration cannot include self, zero or null endpoints. + +Log indices are limited to INT64_MAX to match IState's signed indexing API. +AppendEntries checks index addition and nonzero, nondecreasing entry terms no +greater than the RPC term. The whole batch is checked before mutation, including +conflicts with committed entries. Follower commit advancement is clamped to the +prefix matched by that RPC, while retaining already-established commit progress. +SetCommitIndex preserves uint64_t width. Election/log-index exhaustion raises an +overflow error before wrapping or changing the vote/log. + +Each peer retains at most 256 outstanding AppendEntries sequence/end-index +records. Success must identify a retained RPC and exactly its sent end index. +Processing a response discards that record and older records, preventing duplicate +or reordered failures from repeating backoff. NextIndex never falls below known +MatchIndex+1. Evicted replies are ignored; newer retained replies/retries can +still make progress. Very delayed peers under sustained request load can lose +old correlation records; admission control remains future work. + +Eighteen deterministic tests cover identity, index/term bounds, false/unsent +acknowledgements, reordering, duplicates, follower commit bounds, committed-prefix +protection, batch atomic rejection, configuration, bounded response history and +counter exhaustion. All initial fourteen fail against alpha.7. Existing fixtures +with log terms greater than their RPC term, or mismatched candidate/sender IDs, +were corrected while preserving their intended reconciliation/election cases. + +All 130 tests pass locally under ASan/UBSan, with core Debug/Release verification. +Hosted CI runs the same targets. Self-review checked validation ordering, bounded +correlation ownership, prefix safety, full-width arithmetic and progress after +rejection/eviction. No independent approval is claimed. + +These are consistency checks, not peer authentication or Byzantine tolerance: +an untrusted connection can still claim a configured sender ID. Authentication, +application payload validation, deadlines, admission limits, retry deduplication +and broader generated histories remain open. Custom adapters must preserve RPC +sequence numbers and destination IDs. The wire format is unchanged; stricter +validation intentionally rejects formerly tolerated invalid messages. diff --git a/docs/correctness-baseline.md b/docs/correctness-baseline.md new file mode 100644 index 0000000..1c968ba --- /dev/null +++ b/docs/correctness-baseline.md @@ -0,0 +1,83 @@ +# Correctness baseline: v0.1.0-alpha.1 + +This fork starts from upstream a2848b843b3910664216a00605a406b5516e0f8c. +This is an experimental correctness milestone, not a production-readiness claim. + +## Fixes + +* Select the strict-majority replication index for both odd and even cluster + sizes. For four nodes, two replicas must never commit an entry. +* Process request queues from the front, matching std::queue::pop(). This + preserves FIFO replies for reads, strong reads and writes, and allows an + earlier committed write to complete while a later write remains pending. +* Resolve the pinned coroio submodule against its upstream URL so forks clone + successfully. The pinned dependency commit is unchanged. +* Support MINIRAFT_BUILD_NETWORK=OFF and BUILD_TESTING=OFF independently. + +## Deterministic failure harness + +test/test_correctness.cpp uses virtual time and an explicit message queue. +It uses no sockets, wall clock, sleeps or nondeterministic scheduling. +The eight tests cover: + +1. Every replica count for cluster sizes one through eight. +2. FIFO normal-read replies. +3. FIFO strong-read replies. +4. FIFO write replies. +5. Partial write completion without blocking earlier requests. +6. Dropped replication traffic followed by retry, reversal and duplication. +7. Leader isolation, majority election and commit, delayed old-term messages, + conflicting minority-log replacement and partition healing. +8. A four-node partition where two replicas cannot commit, followed by healing. + +Cluster scenarios check commit progress and equality of committed log prefixes. +These are selected deterministic schedules, not exhaustive state-space exploration. +The original baseline did not cover restarts or storage crash injection. +The [application recovery follow-up](application-recovery.md) adds deterministic +node restarts and committed application-history checks; torn-write simulation +and broader generated schedules remain open. + +## Reproduce + +Install a C++20 compiler, CMake, pkg-config and cmocka development files. +For the full build, initialize submodules and install coroio dependencies +(on Ubuntu: liburing-dev, libssl-dev and libsqlite3-dev). + +```sh +cmake -S . -B build/core -DMINIRAFT_BUILD_NETWORK=OFF -DCMAKE_BUILD_TYPE=Debug +cmake --build build/core --parallel 4 +ctest --test-dir build/core --output-on-failure + +git submodule update --init --recursive +cmake -S . -B build/full -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_CXX_FLAGS="-fsanitize=address,undefined -fno-sanitize-recover=all" \ + -DCMAKE_EXE_LINKER_FLAGS="-fsanitize=address,undefined" +cmake --build build/full --parallel 4 +ctest --test-dir build/full --output-on-failure +``` + +## Review and validation + +The implementing agent performed a source self-review of the final diff, +majority arithmetic, queue ownership and completion ordering, build options, +dependency pin and deterministic test schedules. No independent human or +second-agent review has been obtained. The release is explicitly an alpha. + +Local validation: the 27 upstream core tests, eight new correctness tests and +three upstream transport tests pass. The full suite also passes with ASan and +UBSan. The new tests fail against the upstream implementation: majority and +read-order assertions fail, and queued write handling aborts. Local validation +is separate from hosted GitHub Actions results. + +## Remaining release gates before production + +* Replace stream-only persistence with a defined durable write protocol, ordered + synchronization, recovery validation and crash-injection testing. Current + TDiskState is not validated for power-loss durability. +* Audit strong-read quorum freshness, pending operations across leadership + changes, client retry/deduplication semantics and malformed message handling. + FIFO tests do not establish linearizability. +* Add deterministic node restart and broader generated fault schedules, checking + election safety and committed application histories. +* Implement and validate snapshots, compaction and safe membership changes. +* Establish real-network load/failure testing, backpressure and operational APIs. diff --git a/docs/covered-snapshot-identity.md b/docs/covered-snapshot-identity.md new file mode 100644 index 0000000..faa9df9 --- /dev/null +++ b/docs/covered-snapshot-identity.md @@ -0,0 +1,24 @@ +# Covered snapshot identity checks: v0.1.0-alpha.41 + +When a follower already has a committed prefix covering an InstallSnapshot +boundary, it does not need to install the image. Previously the fast path +recognized only the `MRFTS001` magic, then acknowledged the full snapshot as +installed. It skipped checking the snapshot's embedded index, term and voter +IDs against the RPC and the receiver's fixed voter set. + +The fast path waits for and parses the bounded metadata prefix for fixed v1 and +stable v3 snapshots. It acknowledges coverage only when the embedded boundary, +stable configuration identity (when present), and sorted voter list match. It +also checks the encoded application length against the declared total size. It +does not read the application body or verify its CRC because it never uses that +image; the receiver's already committed prefix is the evidence for coverage. +Actual installations validate the complete image, CRC, metadata, voter set and +application payload before replacing state. + +Deterministic regression cases cover mismatched embedded voters, a snapshot +boundary that disagrees with the request, invalid stable metadata, and valid +metadata prefixes split across chunks. Rejected images do not change durable Raft +or application state. Joint snapshots remain unsupported; live voter changes use +the configuration log protocol described in +[live membership](configuration-entries.md). +Self-review completed; no independent reviewer approval is claimed. diff --git a/docs/durable-configuration-snapshots.md b/docs/durable-configuration-snapshots.md new file mode 100644 index 0000000..4fda75c --- /dev/null +++ b/docs/durable-configuration-snapshots.md @@ -0,0 +1,53 @@ +# Durable configuration snapshot metadata: v0.1.0-alpha.45 + +Bound `TState` and `TDurableState` instances can install and recover MRFTS002 +joint and MRFTS003 stable snapshots so offline tools can validate configuration +history after restart. A store must already have a voter binding. For a +configuration-bearing snapshot, `snapshot.Voters` describes the active set at +the snapshot boundary and may differ from that original binding. Durable +recovery preserves the original binding and exposes the active snapshot set +separately. A legacy snapshot without configuration metadata must still match +the binding. An unbound store cannot infer its original voter set from +configuration metadata. The [alpha.50 follow-up](trusted-legacy-voter-binding.md) +adds an explicit durable migration API that accepts a caller-supplied trusted +original set, validates it against the recovered vote, snapshot, and retained +log, then persists the binding before Raft startup. The API does not establish +that the caller's assertion is true; absent an independent trusted baseline, the +store still cannot infer its voter set. + +The current MRFTD004 durable format persists the snapshot, commit boundary and +original voter binding in the same atomic image; it does not reinterpret that +binding from snapshot metadata. Older durable formats remain readable and use +their snapshot boundary as the conservative commit floor. The history scanner +checks configuration-bearing snapshot metadata against the active baseline; +the original binding may refer to an earlier stable set. + +`ScanCommittedConfigurationHistory` uses the recovered commit index directly +and requires both durable commit-index support and a persisted original voter +binding. For configuration-bearing snapshots, the caller supplies the active +history baseline through the snapshot boundary; the scanner verifies it against +the snapshot metadata. Legacy snapshots still require the baseline voters to +match the original binding. This scanner is an offline validation API; snapshot +metadata is not proof of commitment or cluster identity. The separate live log +transition API uses the durable commit boundary and original binding, while +joint snapshot installation remains unsupported. + +In alpha.45 these images were persistence for offline validation only. The +alpha.46 follow-up enables runtime recovery and transfer of MRFTS003 stable +snapshots when the voter list equals the node's existing fixed membership. +Compaction carries forward that stable configuration identity. MRFTS002 joint +snapshots remain disabled in the live runtime; configuration log entries are +now accepted through the guarded joint-consensus path. +Since alpha.58, bound storage can preserve its original voter binding while +installing configuration-bearing snapshots whose active set changed. Raft +startup and network snapshot handling still reject those transition snapshots; +the storage support is only for offline history validation and trusted inputs. +The [alpha.59 follow-up](configuration-snapshot-continuity.md) checks that a +replacement configuration snapshot extends the previously stored phase, +identity, and active voter sets. + +Tests cover in-memory installation, durable joint and stable round-trips, +rejection of unbound configuration snapshots and mismatched legacy snapshots, +offline scan baselines across multiple committed transitions, stable runtime +restore/transfer/compaction, and continued joint snapshot rejection. +No independent reviewer approval is claimed. diff --git a/docs/durable-state.md b/docs/durable-state.md new file mode 100644 index 0000000..7a01151 --- /dev/null +++ b/docs/durable-state.md @@ -0,0 +1,124 @@ +# Linux durable state: v0.1.0-alpha.3 + +## Commit recovery: MRFTD004 + +The current durable image stores `CommitIndex` alongside the current term, vote, +voter binding, snapshot and retained log. Recovery requires +`SnapshotIndex <= CommitIndex <= LastLogIndex`. Raft publishes a commit advance +before it acknowledges the corresponding AppendEntries request or exposes the +new boundary to application processing. Restart therefore replays committed +entries even when the application checkpoint trails them. Installed snapshots +advance the persisted commit floor to at least their included index, and storage +rejects truncation of committed entries. + +MRFTD004 keeps the 20-byte file header: magic[8], payload length[8] and CRC[4]. +Its payload is CurrentTerm[8], VotedFor[4], retained-log count[8], +CommitIndex[8], voter count[4], sorted voter IDs[4 each], snapshot length[8], +optional encoded snapshot, then retained log records. The CRC covers the v4 magic +and payload. MRFTD001, MRFTD002 and MRFTD003 remain readable; their commit floor +is conservatively initialized to SnapshotIndex, since those formats did not +record commit progress. Their next successful publication writes MRFTD004. + +This closes commit-boundary recovery for `TDurableState`; it does not implement +live membership changes or joint consensus. The legacy `TDiskState` format does +not persist this field and cannot provide the same recovery guarantee. + +TDurableState is an opt-in storage backend with synchronous durable publication. +It closes the missing file-sync and atomic-publication gap for new stores, under +the filesystem assumptions below. The legacy TDiskState backend is unchanged +and still does not provide power-loss durability. + +## Publication protocol + +Each Append, RemoveLast and Commit publishes one complete image containing the +current term, vote and selected log prefix: + +1. Serialize the complete state into a temporary file in the same directory. +2. fsync the temporary file and check the result. +3. Rename the temporary file over the published image atomically. +4. fsync the containing directory and check the result. +5. Return to the caller, allowing Raft to acknowledge the update. + +An older image is never overwritten in place. A crash before rename leaves the +previous published image; a process crash after rename exposes the complete new +image. An unacknowledged update may therefore appear after recovery, which is +allowed. No acknowledged operation returns before file and directory sync finish. +Startup syncs the selected image and directory again before serving requests. +If a crash/filesystem error leaves the published name missing, startup rejects +the store rather than inventing a new term or vote. + +Live publication errors terminate the process, since the networking layer catches +ordinary exceptions and would otherwise keep an unsafe node running. An error +after rename has an uncertain outcome; clients must not assume it rolled back. +Startup errors are exceptions and do not fall back to older/temp images. + +## Format and ownership + +The image has version magic, payload length and CRC32. Recovery checks the +checksum before decoding entries and validates bounds, commit metadata, snapshot +identity and term ordering. CRC32 detects accidental corruption but does not +protect against malicious modification or all collisions. + +The files are NAME.durable.ID, NAME.durable.ID.tmp and NAME.durable.ID.lock. +A nonblocking advisory flock holds the lock for the object's lifetime. Use one +object/thread to mutate a store, and do not delete/replace its lock while running. +Returned/appended entries are copied so external mutation cannot change the +selected state behind the publisher's back. Temporary files are never recovery +candidates; the next publication replaces a stale temp file while holding the lock. + +An existing lock with no published image is rejected, including an interrupted +first initialization. This conservative choice can require operator recovery +even if the node had not acknowledged any updates. Do not delete an existing +store's files to force startup. + +Legacy entries/index/state files with the same base name are rejected. There is +no automatic migration or downgrade. Use this backend for fresh clusters until +a separately validated offline migration tool exists; do not replace an existing +node's state with an empty store under the same identity. A corrupt main image is never replaced by a +seemingly valid temp image. + +## Use + +```cpp +#include +auto state = std::make_shared("state", nodeId); +``` + +On Linux, the KV example accepts --durable in place of --persist. Run it in a +dedicated directory without legacy state files. --persist and the SQL example +continue to use the legacy backend. Other platforms build the library but +TDurableState construction throws an explicit unsupported-platform exception. + +## Validation and self-review + +Tests cover publication ordering, process-crash boundaries across +append/truncation/vote/replacement/commit updates, immediate exits after returned +updates, interrupted initializations, a live injected failure, an actual +write error using a child-only file-size limit, corrupted/truncated images, +single-writer exclusion, missing images, legacy-store refusal, entry ownership and Raft election/application replay across +a storage restart. +There are 31 child-process scenarios; none relies on sleeps or random scheduling. + +All 61 tests pass locally under ASan/UBSan, and the core suite passes in Debug +and Release builds. These process-crash tests do not simulate physical power +loss or prove a storage controller honors flush commands. Hosted CI executes the +same targets; its results are separate from local validation. + +Self-review checked sync-before-rename-before-ack ordering, suffix replacement, +directory durability, lock lifetime, failure propagation, format parsing and +recovery's refusal to adopt temporary images. No independent approval is claimed. + +## Limits and assumptions + +This first implementation rewrites the entire log on every update, uses memory +proportional to the log, and blocks the calling thread during sync. It establishes +a conservative protocol baseline, not NuRaft-level throughput. An efficient WAL, +group commit, snapshots and compaction remain necessary for substantial workloads. + +The durability contract assumes Linux, a pre-existing durable parent directory, +and a local filesystem/storage stack honoring fsync and same-directory atomic +rename, with exclusive application ownership of the files. Network filesystems, +arbitrary external file deletion, broken hardware/flush semantics and real power +cut testing are outside this milestone. CRC32 is not a substitute for backups. +Storage durability also does not establish end-to-end linearizability or exactly +once application behavior; those consensus/client audits remain open. diff --git a/docs/forwarding.md b/docs/forwarding.md new file mode 100644 index 0000000..fea4ab3 --- /dev/null +++ b/docs/forwarding.md @@ -0,0 +1,89 @@ +# Forwarded request lifecycle: v0.1.0-alpha.6 + +This milestone separates client cookies from forwarding correlation and gives +forwarded attempts an explicit leader/term/peer lifecycle. + +## Reproduced failures + +Followers previously interpreted every nonzero client cookie as an already +forwarded request and rejected it. For zero-cookie requests they replaced the +cookie with an internal routing token, then returned that token to the client +instead of restoring zero. They also mutated the caller's request buffer and +retained forwarding routes when terms/roles changed, allowing late replies to +complete obsolete attempts. + +## Routing contract + +EForwarded is a dedicated internal request flag. Clients may use any uint32_t +cookie, including zero and UINT32_MAX. A follower copies a request, sets this +one-hop flag and assigns a distinct nonzero routing cookie. A forwarded request +that reaches another follower fails instead of being relayed again. + +The routing record holds the original cookie, client endpoint, target peer, +leader ID and origin term. Responses are accepted only from that configured +outbound peer. Before delivery, the route is removed and the response is copied +with the original cookie restored. Caller-owned request/response buffers are +not changed. Duplicate, unknown, wrong-peer and unattributed responses cannot +complete another attempt. + +Routes are rejected with ErrorCode=1 and their original cookie when the origin +changes term, leaves follower state, observes a different leader, or receives a +peer-unavailable notification. Client disconnect removes only that client's +routes silently. Late responses for removed routes are ignored. Routing cookies +are never reused within a processor, even after success, cancellation or peer +reconnection. Exhausting the 32-bit wire space rejects further forwarding rather +than wrapping; a larger session/correlation protocol remains future work. + +Cancellation is local bookkeeping. A write already sent or buffered by the +transport may still execute. No automatic retry, rollback or exactly-once +execution guarantee is introduced. ErrorCode=1 can indicate an unknown write +outcome, so non-idempotent writes must not be blindly retried. + +## Transport integration and compatibility + +Custom adapters must call OnCommandResponse(response, peer), passing the same +configured INode object used for forwarding, and call OnPeerDisconnected(peer) +when the outbound connection is reported unavailable. The old one-argument +response call still compiles but intentionally ignores unattributed responses. +The included server supplies peer context and reports outbound read/unavailable +errors. Repeated failure notifications are idempotent. + +Configured-peer context is a routing check, not cryptographic authentication or +Byzantine fault tolerance. Untrusted inbound client sockets are not accepted as +the outbound peer that owns a response. Write-side failure notification timing +still depends on the transport detecting/reporting connection loss; blackholed +connections without such notification have no deadline here. + +Construct a new processor only with fresh transport sessions: the cookie counter +is per processor, and this milestone does not provide a durable cross-processor +session generation. Message struct sizes are unchanged, but EForwarded reserves +flag value 0x08. Upgrade custom adapters and all cluster nodes together; mixed-version +operation is not validated. + +## Validation and self-review + +Twelve forwarding tests cover full-range client cookies, zero-cookie restoration, +request-copy/one-hop semantics, out-of-order and duplicate replies, term and +candidate transitions, stale replies arriving before an application poll, +selective client disconnect, leader discovery, wrong/missing peer context, +error propagation and peer failure/reconnection with late replies. Nine of the +initial ten tests fail against alpha.5; the selective-disconnect baseline passes. + +An additional three-node deterministic test forwards a write and strong read +through actual Raft replication/application, including reversed and duplicate +consensus traffic. Together with existing tests, all 94 pass locally under +ASan/UBSan, with core Debug/Release verification. Hosted CI executes the same +targets; its results are separate from local checks. There are no sleeps or +wall-clock dependencies in the new tests. + +Self-review checked original-cookie restoration, copy ownership, distinct token +allocation, route invalidation before response delivery, source-peer matching, +disconnect selectivity and adapter call sites. No independent approval is +claimed. Access remains serialized; callbacks must not re-enter the processor. + +## Remaining gaps + +Durable request IDs/deduplication, deadlines, admission limits, a larger +session/correlation namespace, broader transport validation and generated +history/linearizability testing remain open. This change does not prove +library-wide linearizability or guarantee completion during quorum loss. diff --git a/docs/joint-quorum.md b/docs/joint-quorum.md new file mode 100644 index 0000000..e64a3ac --- /dev/null +++ b/docs/joint-quorum.md @@ -0,0 +1,42 @@ +# Joint quorum arithmetic: v0.1.0-alpha.24 + +This step prepares the election, commit and strong-read quorum calculations for +joint consensus. A joint quorum requires a strict majority of both the old and +new voter sets. Its replicated-index or read-ack watermark is the lower of the +two majority watermarks. Shared voters count in each set; outsiders and learners +do not contribute. Even-sized groups require more than half. + +TQuorum validates positive, sorted, unique, nonempty voter sets and computes +stable or joint election results and watermarks. Its value callback must return +the same value for a voter throughout a call. TVolatileState can apply this +arithmetic to commit advancement, retaining the current-term commit restriction +and monotonic commit index. Validated retained configuration history now selects +the runtime quorum used by elections, commit advancement and strong-read +confirmation. An uncommitted joint entry uses both voter sets; after that entry +commits, the joint quorum remains active until its stable successor commits. +The fixed-membership snapshot runtime still requires stable snapshot voters to +match the node's configured and bound set; joint snapshot recovery remains +disabled. + +Tests exhaust every pair of nonempty old/new subsets of five IDs and every vote +subset, including disjoint and overlapping sets, even-sized groups, absent self +and outsider votes. They independently enumerate thresholds for joint commit/read +watermarks and check current-term gating. Separate stable-set tests compare the +calculation with strict-majority order statistics for sizes 1–8. Existing Raft, +partition, learner, snapshot and read tests exercise the fixed-membership paths. + +This arithmetic now backs the bounded live transition API described in +[configuration entries](configuration-entries.md). Recovered suffix transitions +and live changes require a bound original voter set, a persisted commit index and +endpoints for every active voter. Joint snapshot recovery and snapshot transfer +across a membership transition remain unsupported; compaction is rejected while +the retained log contains a membership change beyond its snapshot baseline. +Leader removal and explicit leadership transfer are not implemented. + +The change adds a public type and changes TRaft's in-memory layout; rebuild the +library and dependents together. Wire and durable formats are unchanged. +Self-review checked overlap, strict majorities, current-term commitment, stable +path equivalence and learner exclusion. Runtime tests verify that elections, +commit advancement and strong-read confirmation each wait for both majorities +under an in-progress joint configuration. No independent reviewer approval is +claimed. The live transition API remains disabled. diff --git a/docs/joint-snapshots.md b/docs/joint-snapshots.md new file mode 100644 index 0000000..0e6f216 --- /dev/null +++ b/docs/joint-snapshots.md @@ -0,0 +1,67 @@ +# Joint snapshot metadata foundation: v0.1.0-alpha.25 + +The [alpha.27 follow-up](configuration-history.md) requires the configuration +term to equal the snapshot term when both refer to the same index. +The [alpha.48 follow-up](unsupported-snapshot-term-preflight.md) checks the +initial snapshot format before accepting a higher RPC term. The [alpha.49 follow-up](snapshot-metadata-term-preflight.md) +also checks the embedded boundary, fixed voters, and known stable configuration +identity. + +This step adds an experimental snapshot representation for a joint configuration +effective at the included snapshot boundary. MRFTS002 records the old and new +sorted voter sets plus the configuration entry's index and term. Its index must +be positive and no greater than the snapshot index; its term must be positive and +no greater than the snapshot term. Both voter sets are nonempty, positive, sorted, +unique and individually bounded. The codec validates structure, not whether the +configuration was committed or belongs to an authenticated cluster. + +The v2 byte layout is magic[8], snapshot index[8], snapshot term[8], old voter +count[4], application data length[8], configuration index[8], configuration +term[8], new voter count[4], old voter IDs[4 each], new voter IDs[4 each], opaque +application data, and CRC32[4] over all preceding bytes. Lengths and counts are +bounded before allocation. Fixed snapshots continue to encode as MRFTS001 with +exactly the previous bytes; a golden v1 image tests compatibility. The standalone +snapshot store can export and load v2 images for testing or offline tooling. + +The [alpha.59 continuity follow-up](configuration-snapshot-continuity.md) checks +that a stored stable configuration advances to a joint image using the same old +voters, and that a joint image can only continue with the same joint identity or +move to a stable successor selecting its new voters. + +This is a format and offline persistence foundation, not live joint consensus or +complete configuration history. A bound memory or durable store can install and +recover a joint snapshot for offline validation while preserving its original +voter binding, which may differ from the snapshot's active old voter set after +earlier transitions. The caller must supply a trusted history baseline; the +snapshot codec and store do not prove that the transition was committed. Raft +startup and RPC paths still reject joint snapshots, and compaction only creates +fixed-voter snapshots. A covered +snapshot is acknowledged only after its prefix identifies supported fixed v1 or +stable v3 metadata, including when the prefix arrives in several chunks. +Unsupported joint images are rejected without replacing application data or +publishing a joint snapshot. Since alpha.48, a higher-term InstallSnapshot can +advance hard state only after its initial chunk identifies a supported MRFTS001 +or MRFTS003 format. Since alpha.49, that chunk must also contain the complete +bounded metadata prefix and match the RPC boundary, fixed voter set, and any +known stable configuration identity. Short, unknown, joint-format, mismatched, +or continuation chunks cannot change the term before rejection. These rules +avoid falsely acknowledging a joint image merely because its index is already +committed locally. + +Older binaries do not understand MRFTS002 and may acknowledge an already-covered +image without checking its version. Do not send or activate joint snapshots in a +mixed-version cluster. Future live use requires an upgrade of every voter, a +defined compatibility protocol, replicated configuration entries, durable and +snapshot recovery of configuration history, transition rollback rules and dual +quorum handling throughout the protocol. This experimental v2 format may change +before those semantics are implemented. A standalone export is not proof that a +configuration is committed and is not an authoritative Raft checkpoint. + +Tests cover a golden v1 image, joint round-trip, every-byte corruption and +truncation, semantic corruption with valid CRC, bounds and ordering, standalone +publication, bound durable round-trip, runtime rejection without rewriting the +image, and network rejection both above and below the commit boundary. They also +cover a supported v1 prefix split across chunks. The snapshot struct's +in-memory layout changes; rebuild the library and dependents together. Self-review +checked version dispatch, bounds, immutable voter binding, covered acknowledgments +and fail-closed recovery. No independent reviewer approval is claimed. diff --git a/docs/learners.md b/docs/learners.md new file mode 100644 index 0000000..0f6808f --- /dev/null +++ b/docs/learners.md @@ -0,0 +1,146 @@ +# Non-voting learners: v0.1.0-alpha.17 + +Raft startup rejects a zero current term or a persisted vote for an ID outside +the configured fixed voter set, including on unbound nonfresh and legacy stores. +It performs this check before voter binding or serving traffic. This check does +not prove that the full voter set matches a legacy store's original cluster; +legacy storage still has no persisted voter binding. + +This milestone separates replication targets from the fixed voting set. Learners +receive log entries, install snapshots and apply committed commands, but never +campaign, vote, lead, propose entries or contribute to election, commit or strong +read quorums. It implements the catch-up phase described in +[Raft section 6](https://raft.github.io/raft.pdf). Promotion, voter removal and +joint-consensus configuration entries remain the next milestone. + +## Configure a learner + +The KV CLI accepts repeatable `--learner id` options, requiring `--server --durable`. +All replicas must agree on the same voter IDs and role list. Include every voter +and learner endpoint in `--node`; mark each learner by ID on every process. For +example, with voters 1/2/3 and learner 4: + +```sh +./build/full/examples/kv --server --id 4 --durable \ + --node 127.0.0.1:8001:1 --node 127.0.0.1:8002:2 \ + --node 127.0.0.1:8003:3 --node 127.0.0.1:8004:4 --learner 4 +``` + +Use the same endpoints and `--learner 4` on voter processes, with their own IDs. +Learners can catch up from a compacted prefix even without local periodic +snapshotting. Clients can connect through them: writes and strong/default reads +are forwarded to the known voter leader; explicit stale reads use local applied +state. A learner does not elect a replacement leader when disconnected. + +The constructor gains an optional fourth argument, the learner-ID vector. +Omitting it preserves all-voter interpretation. IDs must be positive, unique and +present among the local/peer IDs. At least one voter is required. Transport peer +count still includes learners; GetNservers/GetMinVotes describe voters/quorums. +Memory TState supports role binding for library tests/volatile deployments, but +only the durable backend preserves that binding across actual process loss. +Legacy TDiskState does not support learner mode. + +The KV CLI uses static transport discovery. Library embedders can use +[online learner staging](online-learner-staging.md) on the current leader, and +`TRaftServer` can attach its `TNode` endpoint through +[network learner staging](online-learner-network.md). That local, non-voting +replication target is not persisted or cluster membership. +Deleting `--learner 4` to turn it into a voter is rejected for bound stores; it is +not a membership-change procedure. Durable Raft embedders can use the separate +[live joint-membership API](configuration-entries.md) after catch-up and commit +acknowledgement. + +## Quorum and term isolation + +Leaders replicate to every configured peer, tracking normal RPC-correlated +MatchIndex/NextIndex and snapshot progress. Commit advancement selects indices +only for explicit voter IDs. Election RPCs are sent only to voters, learner vote +responses are ignored, and strong-read confirmation collects only voter ACKs. +One voter with multiple learners can still commit/read alone; a voter minority +cannot make decisions merely because learners are available. + +A learner acknowledges vote requests with a denial and never records a vote. +It follows legitimate voter terms but cannot advance its own term by election +timeout. Voters discard learner-origin vote/leader requests before term mutation. +Replication ACKs from learners are accepted only in the voter's current term, so +an outlier learner term cannot depose a healthy voter leader. A learner with an +incompatible future term may remain unable to catch up; this does not force the +voting quorum to adopt that term. This is a trusted-member protocol, not network +authentication against forged voter IDs. + +IsLearnerCaughtUp(id) is a leader-only, point-in-time log watermark: the configured +learner must match the leader's current log end and the leader must already know +a current-term commit. It does not change roles, authorize promotion, attest to +application health or supply an application-applied watermark. Persistence of +ACKed entries depends on the selected storage backend. Unknown IDs and voter IDs +return false. A new unreplicated tail makes the status false again. + +## Durable voter binding and upgrade + +Fresh binding-capable stores record the voter IDs before accepting any vote or +replication traffic, including all-voter deployments. A snapshot-backed v2 store +can adopt exactly the voter set already recorded in its embedded snapshot. Bound +stores reject different voter IDs on restart and reject snapshots that would +replace that set. Learner local stores must not contain a vote. Ordinary role +validation failure does not modify term, vote, log or application data. + +A nonfresh v1 store without an embedded snapshot has no trustworthy record of its +old voter IDs. Learner mode refuses to guess. First run it under its unchanged, +verified all-voter configuration and create an embedded snapshot with compaction; +then enable learner-aware discovery with the same voter IDs. Unbound legacy +all-voter operation retains its previous startup contract until that proof exists. +A standalone alpha.15 checkpoint alone is not an embedded configuration witness. +Never delete voter state or edit votes/terms to work around this validation. + +Current durable images use `MRFTD004` and retain the 20-byte header. The payload +stores CurrentTerm[8], VotedFor[4], retained-log count[8], CommitIndex[8], voter +count[4], sorted voter IDs[4 each], snapshot length[8], optional MRFTS001, +MRFTS002, or MRFTS003 image, then retained log records. The CRC covers version +magic plus payload. Recovery bounds and validates the voter list, commit +boundary. Legacy snapshots must name the original bound voters; configuration- +bearing snapshots can describe a later active set while retaining that original +binding for offline history scans. Raft startup still requires the runtime voter +set and binding to match. MRFTD001 through MRFTD003 remain readable; formats +without commit metadata conservatively use SnapshotIndex as their commit floor. +The next successful publication writes v4 +using the existing atomic write/fsync/rename/directory-fsync protocol. A crash +before first binding leaves either a fresh unbound store with no votes/log, or a +complete bound store. Publication failure is fatal before the node can serve +traffic. + +Upgrade all voters before configuring any learners. Older binaries treat every +peer as a voter and cannot read v4 stores; mixed-role/mixed-version operation is +unsupported. All fresh bootstrap configurations must agree. Persistent binding +does not authenticate initial bootstrap choices or wrong-store copies. No cluster +UUID, role negotiation or automatic migration from unknown membership is added. + +## Validation and review + +Fifteen new deterministic unit tests cover malformed role lists, no learner +campaign/vote/proposal, term isolation, voter-majority arithmetic for sizes 1–8 +with arbitrary learner matches, a single voter with several learners, ignored +learner grants, commit/read isolation during voter partitions, ordinary and +snapshot replication, log catch-up status, restart role preservation, rejected +manual promotion, learner-only transport expansion, unbound legacy rejection, +v2 snapshot-backed adoption, v3 corruption with valid checksums, unsupported +legacy storage, and crashes at all five first-binding publication stages. + +A new real-process learner CLI test starts an offline learner after a compacted +two-voter majority, verifies local catch-up and tail application, restarts the +learner, confirms its persisted vote is zero, and checks that removing the role +flag fails without rewriting its store. The existing checkpoint/InstallSnapshot +smokes also run against v3 stores. Fault schedules are deterministic core tests; +CLI smokes use separately bounded socket/process waits. + +Local validation passes 222 unit tests and three CLI tests under ASan/UBSan across +17 CTest targets; Release core checks pass. Self-review checked every quorum +site, pre-term RPC filtering, local-role guards, immutable voter binding, adoption +proofs, snapshot voter metadata, v1/v2/v3 decoding, publication ordering and the +distinction between log catch-up and promotion. No independent reviewer approval +is claimed. Process crashes do not substitute for hardware power-cut testing. + +This closes the non-voting catch-up phase, not dynamic voting membership. Next +are replicated joint configurations, old/new election/commit/read majorities, +promotion gates, leader removal and configuration history in snapshots. Online +transport discovery, authenticated bootstrap, SQL snapshots, resource quotas and +independent review remain open production gates. diff --git a/docs/legacy-log-terms.md b/docs/legacy-log-terms.md new file mode 100644 index 0000000..bc94536 --- /dev/null +++ b/docs/legacy-log-terms.md @@ -0,0 +1,51 @@ +# Legacy log-term validation: v0.1.0-alpha.14 + +TDiskState recovery checked file sizes, offsets and entry framing but did not +validate the selected entries' term values. It could accept zero terms, a log +newer than CurrentTerm, or a term sequence that went backward. Those invalid +values then fed Raft's vote freshness and log-matching decisions. The opt-in +TDurableState backend already rejected these term sequences on recovery. + +Legacy recovery now validates every selected entry's term: it must be nonzero, +no greater than the stored CurrentTerm and no less than the preceding term. +Startup throws std::runtime_error without modifying any stored bytes. Equal +terms and gaps between terms are valid, including full-width uint64_t values. + +The basic range check is also applied when reading an entry after startup. If a +live read detects a zero term or a term newer than the in-memory CurrentTerm, the +existing Fail path terminates the node. An ordinary exception would be caught by +the network layer and allow a node with detected storage damage to keep serving. +Full sequence ordering is checked at startup, not by scanning all neighbors on +each live Get. + +Only the prefix selected by LastLogIndex is validated. Obsolete suffix bytes left +by logical truncation remain ignored and can be overwritten by a valid append. +There is no file-format or storage-interface change. Previously accepted stores +with impossible term metadata now require verified recovery or safe rebuilding; +the code does not guess corrections or reset the node's term/vote. + +Six new tests cover zero terms at both selected entry positions, future terms or +stale current-term metadata, decreasing terms, valid repeated/skipped/full-width +terms, an invalid obsolete suffix followed by replacement, and fatal live-read +range violations. Four tests failed against the previous implementation; two +positive compatibility tests already passed. Rejection tests compare snapshots +of every storage file before and after startup to verify byte-for-byte preservation. +The live-read subprocess test is excluded on Windows. + +Two old test fixtures appended terms newer than CurrentTerm and only updated the +current term afterward (or never did). They now persist CurrentTerm first, +matching TRaft's existing term-update ordering before AppendEntries application. +Direct storage users remain responsible for consistent fields and valid append +inputs; this change does not add a proposal validator to the legacy writer. + +All 181 tests across 11 CTest targets pass locally with SQLite under ASan/UBSan; +the Release core suite passes as well. Self-review checked full-prefix traversal, +unsigned comparisons, validation before exposing an entry, catchable startup +errors versus fatal live errors, ignored suffixes and fixture ordering. No +independent reviewer approval is claimed. + +This detects impossible metadata rather than all corruption. Valid-looking term +or payload changes remain undetectable without checksums. Legacy TDiskState still +requires one writer and lacks power-loss durability, atomic publication and +crash-safe replacement. It is not upgraded to TDurableState by this validation +change. Snapshots, safe migration and independent review remain production gates. diff --git a/docs/legacy-voter-proof-startup.md b/docs/legacy-voter-proof-startup.md new file mode 100644 index 0000000..81876dc --- /dev/null +++ b/docs/legacy-voter-proof-startup.md @@ -0,0 +1,26 @@ +# Require voter proof before legacy durable startup: v0.1.0-alpha.52 + +The alpha.51 guard prevented a live Raft instance from being rebound after +startup, but it still allowed a recovered log-only MRFTD001/MRFTD002 store to +start unbound using only the process's peer list. That left the persisted node +identity unprotected across restarts. + +Raft now rejects a recovered durable store with no voter binding and no +ordinary snapshot before it can bind peers or serve traffic. The operator must +first call `TDurableState::BindVotersFromTrustedBaseline` using an independently +verified complete original voter set. A recovered unbound store with an +ordinary snapshot retains the existing adoption path, which validates that the +snapshot voter list matches the configured peers before persisting the +binding. Newly initialized stores still bind their initial peer set during +startup. + +The rejection does not rewrite the durable file. The alpha.51 startup marker +also keeps the explicit migration API startup-only. `TDiskState` and in-memory +state behavior are unchanged. This closes unbound durable startup; it does not +implement live membership changes or joint snapshot recovery. + +Tests cover recovered log-only and idle stores failing without file changes, +successful startup after explicit trusted migration, ordinary-snapshot adoption, +and rejection of migration after a successful startup. + +Self-review completed; no independent reviewer approval is claimed. diff --git a/docs/memory-log-ownership.md b/docs/memory-log-ownership.md new file mode 100644 index 0000000..f8d9c7c --- /dev/null +++ b/docs/memory-log-ownership.md @@ -0,0 +1,23 @@ +# In-memory log ownership: v0.1.0-alpha.21 + +The in-memory TState backend retained the mutable buffer supplied to Append and +returned the same buffer from Get. A caller or application could therefore change +an existing log entry, including its term or command, after Raft appended it. +That could break log matching and change a committed application command. The +durable backend already copies entries on append and read. + +TState now copies entry bytes on append, construction and readback. Copies of a +TState also receive independent log entries. This preserves stored terms and data +when the original proposal or a returned entry is modified. Copies cost additional +memory and CPU, especially for large entries. TState is still volatile and does +not provide crash durability. Its public Log member and IState mutation methods +remain test/backend interfaces: modifying them while Raft runs bypasses its safety +rules, and serialized access is still required. + +Deterministic tests cover mutation of the appended buffer, returned buffer, +constructor input and copied states, plus a three-node replication schedule that +changes the proposal after Append and verifies the original command commits on +all nodes. The first two ownership tests failed on the previous implementation. +Self-review covers snapshot suffix retention and the durable backend's existing +copy behavior; no independent reviewer approval is claimed. There is no wire or +durable format change, and existing corrupted state is not repaired. diff --git a/docs/message-extents.md b/docs/message-extents.md new file mode 100644 index 0000000..0d442f3 --- /dev/null +++ b/docs/message-extents.md @@ -0,0 +1,30 @@ +# Message buffer extents: v0.1.0-alpha.23 + +Locally constructed message holders exposed a mutable frame length but did not +remember their allocated buffer size. A caller could enlarge an entry's declared +length while leaving it within the protocol maximum; proposal validation then +accepted it and the in-memory log copied beyond the allocation. A deterministic +ASan regression reproduced a heap-buffer-overflow. + +Holders now carry the buffer extent and payload-array capacity through typed +conversions. Frame and batch validation reject lengths or entry counts beyond +those extents. Proposal and command request/reply admission validate the holder +before accessing or copying the data. The storage clone paths check the extent, +and the legacy disk write path fails closed for an invalid buffer. A malformed +forwarded reply is rejected before its client route is consumed. + +NewHoldedMessage and InitPayload populate extents automatically. Callers that +construct holders directly around an external allocation must supply its actual +byte capacity and, for batches, the actual payload-array capacity to the holder +constructor; the default zero capacity fails validation. Holder layout changed, +so rebuild the library and all dependent binaries together. Wire formats and +durable formats are unchanged. The extents are trusted metadata owned by local +C++ code, not a sandbox against callers that forge pointers or mutate capacities. +Application code that bypasses Raft admission remains responsible for its own +validation. + +Deterministic tests cover the original oversized proposal, forged batch count, +oversized child entry, typed conversion and a malformed forwarded reply followed +by a successful valid reply. Self-review checked framing, admission, clones and +forwarded-route ordering. No independent reviewer approval is claimed. Voting +membership changes and joint consensus remain open. diff --git a/docs/network-frames.md b/docs/network-frames.md new file mode 100644 index 0000000..0879509 --- /dev/null +++ b/docs/network-frames.md @@ -0,0 +1,99 @@ +# Bounded network framing: v0.1.0-alpha.7 + +This milestone fixes fragmented TCP header handling and validates structural +frame bounds before allocation, typed casts or delivery to Raft. + +## Reproduced failures + +The reader assumed each four-byte header field arrived in one ReadSome call, +rejecting valid byte-fragmented TCP input. It trusted the declared length and +AppendEntries count before allocation, accepted unknown/unimplemented types and +short fixed-size messages, and recursively accepted arbitrary child messages. +All six initial deterministic regressions fail against alpha.6. + +The message factory also only asserted an eight-byte allocation minimum even +when placement-constructing a larger derived type. Undersized typed allocations +could overwrite their buffers, particularly when assertions were disabled. + +## Decoder and encoder contract + +Headers and bodies now use exact-read loops. Before allocating, the decoder +checks the type, its minimum/fixed size, the frame cap and remaining batch +budget. Vote/AppendEntries control frames have exact sizes. Log entries and +command request/response frames have typed minimum sizes and variable data. +Unknown types and unimplemented snapshot types are rejected. + +AppendEntries children must be log entries with no nested payloads. Entry count +and minimum required bytes are checked before allocating the holder array; every +child consumes the same aggregate byte budget. Nesting depth is bounded to the +root and one log-entry level. EOF/truncation is an error; a failed reader cannot +resume at a later frame. Caller adapters must discard the failed stream. + +The writer validates the entire outgoing tree before its first byte, including +Nentries/PayloadSize agreement, required payload storage, child types and byte +budgets. Valid consecutive frames retain their boundaries under partial I/O. +The bundled server closes rejected inbound streams and resets failed outbound +connections after the reader unwinds, rather than parsing more bytes from the +damaged stream. Associated forwarded attempts receive the existing failure +notification; write outcomes may remain unknown. + +## Default limits and replication progress + +* Maximum frame size: 16 MiB, including its header. +* Maximum complete AppendEntries batch: 64 MiB, including root and child headers. +* Maximum AppendEntries child count: 4096. + +Raft log proposals are checked against the frame cap before mutating the log or +the proposed entry's term. Batch construction now observes the same aggregate +cap and continues with the remaining entries after acknowledgement. This avoids +building a valid-looking replication batch that every peer would reject. + +Application Prepare implementations can expand commands. If their prepared log +entry exceeds the cap, the proposal is rejected before append; an accepted +command-frame length alone does not guarantee its prepared entry fits. + +TMessageLimits can be supplied to standalone reader/writer adapters. The bundled +Raft sender/admission path uses shared defaults; coordinated deployment-level +limit configuration is not introduced here. Stricter custom adapter limits can +reject the sender's batches. Existing stores with oversized entries are not +supported by the new default replication cap and require review before upgrade. +The wire layout and native-endian representation are otherwise unchanged. + +## Typed message safety + +Factories reject allocations smaller than sizeof(T), including in Release builds. +Cast checks null, declared minimum size and the concrete protocol type; Maybe +and IsEx no longer expose short or empty messages as larger structures. A generic +TMessage factory can still hold opaque bytes for validation. Local callers must +provide genuine buffers and keep frames immutable during I/O: mutable length +fields are not an allocation-capacity capability, and forged local holders are +outside this contract. + +## Validation and self-review + +Fourteen deterministic stream tests exercise one-byte fragmentation, all header +truncation offsets, fixed-frame body truncation offsets, invalid/oversized lengths, +unknown types, wrong/nested children, entry count and aggregate bounds, exact +budget boundaries, reader poisoning, pre-write validation and adjacent frames. +They use immediate fake I/O, with no sockets, sleeps or wall-clock dependencies. + +Four core tests cover typed allocation bounds, checked casts, bounded multi-batch +replication progress and rejection before persistent-state mutation. An older +election test fixture was corrected to store vote requests as requests instead of +casting them to vote responses; it still checks the same expected wire bytes. + +All 112 tests pass locally under ASan/UBSan, with core Debug/Release verification. +Hosted CI runs the same targets; its results are separate from local checks. +Self-review checked subtraction/aggregation bounds, non-recursive child shape, +pre-allocation validation, pre-write validation, batch continuation and failed +connection reset ordering. No independent reviewer approval is claimed. + +## Remaining gaps + +This is structural framing validation, not authentication or complete validation +of consensus fields. Sender/destination membership, index/term semantics and +application payload lengths (KV/SQL/custom commands) still need their own checks. +Per-connection caps do not bound aggregate memory across unlimited connections; +admission limits, deadlines and broader fuzzing remain open. Native wire encoding, +TLS lifecycle stress, retry deduplication and generated history checking also +remain separate work. No library-wide security or linearizability claim is made. diff --git a/docs/online-learner-network.md b/docs/online-learner-network.md new file mode 100644 index 0000000..9ccaffc --- /dev/null +++ b/docs/online-learner-network.md @@ -0,0 +1,36 @@ +# Network server learner staging: v0.1.0-alpha.40 + +`TRaftServer` now exposes `AddLearnerTarget(id, endpoint)` and +`RemoveLearnerTarget(id)`, using a `shared_ptr>` endpoint. Adding +a learner registers it with Raft and the server's drain set, so the normal +server loop can send its replication and snapshot traffic without restarting. +Removing it drops its Raft progress and removes it from future server drains; +the method returns the endpoint so the embedding application can manage its +transport lifetime. The voter list and quorum calculations remain unchanged. + +Construct the remote node with the right address and socket factory, and make +sure its local Raft instance is configured as a learner with the same voter set. +The storage on the current leader must have a bound voter list. The learner +still must have a configured return path to the voters. The server does not +discover, authenticate or configure the remote process. + +These calls must run on the same event-loop thread as `TRaftServer` and be +serialized with Raft processing. They may be called before `Serve()` or from a +serialized callback after serving begins. Calls from other threads and +reentrant calls from transport or application callbacks are unsupported. +After removal, a write already in progress or data already queued in the +endpoint may remain. Keep the returned endpoint alive while that I/O can still +use it, then shut it down according to the embedding application's transport +lifecycle. + +The KV CLI still discovers peers at startup. Staging is leader-local and is +lost on restart or failover; the new leader and the application must reconcile +it. This adds network transport support for non-voting catch-up. Voter promotion +and replicated joint consensus are now available through the bounded durable +transition path in [live membership](configuration-entries.md). Membership-aware +snapshot installation and leader removal remain unsupported. + +The deterministic server test checks target registration, voter/quorum +invariance, removal and rollback after a nonleader rejection. The existing +learner tests cover snapshot catch-up and stale responses after removal. No +independent reviewer approved this milestone. diff --git a/docs/online-learner-staging.md b/docs/online-learner-staging.md new file mode 100644 index 0000000..76b02f9 --- /dev/null +++ b/docs/online-learner-staging.md @@ -0,0 +1,44 @@ +# Online learner replication staging + +`TRaft::AddLearnerTarget(id, endpoint)` lets the current leader add a non-voting +replication target without restarting that Raft object. `RemoveLearnerTarget(id)` +removes a local target and returns its endpoint. The voter set and persisted +binding do not change. Staged learners receive the ordinary log or a snapshot +when needed, but never count toward election, commit, or strong-read quorums. +`IsLearnerCaughtUp` remains a point-in-time log watermark, not an authorization +to promote or a guarantee of application health. +`IsLearnerCommitAcknowledged` separately reports whether a correlated successful +replication reply confirms the leader's current Raft commit boundary while the +learner also matches the leader's log. It does not report application-applied +progress and does not authorize promotion. + +These are serialized embedding APIs, not a replicated cluster configuration or a +promotion API. The caller must ensure the destination is configured as a learner +with the same voter set, arrange and authenticate its transport, and keep its +endpoint alive while asynchronous operations may still use it. After removal, +use the returned endpoint to coordinate safe transport shutdown; queued traffic +can still exist. The caller must serialize add/remove with Raft processing and +must not re-enter them from transport callbacks. Duplicate IDs, voters, the local +ID, null endpoints and calls on nonleaders are rejected. Staging requires storage +with an already-bound matching voter set; unbound and legacy stores cannot stage. + +Registration is local to this Raft object and is not persisted or propagated to +other voters. The embedding application must reconcile targets on a new leader +and after restart; a previous leader retains its local list if it becomes a +follower. Removing a target drops replication progress and pending RPCs, and a +subsequent add starts a fresh probe. Delayed responses cannot satisfy new RPCs. +The KV CLI still uses static transport setup. `TRaftServer` runtime integration +is described in [network learner staging](online-learner-network.md). Older +binaries must not be mixed with learner configurations because they treat +unmarked peers as voters. + +Deterministic tests cover live add, snapshot catch-up, voter partitions and strong +reads, removal and re-addition with delayed append replies, removal during a +multi-chunk snapshot with delayed acknowledgements, invalid calls, and refusal +on unbound or legacy stores. Durable-store tests verify that staging does not +rewrite the binding and is absent on restart. A leader-change test verifies +that an embedding application must re-register the target on the new leader. +A bounded durable-store promotion/finalization path now uses these catch-up +acknowledgements; see [live joint membership](configuration-entries.md). +Persistent endpoint discovery, leader removal and membership-aware snapshot +installation remain open. diff --git a/docs/pending-requests.md b/docs/pending-requests.md new file mode 100644 index 0000000..dd77914 --- /dev/null +++ b/docs/pending-requests.md @@ -0,0 +1,87 @@ +# Pending request lifecycle: v0.1.0-alpha.5 + +This milestone prevents pending writes from receiving a replacement command's +reply after leadership changes. It also releases disconnected clients and +unused application replies, and makes pre-election requests progress when the +local node becomes leader. + +## Reproduced failure + +A leader accepted an uncommitted command at index 3. After a partition, another +leader replaced that index with a different command and committed it. When the +old leader rejoined, its request processor returned the replacement command's +successful reply to the original client. Requests and application replies were +matched only by index, with no leadership term or lifecycle validation. A +replacement no-op could instead leave no application reply and hit an assertion. + +## Completion and cancellation contract + +Local writes, default leader reads and strong reads now belong to the leadership +term in which they were admitted. Losing leadership or changing term rejects +pending operations with ErrorCode=1 and the original cookie. Re-election cannot +redeem an earlier term's request. Pruning runs before admission, application, +completion and leader-wait queue processing; unchanged terms/roles avoid rescans. + +Write application replies are retained only while a matching local request is +pending. Both index and term must match before a success is returned. Replies +for follower/background application are released immediately. Completion, +leadership cancellation and disconnect cleanup remove cached replies and their +pending-write records. The retained reply cache is bounded by pending writes, +although admission limits for the request queues themselves remain open. + +Cancellation concerns the response, not the replicated command. Committed +commands still apply even if their client disconnected or their leader changed. +An error can therefore mean UNKNOWN OUTCOME: the write may already have +committed. It is not a rollback indication and must not trigger a blind retry of +a non-idempotent operation. Request IDs and replicated deduplication are not +implemented. The KV, SQL and debug clients now display errors rather than +printing a successful commit for nonzero ErrorCode responses. + +CleanUp removes a disconnected endpoint from ordinary, strong-read and +leader-wait queues, plus forwarded-request routing. Disconnect cleanup sends no +reply and does not retract entries already appended to the log. + +Explicit stale reads are independent of leadership terms and target the known +committed prefix, rather than an uncommitted tail. Default reads remain weaker +than strong reads but are cancelled if their admitting leadership ends. FIFO +queueing can still delay a stale read behind an earlier pending request. + +## Leader discovery and timer progress + +A newly elected leader records its own ID. Higher-term stepdown clears stale +leader routing until AppendEntries identifies the new leader. Requests waiting +for leader discovery can execute locally after election, including a single-node +election. Invalid or self-directed follower routes remain queued rather than +asserting or dereferencing a missing peer. + +The server's idle timer now runs request-state checks, committed application and +completion processing after Raft timeouts, using the same ordering as inbound +events. Progress therefore does not require an unrelated incoming message. + +## Validation and self-review + +Twelve new deterministic tests cover replacement by another command or no-op, +stepdown before application, already-committed but unanswered writes, re-election +without intermediate application polls, default reads of lost tails, requests +queued before single/three-node elections, disconnects from all local queues, +cached reply release, unused follower replies and stale reads of a committed +prefix. Weak references verify ownership is actually released. The initial ten +tests all fail against alpha.4. + +All 81 tests pass locally under ASan/UBSan, with core Debug/Release verification. +Hosted CI executes the same targets; its results are separate from local checks. +The wire format and public request-processor methods are unchanged. + +Self-review checked index-and-term matching, reply-cache bounds, cancellation +before log application, disconnect ownership, leader routing and timer/inbound +ordering. Access must remain serialized and callbacks must not re-enter the +processor. No independent reviewer approval is claimed. + +## Remaining gaps + +Exactly-once retries, durable request IDs, forwarded-request lifecycle across +leader failures, queue admission limits, deadlines, transport validation and +broader generated history/linearizability testing remain separate work. This +change does not promise successful completion during quorum loss or establish +library-wide linearizability. Unanswered writes after process crashes also have +unknown outcomes. diff --git a/docs/persistence-recovery.md b/docs/persistence-recovery.md new file mode 100644 index 0000000..11b96e2 --- /dev/null +++ b/docs/persistence-recovery.md @@ -0,0 +1,66 @@ +# Persistence recovery validation: v0.1.0-alpha.2 + +This follow-up closes unsafe recovery behavior in the original three-file +TDiskState format. It is still experimental storage, not power-loss durability. + +## Fixed behavior + +Previously an incorrectly sized state file silently initialized fresh metadata, +potentially forgetting the node's current term and vote. Missing files were +recreated, short indexes could crash startup, and entry lengths were trusted +before allocation. Recovery now rejects these conditions with std::runtime_error +without changing or recreating the stored files. + +A new store is initialized only when all three paths are absent. Existing stores +must contain all three files and an exactly sized metadata record with a +nonzero current term. Recovery validates the complete selected log prefix: +index extent, contiguous offsets, entry types, minimum lengths and file bounds. +Unreferenced trailing bytes remain allowed because log truncation is logical. +The file format and public storage interface are unchanged. +The [log-term follow-up](legacy-log-terms.md) additionally validates nonzero, +nondecreasing selected entry terms bounded by CurrentTerm. + +RemoveLast now updates LastLogTerm, including when the log becomes empty. +Append explicitly seeks the write position, so appending after truncation to +zero overwrites the beginning rather than writing into an obsolete suffix. +File-open failures no longer fall back to truncating an existing file. Stream +write/flush errors are detected, and buffered entries/index are flushed before metadata. + +## Validation and review + +Ten new persistence tests cover truncated/oversized metadata, each missing +storage file, short indexes/entries, invalid entry headers and offsets, zero +terms, excessive log counts and truncation/restart/replacement. Rejected recovery +is checked against a byte-for-byte snapshot of every file to verify no mutation. +An additional POSIX subprocess test verifies that a live storage read failure +terminates the node rather than escaping as an ordinary network exception. +The original implementation failed eight of the initial nine tests, including +a caught segmentation fault on a short index and stale LastLogTerm after removal. + +Local Debug/Release core tests and the full ASan/UBSan suite pass: 27 existing +core tests, eight consensus correctness tests, eleven recovery tests and three +transport tests (49 total on POSIX; the subprocess test is excluded on Windows). CI runs the same targets on pushes and PRs. + +Self-review checked recovery's non-mutating rejection paths, length arithmetic +before allocation, signed file positions, offset continuity, existing-format +compatibility, output positions after truncation and fail-stop behavior in the +presence of the network layer's generic exception handlers. No independent reviewer +approval is claimed. + +## Operational contract and remaining gaps + +Use one writer per storage set. Startup validation throws std::runtime_error. +Detected live storage read/write/flush errors call std::terminate: the network +layer catches ordinary exceptions, so throwing alone would allow a node with +unpersisted state to continue processing. +Recovery rejection requires restoring verified state or deliberately rebuilding +the node through a safe cluster procedure; never delete metadata alone to force +startup. Previously created empty or partial file sets are deliberately rejected. + +Validation detects structural damage, not all corruption: there are no checksums +or record generations, so a bit flip that leaves valid fields can go undetected. +The three-file format still lacks fsync, atomic metadata publication, directory +synchronization and a crash-safe replacement protocol. In-place updates can +destroy an older valid version on power failure. This change rejects detectable +damage rather than attempting speculative repair. Crash injection, snapshots, +membership changes and strong-read linearizability remain separate work. diff --git a/docs/proposal-admission.md b/docs/proposal-admission.md new file mode 100644 index 0000000..a6c5ba5 --- /dev/null +++ b/docs/proposal-admission.md @@ -0,0 +1,44 @@ +# Leader-only proposal admission: v0.1.0-alpha.20 + +TRaft::Append previously rejected learners but accepted proposals on voter +followers and candidates. This violated the log-matching assumption that a term +and index identify one command. A follower could append a local command in the +current term at the same index where its leader subsequently proposed another. +AppendEntries compares entry terms, so the follower retained its different +command and acknowledged it. Both commands could then be applied as committed. + +The deterministic three-node reproducer showed the leader applying `valid` and +a follower applying `rogue` at the same committed index/term. Ordinary client +requests through TRequestProcessor already route proposals through the leader; +the missing check affected callers using the public low-level Append API. + +Append now requires the locally known LEADER state as well as a voter role. +Nonleaders receive std::logic_error before entry validation, term assignment or +storage mutation. This includes a former leader after a higher-term message has +made it step down. A candidate's self-vote does not permit proposals. Existing +leader-side frame validation remains in place, and incoming replication still +uses the storage backend through the validated AppendEntries path. + +Two regressions cover both state preservation and the observed consensus failure. +The first rejects initial-follower, candidate and stepped-down-leader calls, +checking all stored fields and the supplied entry term remain unchanged. The +second attempts the conflicting follower proposal, commits a valid leader write +under duplicate/reordered delivery, and checks application histories, successful +client completion and identical committed prefixes on all three nodes. Both +failed on the old code; the convergence test observed `rogue` instead of `valid`. + +Validation: 230 unit tests and three CLI tests under ASan/UBSan across 18 CTest +targets, plus all 12 Release core targets. Self-review checked the guard's position +before mutation, all production Append call sites, learner admission, higher-term +stepdown and the distinction between local proposals and follower replication. +No independent reviewer approval is claimed. There are no wire or durable format +changes and no migration requirement. +Previously divergent logs are not detected or repaired by this admission guard. + +The role check is not a quorum lease: an isolated leader may still accept entries +that remain uncommitted, and a successful Append return is only a local log index. +Callers must serialize Raft access and wait for committed application/client +completion as appropriate. Directly editing IState or forcing states through the +test-facing Become API is outside proposal admission. The current bounded +joint-consensus path is described in [live membership](configuration-entries.md); +leader removal and membership-aware snapshots remain open. diff --git a/docs/snapshot-committed-terms.md b/docs/snapshot-committed-terms.md new file mode 100644 index 0000000..c196990 --- /dev/null +++ b/docs/snapshot-committed-terms.md @@ -0,0 +1,29 @@ +# Snapshot boundary terms and committed prefixes + +An InstallSnapshot request could previously advertise a snapshot at an index +after the follower's committed prefix but with an older boundary term. Such a +history cannot be a Raft log: entry terms do not decrease as indices increase. +The follower could accept that request, update its current term, replace its +snapshot and application state, and acknowledge the impossible boundary. + +Incoming snapshot requests are now checked before term or role changes. A +snapshot after the committed prefix must have a boundary term at least as large +as the term at the follower's commit index. If the requested boundary itself is +committed and still known locally, its term must match. A boundary older than the +local snapshot cannot be checked for equality because its individual term has been +compacted, but its term must not exceed the local snapshot term. Invalid requests +are dropped, consistent with other malformed RPCs. +Each chunk is checked again, including when commitment advances mid-transfer. + +Deterministic tests reproduce the prior mutation with both retained and compacted +committed prefixes, check higher-term invalid requests leave hard state and the +application unchanged, exercise a commit advancing during transfer, and confirm a +valid later snapshot still installs. This check detects these specific committed +term contradictions; it does not authenticate peers or prove snapshot contents +match the committed application state. Membership changes and joint snapshot +installation remain disabled. No wire or storage format changes. + +The in-memory and durable storage APIs also reject a replacement snapshot whose +term is below the existing compacted boundary term. This protects direct storage +callers even when they bypass RPC validation. Rejection leaves the selected +snapshot, retained suffix and durable image unchanged. diff --git a/docs/snapshot-contact-validation.md b/docs/snapshot-contact-validation.md new file mode 100644 index 0000000..866b198 --- /dev/null +++ b/docs/snapshot-contact-validation.md @@ -0,0 +1,29 @@ +# InstallSnapshot leader-contact validation + +InstallSnapshot previously demoted a candidate and refreshed its election timer +as soon as a structurally valid RPC arrived. The runtime only discovered that a +same-term image used an unsupported format or conflicting metadata after those +state changes. A peer sending an unusable snapshot could therefore delay a new +election without providing a usable recovery image. + +The receiver now accumulates the bounded metadata prefix before treating a +same-term InstallSnapshot as leader contact. Partial prefixes may be retained +and acknowledged so chunked transfer can continue, but they do not change the +Raft role, leader ID or election deadline. Unknown formats and invalid metadata +are rejected without those changes. Once the prefix matches a supported fixed +or stable image, normal transfer handling refreshes leader contact. Higher-term +requests retain the existing preflight rule: their first chunk must contain a +complete supported metadata prefix before hard state advances. + +Deterministic regression coverage checks an unsupported joint image split across +chunks while the receiver is a candidate, both when its boundary is already +covered and when it would require full installation. The first partial prefix +does not demote it or move its deadline; the chunk that identifies the unsupported +format is rejected with the same state preserved. Existing valid split-prefix, +snapshot installation, stale-term and higher-term preflight tests continue to +cover the accepted paths. No wire or storage format changed. + +This validates format and boundary metadata before recognizing leader contact; +the complete image CRC and application payload are still checked only when an +image is installed. It does not authenticate peers or enable joint snapshot +installation or live membership changes. diff --git a/docs/snapshot-metadata-term-preflight.md b/docs/snapshot-metadata-term-preflight.md new file mode 100644 index 0000000..c9cde1a --- /dev/null +++ b/docs/snapshot-metadata-term-preflight.md @@ -0,0 +1,26 @@ +# Snapshot metadata term preflight: v0.1.0-alpha.49 + +Alpha.48 checked a higher-term InstallSnapshot's magic before accepting its +term. That left one metadata gap: MRFTS001 or MRFTS003 could have a boundary, +voter list, or stable configuration identity that the receiver would later +reject, after its hard state had already advanced. + +The higher-term preflight now uses the bounded metadata probe before term +processing. The initial chunk must contain the full format header and voter +list. The probe checks that the embedded index and term match the RPC, voter +count and IDs match the receiver's fixed membership, the encoded lengths fit the +declared total, and any existing stable configuration identity is preserved. +The largest metadata prefix is 16,436 bytes, within the sender's 64 KiB initial +chunk. Payload bytes and CRC remain the full install path's responsibility. + +Malformed or conflicting metadata is rejected before changing the term, vote, +or leader state. Same-term handling remains unchanged. This does not install +joint snapshots or enable voter membership changes; those still require +transition-aware recovery and quorum rules. + +Regression tests cover higher-term joint images, wrong voters, conflicting +stable identities, short or continued chunks, and same-term rejection. Release +and ASAN network-enabled CTest passed 21/21 each, and hosted build, Debug, +Release, sanitizer, and unit-test checks passed. + +Self-review completed. No independent reviewer approval is claimed. diff --git a/docs/snapshot-pin-budget.md b/docs/snapshot-pin-budget.md new file mode 100644 index 0000000..cdde3ba --- /dev/null +++ b/docs/snapshot-pin-budget.md @@ -0,0 +1,34 @@ +# Bound pinned outgoing snapshot generations: v0.1.0-alpha.43 + +Per-peer transfer pinning guarantees that periodic compaction cannot continually +restart an in-flight InstallSnapshot. However, different peers can be transferring +different generations. A lagging peer can retain an older image while another +peer pins a newer one, and unbounded compaction can keep adding pinned generations. + +The leader now allows at most two distinct snapshot generations to remain pinned +by outgoing transfers. `TRaft::Compact` checks this before capturing or changing +storage. At the cap it throws a transient `std::runtime_error` without changing +the application, log, snapshot boundary or active transfers. Callers can retry +after a transfer completes; the KV checkpoint loop already retries failed +compaction attempts while the applied-index threshold remains met. Term changes +and target removal continue to release their corresponding transfer contexts. + +After a successful compaction, the leader drops its standalone encoded-snapshot +cache. Active transfers keep their own shared reference to the immutable bytes, +so they continue unchanged; a later transfer encodes the newer local image. +Transfers of the same image still share the same snapshot and encoding. + +A deterministic test pins two different generations, verifies that compaction +fails without mutation, completes one transfer and verifies that compaction then +succeeds while the other peer still owns its older image. Existing tests continue +to verify catch-up progress under compaction during every chunk. + +This caps the number of distinct active transfer generations, not total process +heap usage. Current application/log state, snapshot capture and installation, +transport buffers and other allocations remain outside this limit; a global byte +quota and disk-streamed snapshot transfer remain open. Voter membership changes +and joint snapshot installation are also still unsupported. + +Self-review checked the two-generation accounting, exception ordering before +state mutation, cache ownership and retry path. No independent reviewer approval +is claimed. diff --git a/docs/snapshot-pinning.md b/docs/snapshot-pinning.md new file mode 100644 index 0000000..6d33c59 --- /dev/null +++ b/docs/snapshot-pinning.md @@ -0,0 +1,64 @@ +# Snapshot transfer pinning: v0.1.0-alpha.19 + +Periodic compaction could starve snapshot catch-up even after alpha.18 fixed +retry correlation. Each send selected the leader's newest snapshot and reset the +peer's transfer if its index changed. When compaction outpaced a multi-chunk +transfer, the receiver repeatedly started new images without completing one. +This affected both voters and non-voting learners. + +Each peer now pins the immutable image and encoded bytes selected at the start +of its transfer. New local snapshots do not change its index, offset, content or +outstanding chunk identity. A completed transfer releases that image. If the +leader compacted past the receiver's next index, the next transfer starts from +the newest available image; otherwise ordinary AppendEntries supplies the tail. +An ACK for the older image proves only that image's boundary, never the newer +prefix. Snapshot ACKs still cannot confirm strong-read barriers. + +Failed chunks restart the pinned image from offset zero. Leadership changes still +discard outgoing transfer contexts. One image/encoding pair is retained per +active peer transfer, with sharing between transfers of the same image. The +[alpha.43 pin budget](snapshot-pin-budget.md) later capped active retention to two +distinct image generations and defers further compaction until a transfer +completes. It does not add a byte quota for all Raft, application, transport and +temporary snapshot memory, or a general transfer cancellation API. + +A deterministic regression commits and compacts new data before every chunk ACK, +including the final ACK, for both voter and learner receivers. The old code fails +on the first replacement (snapshot 5 instead of 4). The fixed code completes the +original image without resetting offsets, releases its snapshot object, then +installs the newer image and replicates a committed tail. Existing tests cover +timeouts, read traffic, duplicate/stale replies, receiver restart, leader change, +corruption, atomic durable publication and strong-read isolation. + +Sanitizer CI also exposed a second reconnect deadline race after alpha.18's +generation checks. The dependency's finite-deadline connect registers both I/O +and timer wakeups. After timeout, a late I/O event can resume the retry sleep; +that sleep's timer may remain queued even after the connector finishes. Destroying +the completed connector then leaves a timer pointing to freed memory. Preserving +active connectors alone cannot prevent this case. + +MiniRaft now calls the dependency's default connect operation, which registers no +connect deadline timer. TCP connection failure timing follows the operating +system, so a failed or blackholed connection may take longer than the former +100 ms application deadline. A safe bounded application-level connect deadline, +including cancellation of both wakeup sources, remains open. The coroio dependency +is unchanged. This is a targeted avoidance of the unsafe deadline path, not a +general asynchronous cancellation or shutdown implementation. + +A deterministic lifecycle regression models a second deadline wakeup surviving +completion and disconnect; it reproduced the ASan use-after-free with the former +finite-deadline call and passes with the default connect operation. + +Validation: 228 unit tests and three CLI tests across 18 CTest targets under +ASan/UBSan, plus 12 Release core targets. Self-review checked image ownership, +encoding reuse, completed-boundary accounting, follow-up transfer selection and +term replacement, connection wakeup sources and deadline behavior. No independent +reviewer approval is claimed. Wire messages and +durable formats are unchanged; no additional migration is required. + +This guarantees progress of the selected finite image under eventual chunk +delivery and ACKs. Reaching the live log end still requires enough replication +capacity relative to writes/compaction and sufficiently stable leadership. +The bounded promotion and joint-consensus path is described in +[live membership](configuration-entries.md). Membership-aware snapshots and +leader removal remain open. diff --git a/docs/snapshot-retries.md b/docs/snapshot-retries.md new file mode 100644 index 0000000..878cbce --- /dev/null +++ b/docs/snapshot-retries.md @@ -0,0 +1,64 @@ +# Snapshot retry progress: v0.1.0-alpha.18 + +Follow-up: partial acknowledgements must report exactly the end of the +outstanding chunk. Previously a larger offset could skip unsent bytes. An +invalid acknowledgement now leaves the chunk outstanding for retry; a +deterministic regression verifies retry and eventual installation. + +The [alpha.19 follow-up](snapshot-pinning.md) also pins an active transfer across +local compaction, replacing the image-switching behavior described below. + +Timeout retries and strong-read requests could indefinitely stall InstallSnapshot +catch-up for a healthy follower or learner. Each send replaced the outstanding +chunk's sequence number. If an ACK arrived after another send, the leader rejected +it, retransmitted the same offset with another sequence number, and repeated the +cycle. This could happen without any corrupt messages or failed storage. + +Retries now preserve the identity of the same immutable chunk. A valid partial +ACK consumes that identity; the next chunk receives a fresh sequence number. A +failure also consumes it before restarting from offset zero. Duplicate ACKs stay +stale both before and after the next send. A changed snapshot or leadership term +continues to discard the old transfer context. Invalid replies do not consume the +outstanding identity. + +This reuse applies only to snapshot chunks. Snapshot ACKs still cannot confirm a +strong-read barrier; a subsequent fresh AppendEntries ACK is required. No wire +layout, durable format, role configuration or migration procedure changes. + +Two deterministic regressions run against both voter and learner receivers. Each +ACK is held while the leader emits three retries, driven by either timeouts or +repeated ApproveRead calls; retry packets are dropped and the original ACK is +then delivered. Before the fix both tests failed at the first 64 KiB boundary: +the leader retransmitted offset zero instead of advancing to 65536. After the fix +the multi-chunk image completes, the application loads once, duplicate ACKs do +not consume subsequent chunks, and a new committed tail reaches the receiver. + +The release also fixes a reconnect use-after-free discovered by the sanitizer CI +snapshot CLI test. A writer could start reconnecting before the old reader +reported its failure. Disconnecting that reader destroyed the pending connector, +but its socket connect timer still held the coroutine handle. When the timer +fired, it resumed freed memory. Disconnects now identify the reader's connection +generation and preserve active connectors; an old reader cannot destroy a newer +pending or established connection. This is not a general asynchronous shutdown +or cancellation API. + +Two deterministic node lifecycle tests use the real node wrapper and poller with +a controllable socket awaiter. They cover disconnect during a pending connect +timer and an old reader failure racing a writer-triggered reconnect, both before +and after replacement establishment. The first reproduced the same ASan +use-after-free before the fix. Node template definitions are now in node_impl.h +so alternate sockets can exercise the production implementation without real +network timing. The coroio dependency is unchanged. + +The full validation set is 226 unit tests plus three CLI tests across 18 CTest +targets under ASan/UBSan, with 12 Release core targets. Existing tests cover +partial-transfer restart, stale replies after a reset, leader replacement, +duplicate installation, corruption, durable publication crashes and strong-read +isolation. Self-review checked connection generation ownership and outstanding +connector lifetime as well as chunk identity lifetime. No independent approval +is claimed. + +Snapshot catch-up is a prerequisite for promotion, not authorization by itself. +The bounded durable joint-consensus API is described in +[live membership](configuration-entries.md); leader removal and +membership-aware snapshot transfer remain open. diff --git a/docs/snapshot-transfer-expiry.md b/docs/snapshot-transfer-expiry.md new file mode 100644 index 0000000..a42fc55 --- /dev/null +++ b/docs/snapshot-transfer-expiry.md @@ -0,0 +1,29 @@ +# Idle InstallSnapshot transfer expiry: v0.1.0-alpha.42 + +InstallSnapshot is chunked and bounded to 64 MiB per image, but a receiver held +the accumulated partial image until a replacement transfer, a completed +transfer, or a voter election timeout. Learners do not campaign, so a learner +could retain that allocation indefinitely after an interrupted transfer. Normal +heartbeats could also keep a voter from reaching its election timeout while an +image remained abandoned. + +Incoming snapshot state now records the time of the last valid chunk. The +receiver discards an incomplete image after 30 seconds without a valid chunk, +regardless of role or heartbeat traffic. Matching retries count as activity so +lost acknowledgements remain safe; malformed, conflicting, and out-of-order +chunks do not extend the deadline. A continuation after expiry receives a +failure with offset zero, which tells the sender to restart the pinned image +from its first chunk. Completed installations and the covered-prefix metadata +fast path release the partial state as before. + +A deterministic learner test starts a partial transfer, continues ordinary +leader heartbeats, advances the injected clock beyond the idle limit, and +verifies that the old continuation is rejected and a fresh transfer succeeds. +The test also verifies the application and Raft snapshot boundary are unchanged +until the fresh image completes. + +Self-review checked both receiving paths, duplicate chunk handling, role +independence and restart-from-zero behavior. This bounds the lifetime of one +partial in-memory image; it does not impose a cluster-wide memory quota or stream +snapshot content to disk. Live voter reconfiguration and joint snapshot adoption +remain separate open features. diff --git a/docs/snapshots.md b/docs/snapshots.md new file mode 100644 index 0000000..1c7b13c --- /dev/null +++ b/docs/snapshots.md @@ -0,0 +1,126 @@ +# Snapshot foundation: durable KV checkpoints, v0.1.0-alpha.15 + +The [alpha.16 follow-up](compaction.md) adds compaction and InstallSnapshot and +changes the behavior of the same `--snapshots` option. This document describes +the original checkpoint-only baseline. + +This milestone adds local application checkpoint creation and restart recovery. +It retains the entire Raft log. InstallSnapshot RPCs and prefix compaction remain +unimplemented, so checkpoints improve application recovery but do not bound log +storage or bootstrap a node whose local log is missing. + +## Use + +On Linux, start a KV server with its existing fixed node list and add: + +```sh +./build/full/examples/kv --server --id 1 --node 127.0.0.1:8001:1 \ + --durable --snapshots 1000 +``` + +Use the complete, identical voter-ID configuration on each node as before. +`--snapshots N` requires a positive interval and `--server --durable`; it is off +by default. A one-second polling coroutine captures a checkpoint after at least +N additional applied log indices, including internal no-ops. Creation, encoding +and publication execute synchronously on the serialized event loop. Large maps +and slow storage can pause processing; asynchronous/COW capture is future work. + +The server writes `state.snapshot.` beside `state.durable.`, with separate +`.lock` and `.tmp` paths. Startup restores an existing checkpoint before creating +the request processor or accepting traffic. The recovered application marker +establishes the existing commit floor, and later commitment replays only the tail. +Without a checkpoint, the application rebuilds from the retained log. Disabling +the option also rebuilds from that log, leaving checkpoint files unused. + +## Contracts and storage + +IRsm gains optional SaveSnapshot/LoadSnapshot callbacks; the defaults throw an +unsupported-operation error. KV implements canonical, sorted, binary-safe state +encoding. Restore validates the entire image into a temporary map before swapping +state and progress. Key/value lengths retain the KV wire format's 65,535-byte +limits. SQL snapshot support is described in [its follow-up](sql-snapshots.md); +the dummy application still does not implement snapshots. + +CaptureSnapshot requires a nonzero applied index no greater than the known commit +index or retained log end. Metadata records its index, term and sorted fixed voter +IDs. RestoreSnapshot requires an application behind the snapshot and an exact voter-set match; +the boundary index must exist in the local log with the same term, not newer than +CurrentTerm. Startup restore can recover a process stopped after durable Raft +snapshot publication but before application replacement. Restore does not +overwrite current term or votedFor. These APIs must run serialized, with restore +only during startup before request processing. + +The versioned image contains little-endian fields in this order: + +| Field | Bytes | +| --- | ---: | +| Magic/version `MRFTS001` | 8 | +| Last included index and term | 8 each | +| Voter count | 4 | +| Application byte length | 8 | +| Sorted unique positive voter IDs | 4 each | +| Application bytes | declared length | +| CRC32 covering every preceding byte | 4 | + +The complete image is capped at 64 MiB and the voter count at 4,096. Decoding +checks lengths/counts against remaining input before allocation/copying, rejects +unknown versions and trailing bytes, and validates metadata after CRC verification. +The KV payload has its own `MRKV0001` version, count and length-prefixed entries. + +TSnapshotStore takes an exclusive advisory writer lock and publishes through temp +creation, complete write, file fsync, atomic rename and directory fsync. Startup +stabilizes the selected image/name with fsync. Temp files are never recovery +candidates. A missing published image is safe because the complete log remains; +an existing corrupt or mismatched image fails startup without speculative repair. +The directory and local writers are trusted; CRC32 detects damage, not forgery or +wrong-store copies with otherwise matching boundary metadata and voter IDs. + +Snapshot-save failures are catchable and logged/retried by the example. No Raft +entry is discarded and the selected checkpoint is a complete committed image, +so consensus can continue using its retained log. A failure around rename can +leave either complete checkpoint selected after a crash. Never delete Raft term, +vote or log files to work around a checkpoint recovery error. + +## Validation and review + +Eleven new unit tests cover codec round trips, every-byte corruption/truncation, +semantic corruption with a valid checksum, bounded file reads, temp symlink and +concurrent-writer rejection, canonical binary KV encoding, atomic malformed KV +restore rejection, capture before commitment, log/configuration mismatch, +publication errors, crashes at all five publication stages on first/replacement +save, and durable Raft/KV restart with tail-only application. Crash tests use +explicit stage hooks and subprocess exit, with no timing-dependent schedules. + +A separate real-process CLI smoke test covers option validation, publication, +restart and strong read, changed voter IDs, corrupt checkpoint rejection and +unchanged Raft storage. It uses bounded waits/sockets and is distinct from the +deterministic fault tests. Linux full CTest runs it when Python 3 is available. +All 192 unit tests and the CLI test pass locally across 13 CTest targets under +ASan/UBSan; the Release core suite passes too. + +Self-review checked serialized capture boundaries, metadata and allocation limits, +checksum coverage, version rejection, atomic map replacement, restore before +commit-floor initialization, fsync/rename order, publication failure semantics, +and retention of all logs. No independent reviewer approval is claimed. Process +crash injection does not substitute for hardware power-cut/storage fault testing. + +## Next snapshot and membership milestones + +1. Introduce snapshot-boundary-aware log indexing and atomic durable publication + of snapshot plus retained suffix. Validate votes, AppendEntries consistency, + current-term commitment and strong reads at the compacted boundary. +2. Add bounded, resumable InstallSnapshot transfer. Test interrupted/reordered/ + duplicate chunks, leadership changes, concurrent tail replication, stale + snapshots and follower restart. Enable prefix deletion only once lagging + followers have a verified recovery path. +3. Add non-voting learners and catch-up gates, then replicated configuration + entries with joint consensus. Elections, commits and strong-read confirmation + must honor both old and new majorities during transition. Persist configuration + state in logs/snapshots and test rollback, restart and leader removal. + +These follow the snapshot metadata/state-transfer requirements in +[Raft section 7](https://raft.github.io/raft.pdf) and the overlapping-quorum +configuration protocol in section 6. Editing the static `--node` list is not a +membership-change protocol. Snapshots do not make such edits safe. This release +keeps fixed membership. Background capture and independent review remain open +release gates. diff --git a/docs/sql-application.md b/docs/sql-application.md new file mode 100644 index 0000000..ed102a1 --- /dev/null +++ b/docs/sql-application.md @@ -0,0 +1,64 @@ +# SQL transaction and progress handling: v0.1.0-alpha.11 + +The previous SQL write path concatenated client text, progress metadata and COMMIT +into one script. Client COMMIT/END could publish a prefix before a later statement +failed; metadata update errors were ignored on the failure path. Rejected SQL +also left LastAppliedIndex stale in memory even when its persisted marker advanced. +Restart used stoi, rejecting indices above INT_MAX and accepting malformed numeric +prefixes. Regression tests reproduced these failures before the fix; the old test +run aborted when the full-width-index restart threw out_of_range. + +Writes now begin an IMMEDIATE transaction and execute the client script separately +under a SQLite authorizer. The policy blocks transaction/savepoint controls, +PRAGMAs, ATTACH/DETACH, temporary objects, virtual-table creation/deletion and +ALTER TABLE. It permits +ordinary main-database DML, table/index/view/trigger creation and deletion, REINDEX, +ANALYZE and the same function allowlist as reads. This deliberately narrows the +previous unrestricted SQL interface. Scripts may still contain multiple ordinary +statements, and comments cannot swallow internal metadata SQL. + +Payload access to raft_metadata_ is denied, including through compiled views and +triggers. Payloads cannot create triggers or indexes on that table. The authorizer +is removed before the application's own metadata and transaction statements run. +The policy stays active throughout payload execution and automatic recompilation. +See SQLite's [authorizer contract](https://www.sqlite.org/c3ref/set_authorizer.html). + +Successful data changes and their progress marker commit in the same transaction. +SQLITE_ERROR, AUTH, CONSTRAINT and MISMATCH are treated as application rejections: +all payload effects are rolled back, then a checked transaction records that the +entry was consumed. The original SQL error text is returned using the existing +write reply format. Memory advances only after COMMIT succeeds. An already applied +index is skipped; original replies are not persisted for retry deduplication. + +Other execution errors, and any BEGIN, rollback, metadata or COMMIT failure, throw +without advancing memory. The committed-entry handler terminates on such an +exception, allowing restart/replay rather than acknowledging uncertain application. +A scope guard attempts rollback during unwinding. SQLite can itself roll back a +transaction, so cleanup checks [autocommit state](https://www.sqlite.org/c3ref/get_autocommit.html) +before issuing ROLLBACK. A crash between rolling back a rejected payload and +recording its marker can replay that rejection; no payload effects survived. + +Startup checks initialization and progress queries, closes the database on +constructor failure, and parses the complete stored text as uint64_t. NULL, empty, +negative, overflowing or partially numeric markers (including embedded NULs) are +rejected. An absent marker continues to mean zero for legacy empty databases; +this does not detect a marker deleted by external tampering or previous releases. + +Ten new tests cover transaction escape, direct/indirect metadata access, rollback +and restart, trailing comments, OR ROLLBACK, metadata-update failure, full-width +indices, malformed metadata, lock failures at BEGIN and COMMIT, and NUL rejection +before admission/preparation. All 162 tests across 11 CTest targets pass locally +with ASan/UBSan and SQLite; the Release core suite passes as well. + +Self-review checked authorizer arguments and lifetime, separate internal SQL, +rollback/automatic-rollback behavior, commit ordering, error classification, +constructor ownership and bounded full-width parsing. No independent reviewer +approval is claimed. These are deterministic logical/locking failure tests, +not new power-loss or filesystem fault-injection coverage for SQLite. + +This remains an experimental example using trusted SQLite builds/modules and a +private serialized connection. Existing unsafe schemas or legacy log scripts need +review before migration. Arbitrary extensions/virtual tables are not sandboxed. +Deterministic SQL across replicas, SQLite durability configuration, schema/metadata +migration and external modification, result/CPU quotas, authentication, retry +deduplication and independent review remain open production gates. diff --git a/docs/sql-read-budgets.md b/docs/sql-read-budgets.md new file mode 100644 index 0000000..dc9d951 --- /dev/null +++ b/docs/sql-read-budgets.md @@ -0,0 +1,57 @@ +# SQL read budgets: v0.1.0-alpha.12 + +SQL reads previously had no execution budget and collected all rows before +serializing and checking the response frame size. Even a small recursive query +could block the single event loop indefinitely; a large result could exhaust +memory before the final size check. Six regression tests failed with the previous +read implementation (with the new limit configuration accepted but unenforced). + +TSql now accepts an optional third constructor argument, TSqlReadLimits. Defaults +apply to the existing SQL server without CLI changes: + +* MaxVmSteps defaults to 1,000,000. SQLite progress callbacks interrupt long reads + during preparation or stepping. This is an approximate instruction budget, + checked at intervals of at most 1,000 VM instructions, not a wall-clock deadline + or an exact cross-version instruction counter. Zero is rejected. +* MaxResultBytes defaults to the existing 16 MiB frame cap minus the response + header. A single serialized buffer is built while stepping; headers, commas, + NULL markers, data bytes and newlines count toward the limit before each append. + Partial output is discarded on failure. A smaller nonnegative limit is allowed; + exceeding the response payload cap is rejected at construction. +* SQLITE_LIMIT_LENGTH is temporarily lowered to 16 MiB for reads, restricting + individual strings, blobs and encoded rows before application copying. The + previous connection limit is restored afterward. Oversized stored rows may + therefore fail reads even when the requested output is small. + +Prepared statements finalize before the authorizer, progress callback and length +limit are removed/restored, on both success and exceptions. Writes retain their +existing execution policy and length limit. Rejected reads do not change applied +progress. The existing request processor maps read exceptions to ErrorCode=2 and +preserves client cookies; subsequent requests continue normally. + +Removing the intermediate row collection also fixes silent truncation at embedded +NUL bytes: value serialization uses SQLite's byte length after text conversion. +The reply remains the example's simple comma/newline format, including a single +newline for an empty result. It does not add CSV escaping or typed blob encoding. + +Seven new tests cover finite expensive aggregate queries with explicit/default +budgets, exact output boundaries (including empty results), accumulated rows, +recovery through ordinary/strong request processing, binary text lengths, invalid +configuration and restoration of SQLite's value limit before writes. Tests use +SQL computation and fixed byte limits; they need no timing assumptions or sleeps. +All 169 tests across 11 CTest targets pass locally with ASan/UBSan and SQLite; +the Release core suite passes as well. + +Self-review checked callback state and lifetime, constructor validation, cleanup +order, subtraction before append, NULL versus conversion failure, SQLite text/byte +conversion order and the unchanged write path. No independent reviewer approval +is claimed. Relevant SQLite contracts are [progress callbacks](https://www.sqlite.org/c3ref/progress_handler.html), +[runtime limits](https://www.sqlite.org/c3ref/limit.html), and +[column conversion and byte lengths](https://www.sqlite.org/c3ref/column_blob.html). + +This closes unbounded read VM execution and application result accumulation, not +all resource-exhaustion risks. A VM instruction or built-in function can itself +be expensive; temporary sorting/storage, preparation memory, total SQLite heap, +database size and committed-write execution need separate controls. There is no +hard CPU-time or process-memory bound. Authentication, admission/backpressure, +deterministic SQL, SQLite durability policy and independent review remain open. diff --git a/docs/sql-readonly.md b/docs/sql-readonly.md new file mode 100644 index 0000000..088f66d --- /dev/null +++ b/docs/sql-readonly.md @@ -0,0 +1,39 @@ +# SQL read policy: v0.1.0-alpha.10 + +The SQL example previously executed read requests through unrestricted +sqlite3_exec. A client could submit UPDATE, schema changes or multiple statements +as a read and mutate one replica without a Raft entry. Six new regression tests +failed against that implementation; the genuine-read positive test already passed. + +Reads now prepare exactly one statement under a deny-by-default SQLite authorizer. +The policy permits SELECT, column reads, recursive queries and an explicit list +of trusted built-in scalar/aggregate functions in sql_rsm.cpp. It rejects writes, +DDL, PRAGMA, transaction/savepoint controls, ATTACH/DETACH and unknown functions. +sqlite3_stmt_readonly provides an additional check. The entire trailing input is +prepared and checked before execution, so a second statement is rejected even +if it is another SELECT. Comments, trailing semicolons and semicolons within +strings work. Empty/comment-only input and embedded NUL bytes are rejected. + +The policy remains installed through stepping and finalization, including SQLite +automatic recompilation, and RAII removes it on success or failure. Read failures +use the existing ErrorCode=2 response path. Replicated writes retain their existing +execution path and continue working after rejected reads. + +SQLite documents why [stmt_readonly alone is insufficient](https://www.sqlite.org/c3ref/stmt_readonly.html) +and how [authorizers apply during preparation and recompilation](https://www.sqlite.org/c3ref/set_authorizer.html). +This policy assumes the example's private connection and trusted SQLite built-ins +and modules. It is not a sandbox for arbitrary registered extensions or virtual +tables. Functions outside the explicit allowlist require a deliberate policy +change; previously accepted read scripts with multiple statements now fail. + +Seven tests cover DML/schema rejection, multi-statement rejection before mutation, +connection controls, unsafe functions, real SELECT/CTE behavior, malformed text +and restoration before a subsequent write. All 152 tests pass locally across 11 +CTest targets under ASan/UBSan; the Release core suite also passes with SQLite. +Self-review checked policy lifetime, preparation before stepping, function-name +handling, statement cleanup and unchanged replicated-write routing. No independent +reviewer approval is claimed. + +SQL write transaction escape, metadata/progress semantics, deterministic execution, +result memory/CPU quotas, authentication and retry deduplication remain open. +The project remains experimental and does not yet provide NuRaft-level assurance. diff --git a/docs/sql-snapshots.md b/docs/sql-snapshots.md new file mode 100644 index 0000000..fcc07d5 --- /dev/null +++ b/docs/sql-snapshots.md @@ -0,0 +1,74 @@ +# SQL application snapshots: v0.1.0-alpha.44 + +The [snapshot foundation](snapshots.md) and [compaction follow-up](compaction.md) +initially supported only the KV example. SQL now supplies the same application +snapshot callbacks, so a Linux SQL server can use the durable backend's +compaction and InstallSnapshot paths. + +## Use + +Start each SQL server with its unchanged fixed voter list and add +`--durable --snapshots N`: + +```sh +./build/full/examples/sql --server --id 1 --node 127.0.0.1:8001:1 \ + --durable --snapshots 1000 +``` + +`N` must be positive. The SQL CLI requires this option pair for periodic +compaction; snapshots remain off otherwise. Each node uses its own +`sql_state.durable.` Raft store and `sql_file.db.` SQLite database. +Every voter must run a binary that understands compacted durable images and +InstallSnapshot. Existing SQL and legacy Raft files remain in place, but old +binaries cannot read the compacted store. There is no automatic format +migration or mixed-version support. + +## Image and recovery contract + +The application image is captured with SQLite's backup API into an in-memory +database and then serialized. This includes committed pages still present in a +WAL file. Save requires a committed `LastAppliedIndex` marker and no open +transaction. The SQL payload is capped below 64 MiB to leave room for the Raft +snapshot header and maximum voter metadata. The decoder adds SQLite's recommended +read slack around malformed untrusted images. Before restore or installation, +SQL checks that the image can be opened, +passes `PRAGMA quick_check(1)`, and has exactly one valid progress marker equal +to the Raft snapshot boundary. WAL-mode backup bytes are normalized to SQLite's +standalone rollback-mode format before deserialization. The receiver also +requires the target database page size to match before Raft publishes the +snapshot durably. Invalid or incompatible images do not replace the live +database. + +Restore uses SQLite's backup API to copy the validated in-memory image into the +existing database connection. The application progress marker in memory moves +only after the backup completes. A failed backup leaves the destination +unchanged according to SQLite's backup transaction contract. Snapshot +installation first publishes the Raft snapshot and retained suffix durably, +then replaces the application database, then acknowledges installation. If +the process stops between the first two steps, startup replaces any SQL +checkpoint behind the durable Raft snapshot before request processing begins. + +This recovery only applies when the saved application index is below the +snapshot boundary. An application at or beyond the boundary is preserved; an +application behind the compacted boundary with no matching snapshot still +fails startup. The image does not make SQL files portable across arbitrary +SQLite versions or extensions. Use the same application schema, SQLite build +and database page size across a cluster. + +Serialization runs synchronously on the event loop and can pause request and +replication processing for large databases. This feature does not provide +background or copy-on-write capture, application database quotas below the +64 MiB image cap, or live voter changes. Fixed voter IDs must still match on +every node. + +## Validation + +Deterministic tests cover round-trip capture and restore, malformed/corrupt +images, wrong progress boundaries, preservation of existing state after +validation or backup failure, and restart recovery when durable Raft state is +ahead of the SQL database. The regular release and ASan/UBSan CTest suites +also exercise the library and examples. + +SQLite API details: [serialize](https://www.sqlite.org/c3ref/serialize.html), +[deserialize, including the WAL-mode restriction](https://www.sqlite.org/c3ref/deserialize.html), and +[backup completion](https://www.sqlite.org/c3ref/backup_finish.html). diff --git a/docs/stable-configuration-snapshot-runtime.md b/docs/stable-configuration-snapshot-runtime.md new file mode 100644 index 0000000..86e1cf7 --- /dev/null +++ b/docs/stable-configuration-snapshot-runtime.md @@ -0,0 +1,33 @@ +# Stable configuration snapshot runtime support: v0.1.0-alpha.46 + +The runtime now accepts MRFTS003 snapshots for the node's already configured +fixed voter set. Startup requires the snapshot voter list, supplied Raft peers, +and persisted voter binding to agree. Joint MRFTS002 snapshots remain rejected. + +With that same-set check, the existing snapshot lifecycle can restore a durable +snapshot into the application, receive and persist it over InstallSnapshot, and +compact later application progress. Compaction carries the prior stable +configuration entry index and term into the new image. A follower whose committed +prefix already covers a stable snapshot validates the RPC boundary, voter list, +configuration identity and encoded data length before acknowledging it. + +Once a node has a stable configuration identity, installing a newer MRFTS001 +snapshot carries that identity into the stored MRFTS003 image. An incoming +MRFTS003 image with a different identity is rejected, including on the +already-covered fast path. This prevents a legacy-format refresh from silently +erasing configuration history and prevents a peer from replacing it with a +conflicting baseline. + +The original voter binding still identifies the trusted initial set. The live +joint-consensus log protocol can move the active set without rewriting that +binding, but the snapshot runtime here still requires snapshot voters to match +the configured set. Joint snapshot installation and snapshot transfer across a +membership transition remain open. Upgrade all voters before sending MRFTS003 +images because older binaries may acknowledge an already-covered unknown +snapshot format. + +Tests cover durable application restore, identity preservation through +compaction, restart, and legacy-format refresh, chunked InstallSnapshot, +covered-prefix validation of stable metadata, and rejection of conflicting +identities and joint snapshots. Self-review completed; no independent reviewer +approval is claimed. diff --git a/docs/stable-configuration-snapshots.md b/docs/stable-configuration-snapshots.md new file mode 100644 index 0000000..8d61382 --- /dev/null +++ b/docs/stable-configuration-snapshots.md @@ -0,0 +1,39 @@ +# Stable configuration snapshot metadata + +The experimental MRFTS003 snapshot format preserves a stable voter set and the +index and term of the configuration entry that selected it. This fills a gap in +MRFTS001: its voter list does not establish the identity needed to validate a +later configuration entry. MRFTS003 stores magic[8], included index[8], included +term[8], voter count[4], application length[8], configuration index[8], +configuration term[8], voter IDs[4 each], application data, and CRC32[4]. The +configuration identity must be nonzero, no later than the snapshot boundary, and +have the same term when both indices coincide. Voters are sorted and bounded. +MRFTS001 and MRFTS002 encoding is unchanged. + +`TConfigurationHistory::FromStableSnapshot` initializes offline history from +this identity. The retained-log scanner checks the baseline identity against +the snapshot before checking subsequent joint/stable entries. These checks do +not prove commitment or authenticate the snapshot: the caller must establish a +trusted snapshot and commit boundary independently. MRFTS003 cannot assert that +there was no earlier configuration entry; an explicit trusted baseline remains +necessary for such histories. + +The live runtime accepts MRFTS003 only when its voter list matches the node's +configured fixed voter set and persisted binding. It can recover the image at +startup, restore the application, install it from a peer, and preserve its +configuration identity through later compaction. The covered-snapshot path +checks the stable configuration identity and voter list before acknowledging an +already-covered boundary. These paths do not activate a changed voter set or +enable joint consensus. + +Since alpha.58, bound in-memory and durable storage can also retain an +MRFTS003/MRFTS002 image whose active set differs from the original persisted +binding, for offline history validation. That storage support does not make the +image acceptable to live Raft or authenticate its membership claim; callers must +provide a trusted history baseline. + +Older binaries may falsely acknowledge covered unknown formats; do not send these +images to a mixed-version cluster. Joint snapshot installation, live +configuration activation, rollback, commit rules and membership recovery remain +open. This experimental format may change before those semantics are implemented. +Rebuild dependents because the in-memory snapshot structure has changed. diff --git a/docs/stable-snapshot-checkpoint-continuity.md b/docs/stable-snapshot-checkpoint-continuity.md new file mode 100644 index 0000000..35df207 --- /dev/null +++ b/docs/stable-snapshot-checkpoint-continuity.md @@ -0,0 +1,21 @@ +# Stable identity checks for checkpoint restore: v0.1.0-alpha.55 + +`RestoreSnapshot` can load a standalone checkpoint into the application when its +index and term match the local retained log. It previously checked the fixed +voter list but did not compare an MRFTS003 checkpoint's stable configuration +identity with the authoritative snapshot already stored by Raft. A checkpoint +with a conflicting configuration index or term could therefore replace +application state while Raft retained a different configuration baseline. + +Restore now rejects an explicit stable identity that conflicts with the local +authoritative snapshot before validating or loading application data. Legacy +checkpoints remain usable because they make no stable identity claim. Joint +snapshots remain rejected by the fixed-membership runtime. + +The regression test uses a checkpoint whose index, term and voter set match the +retained state but whose stable identity conflicts. It verifies rejection leaves +the application unchanged, then verifies a legacy checkpoint at the same +boundary can still restore. + +Release and ASan/UBSan network-enabled CTest each passed 21/21. This change is +self-reviewed; no independent reviewer approval is claimed. diff --git a/docs/stable-snapshot-metadata-continuity.md b/docs/stable-snapshot-metadata-continuity.md new file mode 100644 index 0000000..76a1f29 --- /dev/null +++ b/docs/stable-snapshot-metadata-continuity.md @@ -0,0 +1,18 @@ +# Stable snapshot metadata continuity: v0.1.0-alpha.47 + +A node that already stores an MRFTS003 snapshot has a stable configuration +identity tied to its persisted fixed voter set. A newer incoming MRFTS001 image +does not include that identity. The receiver now carries its trusted identity +forward before persisting the new image, so legacy-format transfer cannot erase +the baseline needed by recovery and later compaction. + +Incoming MRFTS003 snapshots must retain the known configuration index and term. +The receiver rejects conflicting identities both when installing a newer image +and when acknowledging an image whose boundary is already covered. These checks +do not enable membership changes through snapshots: the node still requires the +same configured voters, and joint snapshots remain unsupported. Membership now +changes through the separate configuration log protocol. + +The focused compaction test covers a covered-boundary conflict, a legacy-format +refresh followed by restart, and a conflicting newer stable snapshot. The change +was self-reviewed; no independent reviewer approval is claimed. diff --git a/docs/stable-snapshot-storage-continuity.md b/docs/stable-snapshot-storage-continuity.md new file mode 100644 index 0000000..4c06da9 --- /dev/null +++ b/docs/stable-snapshot-storage-continuity.md @@ -0,0 +1,29 @@ +# Stable snapshot storage continuity: v0.1.0-alpha.54 + +The stable snapshot identity checks previously lived in Raft's +`InstallSnapshot` receiver. Callers that used `TState::InstallSnapshot` or +`TDurableState::InstallSnapshot` directly could replace a known MRFTS003 +configuration identity with another stable identity, or refresh it with an +MRFTS001 image that erased the identity. + +The storage backends now apply the same continuity rule as the receiver. A +legacy refresh inherits the previously stored stable configuration index and +term before it is published. A stable image with a different identity is +rejected before the in-memory state or durable file changes. The receiver and +both backends share one helper, so future callers cannot bypass the rule by +installing snapshots below Raft. + +This preserves only the fixed-membership identity already trusted by storage. It +does not enable joint snapshot recovery, live membership changes, or membership +activation from an untrusted snapshot. Joint images remain explicit metadata and +are still available to the existing offline durable format tests. + +The [alpha.59 follow-up](configuration-snapshot-continuity.md) extends this +continuity check to active voter sets, stable-to-joint transitions and joint +snapshot recovery, while retaining the same offline-only trust boundary. + +Tests install a legacy refresh and attempt a conflicting stable replacement +through both memory and durable storage. They check metadata preservation, +rejection atomicity, and durable recovery after restart. Release and ASan/UBSan +network-enabled CTest passed 21/21 each. This change was self-reviewed; no +independent reviewer approval is claimed. diff --git a/docs/strong-reads.md b/docs/strong-reads.md new file mode 100644 index 0000000..0b9cfcf --- /dev/null +++ b/docs/strong-reads.md @@ -0,0 +1,88 @@ +# Fresh-quorum strong reads: v0.1.0-alpha.4 + +This milestone fixes a reproduced stale-leader read bug in EConsistent and the +KV example's --consistent mode. Default reads and explicit stale reads keep +their previous weaker semantics. + +## Reproduced failure + +In a five-node cluster, node 1 had previously replicated its log to all peers. +A partition left it connected only to node 2. Nodes 3, 4 and 5 elected a new +leader and committed another entry. A strong read arriving at node 1 then +received a successful stale reply after one fresh response from node 2. + +The implementation combined historical MatchIndex values with the sequence +number of a single fresh response. Replication quorum was incorrectly treated +as proof that a fresh quorum had confirmed this read's leadership. + +## Read protocol + +Read confirmation now has per-peer acknowledgement watermarks separate from +log replication indices. A strong read reserves a unique nonzero uint64_t +barrier, then sends AppendEntries RPCs whose sequences are newer than it. +Successful responses from configured peers advance their individual watermarks; +duplicate replies cannot add voters. Pre-read replies cannot reach the new +barrier. Sequence allocation checks exhaustion rather than wrapping to zero. + +The confirmed watermark is the strict-majority order statistic of the leader's +own sequence and each peer's acknowledged sequence, including zero for peers +that have not replied. This works for both odd and even cluster sizes. A +five-node read therefore needs the leader plus two distinct fresh peers. + +Confirmation is usable only while the node is leader and an entry from its +current term is committed. The local state machine must also have applied the +read's target log prefix before executing the read. The target remains the +last local log index at read admission, a conservative choice that can wait +behind an uncommitted tail rather than optimizing read latency. + +Each pending strong read records its leadership term. Losing leadership or +changing term rejects it with ErrorCode=1 and its original cookie. This is a +conservative cancellation policy; operations are not automatically replayed. +Term checking also works if the application does not poll between stepping +down and being elected again. A later read cannot inherit an earlier read's +quorum. If EConsistent and EStale are both set, EConsistent takes precedence. + +The argument assumes a fixed configured voter set, correct non-Byzantine peers, +and serialized access to Raft/application state. It uses RPC ordering rather +than clock leases. It does not establish complete end-to-end linearizability +for the library or application. + +## Validation and self-review + +Eight new deterministic tests cover: + +* Fresh-majority thresholds for cluster sizes one through eight, with duplicate + responses (including single-node progress). +* The five-node stale-leader partition and a new majority's committed write. +* A second read requiring fresh confirmation after a previous read completes. +* Leadership loss after confirmation but before application service. +* Conflicting consistent/stale flags. +* Delayed pre-read replies, followed by partial and complete fresh quorums. +* A new leader waiting for its current-term commit and local application. +* An old-term read rejected after stepping down and being re-elected, before + accepting a new read. + +The first five tests all fail against alpha.3. The existing single-node FIFO +test now elects a real leader and commits its initial no-op, satisfying the +strong-read protocol's preconditions instead of merely forcing the role flag. +All 69 tests pass locally under ASan/UBSan, with core Debug/Release verification. +Hosted CI runs the same targets; its results are separate from local checks. + +Self-review checked distinct-voter counting, strict-majority arithmetic, +monotonic nonzero barriers, term isolation, duplicate/delayed replies, current +term commitment and application progress. No independent approval is claimed. + +## Compatibility and remaining work + +The wire layout is unchanged. TRaft::ApproveRead and TRaft::CommitSeqno remain; +CommitSeqno now reports a fresh-majority read watermark rather than the largest +sequence attached to a log acknowledgement. Internal TVolatileState removes +CommitSeqno, adds acknowledgement maps, and CommitAdvance no longer accepts a +sequence argument. This alpha does not promise internal-structure or mixed +version cluster compatibility. + +Pending write/default-read handling across leadership changes, disconnect +cleanup, retry/deduplication, admission limits, transport validation and broader +generated history/linearizability testing remain separate gaps. Quorum loss can +leave a strong read waiting indefinitely; deadlines and cancellation APIs are +not introduced here. diff --git a/docs/term-lookups.md b/docs/term-lookups.md new file mode 100644 index 0000000..281382a --- /dev/null +++ b/docs/term-lookups.md @@ -0,0 +1,26 @@ +# Term-only storage lookup: v0.1.0-alpha.22 + +The in-memory ownership fix made Get return a defensive copy. LogTerm used Get, +so checking the term of a large entry also copied its entire payload. The durable +backend already copied full entries, so it had the same unnecessary cost. Raft +checks terms during log matching, elections, commitment and compaction. + +IState now has a RetainedLogTerm hook. The in-memory and durable backends read +the term from their retained entry without copying its payload. LogTerm still +checks bounds and returns the snapshot boundary term as before. Full Get calls +continue to return independent copies. The default hook uses Get, preserving +source compatibility for external IState backends that do not override it. +An IState vtable changed: rebuild the library and dependent binaries together. +Backends subclassing TState or TDurableState and overriding Get with different +log semantics must also override RetainedLogTerm consistently. + +Deterministic tests count full-entry reads while repeatedly checking 1 MiB log +entries, verify no Get call is needed in either optimized backend, and verify +mutating a full Get result cannot alter stored data. The durable test also checks +snapshot-boundary and out-of-range behavior. The memory test failed before the +change: 20 term checks invoked 20 full Get calls. No throughput benchmark is +claimed. Legacy and external backends can retain the fallback cost. + +Self-review checked retained-suffix offsets, snapshot boundaries and fallback +behavior. There is no wire or durable-format migration. No independent reviewer +approval is claimed. Voting membership changes and joint consensus remain open. diff --git a/docs/trusted-legacy-voter-binding.md b/docs/trusted-legacy-voter-binding.md new file mode 100644 index 0000000..b5db0f0 --- /dev/null +++ b/docs/trusted-legacy-voter-binding.md @@ -0,0 +1,29 @@ +# Trusted legacy voter binding: v0.1.0-alpha.50 + +Older durable MRFTD001/MRFTD002 files do not persist the complete fixed voter +set. Raft cannot infer that set safely from a vote, a legacy snapshot voter +list, or the current process's peer list alone. + +`TDurableState::BindVotersFromTrustedBaseline(voters)` provides an explicit +migration path for an unbound, recovered durable store. The caller must supply +the complete original fixed voter set from an independently trusted source, +such as the cluster's verified deployment record. The operation checks that the +stored vote belongs to the set, any legacy snapshot names the same voters and +has no configuration metadata, retained terms are valid, and no reserved +configuration entries are present. It then atomically republishes the durable +state with its voter binding before the store is passed to `TRaft`. + +This API records the caller's assertion; it cannot prove the assertion is +correct, discover prior membership transitions hidden by a legacy snapshot, or +authorize a membership change. Existing `BindVoters` behavior is unchanged. +Do not call the migration API when the original complete voter set is unknown. +The migration applies to `TDurableState`; legacy `TDiskState` remains unable to +persist voter bindings. Since alpha.52, Raft rejects recovered log-only durable +stores with no binding, so this migration must be completed before startup. + +Tests cover MRFTD001, MRFTD002 and MRFTD003 recovery, rejection +of inconsistent vote/snapshot baselines and configuration log entries without +rewriting the source file, and the current MRFTD004 binding after publication +and restart. Legacy formats recover their commit floor at the snapshot boundary. + +Self-review completed. No independent reviewer approval is claimed. diff --git a/docs/trusted-legacy-voter-lifecycle.md b/docs/trusted-legacy-voter-lifecycle.md new file mode 100644 index 0000000..2f9d5e1 --- /dev/null +++ b/docs/trusted-legacy-voter-lifecycle.md @@ -0,0 +1,25 @@ +# Trusted voter migration startup boundary: v0.1.0-alpha.51 + +The alpha.50 `TDurableState::BindVotersFromTrustedBaseline` migration must run +before a `TRaft` instance uses a recovered legacy store. A nonfresh legacy store +without a persisted binding can still start under its configured fixed peers for +compatibility. If migration were allowed afterward, the durable state could be +bound to a different voter set while the live Raft object continued using the +peer set it received at construction. + +`IState::MarkRaftStarted` lets the runtime notify storage after all startup +validation succeeds. `TDurableState` records that attachment in memory and +rejects trusted legacy migration from then on, before modifying or publishing +the durable file. The marker is process-local: after restart, recovery creates a +new state object and the operator can migrate it before constructing Raft. + +The guard does not infer or prove the original voter set, and it does not add +live voter reconfiguration. Tests verify that a post-start migration to a +different set is rejected without changing the binding or file; existing tests +continue to cover migration before startup and recovered restart behavior. + +The [alpha.52 follow-up](legacy-voter-proof-startup.md) closes the remaining +startup gap: recovered log-only durable stores without a voter binding now fail +before Raft starts, requiring this migration path or another valid voter proof. + +Self-review completed; no independent reviewer approval is claimed. diff --git a/docs/unsupported-snapshot-term-preflight.md b/docs/unsupported-snapshot-term-preflight.md new file mode 100644 index 0000000..20f0b61 --- /dev/null +++ b/docs/unsupported-snapshot-term-preflight.md @@ -0,0 +1,29 @@ +# Unsupported snapshot term preflight: v0.1.0-alpha.48 + +The [alpha.49 follow-up](snapshot-metadata-term-preflight.md) extends this +preflight through the complete snapshot metadata and voter list. + +Before this change, `TRaft::Process` persisted a higher RPC term before the +InstallSnapshot handler had read enough bytes to recognize its format. A joint +MRFTS002 image is intentionally unsupported, but its first chunk could still +advance the follower's durable term before the handler rejected it. + +For a higher-term InstallSnapshot request, the RPC validator now requires an +initial chunk with at least the eight-byte magic and accepts the term only when +that magic identifies a supported MRFTS001 or MRFTS003 image. Higher-term +continuation chunks are rejected because a term change discards partial +transfers. Current-term malformed or joint images continue through the ordinary +fail-closed rejection path. The outbound sender already places the complete +magic in its initial chunk. + +This closes the hard-state mutation gap for snapshots that can be recognized as +unsupported from their first chunk. It does not enable joint snapshot transfer, +joint snapshot recovery, or live membership changes. + +The compaction regression test sends a full joint image, a short first chunk, +and a continuation chunk at a higher term, then verifies that term, vote, log, +snapshot and responses remain unchanged. It also verifies the same-term rejection +path. Release and ASAN network-enabled CTest passed 21/21 each; hosted C++ build, +Debug, Release, sanitizer, and test-result checks passed. + +Self-review completed. No independent reviewer approval is claimed. diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 6c4cc94..9fcecb3 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -14,7 +14,9 @@ target(debug_client client.cpp) target(debug_server server.cpp) target(debug_percentiles percentiles2.cpp) target(kv kv.cpp) +target_sources(kv PRIVATE kv_rsm.cpp) if (SQLite3_FOUND) target(sql sql.cpp) -endif() \ No newline at end of file +target_sources(sql PRIVATE sql_rsm.cpp) +endif() diff --git a/examples/client.cpp b/examples/client.cpp index b666086..9620c54 100644 --- a/examples/client.cpp +++ b/examples/client.cpp @@ -24,8 +24,13 @@ TVoidSuspendedTask ClientReader(TPoller& poller, TSocket& socket) { auto commandResponse = response.template Cast(); auto len = commandResponse->Len - sizeof(TCommandResponse); std::string_view data(commandResponse->Data, len); - std::cout << "Ok, commitIndex: " << commandResponse->Index << " " - << data << " " << dt.count() << "\n"; + if (commandResponse->ErrorCode) { + std::cout << "Request failed (error " << commandResponse->ErrorCode + << "). A write may still have committed.\n"; + } else { + std::cout << "Ok, commitIndex: " << commandResponse->Index << " " + << data << " " << dt.count() << "\n"; + } inflight --; } } catch (const std::exception& ex) { diff --git a/examples/kv.cpp b/examples/kv.cpp index 990d72f..dfc79f6 100644 --- a/examples/kv.cpp +++ b/examples/kv.cpp @@ -2,76 +2,12 @@ #include #include +#include +#include #include "kv.h" -struct TKvEntry { - uint16_t KeySize = 0; - uint16_t ValSize = 0; - char Data[0]; -}; - -struct TKvLogEntry: public TLogEntry, public TKvEntry -{ -}; - -struct TWriteKv: public TCommandRequest, public TKvEntry -{ -}; - -struct TReadKv: public TCommandRequest { - uint16_t KeySize = 0; - char Data[0]; -}; - -TMessageHolder TKv::Read(TMessageHolder message, uint64_t index) { - auto readKv = message.Cast(); - if (readKv->KeySize == 0) { - std::string data; - for (auto& [k, v] : H) { - data += k + " = " + v + "\n"; - } - auto res = NewHoldedMessage(sizeof(TCommandResponse) + data.size()); - memcpy(res->Data, data.data(), data.size()); - res->Index = index; - return res; - } else { - std::string_view k(readKv->Data, readKv->KeySize); - auto it = H.find(std::string(k)); - if (it == H.end()) { - auto res = NewHoldedMessage(sizeof(TCommandResponse)); - res->Index = index; - return res; - } else { - auto res = NewHoldedMessage(sizeof(TCommandResponse)+it->second.size()); - res->Index = index; - memcpy(res->Data, it->second.data(), it->second.size()); - return res; - } - } -} - -TMessageHolder TKv::Write(TMessageHolder message, uint64_t index) { - if (LastAppliedIndex < index) { - auto entry = message.Cast(); - std::string_view k(entry->TKvEntry::Data, entry->KeySize); - if (entry->ValSize) { - std::string_view v(entry->TKvEntry::Data + entry->KeySize, entry->ValSize); - H[std::string(k)] = std::string(v); - } else { - H.erase(std::string(k)); - } - LastAppliedIndex = index; - } - return {}; -} - -TMessageHolder TKv::Prepare(TMessageHolder command) { - auto dataSize = command->Len - sizeof(TCommandRequest); - auto entry = NewHoldedMessage(sizeof(TLogEntry)+dataSize); - memcpy(entry->Data, command->Data, dataSize); - return entry; -} +#include "kv_messages.h" template NNet::TFuture Client(TPoller& poller, TSocket socket, uint32_t flags) { @@ -95,11 +31,13 @@ NNet::TFuture Client(TPoller& poller, TSocket socket, uint32_t flags) { strLine += std::string_view(line.Part1.data(), line.Part1.size()); strLine += std::string_view(line.Part2.data(), line.Part2.size()); auto prefix = strtok((char*)strLine.data(), sep); + if (!prefix) { std::cout << "> " << std::flush; continue; } TMessageHolder req; if (!strcmp(prefix, "set")) { auto key = strtok(nullptr, sep); auto val = strtok(nullptr, sep); + if (!key || !val) { std::cout << "Expected set \n"; continue; } auto keySize = strlen(key); auto valSize = strlen(val); auto mes = NewHoldedMessage(sizeof(TWriteKv) + keySize + valSize); @@ -111,6 +49,7 @@ NNet::TFuture Client(TPoller& poller, TSocket socket, uint32_t flags) { req = mes; } else if (!strcmp(prefix, "get")) { auto key = strtok(nullptr, sep); + if (!key) { std::cout << "Expected a key\n"; continue; } auto size = strlen(key); auto mes = NewHoldedMessage(sizeof(TReadKv) + size); mes->Flags = flags; @@ -123,6 +62,7 @@ NNet::TFuture Client(TPoller& poller, TSocket socket, uint32_t flags) { req = mes; } else if (!strcmp(prefix, "del")) { auto key = strtok(nullptr, sep); + if (!key) { std::cout << "Expected a key\n"; continue; } auto size = strlen(key); auto mes = NewHoldedMessage(sizeof(TWriteKv) + size); mes->Flags = TCommandRequest::EWrite; @@ -132,6 +72,7 @@ NNet::TFuture Client(TPoller& poller, TSocket socket, uint32_t flags) { req = mes; } else { std::cout << "Cannot parse command: " << strLine << "\n"; + continue; } co_await TMessageWriter(socket).Write(std::move(req)); @@ -139,9 +80,12 @@ NNet::TFuture Client(TPoller& poller, TSocket socket, uint32_t flags) { auto res = reply.template Cast(); auto len = res->Len - sizeof(TCommandResponse); std::string_view data(res->Data, len); - std::cout << "Ok, commitIndex: " << res->Index << "\n"; - if (!data.empty()) { - std::cout << data << "\n"; + if (res->ErrorCode) { + std::cout << "Request failed (error " << res->ErrorCode + << "). A write may still have committed.\n"; + } else { + std::cout << "Ok, commitIndex: " << res->Index << "\n"; + if (!data.empty()) std::cout << data << "\n"; } std::cout << "> " << std::flush; } @@ -151,8 +95,26 @@ NNet::TFuture Client(TPoller& poller, TSocket socket, uint32_t flags) { co_return; } +template +NNet::TVoidTask CheckpointLoop(TPoller& poller, std::shared_ptr raft, + std::shared_ptr rsm, std::shared_ptr store, uint64_t interval, uint64_t saved) { + while (true) { + co_await poller.Sleep(std::chrono::seconds(1)); + if (rsm->LastAppliedIndex <= saved || rsm->LastAppliedIndex - saved < interval) continue; + try { + auto snapshot = raft->Compact(); + store->Save(snapshot); + saved = snapshot.Index; + } catch (const std::exception& error) { + // The durable state image already contains the authoritative snapshot. + // A failed standalone checkpoint export can be retried safely. + std::cerr << "Checkpoint failed: " << error.what() << "\n"; + } + } +} + void usage(const char* prog) { - std::cerr << prog << "--client|--server --id myid --node ip:port:id [--node ip:port:id ...] [--persist] [--stale] [--consistent]" << "\n"; + std::cerr << prog << "--client|--server --id myid --node ip:port:id [--node ip:port:id ...] [--persist|--durable] [--snapshots entries] [--learner id ...] [--stale] [--consistent]" << "\n"; exit(0); } @@ -161,9 +123,12 @@ int main(int argc, char** argv) { std::vector hosts; THost myHost; TNodeDict nodes; + std::vector learners; uint32_t id = 0; bool server = false; bool persist = false; + bool durable = false; + uint64_t snapshotInterval = 0; uint32_t flags = 0; for (int i = 1; i < argc; i++) { if (!strcmp(argv[i], "--server")) { @@ -175,6 +140,25 @@ int main(int argc, char** argv) { id = atoi(argv[++i]); } else if (!strcmp(argv[i], "--persist")) { persist = true; + } else if (!strcmp(argv[i], "--durable")) { + persist = true; + durable = true; + } else if (!strcmp(argv[i], "--snapshots")) { + if (i + 1 >= argc) { std::cerr << "--snapshots requires a positive entry interval\n"; return 1; } + const std::string_view value(argv[++i]); + const auto result = std::from_chars(value.data(), value.data() + value.size(), snapshotInterval); + if (result.ec != std::errc{} || result.ptr != value.data() + value.size() || !snapshotInterval) { + std::cerr << "--snapshots requires a positive entry interval\n"; return 1; + } + } else if (!strcmp(argv[i], "--learner")) { + if (i + 1 >= argc) { std::cerr << "--learner requires a positive node ID\n"; return 1; } + const std::string_view value(argv[++i]); + uint32_t learner = 0; + const auto result = std::from_chars(value.data(), value.data() + value.size(), learner); + if (result.ec != std::errc{} || result.ptr != value.data() + value.size() || !learner) { + std::cerr << "--learner requires a positive node ID\n"; return 1; + } + learners.push_back(learner); } else if (!strcmp(argv[i], "--stale")) { flags |= TCommandRequest::EStale; } else if (!strcmp(argv[i], "--consistent")) { @@ -184,12 +168,20 @@ int main(int argc, char** argv) { } } + if (!learners.empty() && (!server || !durable)) { + std::cerr << "--learner requires --server --durable\n"; return 1; + } + if (snapshotInterval && (!server || !durable)) { + std::cerr << "--snapshots requires --server --durable\n"; return 1; + } using TPoller = NNet::TDefaultPoller; std::shared_ptr timeSource = std::make_shared(); NNet::TLoop loop; if (server) { + std::unordered_set hostIds; for (auto& host : hosts) { + if (!hostIds.insert(host.Id).second) { std::cerr << "Duplicate node ID\n"; return 1; } if (!host) { std::cerr << "Empty host\n"; return 1; } @@ -211,14 +203,27 @@ int main(int argc, char** argv) { std::shared_ptr rsm = std::make_shared(); std::shared_ptr state = std::make_shared(); if (persist) { - state = std::make_shared("state", myHost.Id); + if (durable) state = std::make_shared("state", myHost.Id); + else state = std::make_shared("state", myHost.Id); + } + auto raft = std::make_shared(state, myHost.Id, nodes, learners); + std::shared_ptr snapshots; + uint64_t savedSnapshot = state->SnapshotIndex; + if (snapshotInterval) { + snapshots = std::make_shared("state", myHost.Id); + if (auto snapshot = snapshots->Load()) { + if (snapshot->Index > state->SnapshotIndex) { + RestoreSnapshot(*snapshot, *raft, *rsm); + savedSnapshot = snapshot->Index; + } + } } - auto raft = std::make_shared(state, myHost.Id, nodes); TPoller::TSocket socket(NNet::TAddress{myHost.Address, myHost.Port}, loop.Poller()); socket.Bind(); socket.Listen(); TRaftServer server(loop.Poller(), std::move(socket), raft, rsm, nodes, timeSource); server.Serve(); + if (snapshots) CheckpointLoop(loop.Poller(), raft, rsm, snapshots, snapshotInterval, savedSnapshot); loop.Loop(); } else { NNet::TAddress addr{hosts[0].Address, hosts[0].Port}; diff --git a/examples/kv.h b/examples/kv.h index c0e337b..8597ed2 100644 --- a/examples/kv.h +++ b/examples/kv.h @@ -7,9 +7,13 @@ class TKv: public IRsm { public: + void Validate(TMessageHolder message) const override; TMessageHolder Read(TMessageHolder message, uint64_t index) override; TMessageHolder Write(TMessageHolder message, uint64_t index) override; TMessageHolder Prepare(TMessageHolder message) override; + std::string SaveSnapshot() const override; + void ValidateSnapshot(std::string_view bytes, uint64_t index) const override; + void LoadSnapshot(std::string_view bytes, uint64_t index) override; private: std::unordered_map H; diff --git a/examples/kv_messages.h b/examples/kv_messages.h new file mode 100644 index 0000000..27b048d --- /dev/null +++ b/examples/kv_messages.h @@ -0,0 +1,22 @@ +#pragma once +#include + +struct TKvEntry { + uint16_t KeySize = 0; + uint16_t ValSize = 0; + char Data[0]; +}; + +struct TKvLogEntry: public TLogEntry, public TKvEntry +{ +}; + +struct TWriteKv: public TCommandRequest, public TKvEntry +{ +}; + +struct TReadKv: public TCommandRequest { + uint16_t KeySize = 0; + char Data[0]; +}; + diff --git a/examples/kv_rsm.cpp b/examples/kv_rsm.cpp new file mode 100644 index 0000000..ff03837 --- /dev/null +++ b/examples/kv_rsm.cpp @@ -0,0 +1,122 @@ +#include "kv.h" +#include "kv_messages.h" +#include "payload_validation.h" +#include +#include +#include +#include + +void TKv::Validate(TMessageHolder message) const { + if (!message || message->Len < sizeof(TCommandRequest)) throw TMessageError("Short KV command"); + if (message->Flags & TCommandRequest::EWrite) { + auto write = message.Cast(); + ValidatePayloadRange(message, write->TKvEntry::Data, uint64_t(write->KeySize) + write->ValSize); + } else { + auto read = message.Cast(); + ValidatePayloadRange(message, read->Data, read->KeySize); + } +} + +TMessageHolder TKv::Read(TMessageHolder message, uint64_t index) { + auto readKv = message.Cast(); + ValidatePayloadRange(message, readKv->Data, readKv->KeySize); + if (readKv->KeySize == 0) { + std::string data; + for (auto& [k, v] : H) { + if (k.size() + v.size() + 4 > TMessageLimits{}.MaxFrameBytes - sizeof(TCommandResponse) - data.size()) + throw TMessageError("KV list response exceeds frame limit"); + data += k + " = " + v + "\n"; + } + auto res = NewHoldedMessage(sizeof(TCommandResponse) + data.size()); + memcpy(res->Data, data.data(), data.size()); + res->Index = index; + return res; + } else { + std::string_view k(readKv->Data, readKv->KeySize); + auto it = H.find(std::string(k)); + if (it == H.end()) { + auto res = NewHoldedMessage(sizeof(TCommandResponse)); + res->Index = index; + return res; + } else { + auto res = NewHoldedMessage(sizeof(TCommandResponse)+it->second.size()); + res->Index = index; + memcpy(res->Data, it->second.data(), it->second.size()); + return res; + } + } +} + +TMessageHolder TKv::Write(TMessageHolder message, uint64_t index) { + if (LastAppliedIndex < index) { + auto entry = message.Cast(); + ValidatePayloadRange(message, entry->TKvEntry::Data, uint64_t(entry->KeySize) + entry->ValSize); + std::string_view k(entry->TKvEntry::Data, entry->KeySize); + if (entry->ValSize) { + std::string_view v(entry->TKvEntry::Data + entry->KeySize, entry->ValSize); + H[std::string(k)] = std::string(v); + } else { + H.erase(std::string(k)); + } + LastAppliedIndex = index; + } + return {}; +} + +TMessageHolder TKv::Prepare(TMessageHolder command) { + auto write = command.Cast(); + ValidatePayloadRange(command, write->TKvEntry::Data, uint64_t(write->KeySize) + write->ValSize); + auto dataSize = command->Len - sizeof(TCommandRequest); + auto entry = NewHoldedMessage(sizeof(TLogEntry)+dataSize); + memcpy(entry->Data, command->Data, dataSize); + return entry; +} + +std::string TKv::SaveSnapshot() const { + using namespace NSnapshotCodec; + std::vector keys; + for (const auto& [key, _] : H) keys.push_back(key); + std::sort(keys.begin(), keys.end()); + std::string bytes = "MRKV0001"; + Put(bytes, H.size(), 8); + for (auto key : keys) { + const auto& value = H.at(std::string(key)); + if (key.size() > UINT16_MAX || value.size() > UINT16_MAX) + throw std::runtime_error("KV snapshot field too large"); + Put(bytes, key.size(), 4); + Put(bytes, value.size(), 4); + Append(bytes, key); + Append(bytes, value); + } + return bytes; +} + +void TKv::LoadSnapshot(std::string_view bytes, uint64_t index) { + using namespace NSnapshotCodec; + if (!index || index > uint64_t(INT64_MAX) || index < LastAppliedIndex || bytes.size() > TSnapshot::MaxBytes) + throw std::runtime_error("Invalid KV snapshot progress/size"); + TReader reader{bytes}; + if (reader.Read(8) != "MRKV0001") throw std::runtime_error("Unknown KV snapshot format"); + const auto count = reader.Take(8); + if (count > reader.Input.size() / 8) throw std::runtime_error("Invalid KV snapshot count"); + std::unordered_map candidate; + std::string_view previous; + for (uint64_t i = 0; i < count; ++i) { + const auto keyLength = reader.Take(4), valueLength = reader.Take(4); + if (keyLength > UINT16_MAX || valueLength > UINT16_MAX) + throw std::runtime_error("KV snapshot field exceeds application limits"); + const auto key = reader.Read(keyLength), value = reader.Read(valueLength); + if (!valueLength || (i && key <= previous)) throw std::runtime_error("Invalid KV snapshot order/value"); + candidate.emplace(std::string(key), std::string(value)); + previous = key; + } + if (!reader.Input.empty()) throw std::runtime_error("Trailing KV snapshot bytes"); + H.swap(candidate); + LastAppliedIndex = index; +} + +void TKv::ValidateSnapshot(std::string_view bytes, uint64_t index) const { + TKv candidate; + candidate.LoadSnapshot(bytes, index); +} + diff --git a/examples/payload_validation.h b/examples/payload_validation.h new file mode 100644 index 0000000..349716e --- /dev/null +++ b/examples/payload_validation.h @@ -0,0 +1,11 @@ +#pragma once +#include + +// Derived message structs may contain tail padding. Bound from the actual +// payload offset, preserving the existing native-layout wire representation. +template +void ValidatePayloadRange(const TMessageHolder& message, const char* data, uint64_t size) { + const auto offset = size_t(data - reinterpret_cast(message.Mes)); + if (offset > message->Len || size > message->Len - offset) + throw TMessageError("Application payload length exceeds message"); +} diff --git a/examples/sql.cpp b/examples/sql.cpp index 356089f..09a5a8f 100644 --- a/examples/sql.cpp +++ b/examples/sql.cpp @@ -1,195 +1,14 @@ #include +#include #include #include #include +#include #include +#include "payload_validation.h" -struct TSqlEntry { - uint32_t QuerySize = 0; - char Query[0]; -}; - -struct TSqlLogEntry: public TLogEntry, public TSqlEntry -{ -}; - -struct TWriteSql: public TCommandRequest, public TSqlEntry -{ -}; - -struct TReadSql: public TCommandRequest, public TSqlEntry -{ -}; - -struct TRow { - std::vector> Values; -}; - -struct TResult { - std::vector Cols; - std::vector Rows; - - void Clear() { - Cols.clear(); - Rows.clear(); - } - - bool Empty() { - return Cols.empty(); - } -}; - -class TSql: public IRsm { -public: - TSql(const std::string& path, int serverId); - ~TSql(); - - // select - TMessageHolder Read(TMessageHolder message, uint64_t index) override; - // insert, update, create - TMessageHolder Write(TMessageHolder message, uint64_t index) override; - // convert request to log message - TMessageHolder Prepare(TMessageHolder message) override; - -private: - bool Execute(const std::string& q); - static int Process(void* self, int ncols, char** values, char** cols); - TMessageHolder Reply(const std::string& ans, uint64_t index); - - TResult Result; - std::string LastError; - sqlite3* Db = nullptr; -}; - -TSql::TSql(const std::string& path, int serverId) -{ - std::string dbPath = path + "." + std::to_string(serverId); - if (sqlite3_open(dbPath.c_str(), &Db) != SQLITE_OK) { - std::cerr << "Cannot open db: `" << dbPath << "', " - << "error: " << sqlite3_errmsg(Db) - << std::endl; - throw std::runtime_error("Cannot open db"); - } - Execute(R"__(CREATE TABLE IF NOT EXISTS raft_metadata_ (key TEXT PRIMARY KEY, value TEXT))__"); - Execute(R"__(SELECT value FROM raft_metadata_ WHERE key = 'LastAppliedIndex')__"); - if (!Result.Empty()) { - LastAppliedIndex = std::stoi(*Result.Rows[0].Values[0]); - } - std::cerr << "LastAppliedIndex: " << LastAppliedIndex << std::endl; -} - -TSql::~TSql() -{ - if (Db) { - if (sqlite3_close(Db) != SQLITE_OK) { - std::cerr << "Failed to close db, error:" << sqlite3_errmsg(Db) << std::endl; - } - } -} - -int TSql::Process(void* self, int ncols, char** values, char** cols) { - TSql* this_ = (TSql*)self; - if (this_->Result.Cols.empty()) { - for (int i = 0; i < ncols; i++) { - this_->Result.Cols.emplace_back(cols[i]); - } - } - TRow row; - for (int i = 0; i < ncols; i++) { - if (values[i]) { - row.Values.emplace_back(values[i]); - } else { - row.Values.emplace_back(std::nullopt); - } - } - this_->Result.Rows.emplace_back(std::move(row)); - return 0; -} - -bool TSql::Execute(const std::string& q) { - char* err = nullptr; - std::cerr << "Execute: " << q << std::endl; - Result.Clear(); - LastError.clear(); - if (sqlite3_exec(Db, q.c_str(), Process, this, &err) != SQLITE_OK) { - std::cerr << "Cannot apply changes, error: " << err << std::endl; - LastError = err; - sqlite3_free(err); - return false; - } - std::cerr << "OK" << std::endl; - return true; -} - -TMessageHolder TSql::Read(TMessageHolder message, uint64_t index) { - auto readSql = message.Cast(); - if (!Execute(std::string(readSql->Query, readSql->QuerySize))) { - return Reply(LastError, index); - } else { - std::string text; - for (int i = 0; i < Result.Cols.size(); i++) { - text += Result.Cols[i]; - if (i != Result.Cols.size() - 1) { - text += ","; - } - } - text += "\n"; - for (int j = 0; j < Result.Rows.size(); j++) { - for (int i = 0; i < Result.Cols.size(); i++) { - text += Result.Rows[j].Values[i] ? *Result.Rows[j].Values[i] : "null"; - if (i != Result.Cols.size() - 1) { - text += ","; - } - } - text += "\n"; - } - return Reply(text, index); - } -} - -TMessageHolder TSql::Write(TMessageHolder message, uint64_t index) { - // TODO: index + 1 == LastAppliedIndex - - std::string err; - if (LastAppliedIndex < index) { - auto entry = message.Cast(); - std::cerr << "Execute write of size: " << entry->QuerySize << std::endl; - std::string updateLastApplied; - updateLastApplied += "INSERT INTO raft_metadata_ (key, value) VALUES ('LastAppliedIndex','" + std::to_string(index) + "')\n"; - updateLastApplied += "ON CONFLICT(key) DO UPDATE SET value = '" + std::to_string(index) + "';\n"; - std::string q = "BEGIN TRANSACTION;\n"; - q += std::string(entry->Query, entry->QuerySize); - if (q.back() != ';') { - q += ";\n"; - } - q += updateLastApplied; - q += "COMMIT;"; - if (Execute(q)) { - LastAppliedIndex = index; - } else { - err = LastError; - Execute("ROLLBACK;"); - Execute(updateLastApplied); // need to update LastAppliedIndex in order not to execute failed query aqain - } - } - return Reply(err, index); -} - -TMessageHolder TSql::Reply(const std::string& ans, uint64_t index) -{ - auto res = NewHoldedMessage(sizeof(TCommandResponse)+ans.size()); - res->Index = index; - memcpy(res->Data, ans.data(), ans.size()); - return res; -} - -TMessageHolder TSql::Prepare(TMessageHolder command) { - auto dataSize = command->Len - sizeof(TCommandRequest); - auto entry = NewHoldedMessage(sizeof(TLogEntry)+dataSize); - memcpy(entry->Data, command->Data, dataSize); - return entry; -} +#include "sql_rsm.h" template NNet::TFuture Client(TPoller& poller, TSocket socket) { @@ -236,9 +55,12 @@ NNet::TFuture Client(TPoller& poller, TSocket socket) { auto res = reply.template Cast(); auto len = res->Len - sizeof(TCommandResponse); std::string_view data(res->Data, len); - std::cout << "commitIndex: " << res->Index << "\n"; - if (!data.empty()) { - std::cout << data << "\n"; + if (res->ErrorCode) { + std::cout << "Request failed (error " << res->ErrorCode + << "). A write may still have committed.\n"; + } else { + std::cout << "commitIndex: " << res->Index << "\n"; + if (!data.empty()) std::cout << data << "\n"; } } } catch (const std::exception& ex) { @@ -247,8 +69,22 @@ NNet::TFuture Client(TPoller& poller, TSocket socket) { co_return; } +template +NNet::TVoidTask CompactLoop(TPoller& poller, std::shared_ptr raft, + std::shared_ptr rsm, uint64_t interval, uint64_t saved) { + while (true) { + co_await poller.Sleep(std::chrono::seconds(1)); + if (rsm->LastAppliedIndex <= saved || rsm->LastAppliedIndex - saved < interval) continue; + try { + saved = raft->Compact().Index; + } catch (const std::exception& error) { + std::cerr << "SQL compaction deferred/failed: " << error.what() << "\n"; + } + } +} + void usage(const char* prog) { - std::cerr << prog << "--client|--server --id myid --node ip:port:id [--node ip:port:id ...]" << "\n"; + std::cerr << prog << "--client|--server --id myid --node ip:port:id [--node ip:port:id ...] [--durable] [--snapshots entries]" << "\n"; exit(0); } @@ -260,6 +96,8 @@ int main(int argc, char** argv) TNodeDict nodes; uint32_t id = 0; bool server = false; + bool durable = false; + uint64_t snapshotInterval = 0; for (int i = 1; i < argc; i++) { if (!strcmp(argv[i], "--server")) { server = true; @@ -268,11 +106,24 @@ int main(int argc, char** argv) hosts.push_back(THost{argv[++i]}); } else if (!strcmp(argv[i], "--id") && i < argc - 1) { id = atoi(argv[++i]); + } else if (!strcmp(argv[i], "--durable")) { + durable = true; + } else if (!strcmp(argv[i], "--snapshots")) { + if (i + 1 >= argc) { std::cerr << "--snapshots requires a positive entry interval\n"; return 1; } + const std::string_view value(argv[++i]); + const auto result = std::from_chars(value.data(), value.data() + value.size(), snapshotInterval); + if (result.ec != std::errc{} || result.ptr != value.data() + value.size() || !snapshotInterval) { + std::cerr << "--snapshots requires a positive entry interval\n"; return 1; + } } else if (!strcmp(argv[i], "--help")) { usage(argv[0]); } } + if (snapshotInterval && (!server || !durable)) { + std::cerr << "--snapshots requires --server --durable\n"; return 1; + } + using TPoller = NNet::TDefaultPoller; std::shared_ptr timeSource = std::make_shared(); NNet::TLoop loop; @@ -298,13 +149,17 @@ int main(int argc, char** argv) } std::shared_ptr rsm = std::make_shared("sql_file.db", myHost.Id); - auto state = std::make_shared("sql_state", myHost.Id); + std::shared_ptr state = durable + ? std::static_pointer_cast(std::make_shared("sql_state", myHost.Id)) + : std::static_pointer_cast(std::make_shared("sql_state", myHost.Id)); auto raft = std::make_shared(state, myHost.Id, nodes); + const auto savedSnapshot = state->SnapshotIndex; TPoller::TSocket socket(NNet::TAddress{myHost.Address, myHost.Port}, loop.Poller()); socket.Bind(); socket.Listen(); TRaftServer server(loop.Poller(), std::move(socket), raft, rsm, nodes, timeSource); server.Serve(); + if (snapshotInterval) CompactLoop(loop.Poller(), raft, rsm, snapshotInterval, savedSnapshot); loop.Loop(); } else { NNet::TAddress addr{hosts[0].Address, hosts[0].Port}; diff --git a/examples/sql_rsm.cpp b/examples/sql_rsm.cpp new file mode 100644 index 0000000..3f514a1 --- /dev/null +++ b/examples/sql_rsm.cpp @@ -0,0 +1,418 @@ +#include "sql_rsm.h" +#include "payload_validation.h" +#include +#include +#include +#include +#include + +namespace { +int ReadAuthorization(void*, int action, const char*, const char* second, const char*, const char*) { + if (action == SQLITE_SELECT || action == SQLITE_READ || action == SQLITE_RECURSIVE) return SQLITE_OK; + if (action == SQLITE_FUNCTION && second) { + // Keep this allowlist limited to trusted SQLite built-ins. In particular, + // readonly statements can still call extension functions with side effects. + for (const char* function : {"abs", "avg", "coalesce", "count", "glob", "hex", "ifnull", + "instr", "length", "like", "lower", "ltrim", "max", "min", "nullif", "quote", + "replace", "round", "rtrim", "substr", "substring", "sum", "total", "trim", + "typeof", "unicode", "upper", "group_concat"}) + if (sqlite3_stricmp(function, second) == 0) return SQLITE_OK; + } + return SQLITE_DENY; +} +int WriteAuthorization(void*, int action, const char* first, const char* second, const char* database, const char* context) { + const auto metadata = [](const char* name) { + return name && sqlite3_stricmp(name, "raft_metadata_") == 0; + }; + if (metadata(first) || ((action == SQLITE_CREATE_TRIGGER || action == SQLITE_DROP_TRIGGER || + action == SQLITE_ALTER_TABLE || action == SQLITE_CREATE_INDEX || action == SQLITE_DROP_INDEX) && metadata(second))) return SQLITE_DENY; + if (database && sqlite3_stricmp(database, "main") != 0) return SQLITE_DENY; + switch (action) { + case SQLITE_INSERT: case SQLITE_UPDATE: case SQLITE_DELETE: + case SQLITE_CREATE_TABLE: case SQLITE_CREATE_INDEX: case SQLITE_CREATE_VIEW: case SQLITE_CREATE_TRIGGER: + case SQLITE_DROP_TABLE: case SQLITE_DROP_INDEX: case SQLITE_DROP_VIEW: case SQLITE_DROP_TRIGGER: + case SQLITE_REINDEX: case SQLITE_ANALYZE: + return SQLITE_OK; + default: + return ReadAuthorization(nullptr, action, first, second, database, context); + } +} +using TAuthorizer = int (*)(void*, int, const char*, const char*, const char*, const char*); +struct TSqlPolicy { + sqlite3* Db; + explicit TSqlPolicy(sqlite3* db, TAuthorizer authorize) : Db(db) { + if (sqlite3_set_authorizer(Db, authorize, nullptr) != SQLITE_OK) + throw TMessageError("Cannot install SQL policy"); + } + ~TSqlPolicy() { sqlite3_set_authorizer(Db, nullptr, nullptr); } +}; +struct TTransactionCleanup { + sqlite3* Db; + ~TTransactionCleanup() { + if (!sqlite3_get_autocommit(Db)) sqlite3_exec(Db, "ROLLBACK", nullptr, nullptr, nullptr); + } +}; +struct TReadBudget { + sqlite3* Db; + uint32_t Remaining; + const int Interval; + const int PreviousLength; + TReadBudget(sqlite3* db, uint32_t maxSteps) + : Db(db), Remaining(maxSteps), Interval(std::min(maxSteps, 1000)), + PreviousLength(sqlite3_limit(db, SQLITE_LIMIT_LENGTH, TMessageLimits{}.MaxFrameBytes)) { + sqlite3_progress_handler(Db, Interval, Progress, this); + } + ~TReadBudget() { + sqlite3_progress_handler(Db, 0, nullptr, nullptr); + sqlite3_limit(Db, SQLITE_LIMIT_LENGTH, PreviousLength); + } + static int Progress(void* context) noexcept { + auto& budget = *static_cast(context); + if (budget.Remaining <= uint32_t(budget.Interval)) return 1; + budget.Remaining -= budget.Interval; + return 0; + } +}; +using TStatement = std::unique_ptr; +constexpr size_t MaxSqlSnapshotBytes = TSnapshot::MaxBytes - 40 - 4 * TSnapshot::MaxVoters; + +struct TSnapshotDatabase { + // Keep the deserialize buffer alive until after the connection closes. + std::unique_ptr Buffer{nullptr, sqlite3_free}; + std::unique_ptr Db{nullptr, sqlite3_close_v2}; +}; + +TSnapshotDatabase OpenSnapshotDatabase(std::string_view bytes) { + if (bytes.size() < 100 || bytes.size() > MaxSqlSnapshotBytes) + throw TMessageError("Invalid SQL snapshot size"); + TSnapshotDatabase result; + constexpr size_t MalformedImageReadSlack = 20; + result.Buffer.reset(static_cast(sqlite3_malloc64(bytes.size() + MalformedImageReadSlack))); + if (!result.Buffer) throw std::bad_alloc(); + memcpy(result.Buffer.get(), bytes.data(), bytes.size()); + memset(result.Buffer.get() + bytes.size(), 0, MalformedImageReadSlack); + if (result.Buffer.get()[18] != 1 || result.Buffer.get()[19] != 1) + throw TMessageError("SQL snapshot is not a standalone rollback-mode image"); + + sqlite3* raw = nullptr; + const int opened = sqlite3_open_v2(":memory:", &raw, + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_MEMORY, nullptr); + result.Db.reset(raw); + if (opened != SQLITE_OK) throw TMessageError("Cannot open SQL snapshot database"); + const int rc = sqlite3_deserialize(result.Db.get(), "main", result.Buffer.get(), + sqlite3_int64(bytes.size()), sqlite3_int64(bytes.size()), SQLITE_DESERIALIZE_READONLY); + if (rc != SQLITE_OK) throw TMessageError("Invalid SQL snapshot database image"); + return result; +} + +uint64_t SnapshotAppliedIndex(sqlite3* db) { + sqlite3_stmt* raw = nullptr; + const int prepared = sqlite3_prepare_v2(db, + "SELECT value FROM raft_metadata_ WHERE key='LastAppliedIndex'", -1, &raw, nullptr); + TStatement statement(raw, sqlite3_finalize); + if (prepared != SQLITE_OK || !statement) throw TMessageError("SQL snapshot has no progress metadata"); + int rc = sqlite3_step(statement.get()); + if (rc != SQLITE_ROW || sqlite3_column_type(statement.get(), 0) != SQLITE_TEXT) + throw TMessageError("Invalid SQL snapshot progress metadata"); + const auto* text = reinterpret_cast(sqlite3_column_text(statement.get(), 0)); + const auto length = sqlite3_column_bytes(statement.get(), 0); + if (!text || length <= 0) throw TMessageError("Invalid SQL snapshot progress metadata"); + uint64_t index = 0; + const auto parsed = std::from_chars(text, text + length, index); + if (parsed.ec != std::errc{} || parsed.ptr != text + length || !index || index > uint64_t(INT64_MAX)) + throw TMessageError("Invalid SQL snapshot progress metadata"); + if (sqlite3_step(statement.get()) != SQLITE_DONE) + throw TMessageError("Duplicate SQL snapshot progress metadata"); + return index; +} + +void ValidateSnapshotDatabase(sqlite3* db, uint64_t index) { + if (!index || index > uint64_t(INT64_MAX)) throw TMessageError("Invalid SQL snapshot index"); + sqlite3_stmt* raw = nullptr; + const int prepared = sqlite3_prepare_v2(db, "PRAGMA quick_check(1)", -1, &raw, nullptr); + TStatement statement(raw, sqlite3_finalize); + if (prepared != SQLITE_OK || !statement) + throw TMessageError(std::string("Cannot run SQL snapshot integrity check: ") + sqlite3_errmsg(db)); + const int check = sqlite3_step(statement.get()); + if (check != SQLITE_ROW || sqlite3_column_type(statement.get(), 0) != SQLITE_TEXT) + throw TMessageError(std::string("Cannot read SQL snapshot integrity check: ") + sqlite3_errmsg(db)); + const auto* result = reinterpret_cast(sqlite3_column_text(statement.get(), 0)); + const auto length = sqlite3_column_bytes(statement.get(), 0); + if (!result || std::string_view(result, size_t(length)) != "ok") + throw TMessageError("SQL snapshot integrity check failed: " + + std::string(result ? result : "", result ? size_t(length) : 6)); + if (sqlite3_step(statement.get()) != SQLITE_DONE) + throw TMessageError("SQL snapshot integrity check failed"); + if (SnapshotAppliedIndex(db) != index) + throw TMessageError("SQL snapshot application progress does not match its boundary"); +} + +uint64_t ReadPragmaInteger(sqlite3* db, const char* pragma) { + sqlite3_stmt* raw = nullptr; + const std::string query = std::string("PRAGMA ") + pragma; + const int prepared = sqlite3_prepare_v2(db, query.c_str(), -1, &raw, nullptr); + TStatement statement(raw, sqlite3_finalize); + if (prepared != SQLITE_OK || !statement || sqlite3_step(statement.get()) != SQLITE_ROW || + sqlite3_column_type(statement.get(), 0) != SQLITE_INTEGER) + throw TMessageError("Cannot inspect SQL snapshot size"); + const auto value = sqlite3_column_int64(statement.get(), 0); + if (value <= 0 || sqlite3_step(statement.get()) != SQLITE_DONE) + throw TMessageError("Invalid SQL snapshot size metadata"); + return uint64_t(value); +} + +void ValidateSnapshotForTarget(sqlite3* snapshot, sqlite3* target, uint64_t index) { + ValidateSnapshotDatabase(snapshot, index); + if (!sqlite3_get_autocommit(target)) + throw TMessageError("Cannot replace SQL state during a transaction"); + if (ReadPragmaInteger(snapshot, "page_size") != ReadPragmaInteger(target, "page_size")) + throw TMessageError("SQL snapshot page size does not match the target database"); +} + +TSnapshotDatabase CopyDatabaseToMemory(sqlite3* source, uint64_t pageSize) { + TSnapshotDatabase result; + sqlite3* raw = nullptr; + const int opened = sqlite3_open_v2(":memory:", &raw, + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_MEMORY, nullptr); + result.Db.reset(raw); + if (opened != SQLITE_OK) throw TMessageError("Cannot open SQL snapshot database"); + + // SQLite's backup API requires matching page sizes for an in-memory + // destination. Set it before copying any pages. + const auto setPageSize = std::string("PRAGMA page_size = ") + std::to_string(pageSize); + if (sqlite3_exec(result.Db.get(), setPageSize.c_str(), nullptr, nullptr, nullptr) != SQLITE_OK) + throw TMessageError("Cannot configure SQL snapshot page size"); + + sqlite3_backup* backup = sqlite3_backup_init(result.Db.get(), "main", source, "main"); + if (!backup) throw TMessageError("Cannot begin SQL snapshot capture"); + const int copied = sqlite3_backup_step(backup, -1); + const int finished = sqlite3_backup_finish(backup); + if (copied != SQLITE_DONE || finished != SQLITE_OK) + throw TMessageError("Cannot complete SQL snapshot capture"); + return result; +} +} + +std::string TSql::ExecuteRead(const std::string& q) { + if (q.size() > INT_MAX || q.find('\0') != std::string::npos) + throw TMessageError("Invalid SQL read text"); + TReadBudget budget(Db, ReadLimits.MaxVmSteps); + TSqlPolicy policy(Db, ReadAuthorization); // retained through prepare, stepping and finalization + sqlite3_stmt* raw = nullptr; + const char* tail = nullptr; + int rc = sqlite3_prepare_v2(Db, q.data(), int(q.size()), &raw, &tail); + TStatement statement(raw, sqlite3_finalize); + if (rc != SQLITE_OK || !statement || !sqlite3_stmt_readonly(statement.get())) + throw TMessageError("SQL read must be one permitted read-only statement"); + raw = nullptr; + rc = sqlite3_prepare_v2(Db, tail, int(q.data() + q.size() - tail), &raw, nullptr); + TStatement extra(raw, sqlite3_finalize); + if (rc != SQLITE_OK || extra) throw TMessageError("Multiple SQL read statements are not permitted"); + std::string text; + const auto append = [&](std::string_view part) { + if (part.size() > ReadLimits.MaxResultBytes - text.size()) + throw TMessageError("SQL read output budget exceeded"); + text.append(part); + }; + bool firstRow = true; + while ((rc = sqlite3_step(statement.get())) == SQLITE_ROW) { + const int count = sqlite3_column_count(statement.get()); + if (firstRow) { + for (int i = 0; i < count; ++i) { + const char* name = sqlite3_column_name(statement.get(), i); + if (!name) throw TMessageError("Cannot read SQL column name"); + append(name); + if (i + 1 < count) append(","); + } + append("\n"); + firstRow = false; + } + for (int i = 0; i < count; ++i) { + if (sqlite3_column_type(statement.get(), i) == SQLITE_NULL) { + append("null"); + } else { + const auto* value = reinterpret_cast(sqlite3_column_text(statement.get(), i)); + if (!value) throw TMessageError("Cannot read SQL column value"); + const auto length = sqlite3_column_bytes(statement.get(), i); + append(std::string_view(value, length)); + } + if (i + 1 < count) append(","); + } + append("\n"); + } + if (rc != SQLITE_DONE) throw TMessageError("SQL read failed or exceeded its execution budget"); + if (firstRow) append("\n"); // Preserve the legacy empty-result format. + return text; +} + +void TSql::Validate(TMessageHolder message) const { + auto query = message.Cast(); + ValidatePayloadRange(message, query->Query, query->QuerySize); + if (memchr(query->Query, 0, query->QuerySize)) throw TMessageError("Invalid SQL query text"); +} + +TSql::TSql(const std::string& path, int serverId, TSqlReadLimits readLimits) + : ReadLimits(readLimits) +{ + if (!ReadLimits.MaxVmSteps || + ReadLimits.MaxResultBytes > TMessageLimits{}.MaxFrameBytes - sizeof(TCommandResponse)) + throw std::invalid_argument("Invalid SQL read limits"); + std::string dbPath = path == ":memory:" ? path : path + "." + std::to_string(serverId); + const int opened = sqlite3_open(dbPath.c_str(), &Db); + // Constructors that fail must also close the connection and release its locks. + std::unique_ptr cleanup(Db, sqlite3_close); + if (opened != SQLITE_OK) throw std::runtime_error("Cannot open SQL database"); + RequireExecute("CREATE TABLE IF NOT EXISTS raft_metadata_ (key TEXT PRIMARY KEY, value TEXT)"); + sqlite3_stmt* raw = nullptr; + const int prepared = sqlite3_prepare_v2(Db, + "SELECT value FROM raft_metadata_ WHERE key='LastAppliedIndex'", -1, &raw, nullptr); + TStatement statement(raw, sqlite3_finalize); + if (prepared != SQLITE_OK || !statement) throw std::runtime_error("Cannot read SQL application progress"); + int rc = sqlite3_step(statement.get()); + if (rc == SQLITE_ROW) { + if (sqlite3_column_type(statement.get(), 0) != SQLITE_TEXT) + throw std::runtime_error("Invalid SQL application progress"); + const auto* text = reinterpret_cast(sqlite3_column_text(statement.get(), 0)); + if (!text) throw std::runtime_error("Cannot read SQL application progress"); + const auto length = sqlite3_column_bytes(statement.get(), 0); + uint64_t index = 0; + const auto parsed = std::from_chars(text, text + length, index); + if (parsed.ec != std::errc{} || parsed.ptr != text + length) + throw std::runtime_error("Invalid SQL application progress"); + LastAppliedIndex = index; + rc = sqlite3_step(statement.get()); + } + if (rc != SQLITE_DONE) throw std::runtime_error("Invalid SQL application progress"); + cleanup.release(); + std::cerr << "LastAppliedIndex: " << LastAppliedIndex << std::endl; +} + +TSql::~TSql() +{ + if (Db) { + if (sqlite3_close(Db) != SQLITE_OK) { + std::cerr << "Failed to close db, error:" << sqlite3_errmsg(Db) << std::endl; + } + } +} + +int TSql::Execute(const std::string& q) { + char* err = nullptr; + LastError.clear(); + const int rc = sqlite3_exec(Db, q.c_str(), nullptr, nullptr, &err); + if (rc != SQLITE_OK) LastError = err ? err : sqlite3_errmsg(Db); + sqlite3_free(err); + return rc; +} + +void TSql::RequireExecute(const std::string& q) { + if (Execute(q) != SQLITE_OK) throw std::runtime_error("SQL state update failed: " + LastError); +} + +TMessageHolder TSql::Read(TMessageHolder message, uint64_t index) { + auto readSql = message.Cast(); + ValidatePayloadRange(message, readSql->Query, readSql->QuerySize); + return Reply(ExecuteRead(std::string(readSql->Query, readSql->QuerySize)), index); +} + +TMessageHolder TSql::Write(TMessageHolder message, uint64_t index) { + if (index <= LastAppliedIndex) return Reply({}, index); + auto entry = message.Cast(); + ValidatePayloadRange(message, entry->Query, entry->QuerySize); + const std::string query(entry->Query, entry->QuerySize); + if (query.find('\0') != std::string::npos) throw TMessageError("Invalid SQL write text"); + if (!sqlite3_get_autocommit(Db)) throw std::runtime_error("SQL application transaction already active"); + TTransactionCleanup transaction{Db}; + RequireExecute("BEGIN IMMEDIATE"); + int rc; + std::string error; + { + // Only the payload runs under this policy. It cannot end the transaction + // or access the progress table, including through views and triggers. + TSqlPolicy policy(Db, WriteAuthorization); + rc = Execute(query); + error = LastError; + } + if (rc != SQLITE_OK) { + if (!sqlite3_get_autocommit(Db)) RequireExecute("ROLLBACK"); + switch (rc & 0xff) { + case SQLITE_ERROR: case SQLITE_AUTH: case SQLITE_CONSTRAINT: case SQLITE_MISMATCH: + break; // SQL/application rejection: consume this entry without its effects. + default: + throw std::runtime_error("SQL application execution failed: " + error); + } + // Some errors (including OR ROLLBACK) already rolled back the transaction. + // Start a new one to record the rejected entry; never ignore failure here. + RequireExecute("BEGIN IMMEDIATE"); + } + const auto progress = std::to_string(index); + RequireExecute("INSERT INTO raft_metadata_ (key, value) VALUES ('LastAppliedIndex','" + progress + + "') ON CONFLICT(key) DO UPDATE SET value='" + progress + "'"); + RequireExecute("COMMIT"); + LastAppliedIndex = index; + return Reply(error, index); +} + +TMessageHolder TSql::Reply(const std::string& ans, uint64_t index) +{ + if (ans.size() > TMessageLimits{}.MaxFrameBytes - sizeof(TCommandResponse)) + throw TMessageError("SQL response exceeds frame limit"); + auto res = NewHoldedMessage(sizeof(TCommandResponse)+ans.size()); + res->Index = index; + memcpy(res->Data, ans.data(), ans.size()); + return res; +} + +TMessageHolder TSql::Prepare(TMessageHolder command) { + auto write = command.Cast(); + ValidatePayloadRange(command, write->Query, write->QuerySize); + if (memchr(write->Query, 0, write->QuerySize)) throw TMessageError("Invalid SQL write text"); + auto dataSize = command->Len - sizeof(TCommandRequest); + auto entry = NewHoldedMessage(sizeof(TLogEntry)+dataSize); + memcpy(entry->Data, command->Data, dataSize); + return entry; +} + +std::string TSql::SaveSnapshot() const { + if (!LastAppliedIndex || LastAppliedIndex > uint64_t(INT64_MAX) || !sqlite3_get_autocommit(Db)) + throw TMessageError("SQL snapshot requires committed application progress"); + const auto pages = ReadPragmaInteger(Db, "page_count"); + const auto pageSize = ReadPragmaInteger(Db, "page_size"); + if (pages > MaxSqlSnapshotBytes / pageSize) + throw TMessageError("SQL snapshot exceeds the maximum image size"); + auto consistent = CopyDatabaseToMemory(Db, pageSize); + sqlite3_int64 size = 0; + auto* bytes = sqlite3_serialize(consistent.Db.get(), "main", &size, 0); + std::unique_ptr image(bytes, sqlite3_free); + if (!bytes || size <= 0 || uint64_t(size) > MaxSqlSnapshotBytes) + throw TMessageError("Cannot serialize SQL snapshot within the size limit"); + std::string result(reinterpret_cast(bytes), size_t(size)); + if (result.size() < 100 || + !((static_cast(result[18]) == 1 && static_cast(result[19]) == 1) || + (static_cast(result[18]) == 2 && static_cast(result[19]) == 2))) + throw TMessageError("Unsupported SQL snapshot database format"); + // SQLite documents this conversion for deserializing a complete database + // image. Backup copied all WAL-visible pages above, so rollback mode is now + // the correct self-contained representation. + result[18] = 1; + result[19] = 1; + return result; +} + +void TSql::ValidateSnapshot(std::string_view bytes, uint64_t index) const { + auto image = OpenSnapshotDatabase(bytes); + ValidateSnapshotForTarget(image.Db.get(), Db, index); +} + +void TSql::LoadSnapshot(std::string_view bytes, uint64_t index) { + auto image = OpenSnapshotDatabase(bytes); + ValidateSnapshotForTarget(image.Db.get(), Db, index); + sqlite3_backup* backup = sqlite3_backup_init(Db, "main", image.Db.get(), "main"); + if (!backup) throw TMessageError("Cannot begin SQL snapshot installation"); + const int copied = sqlite3_backup_step(backup, -1); + const int finished = sqlite3_backup_finish(backup); + if (copied != SQLITE_DONE || finished != SQLITE_OK) + throw TMessageError("Cannot atomically install SQL snapshot"); + LastAppliedIndex = index; +} + diff --git a/examples/sql_rsm.h b/examples/sql_rsm.h new file mode 100644 index 0000000..b2a6f54 --- /dev/null +++ b/examples/sql_rsm.h @@ -0,0 +1,54 @@ +#pragma once +#include +#include + +struct TSqlEntry { + uint32_t QuerySize = 0; + char Query[0]; +}; + +struct TSqlLogEntry: public TLogEntry, public TSqlEntry +{ +}; + +struct TWriteSql: public TCommandRequest, public TSqlEntry +{ +}; + +struct TReadSql: public TCommandRequest, public TSqlEntry +{ +}; + +struct TSqlReadLimits { + // Approximate SQLite VM instruction budget, checked at intervals <= 1000. + uint32_t MaxVmSteps = 1'000'000; + size_t MaxResultBytes = TMessageLimits{}.MaxFrameBytes - sizeof(TCommandResponse); +}; + +class TSql: public IRsm { +public: + void Validate(TMessageHolder message) const override; + TSql(const std::string& path, int serverId, TSqlReadLimits readLimits = {}); + ~TSql(); + + // select + TMessageHolder Read(TMessageHolder message, uint64_t index) override; + // insert, update, create + TMessageHolder Write(TMessageHolder message, uint64_t index) override; + // convert request to log message + TMessageHolder Prepare(TMessageHolder message) override; + std::string SaveSnapshot() const override; + void ValidateSnapshot(std::string_view bytes, uint64_t index) const override; + void LoadSnapshot(std::string_view bytes, uint64_t index) override; + +private: + int Execute(const std::string& q); + void RequireExecute(const std::string& q); + std::string ExecuteRead(const std::string& q); + TMessageHolder Reply(const std::string& ans, uint64_t index); + + TSqlReadLimits ReadLimits; + std::string LastError; + sqlite3* Db = nullptr; +}; + diff --git a/miniraft/CMakeLists.txt b/miniraft/CMakeLists.txt index a26c8c4..58a1a6f 100644 --- a/miniraft/CMakeLists.txt +++ b/miniraft/CMakeLists.txt @@ -2,7 +2,10 @@ add_library(miniraft messages.cpp raft.cpp persist.cpp + durable.cpp + snapshot.cpp + configuration.cpp ) target_include_directories(miniraft PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/..) -target_compile_features(miniraft PUBLIC cxx_std_20) \ No newline at end of file +target_compile_features(miniraft PUBLIC cxx_std_20) diff --git a/miniraft/configuration.cpp b/miniraft/configuration.cpp new file mode 100644 index 0000000..fa311ae --- /dev/null +++ b/miniraft/configuration.cpp @@ -0,0 +1,223 @@ +#include "configuration.h" +#include "snapshot_codec.h" +#include "state.h" +#include +#include +#include +#include +#include + +namespace { +using namespace NSnapshotCodec; +constexpr std::string_view Magic = "MRFTC001"; +constexpr size_t MaxPayload = 48 + 8 * TSnapshot::MaxVoters; +void Validate(const TConfigurationRecord& record) { + if ((record.Phase != EConfigurationPhase::Joint && record.Phase != EConfigurationPhase::Stable) || + !record.Term || record.PreviousIndex > uint64_t(INT64_MAX) || + ((record.PreviousIndex == 0) != (record.PreviousTerm == 0)) || + record.PreviousTerm > record.Term || + (record.Phase == EConfigurationPhase::Stable && !record.PreviousIndex)) + throw std::invalid_argument("Invalid configuration entry metadata"); + ValidateFixedVoters(record.OldVoters); + ValidateFixedVoters(record.NewVoters); +} +} + +TMessageHolder EncodeConfigurationEntry(const TConfigurationRecord& record) { + Validate(record); + std::string bytes(Magic); + Put(bytes, uint32_t(record.Phase), 4); + Put(bytes, record.Term, 8); + Put(bytes, record.PreviousIndex, 8); + Put(bytes, record.PreviousTerm, 8); + Put(bytes, record.OldVoters.size(), 4); + Put(bytes, record.NewVoters.size(), 4); + for (auto id : record.OldVoters) Put(bytes, id, 4); + for (auto id : record.NewVoters) Put(bytes, id, 4); + Put(bytes, Checksum(bytes), 4); + auto entry = NewHoldedMessage(sizeof(TLogEntry) + bytes.size()); + entry->Term = record.Term; entry->Flags = TLogEntry::EConfiguration; + memcpy(entry->Data, bytes.data(), bytes.size()); + return entry; +} + +TConfigurationRecord DecodeConfigurationEntry(TMessageHolder entry) { + if (!entry.HasValidExtent() || entry->Type != uint32_t(EMessageType::LOG_ENTRY) || + entry->Len < sizeof(TLogEntry) || entry->Flags != TLogEntry::EConfiguration || entry.PayloadSize) + throw std::invalid_argument("Invalid configuration entry frame"); + const auto length = entry->Len - sizeof(TLogEntry); + if (length < 56 || length > MaxPayload) throw std::invalid_argument("Invalid configuration entry size"); + const std::string_view bytes(entry->Data, length); + TReader trailer{bytes.substr(length - 4)}; + if (Checksum(bytes.substr(0, length - 4)) != trailer.Take(4)) + throw std::invalid_argument("Configuration entry checksum mismatch"); + TReader reader{bytes.substr(0, length - 4)}; + if (reader.Read(Magic.size()) != Magic) throw std::invalid_argument("Unknown configuration entry format"); + TConfigurationRecord record; + record.Phase = EConfigurationPhase(reader.Take(4)); + record.Term = reader.Take(8); record.PreviousIndex = reader.Take(8); record.PreviousTerm = reader.Take(8); + const auto oldCount = reader.Take(4), newCount = reader.Take(4); + if (oldCount > TSnapshot::MaxVoters || newCount > TSnapshot::MaxVoters || + oldCount + newCount > reader.Input.size() / 4) + throw std::invalid_argument("Invalid configuration voter counts"); + for (uint64_t i = 0; i < oldCount; ++i) record.OldVoters.push_back(reader.Take(4)); + for (uint64_t i = 0; i < newCount; ++i) record.NewVoters.push_back(reader.Take(4)); + if (!reader.Input.empty() || entry->Term != record.Term) + throw std::invalid_argument("Configuration entry term or length mismatch"); + Validate(record); + return record; +} + +TConfigurationHistory TConfigurationHistory::Stable(std::vector voters, + uint64_t configurationIndex, uint64_t configurationTerm, uint64_t throughIndex, uint64_t throughTerm) { + ValidateFixedVoters(voters); + if (throughIndex > uint64_t(INT64_MAX) || configurationIndex > throughIndex || + ((configurationIndex == 0) != (configurationTerm == 0)) || + ((throughIndex == 0) != (throughTerm == 0)) || configurationTerm > throughTerm || + (configurationIndex && configurationIndex == throughIndex && configurationTerm != throughTerm)) + throw std::invalid_argument("Invalid configuration history baseline"); + TConfigurationHistory history; + history.Old = std::move(voters); history.ConfigIndex = configurationIndex; + history.ConfigTerm = configurationTerm; history.Through = throughIndex; history.ThroughLogTerm = throughTerm; + return history; +} + +TConfigurationHistory TConfigurationHistory::FromJointSnapshot(const TSnapshot& snapshot) { + ValidateSnapshot(snapshot); + if (!snapshot.IsJoint() || snapshot.StableConfiguration) throw std::invalid_argument("Snapshot does not identify a joint configuration"); + TConfigurationHistory history; + history.Old = snapshot.Voters; history.New = snapshot.JointVoters; + history.ConfigIndex = snapshot.ConfigurationIndex; history.ConfigTerm = snapshot.ConfigurationTerm; + history.Through = snapshot.Index; history.ThroughLogTerm = snapshot.Term; + return history; +} + +TConfigurationHistory TConfigurationHistory::FromStableSnapshot(const TSnapshot& snapshot) { + ValidateSnapshot(snapshot); + if (!snapshot.StableConfiguration) throw std::invalid_argument("Snapshot does not identify a stable configuration"); + return Stable(snapshot.Voters, snapshot.ConfigurationIndex, snapshot.ConfigurationTerm, snapshot.Index, snapshot.Term); +} + +TSnapshot TConfigurationHistory::MakeSnapshot(std::string data) const { + TSnapshot snapshot{Through, ThroughLogTerm, Old, std::move(data)}; + if (IsJoint()) { + snapshot.JointVoters = New; + snapshot.ConfigurationIndex = ConfigIndex; snapshot.ConfigurationTerm = ConfigTerm; + } else if (ConfigIndex) { + snapshot.StableConfiguration = true; + snapshot.ConfigurationIndex = ConfigIndex; snapshot.ConfigurationTerm = ConfigTerm; + } + ValidateSnapshot(snapshot); + return snapshot; +} + +TConfigurationHistory TConfigurationHistory::Advance(uint64_t index, TMessageHolder entry) const { + const auto record = DecodeConfigurationEntry(std::move(entry)); + if (Through >= uint64_t(INT64_MAX) || index != Through + 1 || record.Term < ThroughLogTerm || + record.PreviousIndex != ConfigIndex || record.PreviousTerm != ConfigTerm) + throw std::invalid_argument("Configuration entry does not extend the trusted history"); + TConfigurationHistory next = *this; + if (!IsJoint()) { + if (record.Phase != EConfigurationPhase::Joint || record.OldVoters != Old || record.NewVoters == Old) + throw std::invalid_argument("Expected a joint configuration from the stable voter set"); + next.New = record.NewVoters; + } else { + if (record.Phase != EConfigurationPhase::Stable || record.OldVoters != Old || record.NewVoters != New) + throw std::invalid_argument("Expected the stable successor of the joint configuration"); + next.Old = New; next.New.clear(); + } + next.ConfigIndex = next.Through = index; + next.ConfigTerm = next.ThroughLogTerm = record.Term; + return next; +} + +TConfigurationHistory TConfigurationHistory::Observe(uint64_t index, TMessageHolder entry) const { + if (!entry.HasValidExtent() || entry->Type != uint32_t(EMessageType::LOG_ENTRY) || + entry->Len < sizeof(TLogEntry) || entry.PayloadSize) + throw std::invalid_argument("Invalid ordinary log entry"); + return ObserveMetadata(index, entry->Term, entry->Flags); +} + +TConfigurationHistory TConfigurationHistory::ObserveMetadata(uint64_t index, uint64_t term, uint64_t flags) const { + if ((flags & TLogEntry::EConfiguration) || !term || + Through >= uint64_t(INT64_MAX) || index != Through + 1 || term < ThroughLogTerm) + throw std::invalid_argument("Ordinary entry does not extend the trusted history"); + TConfigurationHistory next = *this; + next.Through = index; next.ThroughLogTerm = term; + return next; +} + +TConfigurationScan ScanConfigurationHistory(const IState& state, + TConfigurationHistory baseline, uint64_t commitIndex) { + if (!state.CurrentTerm || state.LastLogIndex > uint64_t(INT64_MAX) || + state.SnapshotIndex > state.LastLogIndex || state.SnapshotTerm > state.CurrentTerm || + state.LastLogTerm > state.CurrentTerm || + baseline.ThroughIndex() != state.SnapshotIndex || baseline.ThroughTerm() != state.SnapshotTerm || + commitIndex < state.SnapshotIndex || commitIndex > state.LastLogIndex) + throw std::invalid_argument("Invalid configuration scan boundaries"); + + const auto snapshot = state.GetSnapshot(); + if (state.SnapshotIndex) { + if (!snapshot) throw std::invalid_argument("Missing configuration scan snapshot"); + ValidateSnapshot(*snapshot); + if (snapshot->Index != state.SnapshotIndex || snapshot->Term != state.SnapshotTerm) + throw std::invalid_argument("Configuration scan snapshot boundary mismatch"); + if (snapshot->IsJoint()) { + const auto expected = TConfigurationHistory::FromJointSnapshot(*snapshot); + if (!baseline.IsJoint() || baseline.OldVoters() != expected.OldVoters() || + baseline.NewVoters() != expected.NewVoters() || + baseline.ConfigurationIndex() != expected.ConfigurationIndex() || + baseline.ConfigurationTerm() != expected.ConfigurationTerm()) + throw std::invalid_argument("Joint snapshot baseline mismatch"); + } else if (baseline.IsJoint() || baseline.OldVoters() != snapshot->Voters || + (snapshot->StableConfiguration && (baseline.ConfigurationIndex() != snapshot->ConfigurationIndex || + baseline.ConfigurationTerm() != snapshot->ConfigurationTerm))) { + throw std::invalid_argument("Fixed snapshot baseline mismatch"); + } + } else if (snapshot || state.SnapshotTerm || baseline.IsJoint()) { + throw std::invalid_argument("Invalid empty snapshot boundary"); + } + const auto bound = state.GetBoundVoters(); + // The binding is the trusted original set. Configuration-bearing snapshots + // carry the active set and its identity after a transition, so those voters + // may legitimately differ. Legacy snapshots still require the original + // set because they cannot describe or validate that transition boundary. + if (!bound.empty() && bound != baseline.OldVoters() && + (!snapshot || !snapshot->HasConfigurationMetadata())) + throw std::invalid_argument("Configuration baseline differs from the trusted voter history"); + + auto latest = baseline; + auto committed = baseline; + for (uint64_t index = state.SnapshotIndex + 1; index <= state.LastLogIndex; ++index) { + const auto term = state.RetainedLogTerm(index); + const auto flags = state.RetainedLogFlags(index); + if (!term || term > state.CurrentTerm) throw std::invalid_argument("Invalid retained log term"); + if (flags & TLogEntry::EConfiguration) { + // Do not accept a second transition until the preceding config has + // committed. Otherwise an uncommitted joint entry could be followed + // by a stable entry and make Latest appear to leave joint mode + // without ever establishing the required old/new quorum. + if (latest.ConfigurationIndex() > commitIndex) + throw std::invalid_argument("Configuration transition precedes predecessor commitment"); + auto entry = state.Get(index); + if (!entry.HasValidExtent() || entry->Len < sizeof(TLogEntry) || + entry->Term != term || entry->Flags != flags) + throw std::invalid_argument("Inconsistent configuration entry metadata"); + latest = latest.Advance(index, entry); + } else { + latest = latest.ObserveMetadata(index, term, flags); + } + if (index == commitIndex) committed = latest; + } + if (latest.ThroughTerm() != state.LastLogTerm) + throw std::invalid_argument("Configuration scan last term mismatch"); + return {committed, latest}; +} + +TConfigurationScan ScanCommittedConfigurationHistory(const IState& state, + TConfigurationHistory baseline) { + if (!state.SupportsCommitIndexPersistence() || !state.SupportsVoterBinding() || + state.GetBoundVoters().empty()) + throw std::invalid_argument("Committed configuration scan requires durable commit and voter metadata"); + return ScanConfigurationHistory(state, std::move(baseline), state.CommitIndex); +} diff --git a/miniraft/configuration.h b/miniraft/configuration.h new file mode 100644 index 0000000..8405220 --- /dev/null +++ b/miniraft/configuration.h @@ -0,0 +1,80 @@ +#pragma once + +#include "messages.h" +#include "snapshot.h" +#include +#include + +// Configuration record encoding for the live joint-consensus log protocol. +// Snapshot installation across membership transitions remains unsupported. +enum class EConfigurationPhase : uint32_t { Joint = 1, Stable = 2 }; +struct TConfigurationRecord { + EConfigurationPhase Phase = EConfigurationPhase::Joint; + uint64_t Term = 0, PreviousIndex = 0, PreviousTerm = 0; + std::vector OldVoters, NewVoters; + bool operator==(const TConfigurationRecord&) const = default; +}; + +TMessageHolder EncodeConfigurationEntry(const TConfigurationRecord& record); +TConfigurationRecord DecodeConfigurationEntry(TMessageHolder entry); + +// Validates transition ordering and derives configuration identity from a +// trusted baseline and commit boundary. Runtime callers use this before +// activating roles or accepting a candidate AppendEntries suffix. +class TConfigurationHistory { +public: + TConfigurationHistory(const TConfigurationHistory&) = default; + TConfigurationHistory& operator=(const TConfigurationHistory&) = default; + // The caller must supply a trusted identity and scanned-prefix boundary. + // A zero configuration identity asserts that no configuration entry exists + // in that prefix; a v1 snapshot alone does not prove this assertion. + static TConfigurationHistory Stable(std::vector voters, + uint64_t configurationIndex, uint64_t configurationTerm, + uint64_t throughIndex, uint64_t throughTerm); + static TConfigurationHistory FromJointSnapshot(const TSnapshot& snapshot); + static TConfigurationHistory FromStableSnapshot(const TSnapshot& snapshot); + // Visit every retained entry in index order; neither method changes *this. + TConfigurationHistory Observe(uint64_t index, TMessageHolder entry) const; + // Metadata must come from a trusted, already validated storage backend. + TConfigurationHistory ObserveMetadata(uint64_t index, uint64_t term, uint64_t flags) const; + TConfigurationHistory Advance(uint64_t index, TMessageHolder entry) const; + // Offline projection only. The caller must establish that ThroughIndex is + // committed and that data is the application image at exactly that index. + TSnapshot MakeSnapshot(std::string data) const; + + bool IsJoint() const { return !New.empty(); } + const std::vector& OldVoters() const { return Old; } + const std::vector& NewVoters() const { return New; } + uint64_t ConfigurationIndex() const { return ConfigIndex; } + uint64_t ConfigurationTerm() const { return ConfigTerm; } + uint64_t ThroughIndex() const { return Through; } + uint64_t ThroughTerm() const { return ThroughLogTerm; } +private: + TConfigurationHistory() = default; + std::vector Old, New; + uint64_t ConfigIndex = 0, ConfigTerm = 0, Through = 0, ThroughLogTerm = 0; +}; + +struct IState; +struct TConfigurationScan { + TConfigurationHistory Committed; + TConfigurationHistory Latest; + bool HasUncommittedConfiguration() const { + return Committed.ConfigurationIndex() != Latest.ConfigurationIndex() || + Committed.ConfigurationTerm() != Latest.ConfigurationTerm() || + Committed.IsJoint() != Latest.IsJoint() || + Committed.OldVoters() != Latest.OldVoters() || + Committed.NewVoters() != Latest.NewVoters(); + } +}; + +// Read-only validation. The caller supplies and must substantiate both the +// baseline and commit boundary; neither is proven by this scan. +TConfigurationScan ScanConfigurationHistory(const IState& state, + TConfigurationHistory baseline, uint64_t commitIndex); + +// Read the committed boundary from storage instead of accepting a caller's +// guess. Requires voter binding and a backend that persists CommitIndex; the +// caller must still supply a trusted baseline through SnapshotIndex. +TConfigurationScan ScanCommittedConfigurationHistory(const IState& state, + TConfigurationHistory baseline); diff --git a/miniraft/durable.cpp b/miniraft/durable.cpp new file mode 100644 index 0000000..fcc7ae1 --- /dev/null +++ b/miniraft/durable.cpp @@ -0,0 +1,432 @@ +#include "durable.h" +#include +#include +#include +#include +#include +#include +#include +#ifdef __linux__ +#include +#include +#include +#include +#include +#include +#endif + +namespace { +using TBytes = std::vector; +constexpr std::array Magic{'M','R','F','T','D','0','0','1'}; +constexpr std::array SnapshotMagic{'M','R','F','T','D','0','0','2'}; +constexpr std::array ConfigurationMagic{'M','R','F','T','D','0','0','3'}; +constexpr std::array CommitMagic{'M','R','F','T','D','0','0','4'}; +void Put(TBytes& bytes, uint64_t value, unsigned width) { + for (unsigned i = 0; i < width; ++i) bytes.push_back(uint8_t(value >> (8 * i))); +} +uint64_t Take(const TBytes& bytes, size_t& position, unsigned width) { + if (position > bytes.size() || width > bytes.size() - position) + throw std::runtime_error("Truncated durable state"); + uint64_t value = 0; + for (unsigned i = 0; i < width; ++i) value |= uint64_t(bytes[position++]) << (8 * i); + return value; +} +uint32_t Checksum(const TBytes& bytes, size_t start, const std::array* version = nullptr) { + uint32_t crc = UINT32_MAX; + const auto update = [&](uint8_t byte) { + crc ^= byte; + for (int bit = 0; bit < 8; ++bit) crc = (crc >> 1) ^ (0xedb88320u & (0u - (crc & 1u))); + }; + if (version) for (auto byte : *version) update(byte); + for (size_t i = start; i < bytes.size(); ++i) update(bytes[i]); + return ~crc; +} +TMessageHolder Clone(TMessageHolder entry) { + if (!entry.HasValidExtent() || entry->Len < sizeof(TLogEntry) || entry->Type != uint32_t(EMessageType::LOG_ENTRY)) + throw std::runtime_error("Invalid durable log entry"); + auto copy = NewHoldedMessage(entry->Len); + memcpy(copy.Mes, entry.Mes, entry->Len); + return copy; +} +#ifdef __linux__ +struct TFd { + int Value = -1; + TFd() = default; + explicit TFd(int fd) : Value(fd) {} + ~TFd() { if (Value >= 0) close(Value); } + TFd(const TFd&) = delete; + TFd& operator=(const TFd&) = delete; +}; +void Require(bool ok, const char* operation) { + if (!ok) throw std::system_error(errno, std::generic_category(), operation); +} +void Sync(int fd) { + int result; + do { result = fsync(fd); } while (result < 0 && errno == EINTR); + Require(result == 0, "fsync durable state"); +} +void WriteAll(int fd, const TBytes& bytes) { + size_t done = 0; + while (done < bytes.size()) { + auto n = write(fd, bytes.data() + done, bytes.size() - done); + if (n < 0 && errno == EINTR) continue; + Require(n > 0, "write durable state"); + done += size_t(n); + } +} +TBytes ReadAll(int fd) { + struct stat info; + Require(fstat(fd, &info) == 0, "stat durable state"); + if (!S_ISREG(info.st_mode) || info.st_size < 20 || uint64_t(info.st_size) > SIZE_MAX) + throw std::runtime_error("Invalid durable state file size/type"); + TBytes bytes(size_t(info.st_size)); + size_t done = 0; + while (done < bytes.size()) { + auto n = read(fd, bytes.data() + done, bytes.size() - done); + if (n < 0 && errno == EINTR) continue; + if (n == 0) throw std::runtime_error("Truncated durable state file"); + Require(n > 0, "read durable state"); + done += size_t(n); + } + return bytes; +} +#endif +} + +struct TDurableState::TImpl { + std::vector> Log; + std::shared_ptr Snapshot; + // Original trusted voter set; configuration snapshots can carry a later + // active set without rewriting this binding. + std::vector Voters; + bool Recovered = false; + bool RaftStarted = false; + THook Hook; + std::string File, Temp; +#ifdef __linux__ + TFd Directory, Lock; +#endif + void Stage(EStage stage) { if (Hook) Hook(stage); } +}; + +TDurableState::TDurableState(const std::string& name, uint32_t id, THook hook) + : Impl(std::make_unique()) +{ +#ifndef __linux__ + throw std::runtime_error("TDurableState currently requires Linux"); +#else + const auto base = std::filesystem::path(name); + auto directory = base.parent_path(); + if (directory.empty()) directory = "."; + Impl->File = base.filename().string() + ".durable." + std::to_string(id); + Impl->Temp = Impl->File + ".tmp"; + Impl->Hook = std::move(hook); + Impl->Directory.Value = open(directory.c_str(), O_RDONLY | O_DIRECTORY | O_CLOEXEC); + Require(Impl->Directory.Value >= 0, "open durable directory"); + // Never interpret a legacy store as a fresh node. Migration must be explicit. + for (const auto* kind : {"entries", "index", "state"}) { + if (std::filesystem::exists(name + "." + kind + "." + std::to_string(id))) + throw std::runtime_error("Legacy storage exists; durable migration is not automatic"); + } + const auto lockName = Impl->File + ".lock"; + Impl->Lock.Value = openat(Impl->Directory.Value, lockName.c_str(), + O_RDWR | O_CREAT | O_EXCL | O_CLOEXEC | O_NOFOLLOW, 0600); + const bool newLock = Impl->Lock.Value >= 0; + if (!newLock && errno == EEXIST) + Impl->Lock.Value = openat(Impl->Directory.Value, lockName.c_str(), O_RDWR | O_CLOEXEC | O_NOFOLLOW); + Require(Impl->Lock.Value >= 0, "open durable lock"); + Require(flock(Impl->Lock.Value, LOCK_EX | LOCK_NB) == 0, "lock durable state"); + TFd file(openat(Impl->Directory.Value, Impl->File.c_str(), O_RDONLY | O_CLOEXEC | O_NOFOLLOW)); + if (file.Value < 0) { + Require(errno == ENOENT, "open durable state"); + if (!newLock) throw std::runtime_error("Published durable state is missing"); + Publish(); + return; + } + const auto bytes = ReadAll(file.Value); + const bool commitAware = std::equal(CommitMagic.begin(), CommitMagic.end(), bytes.begin()); + const bool configured = std::equal(ConfigurationMagic.begin(), ConfigurationMagic.end(), bytes.begin()); + const bool compacted = std::equal(SnapshotMagic.begin(), SnapshotMagic.end(), bytes.begin()); + const auto* version = commitAware ? &CommitMagic : configured ? &ConfigurationMagic : compacted ? &SnapshotMagic : nullptr; + if (!version && !std::equal(Magic.begin(), Magic.end(), bytes.begin())) + throw std::runtime_error("Unknown durable state format"); + size_t position = Magic.size(); + const auto length = Take(bytes, position, 8); + const auto crc = Take(bytes, position, 4); + if (length != bytes.size() - position || crc != Checksum(bytes, position, version)) + throw std::runtime_error("Durable state length/checksum mismatch"); + CurrentTerm = Take(bytes, position, 8); + VotedFor = Take(bytes, position, 4); + const auto count = Take(bytes, position, 8); + if (commitAware) CommitIndex = Take(bytes, position, 8); + if (configured) { + const auto voters = Take(bytes, position, 4); + if (!voters || voters > TSnapshot::MaxVoters || voters > (bytes.size() - position) / 4) + throw std::runtime_error("Invalid durable voter count"); + for (uint32_t i = 0; i < voters; ++i) Impl->Voters.push_back(Take(bytes, position, 4)); + try { ValidateFixedVoters(Impl->Voters); } + catch (const std::invalid_argument&) { throw std::runtime_error("Invalid durable voter configuration"); } + } + if (commitAware) { + const auto voters = Take(bytes, position, 4); + if (voters > TSnapshot::MaxVoters || voters > (bytes.size() - position) / 4) + throw std::runtime_error("Invalid durable voter count"); + for (uint32_t i = 0; i < voters; ++i) Impl->Voters.push_back(Take(bytes, position, 4)); + if (voters) { + try { ValidateFixedVoters(Impl->Voters); } + catch (const std::invalid_argument&) { throw std::runtime_error("Invalid durable voter configuration"); } + } + const auto length = Take(bytes, position, 8); + if (length > TSnapshot::MaxBytes || length > bytes.size() - position) + throw std::runtime_error("Invalid embedded snapshot size"); + if (length) { + auto snapshot = DecodeSnapshot(std::string_view(reinterpret_cast(bytes.data() + position), size_t(length))); + if (!SnapshotCompatibleWithVoterBinding(snapshot, Impl->Voters)) + throw std::runtime_error("Durable snapshot is incompatible with the persisted voter binding"); + if (snapshot.Term > CurrentTerm) throw std::runtime_error("Future durable snapshot term"); + position += size_t(length); + SnapshotIndex = snapshot.Index; + SnapshotTerm = snapshot.Term; + LastLogTerm = SnapshotTerm; + Impl->Snapshot = std::make_shared(std::move(snapshot)); + } + } else if (compacted || configured) { + const auto length = Take(bytes, position, 8); + if (length > TSnapshot::MaxBytes || length > bytes.size() - position) + throw std::runtime_error("Invalid embedded snapshot size"); + if (!length && compacted) throw std::runtime_error("Missing embedded snapshot"); + if (length) { + auto snapshot = DecodeSnapshot(std::string_view(reinterpret_cast(bytes.data() + position), size_t(length))); + if (snapshot.HasConfigurationMetadata() && !configured) + throw std::runtime_error("Configuration snapshot recovery requires a bound voter set"); + if (snapshot.Term > CurrentTerm) throw std::runtime_error("Future durable snapshot term"); + if (configured && !SnapshotCompatibleWithVoterBinding(snapshot, Impl->Voters)) + throw std::runtime_error("Durable snapshot is incompatible with the persisted voter binding"); + position += size_t(length); + SnapshotIndex = snapshot.Index; + SnapshotTerm = snapshot.Term; + LastLogTerm = SnapshotTerm; + Impl->Snapshot = std::make_shared(std::move(snapshot)); + } + } + if (!commitAware) CommitIndex = SnapshotIndex; + if (!CurrentTerm || count > uint64_t(INT64_MAX) - SnapshotIndex || count > (bytes.size() - position) / 20) + throw std::runtime_error("Invalid durable state metadata"); + for (uint64_t i = 0; i < count; ++i) { + const auto term = Take(bytes, position, 8); + const auto flags = Take(bytes, position, 8); + const auto length = Take(bytes, position, 4); + if (length > bytes.size() - position || length > UINT32_MAX - sizeof(TLogEntry) || + !term || term < LastLogTerm || term > CurrentTerm) + throw std::runtime_error("Invalid durable log entry"); + auto entry = NewHoldedMessage(sizeof(TLogEntry) + length); + entry->Term = term; + entry->Flags = flags; + memcpy(entry->Data, bytes.data() + position, length); + position += length; + Impl->Log.push_back(entry); + LastLogTerm = term; + } + if (position != bytes.size()) throw std::runtime_error("Trailing durable state data"); + LastLogIndex = SnapshotIndex + count; + if (CommitIndex < SnapshotIndex || CommitIndex > LastLogIndex) + throw std::runtime_error("Invalid durable commit index"); + // A crash after rename but before directory fsync may leave a visible new + // image. Stabilize the selected image and its name before serving requests. + Sync(file.Value); + Sync(Impl->Directory.Value); + Impl->Recovered = true; + // A leftover temp is never a recovery candidate, even if its checksum is valid. +#endif +} + +TDurableState::~TDurableState() = default; + +void TDurableState::Publish() +{ +#ifndef __linux__ + throw std::runtime_error("TDurableState currently requires Linux"); +#else + if (!CurrentTerm || LastLogIndex > uint64_t(INT64_MAX) || LastLogIndex != SnapshotIndex + Impl->Log.size() || + CommitIndex < SnapshotIndex || CommitIndex > LastLogIndex || + LastLogTerm != (Impl->Log.empty() ? SnapshotTerm : Impl->Log.back()->Term) || + (Impl->Snapshot ? (Impl->Snapshot->Index != SnapshotIndex || Impl->Snapshot->Term != SnapshotTerm || + SnapshotTerm > CurrentTerm) : (SnapshotIndex != 0 || SnapshotTerm != 0))) + throw std::runtime_error("Invalid durable state fields"); + TBytes payload; + Put(payload, CurrentTerm, 8); + Put(payload, VotedFor, 4); + Put(payload, Impl->Log.size(), 8); + Put(payload, CommitIndex, 8); + Put(payload, Impl->Voters.size(), 4); + if (!Impl->Voters.empty()) { + ValidateFixedVoters(Impl->Voters); + for (auto id : Impl->Voters) Put(payload, id, 4); + } + if (Impl->Snapshot && !SnapshotCompatibleWithVoterBinding(*Impl->Snapshot, Impl->Voters)) + throw std::runtime_error("Durable snapshot is incompatible with the persisted voter binding"); + const auto image = Impl->Snapshot ? EncodeSnapshot(*Impl->Snapshot) : std::string{}; + Put(payload, image.size(), 8); + payload.insert(payload.end(), image.begin(), image.end()); + uint64_t previousTerm = SnapshotTerm; + for (auto entry : Impl->Log) { + if (!entry->Term || entry->Term < previousTerm || entry->Term > CurrentTerm) + throw std::runtime_error("Invalid durable log term"); + previousTerm = entry->Term; + Put(payload, entry->Term, 8); + Put(payload, entry->Flags, 8); + Put(payload, entry->Len - sizeof(TLogEntry), 4); + payload.insert(payload.end(), entry->Data, entry->Data + entry->Len - sizeof(TLogEntry)); + } + const auto& magic = CommitMagic; + TBytes bytes(magic.begin(), magic.end()); + Put(bytes, payload.size(), 8); + Put(bytes, Checksum(payload, 0, &CommitMagic), 4); + bytes.insert(bytes.end(), payload.begin(), payload.end()); + TFd temp(openat(Impl->Directory.Value, Impl->Temp.c_str(), + O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC | O_NOFOLLOW, 0600)); + Require(temp.Value >= 0, "create durable temporary image"); + Impl->Stage(EStage::TempCreated); + WriteAll(temp.Value, bytes); + Impl->Stage(EStage::Written); + Sync(temp.Value); + Impl->Stage(EStage::FileSynced); + Require(renameat(Impl->Directory.Value, Impl->Temp.c_str(), + Impl->Directory.Value, Impl->File.c_str()) == 0, "publish durable image"); + Impl->Stage(EStage::Renamed); + Sync(Impl->Directory.Value); + Impl->Stage(EStage::DirectorySynced); +#endif +} + +void TDurableState::Commit() +{ + try { Publish(); } + catch (const std::exception& error) { + std::cerr << "Fatal durable storage error: " << error.what() << '\n'; + std::terminate(); + } catch (...) { std::terminate(); } +} + +void TDurableState::Append(TMessageHolder entry) +{ + if (LastLogIndex >= uint64_t(INT64_MAX)) throw std::overflow_error("Raft log index exhausted"); + Impl->Log.push_back(Clone(entry)); + LastLogIndex = SnapshotIndex + Impl->Log.size(); + LastLogTerm = Impl->Log.back()->Term; + Commit(); +} + +void TDurableState::RemoveLast() +{ + if (Impl->Log.empty()) return; + if (LastLogIndex <= CommitIndex) + throw std::logic_error("Cannot remove a committed log entry"); + Impl->Log.pop_back(); + LastLogIndex = SnapshotIndex + Impl->Log.size(); + LastLogTerm = Impl->Log.empty() ? SnapshotTerm : Impl->Log.back()->Term; + Commit(); +} + +TMessageHolder TDurableState::Get(int64_t index) const +{ + if (index < 1 || uint64_t(index) <= SnapshotIndex || uint64_t(index) > LastLogIndex) return {}; + return Clone(Impl->Log[index - SnapshotIndex - 1]); +} + +uint64_t TDurableState::RetainedLogTerm(int64_t index) const { + if (index < 1 || uint64_t(index) <= SnapshotIndex || uint64_t(index) > LastLogIndex) return 0; + return Impl->Log[index - SnapshotIndex - 1]->Term; +} +uint64_t TDurableState::RetainedLogFlags(int64_t index) const { + if (index < 1 || uint64_t(index) <= SnapshotIndex || uint64_t(index) > LastLogIndex) + throw std::out_of_range("Missing retained log entry"); + return Impl->Log[index - SnapshotIndex - 1]->Flags; +} + +std::shared_ptr TDurableState::GetSnapshot() const { return Impl->Snapshot; } + +void TDurableState::InstallSnapshot(const TSnapshot& snapshot) { + ValidateSnapshot(snapshot); + TSnapshot image = snapshot; + PreserveStableSnapshotIdentity(Impl->Snapshot.get(), image); + ValidateSnapshot(image); + if (!SnapshotCompatibleWithVoterBinding(image, Impl->Voters)) + throw std::invalid_argument("Snapshot is incompatible with the persisted voter binding"); + if (image.Index <= SnapshotIndex || image.Term < SnapshotTerm || image.Term > CurrentTerm) + throw std::invalid_argument("Obsolete or future durable snapshot"); + if (image.Index <= CommitIndex && + (image.Index > LastLogIndex || LogTerm(image.Index) != image.Term)) + throw std::invalid_argument("Snapshot would discard committed log entries"); + auto saved = std::make_shared(std::move(image)); + std::vector> suffix; + if (saved->Index <= LastLogIndex && LogTerm(saved->Index) == saved->Term) + suffix.assign(Impl->Log.begin() + (saved->Index - SnapshotIndex), Impl->Log.end()); + Impl->Log.swap(suffix); + Impl->Snapshot = std::move(saved); + SnapshotIndex = Impl->Snapshot->Index; + SnapshotTerm = Impl->Snapshot->Term; + LastLogIndex = SnapshotIndex + Impl->Log.size(); + LastLogTerm = Impl->Log.empty() ? SnapshotTerm : Impl->Log.back()->Term; + CommitIndex = std::max(CommitIndex, SnapshotIndex); + Commit(); // Atomically publishes hard state, snapshot and the selected suffix. +} + +std::vector TDurableState::GetBoundVoters() const { return Impl->Voters; } + +bool TDurableState::HasUnboundRecoveredState() const { + return Impl->Recovered && Impl->Voters.empty(); +} + +void TDurableState::MarkRaftStarted() { Impl->RaftStarted = true; } + +void TDurableState::BindVoters(const std::vector& voters) { + ValidateVoterBinding(*this, Impl->Voters, voters); + if (Impl->Voters == voters) return; + Impl->Voters = voters; + Commit(); // Bind before acknowledging any vote/replication traffic. +} + +void TDurableState::BindVotersFromTrustedBaseline(const std::vector& voters) { + ValidateFixedVoters(voters); + if (Impl->RaftStarted) + throw std::invalid_argument("Trusted legacy voter migration must happen before Raft startup"); + if (!Impl->Voters.empty()) { + if (Impl->Voters == voters) return; + throw std::invalid_argument("Durable voter binding already exists"); + } + if (!CurrentTerm || LastLogIndex > uint64_t(INT64_MAX) || SnapshotIndex > LastLogIndex || + LastLogIndex - SnapshotIndex != Impl->Log.size()) + throw std::invalid_argument("Invalid legacy state bounds"); + if (!Impl->Recovered) + throw std::invalid_argument("Trusted legacy baseline is only for recovered state"); + if (VotedFor && !std::binary_search(voters.begin(), voters.end(), VotedFor)) + throw std::invalid_argument("Stored vote is outside the trusted voter baseline"); + + if (Impl->Snapshot) { + const auto& snapshot = *Impl->Snapshot; + ValidateSnapshot(snapshot); + if (snapshot.HasConfigurationMetadata() || snapshot.Index != SnapshotIndex || + snapshot.Term != SnapshotTerm || snapshot.Term > CurrentTerm || snapshot.Voters != voters) + throw std::invalid_argument("Legacy snapshot does not match the trusted voter baseline"); + } else if (SnapshotIndex || SnapshotTerm) { + throw std::invalid_argument("Legacy compacted state is missing its snapshot"); + } + + auto previousTerm = SnapshotTerm; + for (uint64_t index = SnapshotIndex + 1; index <= LastLogIndex; ++index) { + if (RetainedLogFlags(index) & TLogEntry::EConfiguration) + throw std::invalid_argument("Configuration log migration requires an explicit history baseline"); + const auto term = RetainedLogTerm(index); + if (!term || term < previousTerm || term > CurrentTerm) + throw std::invalid_argument("Invalid legacy log term sequence"); + previousTerm = term; + } + if (LastLogTerm != previousTerm) + throw std::invalid_argument("Legacy log last term mismatch"); + + // Persist the operator-supplied baseline before Raft can serve votes or + // install configuration snapshots against it. + Impl->Voters = voters; + Commit(); +} diff --git a/miniraft/durable.h b/miniraft/durable.h new file mode 100644 index 0000000..3c5891c --- /dev/null +++ b/miniraft/durable.h @@ -0,0 +1,42 @@ +#pragma once +#include "state.h" +#include +#include +#include + +// Linux: atomic hard-state + snapshot + suffix replacement, with file/directory +// fsync. O(snapshot + retained log size) per update: a durability baseline. +class TDurableState : public IState { +public: + enum class EStage { TempCreated, Written, FileSynced, Renamed, DirectorySynced }; + // Diagnostics/fault injection. Throwing from this hook during a live + // operation is fatal, just like an I/O failure. + using THook = std::function; + TDurableState(const std::string& name, uint32_t id, THook hook = {}); + ~TDurableState() override; + TDurableState(const TDurableState&) = delete; + TDurableState& operator=(const TDurableState&) = delete; + void RemoveLast() override; + void Append(TMessageHolder entry) override; + TMessageHolder Get(int64_t index) const override; + uint64_t RetainedLogTerm(int64_t index) const override; + uint64_t RetainedLogFlags(int64_t index) const override; + void Commit() override; + bool SupportsCommitIndexPersistence() const override { return true; } + bool SupportsSnapshots() const override { return true; } + std::shared_ptr GetSnapshot() const override; + void InstallSnapshot(const TSnapshot& snapshot) override; + bool SupportsVoterBinding() const override { return true; } + bool HasUnboundRecoveredState() const override; + std::vector GetBoundVoters() const override; + void BindVoters(const std::vector& voters) override; + void MarkRaftStarted() override; + // Explicit legacy migration. Call before constructing TRaft. The caller + // asserts that voters is the complete original fixed set, verified outside + // this store. + void BindVotersFromTrustedBaseline(const std::vector& voters); +private: + struct TImpl; + std::unique_ptr Impl; + void Publish(); +}; diff --git a/miniraft/messages.cpp b/miniraft/messages.cpp index f6152ec..93a421e 100644 --- a/miniraft/messages.cpp +++ b/miniraft/messages.cpp @@ -1 +1,52 @@ #include "messages.h" + +void ValidateMessageLimits(const TMessageLimits& limits) { + if (limits.MaxFrameBytes < sizeof(TMessage) || limits.MaxBatchBytes < sizeof(TMessage)) + throw std::invalid_argument("Message byte limits cannot hold a header"); +} + +void ValidateMessageHeader(uint32_t type, uint32_t length, const TMessageLimits& limits) { + uint32_t minimum; + bool fixed = true; + switch (EMessageType(type)) { + case EMessageType::LOG_ENTRY: minimum = sizeof(TLogEntry); fixed = false; break; + case EMessageType::REQUEST_VOTE_REQUEST: minimum = sizeof(TRequestVoteRequest); break; + case EMessageType::REQUEST_VOTE_RESPONSE: minimum = sizeof(TRequestVoteResponse); break; + case EMessageType::APPEND_ENTRIES_REQUEST: minimum = sizeof(TAppendEntriesRequest); break; + case EMessageType::APPEND_ENTRIES_RESPONSE: minimum = sizeof(TAppendEntriesResponse); break; + case EMessageType::INSTALL_SNAPSHOT_REQUEST: + minimum = sizeof(TInstallSnapshotRequest) + 1; + if (length > sizeof(TInstallSnapshotRequest) + TInstallSnapshotRequest::MaxChunkBytes) + throw TMessageError("Snapshot chunk exceeds byte limit"); + fixed = false; break; + case EMessageType::INSTALL_SNAPSHOT_RESPONSE: minimum = sizeof(TInstallSnapshotResponse); break; + case EMessageType::COMMAND_REQUEST: minimum = sizeof(TCommandRequest); fixed = false; break; + case EMessageType::COMMAND_RESPONSE: minimum = sizeof(TCommandResponse); fixed = false; break; + default: throw TMessageError("Unknown or unsupported message type"); + } + if (length < minimum || (fixed && length != minimum) || length > limits.MaxFrameBytes) + throw TMessageError("Invalid message length"); +} + +void ValidateMessageTree(const TMessageHolder& message, const TMessageLimits& limits) { + ValidateMessageLimits(limits); + uint64_t remaining = limits.MaxBatchBytes; + auto validate = [&](const auto& frame, bool child) { + if (!frame) throw TMessageError("Missing message"); + if (!frame.HasValidExtent()) throw TMessageError("Message length exceeds its buffer"); + ValidateMessageHeader(frame->Type, frame->Len, limits); + if (frame->Len > remaining) throw TMessageError("Message batch exceeds byte limit"); + remaining -= frame->Len; + if (child && frame->Type != uint32_t(EMessageType::LOG_ENTRY)) + throw TMessageError("AppendEntries children must be log entries"); + if (frame->Type != uint32_t(EMessageType::APPEND_ENTRIES_REQUEST) && frame.PayloadSize) + throw TMessageError("Unexpected message payload"); + }; + validate(message, false); + if (message->Type != uint32_t(EMessageType::APPEND_ENTRIES_REQUEST)) return; + auto copy = message; + const auto count = copy.Cast()->Nentries; + if (count > limits.MaxEntries || count != message.PayloadSize || count > message.PayloadCapacity || (count && !message.Payload)) + throw TMessageError("Invalid AppendEntries entry count"); + for (uint32_t i = 0; i < count; ++i) validate(message.Payload[i], true); +} diff --git a/miniraft/messages.h b/miniraft/messages.h index 830030b..625dd04 100644 --- a/miniraft/messages.h +++ b/miniraft/messages.h @@ -1,5 +1,6 @@ #pragma once #include +#include #include #include @@ -7,6 +8,17 @@ #include #include #include +#include + +struct TMessageLimits { + uint32_t MaxFrameBytes = 16 * 1024 * 1024; + uint64_t MaxBatchBytes = 64 * 1024 * 1024; + uint32_t MaxEntries = 4096; +}; + +struct TMessageError : std::runtime_error { using std::runtime_error::runtime_error; }; +void ValidateMessageLimits(const TMessageLimits& limits); +void ValidateMessageHeader(uint32_t type, uint32_t length, const TMessageLimits& limits = {}); enum class EMessageType : uint32_t { NONE = 0, @@ -15,8 +27,8 @@ enum class EMessageType : uint32_t { REQUEST_VOTE_RESPONSE = 3, APPEND_ENTRIES_REQUEST = 4, APPEND_ENTRIES_RESPONSE = 5, - INSTALL_SNAPSHOT_REQUEST = 6, // TODO: not implemented it - INSTALL_SNAPSHOT_RESPONSE = 7, // TODO: not implemented it + INSTALL_SNAPSHOT_REQUEST = 6, + INSTALL_SNAPSHOT_RESPONSE = 7, COMMAND_REQUEST = 8, COMMAND_RESPONSE = 9, }; @@ -32,9 +44,11 @@ static_assert(sizeof(TMessage) == 8); struct TLogEntry: public TMessage { static constexpr EMessageType MessageType = EMessageType::LOG_ENTRY; - enum EFlags { + enum EFlags : uint64_t { ENone = 0, EStub = 1, + // Reserved for configuration entries; live use remains disabled. + EConfiguration = uint64_t(1) << 63, }; uint64_t Term = 1; uint64_t Flags = 0; @@ -81,6 +95,8 @@ static_assert(sizeof(TAppendEntriesRequest) == sizeof(TMessageEx) + 32); struct TAppendEntriesResponse: public TMessageEx { static constexpr EMessageType MessageType = EMessageType::APPEND_ENTRIES_RESPONSE; + // Success: matched index. Failure: zero or a compacted-boundary probe hint; + // a hint never advances MatchIndex or confirms a read. uint64_t MatchIndex; uint32_t Success; uint32_t Padding = 0; @@ -88,6 +104,21 @@ struct TAppendEntriesResponse: public TMessageEx { static_assert(sizeof(TAppendEntriesResponse) == sizeof(TMessageEx) + 16); +struct TInstallSnapshotRequest: public TMessageEx { + static constexpr EMessageType MessageType = EMessageType::INSTALL_SNAPSHOT_REQUEST; + static constexpr uint32_t MaxChunkBytes = 64 * 1024; + uint64_t SnapshotIndex = 0, SnapshotTerm = 0, TotalBytes = 0, Offset = 0; + uint32_t LeaderId = 0, Done = 0; + char Data[0]; +}; +static_assert(sizeof(TInstallSnapshotRequest) == sizeof(TMessageEx) + 40); +struct TInstallSnapshotResponse: public TMessageEx { + static constexpr EMessageType MessageType = EMessageType::INSTALL_SNAPSHOT_RESPONSE; + uint64_t SnapshotIndex = 0, NextOffset = 0; + uint32_t Success = 0, Installed = 0; +}; +static_assert(sizeof(TInstallSnapshotResponse) == sizeof(TMessageEx) + 24); + struct TCommandRequest: public TMessage { static constexpr EMessageType MessageType = EMessageType::COMMAND_REQUEST; enum EFlags { @@ -97,6 +128,7 @@ struct TCommandRequest: public TMessage { // read semantics, default: read from leader w/o ping check, possible stale reads if there are 2 leaders EStale = 2, // stale read, can read from follower EConsistent = 4, // strong consistent read (wait for pings, low latency, no stale read) + EForwarded = 8, // internal one-hop marker, independent of client cookies }; uint32_t Flags = ENone; uint32_t Cookie = 0; @@ -119,6 +151,9 @@ struct TTimeout { static constexpr std::chrono::milliseconds Election = std::chrono::milliseconds(5000); static constexpr std::chrono::milliseconds Heartbeat = std::chrono::milliseconds(1000); static constexpr std::chrono::milliseconds Rpc = std::chrono::milliseconds(10000); + // Keep partial snapshot transfers across normal RPC retries, but bound how + // long an idle receiver retains the accumulated image. + static constexpr std::chrono::milliseconds SnapshotTransferIdle = std::chrono::seconds(30); }; template @@ -126,9 +161,11 @@ requires std::derived_from struct TMessageHolder { T* Mes; std::shared_ptr RawData; + size_t Capacity = 0; // Allocated bytes accessible starting at Mes. - uint32_t PayloadSize; + uint32_t PayloadSize = 0; std::shared_ptr[]> Payload; + size_t PayloadCapacity = 0; TMessageHolder() : Mes(nullptr) @@ -139,11 +176,14 @@ struct TMessageHolder { TMessageHolder(U* u, const std::shared_ptr& rawData, uint32_t payloadSize = 0, - const std::shared_ptr[]>& payload = {}) + const std::shared_ptr[]>& payload = {}, size_t capacity = 0, + size_t payloadCapacity = 0) : Mes(u) , RawData(rawData) + , Capacity(capacity) , PayloadSize(payloadSize) , Payload(payload) + , PayloadCapacity(payloadCapacity) { } template @@ -151,13 +191,16 @@ struct TMessageHolder { TMessageHolder(const TMessageHolder& other) : Mes(other.Mes) , RawData(other.RawData) + , Capacity(other.Capacity) , PayloadSize(other.PayloadSize) , Payload(other.Payload) + , PayloadCapacity(other.PayloadCapacity) { } void InitPayload(uint32_t size) { PayloadSize = size; Payload = std::shared_ptr[]>(new TMessageHolder[size]); + PayloadCapacity = size; } T* operator->() { @@ -172,14 +215,22 @@ struct TMessageHolder { return !!Mes; } + bool HasValidExtent() const { + return Mes && RawData && Capacity >= sizeof(TMessage) && + Mes->Len >= sizeof(TMessage) && Mes->Len <= Capacity; + } + bool IsEx() const { - return (2 <= Mes->Type && Mes->Type <= 5); + return HasValidExtent() && Mes->Len >= sizeof(TMessageEx) && (2 <= Mes->Type && Mes->Type <= 7); } template requires std::derived_from TMessageHolder Cast() { - return TMessageHolder(static_cast(Mes), RawData, PayloadSize, Payload); + if (!HasValidExtent() || Mes->Len < sizeof(U) || + (U::MessageType != EMessageType::NONE && Mes->Type != uint32_t(U::MessageType))) + throw std::bad_cast(); + return TMessageHolder(static_cast(Mes), RawData, PayloadSize, Payload, Capacity, PayloadCapacity); } template @@ -200,16 +251,19 @@ struct TMessageHolder { } }; - return Mes->Type == static_cast(U::MessageType) + return HasValidExtent() && Mes->Len >= sizeof(U) && Mes->Type == static_cast(U::MessageType) ? Maybe { *this } : Maybe { }; } }; +// Validate the complete frame/batch before emitting any bytes. +void ValidateMessageTree(const TMessageHolder& message, const TMessageLimits& limits = {}); + template requires std::derived_from -T* NewMessage(uint32_t type, uint32_t len) { - assert(len >= sizeof(TMessage)); +T* NewMessage(uint32_t type, size_t len) { + if (len < sizeof(T) || len > UINT32_MAX) throw std::invalid_argument("Invalid message allocation size"); char* data = new char[len]; T* mes = new (data) T; mes->Type = type; @@ -219,10 +273,10 @@ T* NewMessage(uint32_t type, uint32_t len) { template requires std::derived_from -TMessageHolder NewHoldedMessage(uint32_t type, uint32_t len) +TMessageHolder NewHoldedMessage(uint32_t type, size_t len) { T* mes = NewMessage(type, len); - return TMessageHolder(mes, std::shared_ptr(reinterpret_cast(mes))); + return TMessageHolder(mes, std::shared_ptr(reinterpret_cast(mes)), 0, {}, len); } template @@ -231,8 +285,7 @@ TMessageHolder NewHoldedMessage() { } template -TMessageHolder NewHoldedMessage(uint32_t size) { - assert(size >= sizeof(T)); +TMessageHolder NewHoldedMessage(size_t size) { return NewHoldedMessage(static_cast(T::MessageType), size); } diff --git a/miniraft/net/node_impl.h b/miniraft/net/node_impl.h new file mode 100644 index 0000000..61f08c1 --- /dev/null +++ b/miniraft/net/node_impl.h @@ -0,0 +1,87 @@ +#pragma once + +// Template definitions kept visible for alternative socket implementations. +template +void TNode::Send(TMessageHolder message) { + Messages.emplace_back(std::move(message)); +} + +template +void TNode::Drain() { + if (!Connected) { + Connect(); + return; + } + if (!Drainer || Drainer.done()) { + if (Drainer && Drainer.done()) { + Drainer.destroy(); + } + Drainer = DoDrain(); + } +} + +template +NNet::TVoidSuspendedTask TNode::DoDrain() { + try { + while (!Messages.empty()) { + auto tosend = std::move(Messages); Messages.clear(); + for (auto&& m : tosend) { + co_await TMessageWriter(Socket).Write(std::move(m)); + } + } + } catch (const std::exception& ex) { + std::cout << "Error on write: " << ex.what() << "\n"; + Connect(); + } + co_return; +} + +template +void TNode::Disconnect(uint64_t generation) { + if (generation != ConnectionGeneration || (Connector && !Connector.done())) return; + // DoConnect may already be retrying after a write failure. Its awaiter can + // still be registered with the poller; never destroy that live coroutine + // because an older reader failed. Established connectors are done here. + Connected = false; + if (Drainer) { Drainer.destroy(); Drainer = {}; } + if (Connector) { Connector.destroy(); Connector = {}; } + std::destroy_at(&Socket); + std::construct_at(&Socket); +} + +template +void TNode::Connect() { + if (Address && (!Connector || Connector.done())) { + if (ConnectionGeneration == UINT64_MAX) throw std::overflow_error("Connection generation exhausted"); + if (Connector && Connector.done()) { + Connector.destroy(); + } + ++ConnectionGeneration; + Connector = DoConnect(); + } +} + +template +NNet::TVoidSuspendedTask TNode::DoConnect() { + std::cout << "Connecting " << Name << "\n"; + Connected = false; + while (!Connected) { + try { + Socket = SocketFactory(*Address); + // The dependency's finite deadline can leave I/O and timer wakeups + // registered for the same coroutine after a timeout. Use its default + // I/O-only connect until cancellation of both sources is supported. + // TCP connect failure timing is therefore controlled by the OS. + co_await Socket.Connect(); + std::cout << "Connected " << Name << "\n"; + Connected = true; + } catch (const std::exception& ex) { + std::cout << "Error on connect: " << Name << " " << ex.what() << "\n"; + } + if (!Connected) { + co_await Socket.Poller()->Sleep(std::chrono::milliseconds(1000)); + } + } + co_return; +} + diff --git a/miniraft/net/server.cpp b/miniraft/net/server.cpp index 6064e8b..6af0d10 100644 --- a/miniraft/net/server.cpp +++ b/miniraft/net/server.cpp @@ -9,110 +9,6 @@ #include "server.h" -template -NNet::TValueTask TMessageWriter::Write(TMessageHolder message) { - co_await NNet::TByteWriter(Socket).Write(message.Mes, message->Len); - - auto payload = std::move(message.Payload); - for (uint32_t i = 0; i < message.PayloadSize; ++i) { - co_await Write(std::move(payload[i])); - } - - co_return; -} - -template -NNet::TValueTask> TMessageReader::Read() { - decltype(TMessage::Type) type; - decltype(TMessage::Len) len; - auto s = co_await Socket.ReadSome(&type, sizeof(type)); - if (s != sizeof(type)) { - throw std::runtime_error("Connection closed"); - } - s = co_await Socket.ReadSome(&len, sizeof(len)); - if (s != sizeof(len)) { - throw std::runtime_error("Connection closed"); - } - auto mes = NewHoldedMessage(type, len); - co_await NNet::TByteReader(Socket).Read(mes->Value, len - sizeof(TMessage)); - auto maybeAppendEntries = mes.Maybe(); - if (maybeAppendEntries) { - auto appendEntries = maybeAppendEntries.Cast(); - auto nentries = appendEntries->Nentries; - mes.InitPayload(nentries); - for (uint32_t i = 0; i < nentries; i++) { - mes.Payload[i] = co_await Read(); - } - } - co_return mes; -} - -template -void TNode::Send(TMessageHolder message) { - Messages.emplace_back(std::move(message)); -} - -template -void TNode::Drain() { - if (!Connected) { - Connect(); - return; - } - if (!Drainer || Drainer.done()) { - if (Drainer && Drainer.done()) { - Drainer.destroy(); - } - Drainer = DoDrain(); - } -} - -template -NNet::TVoidSuspendedTask TNode::DoDrain() { - try { - while (!Messages.empty()) { - auto tosend = std::move(Messages); Messages.clear(); - for (auto&& m : tosend) { - co_await TMessageWriter(Socket).Write(std::move(m)); - } - } - } catch (const std::exception& ex) { - std::cout << "Error on write: " << ex.what() << "\n"; - Connect(); - } - co_return; -} - -template -void TNode::Connect() { - if (Address && (!Connector || Connector.done())) { - if (Connector && Connector.done()) { - Connector.destroy(); - } - Connector = DoConnect(); - } -} - -template -NNet::TVoidSuspendedTask TNode::DoConnect() { - std::cout << "Connecting " << Name << "\n"; - Connected = false; - while (!Connected) { - try { - auto deadline = NNet::TClock::now() + std::chrono::milliseconds(100); // TODO: broken timeout in coroio - Socket = SocketFactory(*Address); - co_await Socket.Connect(deadline); - std::cout << "Connected " << Name << "\n"; - Connected = true; - } catch (const std::exception& ex) { - std::cout << "Error on connect: " << Name << " " << ex.what() << "\n"; - } - if (!Connected) { - co_await Socket.Poller()->Sleep(std::chrono::milliseconds(1000)); - } - } - co_return; -} - template NNet::TVoidTask TRaftServer::InboundConnection(TSocket socket) { std::shared_ptr> client; @@ -127,7 +23,7 @@ NNet::TVoidTask TRaftServer::InboundConnection(TSocket socket) { if (auto maybeCommandRequest = mes.template Maybe()) { RequestProcessor->OnCommandRequest(std::move(maybeCommandRequest.Cast()), client); } else if (auto maybeCommandResponse = mes.template Maybe()) { - RequestProcessor->OnCommandResponse(std::move(maybeCommandResponse.Cast())); + RequestProcessor->OnCommandResponse(std::move(maybeCommandResponse.Cast()), client); } else { Raft->Process(TimeSource->Now(), std::move(mes), client); } @@ -147,17 +43,48 @@ NNet::TVoidTask TRaftServer::InboundConnection(TSocket socket) { template void TRaftServer::Serve() { + if (Serving) throw std::logic_error("Raft server is already serving"); + Serving = true; Idle(); InboundServe(); std::vector tasks; - for (auto& node : Nodes) { - auto realNode = std::dynamic_pointer_cast>(node); - if (realNode) { - tasks.emplace_back(OutboundServe(realNode)); - } + for (auto& [id, node] : PeerNodes) { + // Learners only receive replication and reply through their own + // configured voter connections. They cannot receive forwarded client + // commands, so no outbound response reader is needed for them. + if (Raft->IsVoter(id)) tasks.emplace_back(OutboundServe(node)); + } +} + +template +void TRaftServer::AddLearnerTarget(uint32_t id, std::shared_ptr> node) { + if (!node) throw std::invalid_argument("Learner target endpoint is null"); + auto [peer, insertedPeer] = PeerNodes.emplace(id, node); + if (!insertedPeer) throw std::invalid_argument("Duplicate network peer ID"); + + bool insertedNode = false; + try { + insertedNode = Nodes.insert(node).second; + if (!insertedNode) throw std::invalid_argument("Learner endpoint is already registered"); + Raft->AddLearnerTarget(id, node); + } catch (...) { + if (insertedNode) Nodes.erase(node); + PeerNodes.erase(peer); + throw; } } +template +std::shared_ptr> TRaftServer::RemoveLearnerTarget(uint32_t id) { + auto peer = PeerNodes.find(id); + if (peer == PeerNodes.end()) throw std::invalid_argument("Unknown network learner target"); + auto endpoint = peer->second; + Raft->RemoveLearnerTarget(id); // Rejects voters and nonleaders before server state changes. + Nodes.erase(endpoint); + PeerNodes.erase(peer); + return endpoint; +} + template void TRaftServer::DrainNodes() { for (const auto& node : Nodes) { @@ -170,13 +97,15 @@ NNet::TVoidTask TRaftServer::OutboundServe(std::shared_ptrIsConnected(); + const auto generation = node->GetConnectionGeneration(); try { if (!node->IsConnected()) { throw std::runtime_error("Not connected"); } auto mes = co_await TMessageReader(node->Sock()).Read(); if (auto maybeCommandResponse = mes.template Maybe()) { - RequestProcessor->OnCommandResponse(std::move(maybeCommandResponse.Cast())); + RequestProcessor->OnCommandResponse(std::move(maybeCommandResponse.Cast()), node); DrainNodes(); } else { std::cerr << "Wrong message type: " << mes->Type << std::endl; @@ -187,6 +116,9 @@ NNet::TVoidTask TRaftServer::OutboundServe(std::shared_ptrOnPeerDisconnected(node); + if (wasConnected) node->Disconnect(generation); + DrainNodes(); co_await Poller.Sleep(std::chrono::milliseconds(1000)); } } @@ -252,6 +184,9 @@ NNet::TVoidTask TRaftServer::Idle() { auto sleep = std::chrono::milliseconds(100); while (true) { Raft->ProcessTimeout(TimeSource->Now()); + RequestProcessor->CheckStateChange(); + RequestProcessor->ProcessCommitted(); + RequestProcessor->ProcessWaiting(); DrainNodes(); auto t1 = TimeSource->Now(); if (t1 > t0 + dt) { diff --git a/miniraft/net/server.h b/miniraft/net/server.h index 298d6ca..37edcff 100644 --- a/miniraft/net/server.h +++ b/miniraft/net/server.h @@ -16,27 +16,61 @@ template class TMessageReader { public: - TMessageReader(TSocket& socket) - : Socket(socket) - { } - - NNet::TValueTask> Read(); + TMessageReader(TSocket& socket, TMessageLimits limits = {}) + : Socket(socket), Limits(limits) { ValidateMessageLimits(Limits); } + + NNet::TValueTask> Read() { + if (Failed) throw TMessageError("Cannot reuse a failed frame reader"); + try { + uint64_t remaining = Limits.MaxBatchBytes; + co_return co_await ReadFrame(false, remaining); + } catch (...) { + Failed = true; + throw; + } + } private: + NNet::TValueTask> ReadFrame(bool logOnly, uint64_t& remaining) { + TMessage header; + co_await NNet::TByteReader(Socket).Read(&header, sizeof(header)); + ValidateMessageHeader(header.Type, header.Len, Limits); + if (logOnly && header.Type != uint32_t(EMessageType::LOG_ENTRY)) + throw TMessageError("AppendEntries children must be log entries"); + if (header.Len > remaining) throw TMessageError("Message batch exceeds byte limit"); + remaining -= header.Len; + auto message = NewHoldedMessage(header.Type, header.Len); + co_await NNet::TByteReader(Socket).Read(message->Value, header.Len - sizeof(TMessage)); + if (header.Type == uint32_t(EMessageType::APPEND_ENTRIES_REQUEST)) { + const auto count = message.template Cast()->Nentries; + if (count > Limits.MaxEntries || uint64_t(count) * sizeof(TLogEntry) > remaining) + throw TMessageError("AppendEntries count exceeds limits"); + message.InitPayload(count); + for (uint32_t i = 0; i < count; ++i) + message.Payload[i] = co_await ReadFrame(true, remaining); + } + co_return message; + } TSocket& Socket; + TMessageLimits Limits; + bool Failed = false; }; template class TMessageWriter { public: - TMessageWriter(TSocket& socket) - : Socket(socket) - { } - - NNet::TValueTask Write(TMessageHolder message); - + TMessageWriter(TSocket& socket, TMessageLimits limits = {}) + : Socket(socket), Limits(limits) { ValidateMessageLimits(Limits); } + + NNet::TValueTask Write(TMessageHolder message) { + ValidateMessageTree(message, Limits); + co_await NNet::TByteWriter(Socket).Write(message.Mes, message->Len); + for (uint32_t i = 0; i < message.PayloadSize; ++i) + co_await Write(message.Payload[i]); + } private: TSocket& Socket; + TMessageLimits Limits; }; struct THost { @@ -87,6 +121,11 @@ class TNode: public INode { void Send(TMessageHolder message) override; void Drain() override; + // Called after a failed outbound read, once its reader has unwound. A stale + // reader cannot disconnect a newer connection or its pending connector. + void Disconnect() { Disconnect(ConnectionGeneration); } + void Disconnect(uint64_t generation); + uint64_t GetConnectionGeneration() const { return ConnectionGeneration; } bool IsConnected() { return Connected; } TSocket& Sock() { return Socket; @@ -104,6 +143,7 @@ class TNode: public INode { std::function SocketFactory; TSocket Socket; bool Connected = false; + uint64_t ConnectionGeneration = 0; std::coroutine_handle<> Drainer; std::coroutine_handle<> Connector; @@ -127,13 +167,21 @@ class TRaftServer { , RequestProcessor(std::make_shared(raft, rsm, nodes)) , TimeSource(ts) { - for (const auto& [_, node] : nodes) { + for (const auto& [id, node] : nodes) { Nodes.insert(node); + if (auto realNode = std::dynamic_pointer_cast>(node)) { + PeerNodes.emplace(id, std::move(realNode)); + } } } void Serve(); + // Runtime learner targets are replication-only endpoints. Calls must be + // serialized with the poller loop and other Raft processing. + void AddLearnerTarget(uint32_t id, std::shared_ptr> node); + std::shared_ptr> RemoveLearnerTarget(uint32_t id); + private: NNet::TVoidTask InboundServe(); NNet::TVoidTask InboundConnection(TSocket socket); @@ -147,10 +195,13 @@ class TRaftServer { std::shared_ptr Raft; std::shared_ptr RequestProcessor; std::unordered_set> Nodes; + std::unordered_map>> PeerNodes; std::shared_ptr TimeSource; + bool Serving = false; // for debug TStateFields PersistentFields; TVolatileState VolatileFields; }; +#include "node_impl.h" diff --git a/miniraft/persist.cpp b/miniraft/persist.cpp index 7ad69a8..e225190 100644 --- a/miniraft/persist.cpp +++ b/miniraft/persist.cpp @@ -1,70 +1,133 @@ #include "persist.h" +#include +#include +#include #include TDiskState::TDiskState(const std::string& name, uint32_t id) - : Entries(Open(name + ".entries." + std::to_string(id))) - , Index(Open(name + ".index." + std::to_string(id))) - , State(Open(name + ".state." + std::to_string(id))) { - // TODO: read all an validate - State.seekg(0, std::ios_base::end); - auto size = State.tellg(); - if (size == sizeof(LastLogIndex) + sizeof(CurrentTerm) + sizeof(VotedFor)) { - State.seekg(0); - State.read((char*)&LastLogIndex, sizeof(LastLogIndex)); - State.read((char*)&CurrentTerm, sizeof(CurrentTerm)); - State.read((char*)&VotedFor, sizeof(VotedFor)); - } else { + const auto suffix = "." + std::to_string(id); + const auto entriesPath = name + ".entries" + suffix; + const auto indexPath = name + ".index" + suffix; + const auto statePath = name + ".state" + suffix; + const int existing = std::filesystem::exists(entriesPath) + + std::filesystem::exists(indexPath) + std::filesystem::exists(statePath); + // Never turn a partially missing store into a fresh Raft voter. + if (existing != 0 && existing != 3) { + throw std::runtime_error("Incomplete Raft storage: " + name); + } + const bool fresh = existing == 0; + Entries = Open(entriesPath, fresh); + Index = Open(indexPath, fresh); + State = Open(statePath, fresh); + if (fresh) { Commit(); + Recovering = false; + return; } - if (LastLogIndex > 0) { - LastLogTerm = Get(LastLogIndex)->Term; + + State.seekg(0, std::ios::end); + const auto size = State.tellg(); + if (size != std::streamoff(sizeof(LastLogIndex) + sizeof(CurrentTerm) + sizeof(VotedFor))) { + throw std::runtime_error("Invalid Raft metadata size: " + statePath); + } + State.seekg(0); + if (!State.read(reinterpret_cast(&LastLogIndex), sizeof(LastLogIndex)) || + !State.read(reinterpret_cast(&CurrentTerm), sizeof(CurrentTerm)) || + !State.read(reinterpret_cast(&VotedFor), sizeof(VotedFor)) || CurrentTerm == 0) { + throw std::runtime_error("Invalid Raft metadata: " + statePath); } -} -std::fstream TDiskState::Open(const std::string& name) -{ - auto flags = std::ios::in | std::ios::out | std::ios::binary; - std::fstream f(name, flags); - if (!f.is_open()) { - std::ofstream tmp(name); - tmp.close(); - f.open(name, flags); + Index.seekg(0, std::ios::end); + const auto indexSize = Index.tellg(); + if (indexSize < 0 || LastLogIndex > uint64_t(indexSize) / sizeof(uint64_t)) { + throw std::runtime_error("Truncated Raft index: " + indexPath); } + // Validate the entire selected prefix, not only its last entry. Trailing + // bytes can be left by logical truncation and are deliberately ignored. + uint64_t expectedOffset = 0; + for (uint64_t i = 1; i <= LastLogIndex; ++i) { + uint64_t offset; + Index.seekg((i - 1) * sizeof(offset)); + if (!Index.read(reinterpret_cast(&offset), sizeof(offset)) || offset != expectedOffset) { + throw std::runtime_error("Invalid Raft log offset: " + indexPath); + } + auto entry = Get(i); + if (entry->Term < LastLogTerm) { + throw std::runtime_error("Decreasing Raft log terms: " + entriesPath); + } + expectedOffset += entry->Len; + LastLogTerm = entry->Term; + } + Recovering = false; +} - if (!f.is_open()) { - throw std::runtime_error("Cannot open file: " + name); +std::fstream TDiskState::Open(const std::string& name, bool create) +{ + if (create) { + std::ofstream file(name, std::ios::binary); + if (!file) throw std::runtime_error("Cannot create file: " + name); } + std::fstream file(name, std::ios::in | std::ios::out | std::ios::binary); + // An open failure must never fall back to truncating an existing file. + if (!file) throw std::runtime_error("Cannot open file: " + name); + return file; +} - return f; +[[noreturn]] void TDiskState::Fail(const char* message) const +{ + if (Recovering) throw std::runtime_error(message); + // The network layer catches ordinary exceptions and keeps the node alive. + // Continuing after a live storage failure could acknowledge unpersisted + // votes or logs. Fail-stop instead; startup validation remains catchable. + std::cerr << "Fatal Raft storage error: " << message << '\n'; + std::terminate(); } TMessageHolder TDiskState::Read() const { - decltype(TMessage::Type) type; - decltype(TMessage::Len) len; - if (!Entries.read((char*)&type, sizeof(type))) { - throw std::runtime_error("Error on read 1"); + const auto start = Entries.tellg(); + Entries.seekg(0, std::ios::end); + const auto end = Entries.tellg(); + if (start < 0 || end < start || end - start < std::streamoff(sizeof(TLogEntry))) { + Fail("Truncated Raft log entry"); + } + Entries.seekg(start); + uint32_t type, len; + if (!Entries.read(reinterpret_cast(&type), sizeof(type)) || + !Entries.read(reinterpret_cast(&len), sizeof(len)) || + type != uint32_t(EMessageType::LOG_ENTRY) || len < sizeof(TLogEntry) || + uint64_t(len) > uint64_t(end - start)) { + Fail("Invalid Raft log entry header"); } - if (!Entries.read((char*)&len, sizeof(len))) { - throw std::runtime_error("Error on read 2"); + // Bound the allocation and subtraction by the validated file extent. + auto message = NewHoldedMessage(type, len); + if (!Entries.read(reinterpret_cast(message->Value), len - sizeof(TMessage))) { + Fail("Cannot read Raft log entry"); } - auto mes = NewHoldedMessage(type, len); - if (!Entries.read((char*)mes->Value, len - sizeof(TMessage))) { - throw std::runtime_error("Error on read 3"); + const auto term = message.Cast()->Term; + if (!term || term > CurrentTerm) { + Fail("Raft log term outside current term range"); } - return mes; + return message; } void TDiskState::Write(TMessageHolder message) { - Entries.write((char*)message.Mes, message->Len); + if (!message.HasValidExtent()) Fail("Invalid Raft log entry buffer"); + if (!Entries.write(reinterpret_cast(message.Mes), message->Len)) { + Fail("Cannot write Raft log entry"); + } } void TDiskState::RemoveLast() { if (LastLogIndex > 0) { - LastLogIndex--; + if (LastLogIndex <= CommitIndex) + throw std::logic_error("Cannot remove a committed log entry"); + const auto previousTerm = LastLogIndex == 1 ? 0 : Get(LastLogIndex - 1)->Term; + --LastLogIndex; + LastLogTerm = previousTerm; Commit(); } } @@ -72,43 +135,52 @@ void TDiskState::RemoveLast() void TDiskState::Append(TMessageHolder entry) { uint64_t offset = 0; - if (Get(LastLogIndex)) { // TODO: optimize + if (Get(LastLogIndex)) { offset = Entries.tellg(); } - + // In particular, after truncation to zero the stream may still point into + // the old suffix. Always position the output explicitly. + Entries.seekp(offset); Write(entry); - Index.seekg(LastLogIndex * sizeof(offset)); - Index.write((char*)&offset, sizeof(offset)); + Index.seekp(LastLogIndex * sizeof(offset)); + if (!Index.write(reinterpret_cast(&offset), sizeof(offset))) { + Fail("Cannot write Raft log index"); + } LastLogTerm = entry->Term; - LastLogIndex ++; + ++LastLogIndex; Commit(); } TMessageHolder TDiskState::Get(int64_t index) const { - if (index > LastLogIndex || index < 1) { - return {}; + if (index > LastLogIndex || index < 1) return {}; + uint64_t offset; + Index.seekg((index - 1) * sizeof(offset)); + if (!Index.read(reinterpret_cast(&offset), sizeof(offset))) { + Fail("Cannot read Raft log index"); } - - uint64_t offset = 0; - Index.seekg((index-1) * sizeof(offset)); - if (!Index.read((char*)&offset, sizeof(offset))) { - return {}; + Entries.seekg(0, std::ios::end); + const auto end = Entries.tellg(); + if (end < 0 || offset > uint64_t(end)) { + Fail("Raft log offset outside file"); } - Entries.seekg(offset); - auto entry = Read().Cast(); - return entry; + return Read().Cast(); } void TDiskState::Commit() { - State.seekg(0); - if (!State.write((char*)&LastLogIndex, sizeof(LastLogIndex))) { abort(); } - if (!State.write((char*)&CurrentTerm, sizeof(CurrentTerm))) { abort(); } - if (!State.write((char*)&VotedFor, sizeof(VotedFor))) { abort(); } - State.flush(); - Index.flush(); + // Publish buffered log data before metadata. Stream flushing alone is not + // a power-loss durability protocol; fsync/atomic publication remain TODO. Entries.flush(); + Index.flush(); + if (!Entries || !Index) Fail("Cannot flush Raft log"); + State.seekp(0); + if (!State.write(reinterpret_cast(&LastLogIndex), sizeof(LastLogIndex)) || + !State.write(reinterpret_cast(&CurrentTerm), sizeof(CurrentTerm)) || + !State.write(reinterpret_cast(&VotedFor), sizeof(VotedFor))) { + Fail("Cannot write Raft metadata"); + } + State.flush(); + if (!State) Fail("Cannot flush Raft metadata"); } - diff --git a/miniraft/persist.h b/miniraft/persist.h index 71c92b1..202242c 100644 --- a/miniraft/persist.h +++ b/miniraft/persist.h @@ -14,7 +14,9 @@ struct TDiskState : IState { void Commit() override; private: - std::fstream Open(const std::string& name); + bool Recovering = true; + [[noreturn]] void Fail(const char* message) const; + std::fstream Open(const std::string& name, bool create); TMessageHolder Read() const; void Write(TMessageHolder); diff --git a/miniraft/quorum.h b/miniraft/quorum.h new file mode 100644 index 0000000..b86cc6f --- /dev/null +++ b/miniraft/quorum.h @@ -0,0 +1,48 @@ +#pragma once + +#include "state.h" +#include +#include +#include +#include +#include + +// Quorum arithmetic only. This does not activate or persist a membership change. +class TQuorum { +public: + explicit TQuorum(std::vector voters) : Old(std::move(voters)) { + ValidateFixedVoters(Old); + } + TQuorum(std::vector oldVoters, std::vector newVoters) + : Old(std::move(oldVoters)), New(std::move(newVoters)) { + ValidateFixedVoters(Old); + ValidateFixedVoters(New); + } + + // Highest threshold acknowledged by a strict majority of each voter set. + // Missing voters should return zero. Overlapping voters contribute once to + // each set; members outside both sets cannot affect the result. + // The callback must return a stable value for an ID during the call. + template uint64_t Value(F value, std::vector* scratch = nullptr) const { + std::vector local; + auto& values = scratch ? *scratch : local; + const auto majority = [&](const auto& voters) { + values.clear(); + values.reserve(voters.size()); + for (auto id : voters) values.push_back(value(id)); + auto middle = values.begin() + (values.size() - 1) / 2; + std::nth_element(values.begin(), middle, values.end()); + return *middle; + }; + const auto old = majority(Old); + return New.empty() ? old : std::min(old, majority(New)); + } + + bool HasQuorum(const std::unordered_set& votes, uint32_t self = 0) const { + return Value([&](uint32_t id) -> uint64_t { + return (self != 0 && id == self) || votes.contains(id); + }) != 0; + } +private: + std::vector Old, New; +}; diff --git a/miniraft/raft.cpp b/miniraft/raft.cpp index cb72b2e..f218777 100644 --- a/miniraft/raft.cpp +++ b/miniraft/raft.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include @@ -12,6 +13,7 @@ #include "raft.h" #include "messages.h" #include "timesource.h" +#include "configuration.h" namespace { @@ -22,15 +24,89 @@ static uint32_t rand_(uint32_t* seed) { return *seed; } +struct TSnapshotMetadataProbe { + bool Complete = false; + bool Supported = false; + size_t Need = 8; +}; + +bool IsSupportedSnapshotMagicPrefix(std::string_view prefix) { + constexpr std::string_view legacy = "MRFTS001"; + constexpr std::string_view stable = "MRFTS003"; + const auto bytes = prefix.substr(0, std::min(prefix.size(), legacy.size())); + return legacy.substr(0, bytes.size()) == bytes || stable.substr(0, bytes.size()) == bytes; +} + +bool ReadSnapshotInteger(std::string_view bytes, size_t offset, unsigned width, uint64_t& value) { + if (offset > bytes.size() || width > bytes.size() - offset) return false; + value = 0; + for (unsigned i = 0; i < width; ++i) + value |= uint64_t(uint8_t(bytes[offset + i])) << (8 * i); + return true; +} + +TSnapshotMetadataProbe ProbeSnapshotMetadata(std::string_view prefix, uint64_t totalBytes, + uint64_t index, uint64_t term, const std::vector& voters, + const TSnapshot* knownSnapshot) { + if (prefix.size() < 8) return {false, false, 8}; + const auto magic = prefix.substr(0, 8); + const bool stable = magic == "MRFTS003"; + if (!stable && magic != "MRFTS001") return {true, false, 0}; + + // Inspect only the bounded format header and voter list. Callers use this + // before granting replication progress or accepting a higher term; payload + // bytes and CRC are validated by the full install path when needed. + const size_t fixedHeaderBytes = stable ? 52 : 36; + if (prefix.size() < fixedHeaderBytes) return {false, false, fixedHeaderBytes}; + uint64_t embeddedIndex = 0, embeddedTerm = 0, voterCount = 0, dataBytes = 0; + if (!ReadSnapshotInteger(prefix, 8, 8, embeddedIndex) || + !ReadSnapshotInteger(prefix, 16, 8, embeddedTerm) || + !ReadSnapshotInteger(prefix, 24, 4, voterCount) || + !ReadSnapshotInteger(prefix, 28, 8, dataBytes) || + embeddedIndex != index || embeddedTerm != term || voterCount != voters.size()) + return {true, false, 0}; + if (stable) { + uint64_t configurationIndex = 0, configurationTerm = 0; + if (!ReadSnapshotInteger(prefix, 36, 8, configurationIndex) || + !ReadSnapshotInteger(prefix, 44, 8, configurationTerm) || !configurationIndex || + configurationIndex > embeddedIndex || !configurationTerm || configurationTerm > embeddedTerm || + (configurationIndex == embeddedIndex && configurationTerm != embeddedTerm) || + (knownSnapshot && knownSnapshot->StableConfiguration && + (configurationIndex != knownSnapshot->ConfigurationIndex || + configurationTerm != knownSnapshot->ConfigurationTerm))) + return {true, false, 0}; + } + + const uint64_t voterBytes = voterCount * 4; + const uint64_t headerBytes = fixedHeaderBytes + voterBytes; + if (headerBytes + 4 > totalBytes || dataBytes != totalBytes - headerBytes - 4) + return {true, false, 0}; + if (headerBytes > prefix.size()) return {false, false, size_t(headerBytes)}; + for (size_t i = 0; i < voters.size(); ++i) { + uint64_t voter = 0; + if (!ReadSnapshotInteger(prefix, fixedHeaderBytes + i * 4, 4, voter) || voter != voters[i]) + return {true, false, 0}; + } + return {true, true, size_t(headerBytes)}; +} + } // namespace +void TDummyRsm::Validate(TMessageHolder message) const { + if (!message || message->Len < sizeof(TCommandRequest) || + (!(message->Flags & TCommandRequest::EWrite) && message->Len < sizeof(TCommandRequest) + sizeof(int64_t))) + throw TMessageError("Invalid dummy RSM request"); +} + TMessageHolder TDummyRsm::Read(TMessageHolder message, uint64_t index) { + if (!message || message->Len < sizeof(TCommandRequest) + sizeof(int64_t)) + throw TMessageError("Missing log read index"); int64_t readIndex; memcpy(&readIndex, message->Data, sizeof(readIndex)); if (readIndex > 0 && readIndex <= Log.size()) { auto log = Log[readIndex-1]; - auto size = log->Len - sizeof(TMessage); + auto size = log->Len - sizeof(TLogEntry); auto mes = NewHoldedMessage(sizeof(TCommandResponse) + size); mes->Index = index; memcpy(mes->Data, log->Data, size); @@ -51,6 +127,7 @@ TMessageHolder TDummyRsm::Write(TMessageHolder message, uin TMessageHolder TDummyRsm::Prepare(TMessageHolder command) { + if (!command || command->Len < sizeof(TCommandRequest)) throw TMessageError("Short command"); auto dataSize = command->Len - sizeof(TCommandRequest); auto entry = NewHoldedMessage(sizeof(TLogEntry)+dataSize); memcpy(entry->Data, command->Data, dataSize); @@ -68,7 +145,7 @@ TVolatileState& TVolatileState::Vote(uint32_t nodeId) return *this; } -TVolatileState& TVolatileState::CommitAdvance(int nservers, const IState& state, uint64_t seqno) +TVolatileState& TVolatileState::CommitAdvance(int nservers, const IState& state) { auto lastIndex = state.LastLogIndex; Indices.clear(); Indices.reserve(nservers); @@ -80,14 +157,29 @@ TVolatileState& TVolatileState::CommitAdvance(int nservers, const IState& state, Indices.push_back(0); } std::sort(Indices.begin(), Indices.end()); - auto commitIndex = std::max(CommitIndex, Indices[nservers / 2]); + // Ascending order: a strict majority must have replicated this index. + // For even cluster sizes, the upper median would accept only half. + auto commitIndex = std::max(CommitIndex, Indices[(nservers - 1) / 2]); if (state.LogTerm(commitIndex) == state.CurrentTerm) { - CommitSeqno = std::max(CommitSeqno, seqno); CommitIndex = commitIndex; } return *this; } +TVolatileState& TVolatileState::CommitAdvance(const std::vector& voters, uint32_t self, const IState& state) { + return CommitAdvance(TQuorum(voters), self, state); +} + +TVolatileState& TVolatileState::CommitAdvance(const TQuorum& quorum, uint32_t self, const IState& state) { + const auto quorumIndex = quorum.Value([&](uint32_t id) { + auto match = MatchIndex.find(id); + return id == self ? state.LastLogIndex : match == MatchIndex.end() ? uint64_t(0) : match->second; + }, &Indices); + const auto index = std::max(CommitIndex, quorumIndex); + if (state.LogTerm(index) == state.CurrentTerm) CommitIndex = index; + return *this; +} + TVolatileState& TVolatileState::SetNextIndex(uint32_t id, uint64_t nextIndex) { NextIndex[id] = nextIndex; @@ -128,28 +220,189 @@ TVolatileState& TVolatileState::SetLeaderId(uint32_t id) { return *this; } -TVolatileState& TVolatileState::SetCommitIndex(int index) +TVolatileState& TVolatileState::SetCommitIndex(uint64_t index) { CommitIndex = index; return *this; } -TRaft::TRaft(std::shared_ptr state, int node, const TNodeDict& nodes) +TRaft::TRaft(std::shared_ptr state, int node, const TNodeDict& nodes, std::vector learners) : Id(node) , Nodes(nodes) - , MinVotes((nodes.size()+2+nodes.size()%2)/2) + , MinVotes(0) // derived from voting IDs after role validation , Npeers(nodes.size()) - , Nservers(nodes.size()+1) + , Nservers(0) , State(std::move(state)) , VolatileState(std::make_unique()) , StateName(EState::FOLLOWER) { - for (auto [id, _] : Nodes) { + if (node <= 0 || !State) throw std::invalid_argument("Raft requires a positive ID and state"); + // Equal seeds make simultaneously started voters campaign in lockstep. + // An odd multiplier gives every positive node ID a distinct nonzero stream. + Seed = uint32_t(node) * 0x9e3779b9u; + for (const auto& [id, peer] : Nodes) { + if (id == 0 || id == Id || !peer) + throw std::invalid_argument("Invalid Raft peer configuration"); VolatileState->NextIndex[id] = 1; } + Learners = std::move(learners); + std::sort(Learners.begin(), Learners.end()); + uint32_t previous = 0; + for (auto id : Learners) { + if (id <= previous || (id != Id && !Nodes.contains(id))) + throw std::invalid_argument("Invalid or duplicate learner ID"); + previous = id; + } + if (!std::binary_search(Learners.begin(), Learners.end(), Id)) InitialVoters.push_back(Id); + for (const auto& [id, _] : Nodes) + if (!std::binary_search(Learners.begin(), Learners.end(), id)) InitialVoters.push_back(id); + std::sort(InitialVoters.begin(), InitialVoters.end()); + ValidateFixedVoters(InitialVoters); + Voters = InitialVoters; + VotingQuorum.emplace(Voters); + if (!State->CurrentTerm) throw std::invalid_argument("Stored Raft term must be positive"); + Nservers = Voters.size(); MinVotes = Nservers / 2 + 1; + if (!Learners.empty() && !State->SupportsVoterBinding()) + throw std::invalid_argument("Learners require voter-binding-capable storage"); + const auto bound = State->GetBoundVoters(); + if (State->SnapshotIndex > State->LastLogIndex || State->LastLogIndex > uint64_t(INT64_MAX) || + State->CommitIndex > State->LastLogIndex) + throw std::invalid_argument("Invalid retained log bounds"); + const bool commitFloorAdded = State->CommitIndex < State->SnapshotIndex; + const auto snapshot = State->GetSnapshot(); + if (snapshot) { + ValidateSnapshot(*snapshot); + if (snapshot->IsJoint()) + throw std::invalid_argument("Joint snapshot recovery is unsupported"); + if (snapshot->StableConfiguration && bound != InitialVoters) + throw std::invalid_argument("Stable configuration snapshot requires the existing voter binding"); + if (snapshot->Index != State->SnapshotIndex || snapshot->Term != State->SnapshotTerm || + snapshot->Term > State->CurrentTerm || + snapshot->Voters != InitialVoters || + State->LastLogTerm < State->SnapshotTerm || State->LastLogTerm > State->CurrentTerm || + (State->LastLogIndex == State->SnapshotIndex && State->LastLogTerm != State->SnapshotTerm)) + throw std::invalid_argument("Compacted state snapshot boundary mismatch"); + } else if (State->SnapshotIndex || State->SnapshotTerm) { + throw std::invalid_argument("Compacted state is missing its snapshot"); + } + if (State->HasUnboundRecoveredState() && !snapshot) + throw std::invalid_argument("Recovered durable state requires voter proof before Raft startup"); + auto previousTerm = State->SnapshotTerm; + for (uint64_t index = State->SnapshotIndex + 1; index <= State->LastLogIndex; ++index) { + if (snapshot) { + const auto term = State->RetainedLogTerm(index); + if (!term || term < previousTerm || term > State->CurrentTerm) + throw std::invalid_argument("Invalid compacted log term sequence"); + previousTerm = term; + } + } + if (snapshot && previousTerm != State->LastLogTerm) + throw std::invalid_argument("Compacted log last term mismatch"); + const bool fresh = State->CurrentTerm == 1 && !State->VotedFor && !State->LastLogIndex; + if (!bound.empty()) { + if (bound != InitialVoters) + throw std::invalid_argument("Persisted voter configuration differs from the configured peers"); + } + const bool bindInitialVoters = bound.empty() && State->SupportsVoterBinding() && + (fresh || snapshot || !Learners.empty()); + const auto baseline = snapshot && snapshot->StableConfiguration + ? TConfigurationHistory::FromStableSnapshot(*snapshot) + : TConfigurationHistory::Stable(InitialVoters, 0, 0, State->SnapshotIndex, State->SnapshotTerm); + ConfigurationBaseline = baseline; + RefreshConfiguration(std::max(State->CommitIndex, State->SnapshotIndex)); + const bool hasRetainedConfiguration = Configuration->Latest.ConfigurationIndex() > baseline.ConfigurationIndex(); + if (hasRetainedConfiguration && (!State->SupportsVoterBinding() || bound.empty() || + !State->SupportsCommitIndexPersistence())) + throw std::invalid_argument("Retained membership changes require bound voters and a persisted commit index"); + if (State->VotedFor && !IsVoter(State->VotedFor) && !hasRetainedConfiguration) + throw std::invalid_argument("Stored vote does not name a configured voter"); + if (!IsVoter(Id) && State->VotedFor && !hasRetainedConfiguration) + throw std::invalid_argument("A learner store must not contain a vote"); + if (bindInitialVoters) State->BindVoters(InitialVoters); + if (commitFloorAdded) { + State->CommitIndex = State->SnapshotIndex; + State->Commit(); + } + VolatileState->CommitIndex = State->CommitIndex; + State->MarkRaftStarted(); +} + +void TRaft::RefreshConfiguration(uint64_t commitIndex) { + if (!ConfigurationBaseline) + throw std::logic_error("Missing trusted configuration baseline"); + auto scan = ScanConfigurationHistory(*State, *ConfigurationBaseline, commitIndex); + + // An uncommitted joint entry already requires both majorities to commit. + // Once the joint entry commits, keep joint consensus until its stable + // successor commits as well. This makes truncating either suffix restore + // the quorum implied by the surviving log. + const TConfigurationHistory* quorumConfiguration = nullptr; + if (scan.Committed.IsJoint()) quorumConfiguration = &scan.Committed; + else if (scan.Latest.IsJoint()) quorumConfiguration = &scan.Latest; + else quorumConfiguration = &scan.Committed; + + std::vector voters = quorumConfiguration->OldVoters(); + std::optional quorum; + int minVotes = 0; + if (quorumConfiguration->IsJoint()) { + voters.insert(voters.end(), quorumConfiguration->NewVoters().begin(), + quorumConfiguration->NewVoters().end()); + std::sort(voters.begin(), voters.end()); + voters.erase(std::unique(voters.begin(), voters.end()), voters.end()); + quorum.emplace(quorumConfiguration->OldVoters(), quorumConfiguration->NewVoters()); + minVotes = std::max(int(quorumConfiguration->OldVoters().size() / 2 + 1), + int(quorumConfiguration->NewVoters().size() / 2 + 1)); + } else { + quorum.emplace(quorumConfiguration->OldVoters()); + minVotes = int(quorumConfiguration->OldVoters().size() / 2 + 1); + } + for (auto id : voters) + if (id != Id && !Nodes.contains(id)) + throw std::invalid_argument("Active configuration has no replication target for a voter"); + + std::vector learners; + if (!std::binary_search(voters.begin(), voters.end(), Id)) learners.push_back(Id); + for (const auto& [id, _] : Nodes) + if (!std::binary_search(voters.begin(), voters.end(), id)) learners.push_back(id); + std::sort(learners.begin(), learners.end()); + + Voters.swap(voters); + Learners.swap(learners); + VotingQuorum.swap(quorum); + MinVotes = minVotes; + Nservers = int(Voters.size()); + Configuration = std::move(scan); + const auto& baseline = *ConfigurationBaseline; + const auto& latest = Configuration->Latest; + ConfigurationScanRequired = baseline.ConfigurationIndex() != latest.ConfigurationIndex() || + baseline.ConfigurationTerm() != latest.ConfigurationTerm() || baseline.IsJoint() != latest.IsJoint() || + baseline.OldVoters() != latest.OldVoters() || baseline.NewVoters() != latest.NewVoters(); + + if (!IsVoter(Id)) { + VolatileState->Votes.clear(); + if (StateName == EState::CANDIDATE || StateName == EState::LEADER) { + StateName = EState::FOLLOWER; + VolatileState->LeaderId = 0; + } + } +} + +bool TRaft::HasMembershipHistory() const { + if (!ConfigurationBaseline || !Configuration) return false; + const auto& baseline = *ConfigurationBaseline; + const auto& latest = Configuration->Latest; + return baseline.ConfigurationIndex() != latest.ConfigurationIndex() || + baseline.ConfigurationTerm() != latest.ConfigurationTerm() || + baseline.IsJoint() != latest.IsJoint() || baseline.OldVoters() != latest.OldVoters() || + baseline.NewVoters() != latest.NewVoters(); } void TRaft::OnRequestVote(ITimeSource::Time now, TMessageHolder message) { + if (!IsVoter(Id)) { + Nodes.at(message->Src)->Send(NewHoldedMessage( + TMessageEx{.Src = Id, .Dst = message->Src, .Term = State->CurrentTerm}, TRequestVoteResponse{.VoteGranted = false})); + return; + } if (message->Term < State->CurrentTerm) { auto reply = NewHoldedMessage( TMessageEx {.Src = Id, .Dst = message->Src, .Term = State->CurrentTerm}, @@ -209,9 +462,10 @@ void TRaft::OnAppendEntries(ITimeSource::Time now, TMessageHolderCommitIndex; bool success = false; - if (message->PrevLogIndex == 0 || + bool configurationTouched = false; + if (message->PrevLogIndex >= State->SnapshotIndex && (message->PrevLogIndex == 0 || (message->PrevLogIndex <= State->LastLogIndex - && State->LogTerm(message->PrevLogIndex) == message->PrevLogTerm)) + && State->LogTerm(message->PrevLogIndex) == message->PrevLogTerm))) { success = true; auto index = message->PrevLogIndex; @@ -221,24 +475,29 @@ void TRaft::OnAppendEntries(ITimeSource::Time now, TMessageHolderLogTerm(index) != entry->Term) { + if (Configuration && index <= Configuration->Latest.ConfigurationIndex()) + configurationTouched = true; while (State->LastLogIndex > index-1) { State->RemoveLast(); } State->Append(entry); + if (entry->Flags & TLogEntry::EConfiguration) configurationTouched = true; } } matchIndex = index; - commitIndex = std::max(commitIndex, message->LeaderCommit); + commitIndex = std::max(commitIndex, std::min(message->LeaderCommit, matchIndex)); } + if (!success && message->PrevLogIndex < State->SnapshotIndex) matchIndex = State->SnapshotIndex; auto reply = NewHoldedMessage( TMessageEx {.Src = Id, .Dst = message->Src, .Term = State->CurrentTerm, .Seqno = message->Seqno}, TAppendEntriesResponse {.MatchIndex = matchIndex, .Success = success}); + AdvanceCommitIndex(commitIndex); + if (configurationTouched) RefreshConfiguration(VolatileState->CommitIndex); (*VolatileState) .SetLeaderId(message->LeaderId) - .SetCommitIndex(commitIndex) .SetElectionDue(MakeElection(now)); Become(EState::FOLLOWER); @@ -251,22 +510,47 @@ void TRaft::OnAppendEntries(TMessageHolder message) { } auto nodeId = message->Src; + if (!Nodes.contains(nodeId)) return; + auto pending = VolatileState->PendingAppendRpcs.find(nodeId); + if (pending == VolatileState->PendingAppendRpcs.end()) return; + auto& requests = pending->second; + auto request = std::find_if(requests.begin(), requests.end(), [&](const auto& rpc) { + return rpc.Seqno == message->Seqno; + }); + if (request == requests.end() || + (message->Success && message->MatchIndex != request->LastIndex)) return; + const auto acknowledgedLeaderCommit = request->LeaderCommit; + // A newer response supersedes older RPCs. Reordered/duplicate failures + // must not undo progress or repeatedly increase backoff. + requests.erase(requests.begin(), std::next(request)); if (message->Success) { + if (message->Seqno > 0 && message->Seqno <= VolatileState->LastSentSeqno[nodeId]) { + auto& acknowledged = VolatileState->ReadAckSeqno[nodeId]; + acknowledged = std::max(acknowledged, message->Seqno); + } auto matchIndex = std::max(VolatileState->MatchIndex[nodeId], message->MatchIndex); + auto commitAckIndex = std::max(VolatileState->CommitAckIndex[nodeId], + std::min(acknowledgedLeaderCommit, message->MatchIndex)); (*VolatileState) .SetMatchIndex(nodeId, matchIndex) - .SetNextIndex(nodeId, message->MatchIndex+1) + .SetNextIndex(nodeId, matchIndex+1) .SetRpcDue(nodeId, ITimeSource::Time{}) .SetBatchSize(nodeId, 1024) - .SetBackOff(nodeId, 1) - .CommitAdvance(Nservers, *State, message->Seqno); + .SetBackOff(nodeId, 1); + VolatileState->CommitAckIndex[nodeId] = commitAckIndex; + AdvanceLeaderCommit(); + } else if (message->MatchIndex > 0) { + // A compacted-prefix hint is not replication evidence. Probe its boundary + // with AppendEntries before advancing MatchIndex or confirming reads. + (*VolatileState).SetNextIndex(nodeId, std::max(VolatileState->MatchIndex[nodeId] + 1, message->MatchIndex + 1)) + .SetRpcDue(nodeId, ITimeSource::Time{}).SetBatchSize(nodeId, 1).SetBackOff(nodeId, 1); } else { auto backOff = std::max(VolatileState->BackOff[nodeId], 1); auto nextIndex = VolatileState->NextIndex[nodeId] > backOff ? VolatileState->NextIndex[nodeId] - backOff : 0; (*VolatileState) - .SetNextIndex(nodeId, std::max((uint64_t)1, nextIndex)) + .SetNextIndex(nodeId, std::max(VolatileState->MatchIndex[nodeId] + 1, nextIndex)) .SetRpcDue(nodeId, ITimeSource::Time{}) .SetBatchSize(nodeId, 1) .SetBackOff(nodeId, std::min(32768, backOff << 1)); @@ -292,23 +576,37 @@ TMessageHolder TRaft::CreateAppendEntries(uint32_t nodeId lastIndex = prevIndex; } + const TMessageLimits limits; + uint64_t bytes = sizeof(TAppendEntriesRequest); + std::vector> entries; + for (auto i = prevIndex + 1; i <= lastIndex && entries.size() < limits.MaxEntries; ++i) { + auto entry = State->Get(i); + ValidateMessageHeader(entry->Type, entry->Len, limits); + if (entry->Len > limits.MaxBatchBytes - bytes) break; + bytes += entry->Len; + entries.push_back(std::move(entry)); + } + lastIndex = prevIndex + entries.size(); + + const auto leaderCommit = std::min(VolatileState->CommitIndex, lastIndex); auto mes = NewHoldedMessage( - TMessageEx {.Src = Id, .Dst = nodeId, .Term = State->CurrentTerm, .Seqno = Seqno++}, + TMessageEx {.Src = Id, .Dst = nodeId, .Term = State->CurrentTerm, .Seqno = NextSeqno()}, TAppendEntriesRequest { .PrevLogIndex = prevIndex, .PrevLogTerm = State->LogTerm(prevIndex), - .LeaderCommit = std::min(VolatileState->CommitIndex, lastIndex), + .LeaderCommit = leaderCommit, .LeaderId = Id, .Nentries = static_cast(lastIndex - prevIndex), }); - if (lastIndex - prevIndex > 0) { - mes.InitPayload(lastIndex - prevIndex); - uint32_t j = 0; - for (auto i = prevIndex+1; i <= lastIndex; i++) { - mes.Payload[j++] = State->Get(i); - } + if (!entries.empty()) { + mes.InitPayload(entries.size()); + for (uint32_t i = 0; i < entries.size(); ++i) mes.Payload[i] = entries[i]; } + VolatileState->LastSentSeqno[nodeId] = mes->Seqno; + auto& pending = VolatileState->PendingAppendRpcs[nodeId]; + pending.push_back({mes->Seqno, lastIndex, leaderCommit}); + if (pending.size() > TVolatileState::MaxPendingAppendRpcs) pending.pop_front(); return mes; } @@ -317,6 +615,8 @@ void TRaft::Follower(ITimeSource::Time now, TMessageHolder message) { OnRequestVote(now, std::move(maybeRequestVote.Cast())); } else if (auto maybeAppendEntries = message.Maybe()) { OnAppendEntries(now, std::move(maybeAppendEntries.Cast())); + } else if (auto snapshot = message.Maybe()) { + OnInstallSnapshot(now, snapshot.Cast()); } } @@ -327,26 +627,35 @@ void TRaft::Candidate(ITimeSource::Time now, TMessageHolder message) { OnRequestVote(now, std::move(maybeRequestVote.Cast())); } else if (auto maybeAppendEntries = message.Maybe()) { OnAppendEntries(now, std::move(maybeAppendEntries.Cast())); + } else if (auto snapshot = message.Maybe()) { + OnInstallSnapshot(now, snapshot.Cast()); } } void TRaft::Leader(ITimeSource::Time now, TMessageHolder message) { - if (auto maybeAppendEntries = message.Maybe()) { + if (auto snapshot = message.Maybe()) { + OnInstallSnapshot(snapshot.Cast()); + } else if (auto maybeAppendEntries = message.Maybe()) { OnAppendEntries(std::move(maybeAppendEntries.Cast())); } else if (auto maybeVoteRequest = message.Maybe()) { OnRequestVote(now, std::move(maybeVoteRequest.Cast())); } else if (auto maybeAppendEntries = message.Maybe()) { OnAppendEntries(now, std::move(maybeAppendEntries.Cast())); + } else if (auto snapshot = message.Maybe()) { + OnInstallSnapshot(now, snapshot.Cast()); } } void TRaft::Become(EState newStateName) { + if (!IsVoter(Id) && (newStateName == EState::CANDIDATE || newStateName == EState::LEADER)) + throw std::logic_error("A learner cannot campaign or lead"); if (StateName != newStateName) { StateName = newStateName; } } void TRaft::Process(ITimeSource::Time now, TMessageHolder message, const std::shared_ptr& replyTo) { + if (!ValidateRpc(message)) return; if (message.IsEx()) { auto messageEx = message.Cast(); if (messageEx->Term > State->CurrentTerm) { @@ -354,6 +663,11 @@ void TRaft::Process(ITimeSource::Time now, TMessageHolder message, con State->VotedFor = 0; State->Commit(); StateName = EState::FOLLOWER; + IncomingSnapshot.reset(); + VolatileState->SnapshotRpcs.clear(); + EncodedSnapshot.reset(); + EncodedSnapshotIndex = 0; + VolatileState->LeaderId = 0; if (VolatileState->ElectionDue <= now || VolatileState->ElectionDue == ITimeSource::Max) { VolatileState->ElectionDue = MakeElection(now); } @@ -375,6 +689,179 @@ void TRaft::Process(ITimeSource::Time now, TMessageHolder message, con } } +bool TRaft::ValidateRpc(TMessageHolder message) const { + if (!message || !message.IsEx()) return false; + try { ValidateMessageTree(message); } catch (const TMessageError&) { return false; } + auto header = message.Cast(); + if (header->Dst != Id || !Nodes.contains(header->Src)) return false; + if (!IsVoter(header->Src)) { + if ((header->Type != uint32_t(EMessageType::APPEND_ENTRIES_RESPONSE) && + header->Type != uint32_t(EMessageType::INSTALL_SNAPSHOT_RESPONSE)) || header->Term != State->CurrentTerm) + return false; // Nonvoters cannot campaign, lead or force a term change. + } + constexpr uint64_t maxIndex = INT64_MAX; + switch (EMessageType(message->Type)) { + case EMessageType::REQUEST_VOTE_REQUEST: { + auto vote = message.Cast(); + return vote->CandidateId == header->Src && vote->LastLogIndex <= maxIndex && + vote->LastLogTerm <= header->Term && + ((vote->LastLogIndex == 0) == (vote->LastLogTerm == 0)); + } + case EMessageType::REQUEST_VOTE_RESPONSE: + return message.Cast()->VoteGranted <= 1; + case EMessageType::APPEND_ENTRIES_RESPONSE: { + auto reply = message.Cast(); + return reply->Success <= 1 && reply->MatchIndex <= maxIndex && + reply->MatchIndex <= State->LastLogIndex; + } + case EMessageType::INSTALL_SNAPSHOT_REQUEST: { + auto request = message.Cast(); + const auto length = request->Len - sizeof(TInstallSnapshotRequest); + // A later boundary cannot have an older term than the committed prefix. + // For an already committed, retained boundary, the terms must match; + // an older compacted boundary cannot have a newer term than the snapshot. + const bool valid = request->LeaderId == header->Src && request->SnapshotIndex > 0 && request->SnapshotIndex <= maxIndex && + request->SnapshotTerm > 0 && request->SnapshotTerm <= header->Term && request->Done <= 1 && + (request->SnapshotIndex <= VolatileState->CommitIndex || + request->SnapshotTerm >= State->LogTerm(VolatileState->CommitIndex)) && + (request->SnapshotIndex > VolatileState->CommitIndex || request->SnapshotIndex < State->SnapshotIndex || + request->SnapshotTerm == State->LogTerm(request->SnapshotIndex)) && + (request->SnapshotIndex >= State->SnapshotIndex || request->SnapshotTerm <= State->SnapshotTerm) && + request->TotalBytes >= 44 && request->TotalBytes <= TSnapshot::MaxBytes && + request->Offset <= request->TotalBytes && length > 0 && length <= TInstallSnapshotRequest::MaxChunkBytes && + length <= request->TotalBytes - request->Offset && + bool(request->Done) == (request->Offset + length == request->TotalBytes); + if (!valid) return false; + if (header->Term > State->CurrentTerm) { + // Do not persist a higher term based on an InstallSnapshot request + // whose format cannot be accepted by this runtime. A continuation + // cannot be valid across a term change because term changes discard + // the partial transfer; an initial chunk must identify a supported + // fixed/stable format and complete metadata prefix before it can + // establish leader contact. + if (request->Offset != 0) return false; + const auto knownSnapshot = State->GetSnapshot(); + const auto probe = ProbeSnapshotMetadata(std::string_view(request->Data, length), + request->TotalBytes, request->SnapshotIndex, request->SnapshotTerm, + GetVoterIds(), knownSnapshot.get()); + if (!probe.Complete || !probe.Supported) return false; + } + return true; + } + case EMessageType::INSTALL_SNAPSHOT_RESPONSE: { + auto response = message.Cast(); + return response->SnapshotIndex > 0 && response->SnapshotIndex <= maxIndex && + response->NextOffset <= TSnapshot::MaxBytes && response->Success <= 1 && response->Installed <= 1 && + (!response->Installed || response->Success) && (response->Success || response->NextOffset == 0); + } + case EMessageType::APPEND_ENTRIES_REQUEST: { + auto append = message.Cast(); + if (append->LeaderId != header->Src || append->PrevLogIndex > maxIndex || + append->LeaderCommit > maxIndex || append->Nentries > maxIndex - append->PrevLogIndex || + append->PrevLogTerm > header->Term || + ((append->PrevLogIndex == 0) != (append->PrevLogTerm == 0))) return false; + auto previousTerm = append->PrevLogTerm; + for (uint32_t i = 0; i < message.PayloadSize; ++i) { + auto entry = message.Payload[i].Cast(); + if (!entry->Term || entry->Term < previousTerm || entry->Term > header->Term) return false; + previousTerm = entry->Term; + const auto index = append->PrevLogIndex + i + 1; + if (index >= State->SnapshotIndex && index <= VolatileState->CommitIndex && State->LogTerm(index) != entry->Term) return false; + } + return ValidateAppendConfiguration(append); + } + default: return false; + } +} + +bool TRaft::ValidateAppendConfiguration(TMessageHolder message) const { + if (message->Nentries != message.PayloadSize) return false; + bool hasConfigurationEntry = false; + for (uint32_t i = 0; i < message.PayloadSize; ++i) { + auto entry = message.Payload[i].Cast(); + if (!(entry->Flags & TLogEntry::EConfiguration)) continue; + try { + const auto record = DecodeConfigurationEntry(entry); + // A transition leader must be able to remain a voter through the + // stable successor. Leader removal needs an explicit transfer and + // commit-propagation protocol, which this runtime does not yet have. + if (!std::binary_search(record.NewVoters.begin(), record.NewVoters.end(), message->LeaderId)) + return false; + } catch (const std::exception&) { + return false; + } + hasConfigurationEntry = true; + } + if (hasConfigurationEntry && (!ConfigurationBaseline || !State->SupportsVoterBinding() || + !State->SupportsCommitIndexPersistence() || State->GetBoundVoters() != InitialVoters)) + return false; + + // A mismatched previous boundary cannot change this log, so after checking + // the entry frames we can safely return the normal AppendEntries conflict. + if (message->Term < State->CurrentTerm || message->PrevLogIndex < State->SnapshotIndex || + (message->PrevLogIndex && (message->PrevLogIndex > State->LastLogIndex || + State->LogTerm(message->PrevLogIndex) != message->PrevLogTerm))) + return true; + + bool scanRequired = hasConfigurationEntry; + for (uint32_t i = 0; i < message.PayloadSize; ++i) { + const auto index = message->PrevLogIndex + i + 1; + auto incoming = message.Payload[i].Cast(); + const auto localTerm = State->LogTerm(index); + const bool incomingConfiguration = incoming->Flags & TLogEntry::EConfiguration; + const bool localConfiguration = index <= State->LastLogIndex && + (State->RetainedLogFlags(index) & TLogEntry::EConfiguration); + if (localTerm == incoming->Term && (incomingConfiguration || localConfiguration)) { + auto local = State->Get(index); + if (!local || local->Flags != incoming->Flags || local->Len != incoming->Len || + memcmp(local.Mes, incoming.Mes, incoming->Len) != 0) + return false; + } + if (localTerm != incoming->Term && Configuration && + index <= Configuration->Latest.ConfigurationIndex()) + scanRequired = true; + } + if (!scanRequired) return true; + if (!ConfigurationBaseline || !State->SupportsVoterBinding() || + !State->SupportsCommitIndexPersistence() || State->GetBoundVoters() != InitialVoters) + return false; + + try { + TState shadow; + static_cast(shadow) = static_cast(*State); + shadow.Snapshot = State->GetSnapshot(); + shadow.BoundVoters = State->GetBoundVoters(); + shadow.LastLogIndex = State->SnapshotIndex; + shadow.LastLogTerm = State->SnapshotTerm; + for (uint64_t index = State->SnapshotIndex + 1; index <= State->LastLogIndex; ++index) { + auto entry = State->Get(index); + if (!entry) return false; + shadow.Append(std::move(entry)); + } + shadow.CurrentTerm = message->Term; + auto index = message->PrevLogIndex; + for (uint32_t i = 0; i < message.PayloadSize; ++i) { + auto entry = message.Payload[i].Cast(); + ++index; + if (shadow.LogTerm(index) == entry->Term) continue; + while (shadow.LastLogIndex > index - 1) shadow.RemoveLast(); + shadow.Append(std::move(entry)); + } + const auto commitIndex = std::max(VolatileState->CommitIndex, + std::min(message->LeaderCommit, index)); + const auto scan = ScanConfigurationHistory(shadow, *ConfigurationBaseline, commitIndex); + const auto* active = scan.Committed.IsJoint() ? &scan.Committed : + scan.Latest.IsJoint() ? &scan.Latest : &scan.Committed; + for (auto id : active->OldVoters()) + if (id != Id && !Nodes.contains(id)) return false; + for (auto id : active->NewVoters()) + if (id != Id && !Nodes.contains(id)) return false; + } catch (const std::exception&) { + return false; + } + return true; +} + void TRaft::FollowerTimeout(ITimeSource::Time now) { if (VolatileState->ElectionDue <= now) { Become(EState::CANDIDATE); @@ -383,6 +870,7 @@ void TRaft::FollowerTimeout(ITimeSource::Time now) { void TRaft::CandidateTimeout(ITimeSource::Time now) { for (auto& [id, node] : Nodes) { + if (!IsVoter(id)) continue; if (VolatileState->RpcDue[id] <= now) { VolatileState->RpcDue[id] = now + TTimeout::Rpc; node->Send(CreateVote(id)); @@ -398,30 +886,55 @@ void TRaft::LeaderTimeout(ITimeSource::Time now) { { VolatileState->HeartbeatDue[id] = now + TTimeout::Election / 2; VolatileState->RpcDue[id] = now + TTimeout::Rpc; - node->Send(CreateAppendEntries(id)); + if (VolatileState->NextIndex[id] <= State->SnapshotIndex) node->Send(CreateInstallSnapshot(id)); + else node->Send(CreateAppendEntries(id)); } } if (Nservers == 1) { - VolatileState->CommitAdvance(Nservers, *State); + AdvanceLeaderCommit(); } } uint64_t TRaft::ApproveRead() { - int seqno = Seqno; + if (StateName != EState::LEADER) throw std::logic_error("Only a leader can approve a read"); + // Reserve a unique barrier even for a single-node cluster. Every request + // emitted below is newer than this barrier; old replies cannot satisfy it. + const uint64_t seqno = NextSeqno(); for (auto& [id, node] : Nodes) { - node->Send(CreateAppendEntries(id)); + if (VolatileState->NextIndex[id] <= State->SnapshotIndex) node->Send(CreateInstallSnapshot(id)); + else node->Send(CreateAppendEntries(id)); } return seqno; } uint64_t TRaft::CommitSeqno() const { - return VolatileState->CommitSeqno; + if (StateName != EState::LEADER || VolatileState->CommitIndex == 0 || + State->LogTerm(VolatileState->CommitIndex) != State->CurrentTerm) return 0; + return VotingQuorum->Value([&](uint32_t id) -> uint64_t { + if (id == Id) return Seqno - 1; // leader's own confirmation + auto it = VolatileState->ReadAckSeqno.find(id); + return it == VolatileState->ReadAckSeqno.end() ? 0 : it->second; + }); +} + +uint64_t TRaft::NextSeqno() { + if (Seqno == UINT64_MAX) throw std::overflow_error("Raft RPC sequence exhausted"); + return Seqno++; } void TRaft::ProcessTimeout(ITimeSource::Time now) { + // Learners do not campaign, so their election timeout cannot be used to + // reclaim an interrupted InstallSnapshot transfer. Expire partial images + // independently of role and ordinary heartbeat activity. + if (IncomingSnapshot && now >= IncomingSnapshot->LastProgress && + now - IncomingSnapshot->LastProgress >= TTimeout::SnapshotTransferIdle) + IncomingSnapshot.reset(); + if (!IsVoter(Id)) return; if (StateName == EState::CANDIDATE || StateName == EState::FOLLOWER) { if (VolatileState->ElectionDue <= now) { + if (State->CurrentTerm == UINT64_MAX) throw std::overflow_error("Raft term exhausted"); + IncomingSnapshot.reset(); auto nextVolatileState = std::make_unique(); for (auto [id, _] : Nodes) { nextVolatileState->NextIndex[id] = 1; @@ -437,8 +950,8 @@ void TRaft::ProcessTimeout(ITimeSource::Time now) { } if (StateName == EState::CANDIDATE) { - int nvotes = VolatileState->Votes.size()+1; - if (nvotes >= MinVotes) { + if (VotingQuorum->HasQuorum(VolatileState->Votes, Id)) { + if (State->LastLogIndex >= uint64_t(INT64_MAX)) throw std::overflow_error("Raft log index exhausted"); auto value = State->LastLogIndex+1; decltype(VolatileState->NextIndex) nextIndex; decltype(VolatileState->RpcDue) rpcDue; @@ -449,6 +962,7 @@ void TRaft::ProcessTimeout(ITimeSource::Time now) { auto nextVolatileState = std::make_unique(TVolatileState { .CommitIndex = VolatileState->CommitIndex, + .LeaderId = Id, .NextIndex = nextIndex, .RpcDue = rpcDue, .ElectionDue = ITimeSource::Max, @@ -486,11 +1000,73 @@ ITimeSource::Time TRaft::MakeElection(ITimeSource::Time now) { } uint64_t TRaft::Append(TMessageHolder entry) { + if (!IsVoter(Id)) throw std::logic_error("A learner cannot propose log entries"); + if (StateName != EState::LEADER) throw std::logic_error("Only a leader can propose log entries"); + if (State->LastLogIndex >= uint64_t(INT64_MAX)) throw std::overflow_error("Raft log index exhausted"); + if (!entry.HasValidExtent() || entry->Type != uint32_t(EMessageType::LOG_ENTRY)) + throw TMessageError("Invalid proposed log entry"); + ValidateMessageTree(entry); + if (entry->Flags & TLogEntry::EConfiguration) + throw TMessageError("Configuration proposals are unsupported"); entry->Term = State->CurrentTerm; State->Append(std::move(entry)); return State->LastLogIndex; } +void TRaft::RequireLiveConfigurationStorage() const { + if (!State->SupportsVoterBinding() || !State->SupportsCommitIndexPersistence() || + InitialVoters.empty() || State->GetBoundVoters() != InitialVoters) + throw std::logic_error("Live membership changes require persisted commit and voter metadata"); +} + +uint64_t TRaft::ProposeJointConfiguration(std::vector newVoters) { + if (StateName != EState::LEADER) throw std::logic_error("Only a leader can change membership"); + RequireLiveConfigurationStorage(); + ValidateFixedVoters(newVoters); + if (!Configuration || Configuration->HasUncommittedConfiguration() || Configuration->Latest.IsJoint()) + throw std::logic_error("A stable committed configuration is required to start a transition"); + if (State->LastLogIndex >= uint64_t(INT64_MAX)) throw std::overflow_error("Raft log index exhausted"); + if (!VolatileState->CommitIndex || State->LogTerm(VolatileState->CommitIndex) != State->CurrentTerm) + throw std::logic_error("A current-term entry must be committed before changing membership"); + + const auto& oldVoters = Configuration->Committed.OldVoters(); + if (newVoters == oldVoters || !std::binary_search(newVoters.begin(), newVoters.end(), Id)) + throw std::invalid_argument("The new voter set must change membership and retain the current leader"); + for (auto id : newVoters) { + if (id != Id && !Nodes.contains(id)) + throw std::invalid_argument("Every new voter must have a configured replication target"); + if (!std::binary_search(oldVoters.begin(), oldVoters.end(), id) && + !IsLearnerCommitAcknowledged(id)) + throw std::logic_error("Every added voter must be a staged learner with a committed catch-up ACK"); + } + + const auto& latest = Configuration->Latest; + TConfigurationRecord record{EConfigurationPhase::Joint, State->CurrentTerm, + latest.ConfigurationIndex(), latest.ConfigurationTerm(), oldVoters, std::move(newVoters)}; + State->Append(EncodeConfigurationEntry(record)); + RefreshConfiguration(VolatileState->CommitIndex); + return State->LastLogIndex; +} + +uint64_t TRaft::FinalizeConfiguration() { + if (StateName != EState::LEADER) throw std::logic_error("Only a leader can change membership"); + RequireLiveConfigurationStorage(); + if (!Configuration || Configuration->HasUncommittedConfiguration() || !Configuration->Committed.IsJoint()) + throw std::logic_error("The joint configuration must commit before finalization"); + if (State->LastLogIndex >= uint64_t(INT64_MAX)) throw std::overflow_error("Raft log index exhausted"); + if (!VolatileState->CommitIndex || State->LogTerm(VolatileState->CommitIndex) != State->CurrentTerm) + throw std::logic_error("A current-term entry must be committed before finalizing membership"); + + const auto& joint = Configuration->Committed; + if (!std::binary_search(joint.NewVoters().begin(), joint.NewVoters().end(), Id)) + throw std::logic_error("Leadership must transfer to a voter in the new set before finalization"); + TConfigurationRecord record{EConfigurationPhase::Stable, State->CurrentTerm, + joint.ConfigurationIndex(), joint.ConfigurationTerm(), joint.OldVoters(), joint.NewVoters()}; + State->Append(EncodeConfigurationEntry(record)); + RefreshConfiguration(VolatileState->CommitIndex); + return State->LastLogIndex; +} + uint32_t TRaft::GetLeaderId() const { return VolatileState->LeaderId; } @@ -499,7 +1075,304 @@ uint64_t TRaft::GetLastIndex() const { return State->LastLogIndex; } +std::vector TRaft::GetVoterIds() const { return Voters; } + +bool TRaft::IsVoter(uint32_t id) const { return std::binary_search(Voters.begin(), Voters.end(), id); } + +void TRaft::AddLearnerTarget(uint32_t id, std::shared_ptr node) { + if (StateName != EState::LEADER) throw std::logic_error("Only a leader can stage a learner"); + if (!id || id == Id || !node || Nodes.contains(id) || IsVoter(id) || + std::binary_search(Learners.begin(), Learners.end(), id)) + throw std::invalid_argument("Invalid or duplicate learner target"); + if (!State->SupportsVoterBinding() || State->GetBoundVoters() != InitialVoters) + throw std::logic_error("Learner staging requires a bound voter configuration"); + if (Nodes.size() >= size_t(INT_MAX)) throw std::overflow_error("Too many replication targets"); + // Build all allocating changes before publishing any part of the target. + auto nodes = Nodes; + auto learners = Learners; + auto nextIndex = VolatileState->NextIndex; + nodes.emplace(id, std::move(node)); + learners.insert(std::lower_bound(learners.begin(), learners.end(), id), id); + nextIndex.emplace(id, State->LastLogIndex + 1); + Nodes.swap(nodes); Learners.swap(learners); VolatileState->NextIndex.swap(nextIndex); + Npeers = int(Nodes.size()); +} + +std::shared_ptr TRaft::RemoveLearnerTarget(uint32_t id) { + if (StateName != EState::LEADER) throw std::logic_error("Only a leader can remove a learner target"); + auto learner = std::lower_bound(Learners.begin(), Learners.end(), id); + auto peer = Nodes.find(id); + if (id == Id || learner == Learners.end() || *learner != id || peer == Nodes.end() || IsVoter(id)) + throw std::invalid_argument("Unknown learner target"); + auto node = peer->second; + Nodes.erase(peer); Learners.erase(learner); + VolatileState->NextIndex.erase(id); VolatileState->MatchIndex.erase(id); + VolatileState->CommitAckIndex.erase(id); + VolatileState->ReadAckSeqno.erase(id); VolatileState->LastSentSeqno.erase(id); + VolatileState->PendingAppendRpcs.erase(id); VolatileState->SnapshotRpcs.erase(id); + VolatileState->HeartbeatDue.erase(id); VolatileState->RpcDue.erase(id); + VolatileState->BatchSize.erase(id); VolatileState->BackOff.erase(id); + VolatileState->Votes.erase(id); + Npeers = int(Nodes.size()); + return node; +} + +bool TRaft::IsLearnerCaughtUp(uint32_t id) const { + if (StateName != EState::LEADER || !std::binary_search(Learners.begin(), Learners.end(), id) || + VolatileState->CommitIndex == 0 || State->LogTerm(VolatileState->CommitIndex) != State->CurrentTerm) return false; + auto match = VolatileState->MatchIndex.find(id); + return match != VolatileState->MatchIndex.end() && match->second >= State->LastLogIndex; +} + +bool TRaft::IsLearnerCommitAcknowledged(uint32_t id) const { + if (StateName != EState::LEADER || !std::binary_search(Learners.begin(), Learners.end(), id) || + VolatileState->CommitIndex == 0 || State->LogTerm(VolatileState->CommitIndex) != State->CurrentTerm) + return false; + const auto match = VolatileState->MatchIndex.find(id); + const auto commitAck = VolatileState->CommitAckIndex.find(id); + return match != VolatileState->MatchIndex.end() && match->second >= State->LastLogIndex && + commitAck != VolatileState->CommitAckIndex.end() && commitAck->second >= VolatileState->CommitIndex; +} + +TSnapshot TRaft::Compact() { + if (!State->SupportsSnapshots()) throw std::logic_error("Storage cannot compact snapshots"); + if (HasMembershipHistory()) + throw std::logic_error("Compaction after a log membership change requires runtime configuration snapshots"); + auto application = SnapshotApplication.lock(); + if (!application) throw std::logic_error("Compaction requires a bound application"); + if (application->LastAppliedIndex == State->SnapshotIndex && State->GetSnapshot()) return *State->GetSnapshot(); + std::unordered_set pinnedGenerations; + for (const auto& [_, rpc] : VolatileState->SnapshotRpcs) + if (rpc.Snapshot) pinnedGenerations.insert(rpc.Snapshot->Index); + if (pinnedGenerations.size() >= TVolatileState::MaxPinnedSnapshotGenerations) + throw std::runtime_error("Compaction deferred while snapshot transfers pin multiple generations"); + auto snapshot = CaptureSnapshot(*this, *application); + application->ValidateSnapshot(snapshot.Data, snapshot.Index); + AdvanceCommitIndex(VolatileState->CommitIndex); + State->InstallSnapshot(snapshot); + ConfigurationBaseline = snapshot.StableConfiguration + ? TConfigurationHistory::FromStableSnapshot(snapshot) + : TConfigurationHistory::Stable(Voters, 0, 0, snapshot.Index, snapshot.Term); + RefreshConfiguration(VolatileState->CommitIndex); + // Outstanding transfers own their immutable encoded bytes; discard any + // otherwise-retained encoding of the old local snapshot. + EncodedSnapshot.reset(); + EncodedSnapshotIndex = 0; + return snapshot; +} + +TMessageHolder TRaft::CreateInstallSnapshot(uint32_t nodeId) { + auto& rpc = VolatileState->SnapshotRpcs[nodeId]; + if (!rpc.Snapshot) { + auto snapshot = State->GetSnapshot(); + if (!snapshot) throw std::logic_error("Missing snapshot for compacted log"); + if (EncodedSnapshotIndex != snapshot->Index) { + EncodedSnapshot = std::make_shared(EncodeSnapshot(*snapshot)); + EncodedSnapshotIndex = snapshot->Index; + } + rpc = TVolatileState::TSnapshotRpc{snapshot, EncodedSnapshot}; + VolatileState->PendingAppendRpcs[nodeId].clear(); + } + // Pin one immutable image until this peer finishes. Periodic compaction + // must not restart every chunk; after completion a still-lagging peer can + // start a separate transfer of the newest image. + const auto& snapshot = rpc.Snapshot; + const auto count = std::min(TInstallSnapshotRequest::MaxChunkBytes, rpc.Bytes->size() - rpc.Offset); + auto request = NewHoldedMessage(sizeof(TInstallSnapshotRequest) + count); + request->Src = Id; request->Dst = nodeId; request->Term = State->CurrentTerm; + // A retry sends the same immutable chunk. Keep its identity until a reply + // consumes it, otherwise traffic faster than the RTT invalidates every ACK. + // Snapshot ACKs never confirm read barriers, so reuse cannot confirm a read + // that arrived after the first transmission. + if (!rpc.Seqno) rpc.Seqno = NextSeqno(); + request->Seqno = rpc.Seqno; + request->SnapshotIndex = snapshot->Index; request->SnapshotTerm = snapshot->Term; + request->Offset = rpc.Offset; request->TotalBytes = rpc.Bytes->size(); + rpc.SentEnd = rpc.Offset + count; + request->Done = rpc.SentEnd == rpc.Bytes->size(); request->LeaderId = Id; + memcpy(request->Data, rpc.Bytes->data() + rpc.Offset, count); + return request; +} + +void TRaft::OnInstallSnapshot(TMessageHolder response) { + if (response->Term != State->CurrentTerm) return; + auto found = VolatileState->SnapshotRpcs.find(response->Src); + if (found == VolatileState->SnapshotRpcs.end()) return; + auto& rpc = found->second; + if (!rpc.Seqno || response->Seqno != rpc.Seqno || response->SnapshotIndex != rpc.Snapshot->Index) return; + if (!response->Success) { + rpc.Offset = 0; + rpc.Seqno = 0; + } else if (response->Installed) { + if (response->NextOffset != rpc.Bytes->size()) return; + const auto index = std::max(VolatileState->MatchIndex[response->Src], response->SnapshotIndex); + (*VolatileState).SetMatchIndex(response->Src, index).SetNextIndex(response->Src, index + 1) + .SetBatchSize(response->Src, 1024).SetBackOff(response->Src, 1); + VolatileState->CommitAckIndex[response->Src] = std::max( + VolatileState->CommitAckIndex[response->Src], + std::min(VolatileState->CommitIndex, response->SnapshotIndex)); + AdvanceLeaderCommit(); + VolatileState->SnapshotRpcs.erase(found); + // Snapshot completion is not a fresh-read acknowledgement. AppendEntries + // must still establish the next read barrier and replicate the suffix. + } else { + if (response->NextOffset != rpc.SentEnd || response->NextOffset >= rpc.Bytes->size()) return; + rpc.Offset = response->NextOffset; + rpc.Seqno = 0; // Next chunk gets a new identity; old replies stay stale. + } + VolatileState->RpcDue[response->Src] = ITimeSource::Time{}; +} + +void TRaft::OnInstallSnapshot(ITimeSource::Time now, TMessageHolder request) { + auto reply = [&](bool success, bool installed, uint64_t offset) { + Nodes.at(request->Src)->Send(NewHoldedMessage( + TMessageEx{.Src = Id, .Dst = request->Src, .Term = State->CurrentTerm, .Seqno = request->Seqno}, + TInstallSnapshotResponse{.SnapshotIndex = request->SnapshotIndex, .NextOffset = offset, + .Success = uint32_t(success), .Installed = uint32_t(installed)})); + }; + if (request->Term < State->CurrentTerm) { reply(false, false, 0); return; } + // Partial or unsupported image headers are not evidence of leader contact. + // In particular, do not demote a candidate or refresh its election timer + // until the complete bounded metadata prefix is supported. + auto acceptLeaderContact = [&] { + Become(EState::FOLLOWER); + VolatileState->LeaderId = request->LeaderId; + VolatileState->ElectionDue = MakeElection(now); + }; + if (request->SnapshotIndex <= VolatileState->CommitIndex) { + const auto term = State->LogTerm(request->SnapshotIndex); + const bool covered = request->SnapshotIndex < State->SnapshotIndex || term == request->SnapshotTerm; + if (!covered) { reply(false, false, 0); return; } + // A covered image needs no application replacement, but its snapshot + // boundary and fixed voter set must match before it can count as + // replicated. Accumulate only the bounded metadata prefix. + const bool same = IncomingSnapshot && IncomingSnapshot->Term == request->Term && + IncomingSnapshot->Leader == request->LeaderId && IncomingSnapshot->Index == request->SnapshotIndex && + IncomingSnapshot->BoundaryTerm == request->SnapshotTerm && IncomingSnapshot->Total == request->TotalBytes; + if (!same) { + if (request->Offset != 0 || (IncomingSnapshot && IncomingSnapshot->Term == request->Term && + IncomingSnapshot->Index > request->SnapshotIndex)) { reply(false, false, 0); return; } + IncomingSnapshot = TIncomingSnapshot{request->Term, request->SnapshotIndex, request->SnapshotTerm, + request->TotalBytes, request->LeaderId, now, {}}; + } + auto& prefix = IncomingSnapshot->Bytes; + const auto count = request->Len - sizeof(TInstallSnapshotRequest); + if (request->Offset > prefix.size()) { reply(false, false, 0); return; } + if (request->Offset < prefix.size()) { + if (count > prefix.size() - request->Offset || + std::string_view(prefix).substr(request->Offset, count) != std::string_view(request->Data, count)) { + IncomingSnapshot.reset(); reply(false, false, 0); return; + } + } else prefix.append(request->Data, count); + IncomingSnapshot->LastProgress = now; + if (!IsSupportedSnapshotMagicPrefix(prefix)) { + IncomingSnapshot.reset(); reply(false, false, 0); return; + } + const auto knownSnapshot = State->GetSnapshot(); + const auto probe = ProbeSnapshotMetadata(prefix, request->TotalBytes, + request->SnapshotIndex, request->SnapshotTerm, GetVoterIds(), knownSnapshot.get()); + if (!probe.Complete) { reply(true, false, prefix.size()); return; } + IncomingSnapshot.reset(); + if (probe.Supported) acceptLeaderContact(); + reply(probe.Supported, probe.Supported, probe.Supported ? request->TotalBytes : 0); + return; + } + auto application = SnapshotApplication.lock(); + if (!State->SupportsSnapshots() || !application) { reply(false, false, 0); return; } + const bool same = IncomingSnapshot && IncomingSnapshot->Term == request->Term && + IncomingSnapshot->Leader == request->LeaderId && IncomingSnapshot->Index == request->SnapshotIndex && + IncomingSnapshot->BoundaryTerm == request->SnapshotTerm && IncomingSnapshot->Total == request->TotalBytes; + if (!same) { + if (request->Offset != 0 || (IncomingSnapshot && IncomingSnapshot->Term == request->Term && + IncomingSnapshot->Index > request->SnapshotIndex)) { reply(false, false, 0); return; } + IncomingSnapshot = TIncomingSnapshot{request->Term, request->SnapshotIndex, request->SnapshotTerm, + request->TotalBytes, request->LeaderId, now, {}}; + } + auto& bytes = IncomingSnapshot->Bytes; + const auto count = request->Len - sizeof(TInstallSnapshotRequest); + if (request->Offset > bytes.size()) { reply(false, false, 0); return; } + if (request->Offset < bytes.size()) { + if (count > bytes.size() - request->Offset || + std::string_view(bytes).substr(request->Offset, count) != std::string_view(request->Data, count)) { + IncomingSnapshot.reset(); reply(false, false, 0); return; + } + } else bytes.append(request->Data, count); + IncomingSnapshot->LastProgress = now; + if (!IsSupportedSnapshotMagicPrefix(bytes)) { + IncomingSnapshot.reset(); reply(false, false, 0); return; + } + const auto knownSnapshot = State->GetSnapshot(); + const auto probe = ProbeSnapshotMetadata(bytes, request->TotalBytes, + request->SnapshotIndex, request->SnapshotTerm, GetVoterIds(), knownSnapshot.get()); + if (!probe.Complete) { reply(true, false, bytes.size()); return; } + if (!probe.Supported) { IncomingSnapshot.reset(); reply(false, false, 0); return; } + acceptLeaderContact(); + if (bytes.size() < request->TotalBytes) { reply(true, false, bytes.size()); return; } + TSnapshot snapshot; + try { + snapshot = DecodeSnapshot(bytes); + if (snapshot.IsJoint() || snapshot.Index != request->SnapshotIndex || snapshot.Term != request->SnapshotTerm || snapshot.Voters != GetVoterIds()) + throw TMessageError("Snapshot metadata mismatch"); + PreserveStableSnapshotIdentity(knownSnapshot.get(), snapshot); + ValidateSnapshot(snapshot); + application->ValidateSnapshot(snapshot.Data, snapshot.Index); + } catch (const std::exception&) { + IncomingSnapshot.reset(); reply(false, false, 0); return; + } + try { + State->InstallSnapshot(snapshot); // durable snapshot and suffix before application replacement or ACK + application->LoadSnapshot(snapshot.Data, snapshot.Index); + if (application->LastAppliedIndex != snapshot.Index) std::terminate(); + } catch (...) { std::terminate(); } + VolatileState->CommitIndex = State->CommitIndex; + IncomingSnapshot.reset(); + reply(true, true, request->TotalBytes); +} + +void TRaft::RestoreAppliedIndex(uint64_t index) { + if (index > uint64_t(INT64_MAX) || index > State->LastLogIndex) + throw std::invalid_argument("Application progress exceeds recovered Raft log"); + // A correctly recovered application marker is evidence of commitment. + // Protect its prefix immediately, before any AppendEntries can be handled. + AdvanceCommitIndex(std::max(VolatileState->CommitIndex, index)); +} + +void TRaft::AdvanceCommitIndex(uint64_t index) { + if (index > State->LastLogIndex) + throw std::logic_error("Invalid Raft commit index advance"); + const auto target = std::max({index, VolatileState->CommitIndex, State->CommitIndex}); + if (target > State->LastLogIndex) + throw std::logic_error("Invalid persisted Raft commit index"); + if (State->CommitIndex < target) { + State->CommitIndex = target; + State->Commit(); + } + VolatileState->CommitIndex = target; + if (ConfigurationScanRequired) RefreshConfiguration(target); +} + +void TRaft::AdvanceLeaderCommit() { + const auto previous = VolatileState->CommitIndex; + VolatileState->CommitAdvance(*VotingQuorum, Id, *State); + const auto updated = VolatileState->CommitIndex; + VolatileState->CommitIndex = previous; + AdvanceCommitIndex(updated); +} + +TRequestProcessor::TRequestProcessor(std::shared_ptr raft, std::shared_ptr rsm, const TNodeDict& nodes) + : Raft(std::move(raft)), Rsm(std::move(rsm)), Nodes(nodes) +{ + if (!Raft || !Rsm) throw std::invalid_argument("Request processor requires Raft and application state"); + if (auto snapshot = Raft->GetState()->GetSnapshot(); snapshot && Rsm->LastAppliedIndex < snapshot->Index) + RestoreSnapshot(*snapshot, *Raft, *Rsm); + if (Rsm->LastAppliedIndex < Raft->GetState()->SnapshotIndex) + throw std::invalid_argument("Application progress precedes compacted log"); + Raft->RestoreAppliedIndex(Rsm->LastAppliedIndex); + Raft->SnapshotApplication = Rsm; +} + void TRequestProcessor::CheckStateChange() { + PruneRequests(); if (WaitingStateChange.empty()) { return; } @@ -507,7 +1380,8 @@ void TRequestProcessor::CheckStateChange() { auto stateName = Raft->CurrentStateName(); auto leaderId = Raft->GetLeaderId(); - if (stateName == EState::CANDIDATE || leaderId == 0) { + if (stateName == EState::CANDIDATE || (stateName != EState::LEADER && + (leaderId == 0 || leaderId == Raft->GetId() || !Nodes.contains(leaderId)))) { return; } @@ -525,7 +1399,7 @@ void TRequestProcessor::Forward(TMessageHolder command, const s return; } - if (command->Cookie) { + if (command->Flags & TCommandRequest::EForwarded) { // already forwarded replyTo->Send(NewHoldedMessage(TCommandResponse{.Cookie = command->Cookie, .ErrorCode = 1})); return; @@ -533,19 +1407,26 @@ void TRequestProcessor::Forward(TMessageHolder command, const s auto stateName = Raft->CurrentStateName(); auto leaderId = Raft->GetLeaderId(); - if (stateName == EState::CANDIDATE || leaderId == 0) { + if (stateName == EState::CANDIDATE || leaderId == 0 || + leaderId == Raft->GetId() || !Nodes.contains(leaderId)) { WaitingStateChange.emplace(TWaiting{0, 0, std::move(command), replyTo}); return; } if (stateName == EState::FOLLOWER) { - assert(leaderId != Raft->GetId()); - // Forward - command->Cookie = std::max(1, ForwardCookie); - Nodes[leaderId]->Send(std::move(command)); - Cookie2Client[ForwardCookie] = replyTo; - Client2Cookie[replyTo].emplace(ForwardCookie); - ForwardCookie++; + if (NextForwardCookie > UINT32_MAX) { + replyTo->Send(NewHoldedMessage(TCommandResponse{.Cookie = command->Cookie, .ErrorCode = 1})); + return; + } + const auto cookie = uint32_t(NextForwardCookie++); + auto forwarded = NewHoldedMessage(command->Len); + memcpy(forwarded.Mes, command.Mes, command->Len); + forwarded->Cookie = cookie; + forwarded->Flags |= TCommandRequest::EForwarded; + auto peer = Nodes.at(leaderId); + Forwarded.emplace(cookie, TForwarded{replyTo, peer, command->Cookie, + leaderId, Raft->GetState()->CurrentTerm}); + peer->Send(std::move(forwarded)); return; } @@ -559,9 +1440,12 @@ void TRequestProcessor::OnReadRequest(TMessageHolder command, c assert(!(flags & TCommandRequest::EWrite)); // stale read, default read (from leader) - if ((flags & TCommandRequest::EStale) || (!(flags & TCommandRequest::EConsistent) && stateName == EState::LEADER)) { - assert(Waiting.empty() || Waiting.back().Index <= Raft->GetLastIndex()); - Waiting.emplace(TWaiting{Raft->GetLastIndex(), 0, std::move(command), replyTo}); + if (!(flags & TCommandRequest::EConsistent) && + ((flags & TCommandRequest::EStale) || stateName == EState::LEADER)) { + const bool stale = flags & TCommandRequest::EStale; + const auto index = stale ? Raft->GetVolatileState()->CommitIndex : Raft->GetLastIndex(); + const auto term = stale ? 0 : Raft->GetState()->CurrentTerm; + Waiting.emplace(TWaiting{index, 0, std::move(command), replyTo, term}); return; } @@ -570,9 +1454,60 @@ void TRequestProcessor::OnReadRequest(TMessageHolder command, c } // Consistent read + const auto term = Raft->GetState()->CurrentTerm; + const auto index = Raft->GetLastIndex(); auto seqno = Raft->ApproveRead(); - assert(StrongWaiting.empty() || (StrongWaiting.back().Index <= Raft->GetLastIndex() && StrongWaiting.back().Seqno <= seqno)); - StrongWaiting.emplace(TWaiting{Raft->GetLastIndex(), seqno, std::move(command), replyTo}); + assert(StrongWaiting.empty() || (StrongWaiting.back().Index <= index && StrongWaiting.back().Seqno <= seqno)); + StrongWaiting.emplace(TWaiting{index, seqno, std::move(command), replyTo, term}); +} + +void TRequestProcessor::PruneRequests(const std::shared_ptr& disconnected) { + const auto term = Raft->GetState()->CurrentTerm; + const auto state = Raft->CurrentStateName(); + const bool leader = state == EState::LEADER; + const auto leaderId = Raft->GetLeaderId(); + if (!disconnected) { + if (RequestsTerm == term && RequestsState == state && RequestsLeaderId == leaderId) return; + RequestsTerm = term; + RequestsState = state; + RequestsLeaderId = leaderId; + } + for (auto* queue : {&Waiting, &StrongWaiting, &WaitingStateChange}) { + std::queue retained; + while (!queue->empty()) { + auto waiting = std::move(queue->front()); + queue->pop(); + const bool remove = disconnected ? waiting.ReplyTo == disconnected : + waiting.Term != 0 && (!leader || waiting.Term != term); + if (!remove) { + retained.push(std::move(waiting)); + continue; + } + if (waiting.Command->Flags & TCommandRequest::EWrite) { + PendingWriteTerms.erase(waiting.Index); + WriteAnswers.erase(waiting.Index); + } + if (!disconnected && waiting.ReplyTo) waiting.ReplyTo->Send(NewHoldedMessage( + TCommandResponse{.Cookie = waiting.Command->Cookie, .ErrorCode = 1})); + } + queue->swap(retained); + } + CancelForwarded(disconnected); +} + +void TRequestProcessor::CancelForwarded(const std::shared_ptr& disconnected, bool peerFailure) { + for (auto it = Forwarded.begin(); it != Forwarded.end();) { + const auto& route = it->second; + const bool remove = disconnected ? + (peerFailure ? route.Peer == disconnected : route.Client == disconnected) : + Raft->CurrentStateName() != EState::FOLLOWER || + route.Term != Raft->GetState()->CurrentTerm || route.LeaderId != Raft->GetLeaderId(); + if (!remove) { ++it; continue; } + auto pending = std::move(it->second); + it = Forwarded.erase(it); + if (!disconnected || peerFailure) pending.Client->Send(NewHoldedMessage( + TCommandResponse{.Cookie = pending.OriginalCookie, .ErrorCode = 1})); + } } void TRequestProcessor::OnWriteRequest(TMessageHolder command, const std::shared_ptr& replyTo) { @@ -581,11 +1516,11 @@ void TRequestProcessor::OnWriteRequest(TMessageHolder command, assert((flags & TCommandRequest::EWrite)); if (stateName == EState::LEADER) { + const auto term = Raft->GetState()->CurrentTerm; uint64_t index = Raft->Append(std::move(Rsm->Prepare(command))); if (replyTo) { - assert(Waiting.empty() || Waiting.back().Index <= index); - // TODO: cleanup these queues on state change - Waiting.emplace(TWaiting{index, 0, std::move(command), replyTo}); + PendingWriteTerms[index] = term; + Waiting.emplace(TWaiting{index, 0, std::move(command), replyTo, term}); } } else { Forward(std::move(command), replyTo); @@ -593,6 +1528,11 @@ void TRequestProcessor::OnWriteRequest(TMessageHolder command, } void TRequestProcessor::OnCommandRequest(TMessageHolder command, const std::shared_ptr& replyTo) { + if (!command.HasValidExtent() || command->Type != uint32_t(EMessageType::COMMAND_REQUEST)) + throw TMessageError("Invalid command request"); + ValidateMessageTree(command); + Rsm->Validate(command); + PruneRequests(); auto flags = command->Flags; if (!(flags & TCommandRequest::EWrite)) { @@ -602,77 +1542,111 @@ void TRequestProcessor::OnCommandRequest(TMessageHolder command } } -void TRequestProcessor::OnCommandResponse(TMessageHolder command) { - // forwarded - auto it = Cookie2Client.find(command->Cookie); - if (it == Cookie2Client.end()) { - return; - } - it->second->Send(std::move(command)); - auto jt = Client2Cookie.find(it->second); - jt->second.erase(command->Cookie); - if (jt->second.empty()) { - Client2Cookie.erase(jt); - } - Cookie2Client.erase(it); +void TRequestProcessor::OnCommandResponse(TMessageHolder command, + const std::shared_ptr& peer) { + if (!command.HasValidExtent() || command->Type != uint32_t(EMessageType::COMMAND_RESPONSE)) + throw TMessageError("Invalid command response"); + ValidateMessageTree(command); + PruneRequests(); + auto it = Forwarded.find(command->Cookie); + if (it == Forwarded.end() || !peer || it->second.Peer != peer) return; + auto pending = std::move(it->second); + Forwarded.erase(it); + auto reply = NewHoldedMessage(command->Len); + memcpy(reply.Mes, command.Mes, command->Len); + reply->Cookie = pending.OriginalCookie; + pending.Client->Send(std::move(reply)); +} + +void TRequestProcessor::OnPeerDisconnected(const std::shared_ptr& peer) { + if (peer) CancelForwarded(peer, true); } void TRequestProcessor::CleanUp(const std::shared_ptr& replyTo) { - auto jt = Client2Cookie.find(replyTo); - if (jt == Client2Cookie.end()) { - return; - } - for (auto cookie : jt->second) { - Cookie2Client.erase(cookie); - } - Client2Cookie.erase(jt); + if (!replyTo) return; + PruneRequests(replyTo); } void TRequestProcessor::ProcessCommitted() { + PruneRequests(); auto commitIndex = Raft->GetVolatileState()->CommitIndex; + // Never rewind application progress, or overflow LastAppliedIndex + 1. + if (commitIndex <= Rsm->LastAppliedIndex) return; auto& state = Raft->GetState(); for (auto i = Rsm->LastAppliedIndex+1; i <= commitIndex; i++) { auto entry = state->Get(i); + if (entry->Flags & TLogEntry::EConfiguration) { + // Configuration is Raft metadata, never an application command. + // Validate its control record and advance the application boundary + // without exposing it to the state machine. + DecodeConfigurationEntry(entry); + continue; + } if (entry->Flags == TLogEntry::EStub) { continue; } - auto reply = Rsm->Write(entry, i); - WriteAnswers.emplace(TAnswer { - .Index = i, - .Reply = reply ? reply : NewHoldedMessage(TCommandResponse {.Index = i}) - }); + TMessageHolder reply; + try { reply = Rsm->Write(entry, i); } + catch (...) { std::terminate(); } + auto pending = PendingWriteTerms.find(i); + if (pending != PendingWriteTerms.end() && pending->second == entry->Term) { + WriteAnswers.emplace(i, TAnswer { + .Term = entry->Term, + .Reply = reply ? reply : NewHoldedMessage(TCommandResponse {.Index = i}) + }); + } } Rsm->LastAppliedIndex = commitIndex; } void TRequestProcessor::ProcessWaiting() { + PruneRequests(); auto lastApplied = Rsm->LastAppliedIndex; - while (!Waiting.empty() && Waiting.back().Index <= lastApplied) { - auto w = Waiting.back(); Waiting.pop(); + while (!Waiting.empty()) { + if (Waiting.front().Term != 0 && (Raft->CurrentStateName() != EState::LEADER || + Waiting.front().Term != Raft->GetState()->CurrentTerm)) { + PruneRequests(); + continue; + } + if (Waiting.front().Index > lastApplied) break; + auto w = Waiting.front(); Waiting.pop(); TMessageHolder reply; - auto cookie = w.Command->Cookie;; + auto cookie = w.Command->Cookie; if (w.Command->Flags & TCommandRequest::EWrite) { - while (!WriteAnswers.empty() && WriteAnswers.front().Index < w.Index) { - WriteAnswers.pop(); + auto answer = WriteAnswers.find(w.Index); + if (answer != WriteAnswers.end() && answer->second.Term == w.Term) { + reply = answer->second.Reply.Cast(); + } else { + reply = NewHoldedMessage(TCommandResponse{.ErrorCode = 1}); } - assert(!WriteAnswers.empty()); - auto answer = std::move(WriteAnswers.front()); WriteAnswers.pop(); - assert(answer.Index == w.Index); - reply = std::move(answer.Reply.Cast()); + PendingWriteTerms.erase(w.Index); + WriteAnswers.erase(w.Index); } else { - reply = Rsm->Read(std::move(w.Command), w.Index).Cast(); + try { reply = Rsm->Read(std::move(w.Command), w.Index).Cast(); } + catch (const std::exception&) { + reply = NewHoldedMessage(TCommandResponse{.Index = w.Index, .ErrorCode = 2}); + } } reply->Cookie = cookie; w.ReplyTo->Send(std::move(reply)); } - auto seqno = Raft->CommitSeqno(); - while (!StrongWaiting.empty() && StrongWaiting.back().Index <= lastApplied && StrongWaiting.back().Seqno <= seqno) { - auto w = StrongWaiting.back(); StrongWaiting.pop(); + while (!StrongWaiting.empty()) { + if (Raft->CurrentStateName() != EState::LEADER || + StrongWaiting.front().Term != Raft->GetState()->CurrentTerm) { + PruneRequests(); + continue; + } + if (StrongWaiting.front().Index > lastApplied || + StrongWaiting.front().Seqno > Raft->CommitSeqno()) break; + auto w = StrongWaiting.front(); StrongWaiting.pop(); TMessageHolder reply; assert (!(w.Command->Flags & TCommandRequest::EWrite)); auto cookie = w.Command->Cookie; - reply = Rsm->Read(std::move(w.Command), w.Index).Cast(); + try { reply = Rsm->Read(std::move(w.Command), w.Index).Cast(); } + catch (const std::exception&) { + reply = NewHoldedMessage(TCommandResponse{.Index = w.Index, .ErrorCode = 2}); + } reply->Cookie = cookie; w.ReplyTo->Send(std::move(reply)); } diff --git a/miniraft/raft.h b/miniraft/raft.h index c6bb923..bc8b1f2 100644 --- a/miniraft/raft.h +++ b/miniraft/raft.h @@ -2,7 +2,9 @@ #include #include +#include #include +#include #include #include #include @@ -10,6 +12,8 @@ #include "messages.h" #include "timesource.h" #include "state.h" +#include "quorum.h" +#include "configuration.h" struct INode { virtual ~INode() = default; @@ -21,14 +25,25 @@ struct INode { // CommandRequest -> Read? -> CurrentIndex (fixate) >= CommittedIndex -> CommandResponse struct IRsm { virtual ~IRsm() = default; + // Check application wire shape before queueing/forwarding; must not mutate state. + virtual void Validate(TMessageHolder) const {} virtual TMessageHolder Read(TMessageHolder message, uint64_t index) = 0; virtual TMessageHolder Write(TMessageHolder message, uint64_t index) = 0; virtual TMessageHolder Prepare(TMessageHolder message) = 0; - + virtual std::string SaveSnapshot() const { throw std::logic_error("Application snapshots are unsupported"); } + virtual void ValidateSnapshot(std::string_view, uint64_t) const { throw std::logic_error("Application snapshots are unsupported"); } + // Implementations must validate completely before atomically replacing state + // and LastAppliedIndex. Invoked at startup or during serialized installation; + // a failure after durable installation is fatal and requires restart. + virtual void LoadSnapshot(std::string_view, uint64_t) { throw std::logic_error("Application snapshots are unsupported"); } + + // Durable RSMs restore this atomically with application data before constructing + // TRequestProcessor. It must identify committed progress in the same Raft log. uint64_t LastAppliedIndex = 0; }; struct TDummyRsm: public IRsm { + void Validate(TMessageHolder message) const override; TMessageHolder Read(TMessageHolder message, uint64_t index) override; TMessageHolder Write(TMessageHolder message, uint64_t index) override; TMessageHolder Prepare(TMessageHolder message) override; @@ -40,11 +55,30 @@ struct TDummyRsm: public IRsm { using TNodeDict = std::unordered_map>; struct TVolatileState { + struct TAppendRpc { uint64_t Seqno, LastIndex, LeaderCommit; }; + static constexpr size_t MaxPendingAppendRpcs = 256; + // Distinct snapshot images retained by active peer transfers. New + // compaction is deferred when this many generations are already pinned. + static constexpr size_t MaxPinnedSnapshotGenerations = 2; uint64_t CommitIndex = 0; - uint64_t CommitSeqno = 0; + // Successful current-term RPC acknowledgements, separately from log indices. + std::unordered_map ReadAckSeqno; + std::unordered_map LastSentSeqno; + std::unordered_map> PendingAppendRpcs; + struct TSnapshotRpc { + // Pinned for this transfer even if the local store compacts again. + std::shared_ptr Snapshot; + std::shared_ptr Bytes; + // Seqno identifies the outstanding chunk across retries; zero means + // its reply was consumed and the next send needs a fresh identity. + uint64_t Offset = 0, Seqno = 0, SentEnd = 0; + }; + std::unordered_map SnapshotRpcs; uint32_t LeaderId = 0; std::unordered_map NextIndex; std::unordered_map MatchIndex; + // Highest leader commit boundary confirmed by a correlated replication ACK. + std::unordered_map CommitAckIndex; std::unordered_set Votes; std::unordered_map HeartbeatDue; std::unordered_map RpcDue; @@ -55,8 +89,11 @@ struct TVolatileState { std::vector Indices; TVolatileState& Vote(uint32_t id); - TVolatileState& CommitAdvance(int nservers, const IState& state, uint64_t seqno = 0); - TVolatileState& SetCommitIndex(int index); + // Compatibility helper for all-voter fixtures; mixed roles require IDs below. + TVolatileState& CommitAdvance(int nservers, const IState& state); + TVolatileState& CommitAdvance(const std::vector& voters, uint32_t self, const IState& state); + TVolatileState& CommitAdvance(const TQuorum& quorum, uint32_t self, const IState& state); + TVolatileState& SetCommitIndex(uint64_t index); TVolatileState& SetElectionDue(ITimeSource::Time); TVolatileState& SetNextIndex(uint32_t id, uint64_t nextIndex); TVolatileState& SetMatchIndex(uint32_t id, uint64_t matchIndex); @@ -82,15 +119,47 @@ enum class EState: int { class TRaft { public: - TRaft(std::shared_ptr state, int node, const TNodeDict& nodes); + // Nodes contains every configured replication peer; learners may include + // the local ID. Retained validated configuration history determines active + // voter roles and joint quorum arithmetic. + TRaft(std::shared_ptr state, int node, const TNodeDict& nodes, std::vector learners = {}); void Process(ITimeSource::Time now, TMessageHolder message, const std::shared_ptr& replyTo = {}); void ProcessTimeout(ITimeSource::Time now); + // Local proposals require leader state. Role rejection leaves storage and + // the supplied entry unchanged; followers receive entries through Process. uint64_t Append(TMessageHolder entry); + // Start a joint-consensus transition. Every newly added voter must already + // be staged as a learner and have acknowledged this leader's committed + // boundary. This first runtime path requires the current leader to remain + // in the new voter set; leadership transfer must precede leader removal. + uint64_t ProposeJointConfiguration(std::vector newVoters); + // Append the stable successor after the joint entry has committed. + uint64_t FinalizeConfiguration(); uint32_t GetLeaderId() const; uint64_t GetLastIndex() const; + std::vector GetVoterIds() const; + std::vector GetLearnerIds() const { return Learners; } + bool IsVoter(uint32_t id) const; + // Leader-only, point-in-time log catch-up; not permission to promote. + bool IsLearnerCaughtUp(uint32_t id) const; + // True when a correlated ACK confirms the learner's Raft commit boundary + // through this leader's current commit index, and its log matches the leader. + // This does not attest to application progress or authorize promotion. + bool IsLearnerCommitAcknowledged(uint32_t id) const; + // Leader-local replication staging; not a membership change. Caller owns + // endpoint lifecycle, reconciles targets after failover/restart, and must + // serialize these calls with all Raft processing. + void AddLearnerTarget(uint32_t id, std::shared_ptr node); + std::shared_ptr RemoveLearnerTarget(uint32_t id); + // Capture applied state and atomically replace its prefix with a snapshot. + // Throws without mutation while two distinct generations remain pinned by + // outgoing peer transfers; callers may retry after a transfer completes. + TSnapshot Compact(); uint64_t ApproveRead(); + // Highest barrier confirmed by a fresh majority in this leadership term. + // Zero means no confirmation (including before a current-term commit). uint64_t CommitSeqno() const; // ut @@ -113,6 +182,9 @@ class TRaft { } int GetMinVotes() const { + // Stable configurations return their strict-majority threshold. Joint + // configurations return the larger of the two thresholds; callers + // must use protocol outcomes, not this scalar, to decide a quorum. return MinVotes; } @@ -121,10 +193,18 @@ class TRaft { } int GetNservers() const { + // Voting members only; GetNpeers counts all remote replication targets. return Nservers; } private: + friend class TRequestProcessor; + void RestoreAppliedIndex(uint64_t index); + void AdvanceCommitIndex(uint64_t index); + void AdvanceLeaderCommit(); + void OnInstallSnapshot(ITimeSource::Time now, TMessageHolder message); + void OnInstallSnapshot(TMessageHolder message); + TMessageHolder CreateInstallSnapshot(uint32_t nodeId); void Candidate(ITimeSource::Time now, TMessageHolder message); void Follower(ITimeSource::Time now, TMessageHolder message); void Leader(ITimeSource::Time now, TMessageHolder message); @@ -141,15 +221,39 @@ class TRaft { TMessageHolder CreateVote(uint32_t nodeId); TMessageHolder CreateAppendEntries(uint32_t nodeId); ITimeSource::Time MakeElection(ITimeSource::Time now); + uint64_t NextSeqno(); + bool ValidateRpc(TMessageHolder message) const; + bool ValidateAppendConfiguration(TMessageHolder message) const; + void RequireLiveConfigurationStorage() const; + void RefreshConfiguration(uint64_t commitIndex); + bool HasMembershipHistory() const; uint32_t Id; TNodeDict Nodes; + // InitialVoters is the immutable storage binding. Voters is the active + // voting union (the union of both sets while a joint configuration is in + // force); Learners is derived from that active membership. + std::vector InitialVoters, Voters, Learners; + std::optional ConfigurationBaseline; + std::optional Configuration; + bool ConfigurationScanRequired = false; + std::optional VotingQuorum; int MinVotes; int Npeers; int Nservers; std::shared_ptr State; std::unique_ptr VolatileState; - uint64_t Seqno = 0; // for matching responses + uint64_t Seqno = 1; // zero is reserved for an unconfirmed read + std::weak_ptr SnapshotApplication; + struct TIncomingSnapshot { + uint64_t Term, Index, BoundaryTerm, Total; + uint32_t Leader; + ITimeSource::Time LastProgress; + std::string Bytes; + }; + std::optional IncomingSnapshot; + std::shared_ptr EncodedSnapshot; + uint64_t EncodedSnapshotIndex = 0; EState StateName; uint32_t Seed = 31337; @@ -157,15 +261,14 @@ class TRaft { class TRequestProcessor { public: - TRequestProcessor(std::shared_ptr raft, std::shared_ptr rsm, const TNodeDict& nodes) - : Raft(raft) - , Rsm(rsm) - , Nodes(nodes) - { } + // Construct after restoring the RSM and before accepting network traffic. + TRequestProcessor(std::shared_ptr raft, std::shared_ptr rsm, const TNodeDict& nodes); void CheckStateChange(); void OnCommandRequest(TMessageHolder message, const std::shared_ptr& replyTo); - void OnCommandResponse(TMessageHolder message); + // Unattributed responses are ignored; pass the configured outbound peer. + void OnCommandResponse(TMessageHolder message, const std::shared_ptr& peer = {}); + void OnPeerDisconnected(const std::shared_ptr& peer); void ProcessCommitted(); void ProcessWaiting(); void CleanUp(const std::shared_ptr& replyTo); @@ -174,6 +277,8 @@ class TRequestProcessor { void Forward(TMessageHolder message, const std::shared_ptr& replyTo); void OnReadRequest(TMessageHolder message, const std::shared_ptr& replyTo); void OnWriteRequest(TMessageHolder message, const std::shared_ptr& replyTo); + void PruneRequests(const std::shared_ptr& disconnected = {}); + void CancelForwarded(const std::shared_ptr& disconnected = {}, bool peerFailure = false); std::shared_ptr Raft; std::shared_ptr Rsm; @@ -184,18 +289,31 @@ class TRequestProcessor { uint64_t Seqno = 0; TMessageHolder Command; std::shared_ptr ReplyTo; + // Zero is reserved for requests not owned by a leadership term. + uint64_t Term = 0; }; std::queue Waiting; std::queue StrongWaiting; std::queue WaitingStateChange; struct TAnswer { - uint64_t Index; + uint64_t Term; TMessageHolder Reply; }; - std::queue WriteAnswers; - uint32_t ForwardCookie = 1; - std::unordered_map> Cookie2Client; - std::unordered_map, std::unordered_set> Client2Cookie; + // Retain application replies only for live local writes, matched by index + // and term. An overwritten index must never redeem an older request. + std::unordered_map PendingWriteTerms; + std::unordered_map WriteAnswers; + uint64_t RequestsTerm = 0; + EState RequestsState = EState::NONE; + uint32_t RequestsLeaderId = 0; + struct TForwarded { + std::shared_ptr Client, Peer; + uint32_t OriginalCookie, LeaderId; + uint64_t Term; + }; + // Never reuse a wire cookie within this processor, including after failure. + uint64_t NextForwardCookie = 1; + std::unordered_map Forwarded; }; diff --git a/miniraft/snapshot.cpp b/miniraft/snapshot.cpp new file mode 100644 index 0000000..d34c409 --- /dev/null +++ b/miniraft/snapshot.cpp @@ -0,0 +1,296 @@ +#include "snapshot.h" +#include "snapshot_codec.h" +#include "raft.h" +#include +#include +#include +#include +#ifdef __linux__ +#include +#include +#include +#include +#endif + +namespace { +using namespace NSnapshotCodec; +constexpr std::string_view Magic = "MRFTS001"; +constexpr std::string_view JointMagic = "MRFTS002"; +constexpr std::string_view StableMagic = "MRFTS003"; +void Validate(const TSnapshot& snapshot) { + if (!snapshot.Index || snapshot.Index > uint64_t(INT64_MAX) || !snapshot.Term || + snapshot.Voters.empty() || snapshot.Voters.size() > TSnapshot::MaxVoters || + snapshot.JointVoters.size() > TSnapshot::MaxVoters || + snapshot.Data.size() > TSnapshot::MaxBytes - (snapshot.StableConfiguration ? 56 : snapshot.IsJoint() ? 60 : 40) - + 4 * (snapshot.Voters.size() + snapshot.JointVoters.size())) + throw std::runtime_error("Invalid snapshot metadata/size"); + const auto validateVoters = [](const auto& voters) { + uint32_t previous = 0; + for (auto voter : voters) { + if (voter <= previous) throw std::runtime_error("Snapshot voters must be positive, sorted and unique"); + previous = voter; + } + }; + validateVoters(snapshot.Voters); + if (snapshot.HasConfigurationMetadata()) { + if ((snapshot.StableConfiguration ? !snapshot.JointVoters.empty() : snapshot.JointVoters.empty()) || !snapshot.ConfigurationIndex || + snapshot.ConfigurationIndex > snapshot.Index || !snapshot.ConfigurationTerm || + snapshot.ConfigurationTerm > snapshot.Term || + (snapshot.ConfigurationIndex == snapshot.Index && snapshot.ConfigurationTerm != snapshot.Term)) + throw std::runtime_error("Invalid joint snapshot configuration boundary"); + validateVoters(snapshot.JointVoters); + } +} +void CheckSnapshotConfigurationContinuity(const TSnapshot* current, const TSnapshot& incoming) { + if (!current || !current->HasConfigurationMetadata()) return; + if (current->StableConfiguration) { + if (incoming.StableConfiguration) { + if (incoming.ConfigurationIndex != current->ConfigurationIndex || + incoming.ConfigurationTerm != current->ConfigurationTerm) + throw std::invalid_argument("Stable snapshot configuration identity changed"); + if (incoming.Voters != current->Voters) + throw std::invalid_argument("Stable snapshot voters changed without a new configuration identity"); + return; + } + if (incoming.IsJoint()) { + if (incoming.Voters != current->Voters || + incoming.ConfigurationIndex <= current->Index || + incoming.ConfigurationTerm < current->Term) + throw std::invalid_argument("Joint snapshot does not extend the stable configuration"); + return; + } + if (incoming.Voters != current->Voters) + throw std::invalid_argument("Legacy snapshot would change the active voter set"); + return; + } + + if (incoming.IsJoint()) { + if (incoming.Voters != current->Voters || incoming.JointVoters != current->JointVoters || + incoming.ConfigurationIndex != current->ConfigurationIndex || + incoming.ConfigurationTerm != current->ConfigurationTerm) + throw std::invalid_argument("Joint snapshot configuration changed without a stable successor"); + return; + } + if (incoming.StableConfiguration) { + if (incoming.Voters != current->JointVoters || + incoming.ConfigurationIndex <= current->Index || + incoming.ConfigurationTerm < current->Term) + throw std::invalid_argument("Stable snapshot does not extend the joint configuration"); + return; + } + throw std::invalid_argument("Legacy snapshot cannot discard joint configuration metadata"); +} +void CheckLocalLog(const TSnapshot& snapshot, const TRaft& raft) { + Validate(snapshot); + if (snapshot.IsJoint() || snapshot.Voters != raft.GetVoterIds() || snapshot.Index > raft.GetLastIndex() || + snapshot.Term > raft.GetState()->CurrentTerm || + raft.GetState()->LogTerm(snapshot.Index) != snapshot.Term) + throw std::runtime_error("Snapshot does not match retained log and active configuration"); +} +#ifdef __linux__ +struct TFd { + int Value = -1; + explicit TFd(int fd = -1) : Value(fd) {} + ~TFd() { if (Value >= 0) close(Value); } + TFd(const TFd&) = delete; + TFd& operator=(const TFd&) = delete; +}; +void Require(bool ok, const char* message) { + if (!ok) throw std::system_error(errno, std::generic_category(), message); +} +void Sync(int fd) { + int rc; + do { rc = fsync(fd); } while (rc < 0 && errno == EINTR); + Require(rc == 0, "sync snapshot"); +} +#endif +} + +void ValidateSnapshot(const TSnapshot& snapshot) { Validate(snapshot); } + +void PreserveStableSnapshotIdentity(const TSnapshot* current, TSnapshot& incoming) { + CheckSnapshotConfigurationContinuity(current, incoming); + if (current && current->StableConfiguration && !incoming.HasConfigurationMetadata()) { + // Legacy images carry no membership claim, so retain the known stable + // identity after confirming their active voters are unchanged. + incoming.StableConfiguration = true; + incoming.ConfigurationIndex = current->ConfigurationIndex; + incoming.ConfigurationTerm = current->ConfigurationTerm; + } +} + +std::string EncodeSnapshot(const TSnapshot& snapshot) { + Validate(snapshot); + std::string out(snapshot.StableConfiguration ? StableMagic : snapshot.IsJoint() ? JointMagic : Magic); + Put(out, snapshot.Index, 8); + Put(out, snapshot.Term, 8); + Put(out, snapshot.Voters.size(), 4); + Put(out, snapshot.Data.size(), 8); + if (snapshot.HasConfigurationMetadata()) { + Put(out, snapshot.ConfigurationIndex, 8); + Put(out, snapshot.ConfigurationTerm, 8); + if (!snapshot.StableConfiguration) Put(out, snapshot.JointVoters.size(), 4); + } + for (auto voter : snapshot.Voters) Put(out, voter, 4); + for (auto voter : snapshot.JointVoters) Put(out, voter, 4); + Append(out, snapshot.Data); + Put(out, Checksum(out), 4); + return out; +} + +TSnapshot DecodeSnapshot(std::string_view bytes) { + if (bytes.size() < 44 || bytes.size() > TSnapshot::MaxBytes) + throw std::runtime_error("Invalid snapshot size"); + TReader trailer{bytes.substr(bytes.size() - 4)}; + if (Checksum(bytes.substr(0, bytes.size() - 4)) != trailer.Take(4)) + throw std::runtime_error("Snapshot checksum mismatch"); + TReader reader{bytes.substr(0, bytes.size() - 4)}; + const auto magic = reader.Read(Magic.size()); + const bool joint = magic == JointMagic; + const bool stable = magic == StableMagic; + if (!joint && !stable && magic != Magic) throw std::runtime_error("Unknown snapshot format"); + TSnapshot snapshot; + snapshot.StableConfiguration = stable; + snapshot.Index = reader.Take(8); + snapshot.Term = reader.Take(8); + const auto count = reader.Take(4); + const auto length = reader.Take(8); + uint64_t newCount = 0; + if (joint || stable) { + snapshot.ConfigurationIndex = reader.Take(8); + snapshot.ConfigurationTerm = reader.Take(8); + if (joint) newCount = reader.Take(4); + } + if (count > TSnapshot::MaxVoters || newCount > TSnapshot::MaxVoters || + count + newCount > reader.Input.size() / 4) + throw std::runtime_error("Invalid snapshot voter count"); + for (uint32_t i = 0; i < count; ++i) snapshot.Voters.push_back(reader.Take(4)); + for (uint32_t i = 0; i < newCount; ++i) snapshot.JointVoters.push_back(reader.Take(4)); + if (joint && !snapshot.IsJoint()) throw std::runtime_error("Empty joint snapshot configuration"); + if (length > reader.Input.size()) throw std::runtime_error("Invalid snapshot data length"); + snapshot.Data = reader.Read(size_t(length)); + if (!reader.Input.empty()) throw std::runtime_error("Trailing snapshot data"); + Validate(snapshot); + return snapshot; +} + +TSnapshot CaptureSnapshot(const TRaft& raft, const IRsm& rsm) { + const auto index = rsm.LastAppliedIndex; + if (!index || index > raft.GetVolatileState()->CommitIndex || index > raft.GetLastIndex()) + throw std::runtime_error("Snapshot requires applied committed progress"); + TSnapshot snapshot{index, raft.GetState()->LogTerm(index), raft.GetVoterIds(), {}}; + if (const auto previous = raft.GetState()->GetSnapshot(); previous && previous->StableConfiguration) { + snapshot.StableConfiguration = true; + snapshot.ConfigurationIndex = previous->ConfigurationIndex; + snapshot.ConfigurationTerm = previous->ConfigurationTerm; + } + CheckLocalLog(snapshot, raft); + snapshot.Data = rsm.SaveSnapshot(); + Validate(snapshot); + return snapshot; +} + +void RestoreSnapshot(const TSnapshot& snapshot, const TRaft& raft, IRsm& rsm) { + const auto knownSnapshot = raft.GetState()->GetSnapshot(); + ValidateSnapshot(snapshot); + CheckSnapshotConfigurationContinuity(knownSnapshot.get(), snapshot); + CheckLocalLog(snapshot, raft); + if (rsm.LastAppliedIndex >= snapshot.Index) + throw std::runtime_error("Application is not behind the snapshot"); + rsm.ValidateSnapshot(snapshot.Data, snapshot.Index); + rsm.LoadSnapshot(snapshot.Data, snapshot.Index); + if (rsm.LastAppliedIndex != snapshot.Index) throw std::runtime_error("Application failed to restore snapshot progress"); +} + +struct TSnapshotStore::TImpl { + THook Hook; + std::string File, Temp; +#ifdef __linux__ + TFd Directory, Lock; +#endif + void Stage(EStage stage) { if (Hook) Hook(stage); } +}; + +TSnapshotStore::TSnapshotStore(const std::string& name, uint32_t id, THook hook) + : Impl(std::make_unique()) +{ +#ifndef __linux__ + throw std::runtime_error("Snapshot store currently requires Linux"); +#else + if (!id) throw std::invalid_argument("Snapshot store requires a positive node ID"); + const auto base = std::filesystem::path(name); + auto directory = base.parent_path(); + if (directory.empty()) directory = "."; + Impl->File = base.filename().string() + ".snapshot." + std::to_string(id); + Impl->Temp = Impl->File + ".tmp"; + Impl->Hook = std::move(hook); + Impl->Directory.Value = open(directory.c_str(), O_RDONLY | O_DIRECTORY | O_CLOEXEC); + Require(Impl->Directory.Value >= 0, "open snapshot directory"); + const auto lock = Impl->File + ".lock"; + Impl->Lock.Value = openat(Impl->Directory.Value, lock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC | O_NOFOLLOW | O_NONBLOCK, 0600); + Require(Impl->Lock.Value >= 0, "open snapshot lock"); + struct stat lockInfo; + Require(fstat(Impl->Lock.Value, &lockInfo) == 0, "stat snapshot lock"); + if (!S_ISREG(lockInfo.st_mode)) throw std::runtime_error("Invalid snapshot lock file type"); + Require(flock(Impl->Lock.Value, LOCK_EX | LOCK_NB) == 0, "lock snapshot store"); +#endif +} + +TSnapshotStore::~TSnapshotStore() = default; + +std::optional TSnapshotStore::Load() { +#ifndef __linux__ + throw std::runtime_error("Snapshot store currently requires Linux"); +#else + TFd file(openat(Impl->Directory.Value, Impl->File.c_str(), O_RDONLY | O_CLOEXEC | O_NOFOLLOW | O_NONBLOCK)); + if (file.Value < 0 && errno == ENOENT) return {}; + Require(file.Value >= 0, "open snapshot"); + struct stat info; + Require(fstat(file.Value, &info) == 0, "stat snapshot"); + if (!S_ISREG(info.st_mode) || info.st_size < 44 || uint64_t(info.st_size) > TSnapshot::MaxBytes) + throw std::runtime_error("Invalid snapshot file type/size"); + std::string bytes(size_t(info.st_size), '\0'); + size_t done = 0; + while (done < bytes.size()) { + const auto n = read(file.Value, bytes.data() + done, bytes.size() - done); + if (n < 0 && errno == EINTR) continue; + if (!n) throw std::runtime_error("Truncated snapshot file"); + Require(n > 0, "read snapshot"); + done += size_t(n); + } + auto snapshot = DecodeSnapshot(bytes); + Sync(file.Value); + Sync(Impl->Directory.Value); + return snapshot; +#endif +} + +void TSnapshotStore::Save(const TSnapshot& snapshot) { + const auto bytes = EncodeSnapshot(snapshot); +#ifndef __linux__ + throw std::runtime_error("Snapshot store currently requires Linux"); +#else + TFd file(openat(Impl->Directory.Value, Impl->Temp.c_str(), + O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC | O_NOFOLLOW | O_NONBLOCK, 0600)); + Require(file.Value >= 0, "create snapshot temporary image"); + struct stat info; + Require(fstat(file.Value, &info) == 0, "stat snapshot temporary image"); + if (!S_ISREG(info.st_mode)) throw std::runtime_error("Invalid snapshot temporary image type"); + Impl->Stage(EStage::TempCreated); + size_t done = 0; + while (done < bytes.size()) { + const auto n = write(file.Value, bytes.data() + done, bytes.size() - done); + if (n < 0 && errno == EINTR) continue; + Require(n > 0, "write snapshot"); + done += size_t(n); + } + Impl->Stage(EStage::Written); + Sync(file.Value); + Impl->Stage(EStage::FileSynced); + Require(renameat(Impl->Directory.Value, Impl->Temp.c_str(), Impl->Directory.Value, Impl->File.c_str()) == 0, + "publish snapshot"); + Impl->Stage(EStage::Renamed); + Sync(Impl->Directory.Value); + Impl->Stage(EStage::DirectorySynced); +#endif +} diff --git a/miniraft/snapshot.h b/miniraft/snapshot.h new file mode 100644 index 0000000..ece53c2 --- /dev/null +++ b/miniraft/snapshot.h @@ -0,0 +1,62 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include + +class TRaft; +struct IRsm; + +// Versioned application snapshot, shared by local compaction and state transfer. +struct TSnapshot { + static constexpr size_t MaxBytes = 64 * 1024 * 1024; + static constexpr uint32_t MaxVoters = 4096; + uint64_t Index = 0, Term = 0; + std::vector Voters; + std::string Data; + // Optional joint configuration effective at this snapshot boundary. + // Runtime installation is disabled until membership recovery is implemented. + std::vector JointVoters; + uint64_t ConfigurationIndex = 0, ConfigurationTerm = 0; + // Experimental stable-configuration identity. Offline storage may retain + // an active voter set that differs from its original persisted binding; + // live Raft still rejects that configuration until transitions are enabled. + bool StableConfiguration = false; + bool IsJoint() const { return !JointVoters.empty() || (!StableConfiguration && (ConfigurationIndex || ConfigurationTerm)); } + bool HasConfigurationMetadata() const { return StableConfiguration || IsJoint(); } + bool operator==(const TSnapshot&) const = default; +}; +std::string EncodeSnapshot(const TSnapshot& snapshot); +TSnapshot DecodeSnapshot(std::string_view bytes); +// Validate metadata and size without copying the application image. +void ValidateSnapshot(const TSnapshot& snapshot); +// Preserve a trusted stable identity on legacy refresh and validate snapshot +// configuration continuity: stable -> joint -> stable, with voter-set checks. +void PreserveStableSnapshotIdentity(const TSnapshot* current, TSnapshot& incoming); +TSnapshot CaptureSnapshot(const TRaft& raft, const IRsm& rsm); +// Startup only, with an application behind the snapshot and +// a matching local boundary/log. This can recover an app left behind by a crash +// after durable Raft snapshot publication. +void RestoreSnapshot(const TSnapshot& snapshot, const TRaft& raft, IRsm& rsm); + +class TSnapshotStore { +public: + enum class EStage { TempCreated, Written, FileSynced, Renamed, DirectorySynced }; + using THook = std::function; + // Linux atomic publication, exclusive writer lock, bounded reads and CRC32. + TSnapshotStore(const std::string& name, uint32_t id, THook hook = {}); + ~TSnapshotStore(); + TSnapshotStore(const TSnapshotStore&) = delete; + TSnapshotStore& operator=(const TSnapshotStore&) = delete; + std::optional Load(); + // Standalone checkpoint export, not the compaction commit point. Failure is + // catchable when the authoritative snapshot/log remains in the Raft state. + void Save(const TSnapshot& snapshot); +private: + struct TImpl; + std::unique_ptr Impl; +}; diff --git a/miniraft/snapshot_codec.h b/miniraft/snapshot_codec.h new file mode 100644 index 0000000..b819609 --- /dev/null +++ b/miniraft/snapshot_codec.h @@ -0,0 +1,40 @@ +#pragma once +#include "snapshot.h" +#include + +namespace NSnapshotCodec { +inline void Append(std::string& out, std::string_view part) { + if (out.size() > TSnapshot::MaxBytes || part.size() > TSnapshot::MaxBytes - out.size()) + throw std::runtime_error("Snapshot exceeds size limit"); + out.append(part); +} +inline void Put(std::string& out, uint64_t value, unsigned width) { + for (unsigned i = 0; i < width; ++i) { + const char byte = char(value >> (8 * i)); + Append(out, std::string_view(&byte, 1)); + } +} +struct TReader { + std::string_view Input; + std::string_view Read(size_t length) { + if (length > Input.size()) throw std::runtime_error("Truncated snapshot"); + const auto result = Input.substr(0, length); + Input.remove_prefix(length); + return result; + } + uint64_t Take(unsigned width) { + const auto bytes = Read(width); + uint64_t value = 0; + for (unsigned i = 0; i < width; ++i) value |= uint64_t(uint8_t(bytes[i])) << (8 * i); + return value; + } +}; +inline uint32_t Checksum(std::string_view bytes) { + uint32_t crc = UINT32_MAX; + for (uint8_t byte : bytes) { + crc ^= byte; + for (int i = 0; i < 8; ++i) crc = (crc >> 1) ^ (0xedb88320u & (0u - (crc & 1u))); + } + return ~crc; +} +} diff --git a/miniraft/state.h b/miniraft/state.h index cb6c488..10cdd31 100644 --- a/miniraft/state.h +++ b/miniraft/state.h @@ -1,12 +1,19 @@ #pragma once #include "messages.h" +#include "snapshot.h" +#include struct TStateFields { uint64_t CurrentTerm = 1; uint32_t VotedFor = 0; uint64_t LastLogIndex = 0; uint64_t LastLogTerm = 0; + uint64_t SnapshotIndex = 0; + uint64_t SnapshotTerm = 0; + // Persisted commit boundary where the storage backend supports it. Older + // formats conservatively recover this as SnapshotIndex. + uint64_t CommitIndex = 0; bool operator==(const TStateFields& other) const = default; }; @@ -14,30 +21,104 @@ struct TStateFields { struct IState: TStateFields { virtual void RemoveLast() = 0; virtual void Append(TMessageHolder) = 0; - // index starts from 1 + // Absolute indices; only the suffix after SnapshotIndex has log entries. + // The boundary term remains available through LogTerm(SnapshotIndex). virtual TMessageHolder Get(int64_t index) const = 0; + // Backends may read a retained entry's term without copying its payload. + // The default keeps existing external backends source compatible. + virtual uint64_t RetainedLogTerm(int64_t index) const { + auto entry = Get(index); + return entry ? entry->Term : 0; + } + // Used during startup to reject unsupported retained entry kinds. + virtual uint64_t RetainedLogFlags(int64_t index) const { + auto entry = Get(index); + if (!entry) throw std::runtime_error("Missing retained log entry"); + return entry->Flags; + } virtual void Commit() = 0; + // True only when Commit() durably records CommitIndex and recovery restores + // it. Membership protocols can require this before trusting recovered history. + virtual bool SupportsCommitIndexPersistence() const { return false; } + virtual bool SupportsSnapshots() const { return false; } + virtual std::shared_ptr GetSnapshot() const { return {}; } + virtual void InstallSnapshot(const TSnapshot&) { throw std::logic_error("Storage snapshots are unsupported"); } + virtual bool SupportsVoterBinding() const { return false; } + // True when this backend recovered a persistent image without a stored + // voter identity; Raft must obtain proof before serving this state. + virtual bool HasUnboundRecoveredState() const { return false; } + // Original trusted voter set persisted with this store. It may differ from + // the active set in a configuration-bearing offline snapshot. + virtual std::vector GetBoundVoters() const { return {}; } + virtual void BindVoters(const std::vector&) { throw std::logic_error("Storage voter binding is unsupported"); } + // Storage backends can close startup-only migration paths once a Raft + // instance has finished validating and attached to this state. + virtual void MarkRaftStarted() {} virtual ~IState() = default; uint64_t LogTerm(int64_t index = -1) const { if (index < 0) { index = LastLogIndex; } - if (index < 1 || index > LastLogIndex) { + if (index > 0 && uint64_t(index) == SnapshotIndex) return SnapshotTerm; + if (index < 1 || uint64_t(index) < SnapshotIndex || index > LastLogIndex) { return 0; } else { - return Get(index)->Term; + return RetainedLogTerm(index); } } }; +inline void ValidateFixedVoters(const std::vector& voters) { + if (voters.empty() || voters.size() > TSnapshot::MaxVoters) throw std::invalid_argument("Invalid voter count"); + uint32_t previous = 0; + for (auto id : voters) { + if (id <= previous) throw std::invalid_argument("Voters must be positive, sorted and unique"); + previous = id; + } +} +inline void ValidateVoterBinding(const IState& state, const std::vector& old, const std::vector& voters) { + ValidateFixedVoters(voters); + if (state.VotedFor && !std::binary_search(voters.begin(), voters.end(), state.VotedFor)) + throw std::invalid_argument("Stored vote does not name a configured voter"); + if (!old.empty()) { + if (old != voters) throw std::invalid_argument("Persisted voter configuration cannot be changed by restart"); + } else if (auto snapshot = state.GetSnapshot()) { + if (snapshot->HasConfigurationMetadata()) + throw std::invalid_argument("Configuration snapshot recovery requires an existing voter binding"); + if (snapshot->Voters != voters) throw std::invalid_argument("Snapshot voter configuration mismatch"); + } else if (state.CurrentTerm != 1 || state.VotedFor || state.LastLogIndex) { + throw std::invalid_argument("Unbound nonfresh store needs a verified snapshot before enabling learners"); + } +} + +// A persisted voter binding identifies the original, trusted membership. For +// offline history storage, explicit configuration metadata allows the snapshot +// Voters field to describe a later active set while that original binding stays +// put. Metadata still requires an existing binding; it is not proof by itself +// and does not authorize runtime membership changes. +inline bool SnapshotCompatibleWithVoterBinding(const TSnapshot& snapshot, + const std::vector& voters) { + if (snapshot.HasConfigurationMetadata()) return !voters.empty(); + return voters.empty() || snapshot.Voters == voters; +} + struct TState: IState { + static TMessageHolder CloneEntry(const TMessageHolder& entry) { + if (!entry.HasValidExtent() || entry->Type != uint32_t(EMessageType::LOG_ENTRY) || entry->Len < sizeof(TLogEntry)) + throw std::invalid_argument("Invalid in-memory log entry"); + auto copy = NewHoldedMessage(entry->Len); + memcpy(copy.Mes, entry.Mes, entry->Len); + return copy; + } std::vector> Log; + std::shared_ptr Snapshot; + std::vector BoundVoters; TState() = default; TState(uint64_t currentTerm, uint32_t votedFor, const std::vector>& log) - : Log(log) { + for (const auto& entry : log) Log.push_back(CloneEntry(entry)); CurrentTerm = currentTerm; VotedFor = votedFor; if (!log.empty()) { @@ -45,21 +126,79 @@ struct TState: IState { LastLogTerm = log.back()->Term; } } + TState(const TState& other) : IState(other), Snapshot(other.Snapshot), BoundVoters(other.BoundVoters) { + for (const auto& entry : other.Log) Log.push_back(CloneEntry(entry)); + } + TState& operator=(const TState& other) { + if (this != &other) { + TState copy(other); + static_cast(*this) = copy; + Log.swap(copy.Log); Snapshot.swap(copy.Snapshot); BoundVoters.swap(copy.BoundVoters); + } + return *this; + } void RemoveLast() override { + if (Log.empty()) return; + if (LastLogIndex <= CommitIndex) + throw std::logic_error("Cannot remove a committed log entry"); Log.pop_back(); - LastLogIndex = Log.size(); - LastLogTerm = Log.empty() ? 0 : Log.back()->Term; + LastLogIndex = SnapshotIndex + Log.size(); + LastLogTerm = Log.empty() ? SnapshotTerm : Log.back()->Term; } void Append(TMessageHolder entry) override { - Log.emplace_back(std::move(entry)); - LastLogIndex = Log.size(); + if (LastLogIndex >= uint64_t(INT64_MAX)) throw std::overflow_error("Raft log index exhausted"); + Log.emplace_back(CloneEntry(entry)); + LastLogIndex = SnapshotIndex + Log.size(); LastLogTerm = Log.back()->Term; } TMessageHolder Get(int64_t index) const override { - return Log[index-1]; + if (index < 1 || uint64_t(index) <= SnapshotIndex || uint64_t(index) > LastLogIndex) return {}; + return CloneEntry(Log[index - SnapshotIndex - 1]); + } + uint64_t RetainedLogTerm(int64_t index) const override { + if (index < 1 || uint64_t(index) <= SnapshotIndex || uint64_t(index) > LastLogIndex) return 0; + return Log[index - SnapshotIndex - 1]->Term; + } + uint64_t RetainedLogFlags(int64_t index) const override { + if (index < 1 || uint64_t(index) <= SnapshotIndex || uint64_t(index) > LastLogIndex) + throw std::out_of_range("Missing retained log entry"); + return Log[index - SnapshotIndex - 1]->Flags; + } + + bool SupportsSnapshots() const override { return true; } + bool SupportsVoterBinding() const override { return true; } + std::vector GetBoundVoters() const override { return BoundVoters; } + void BindVoters(const std::vector& voters) override { + ValidateVoterBinding(*this, BoundVoters, voters); + BoundVoters = voters; + } + std::shared_ptr GetSnapshot() const override { return Snapshot; } + void InstallSnapshot(const TSnapshot& snapshot) override { + ValidateSnapshot(snapshot); + TSnapshot image = snapshot; + PreserveStableSnapshotIdentity(Snapshot.get(), image); + ValidateSnapshot(image); // Validate before changing any state. + if (!SnapshotCompatibleWithVoterBinding(image, BoundVoters)) + throw std::invalid_argument("Snapshot is incompatible with the persisted voter binding"); + if (image.Index <= SnapshotIndex || image.Term < SnapshotTerm || image.Term > CurrentTerm) + throw std::invalid_argument("Obsolete or future snapshot"); + if (image.Index <= CommitIndex && + (image.Index > LastLogIndex || LogTerm(image.Index) != image.Term)) + throw std::invalid_argument("Snapshot would discard committed log entries"); + auto saved = std::make_shared(std::move(image)); + std::vector> suffix; + if (saved->Index <= LastLogIndex && LogTerm(saved->Index) == saved->Term) + suffix.assign(Log.begin() + (saved->Index - SnapshotIndex), Log.end()); + Log.swap(suffix); + Snapshot = std::move(saved); + SnapshotIndex = Snapshot->Index; + SnapshotTerm = Snapshot->Term; + LastLogIndex = SnapshotIndex + Log.size(); + LastLogTerm = Log.empty() ? SnapshotTerm : Log.back()->Term; + CommitIndex = std::max(CommitIndex, SnapshotIndex); } void Commit() override { } diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index b681e0c..9dfb524 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -13,5 +13,47 @@ endmacro() if (CMOCKA_FOUND) ut(test_raft test_raft.cpp miniraft) +ut(test_quorum test_quorum.cpp miniraft) +ut(test_configuration test_configuration.cpp miniraft) +ut(test_correctness test_correctness.cpp miniraft) +ut(test_forwarding test_forwarding.cpp miniraft) +ut(test_message_safety test_message_safety.cpp miniraft) +ut(test_rpc_validation test_rpc_validation.cpp miniraft) +ut(test_app_payloads "test_app_payloads.cpp;../examples/kv_rsm.cpp" miniraft) +ut(test_snapshots "test_snapshots.cpp;../examples/kv_rsm.cpp" miniraft) +ut(test_compaction "test_compaction.cpp;../examples/kv_rsm.cpp" miniraft) +ut(test_learners "test_learners.cpp;../examples/kv_rsm.cpp" miniraft) +find_package(SQLite3 QUIET) +if(SQLite3_FOUND) +ut(test_sql_payloads "test_sql_payloads.cpp;../examples/sql_rsm.cpp" miniraft) +target_link_libraries(test_sql_payloads PRIVATE SQLite::SQLite3) +endif() +ut(test_persist_recovery test_persist_recovery.cpp miniraft) +if(CMAKE_SYSTEM_NAME STREQUAL "Linux") +ut(test_durable test_durable.cpp miniraft) +endif() +if(MINIRAFT_BUILD_NETWORK) ut(test_read_write test_read_write.cpp miniraft.net) +ut(test_network_frames test_network_frames.cpp miniraft.net) +ut(test_node_lifecycle test_node_lifecycle.cpp miniraft.net) +if(CMAKE_SYSTEM_NAME STREQUAL "Linux") +find_package(Python3 COMPONENTS Interpreter QUIET) +if(Python3_Interpreter_FOUND) +add_test(NAME test_kv_checkpoints_cli COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/smoke_kv_checkpoints.py $) +set_tests_properties(test_kv_checkpoints_cli PROPERTIES TIMEOUT 35) +add_test(NAME test_install_snapshot_cli COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/smoke_install_snapshot.py $) +set_tests_properties(test_install_snapshot_cli PROPERTIES TIMEOUT 120) +add_test(NAME test_learner_cli COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/smoke_install_snapshot.py $ learner) +set_tests_properties(test_learner_cli PROPERTIES TIMEOUT 120) +if(SQLite3_FOUND) +add_test(NAME test_sql_snapshots_cli COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/smoke_sql_snapshots.py $) +set_tests_properties(test_sql_snapshots_cli PROPERTIES TIMEOUT 60) +endif() +endif() +endif() +endif() endif () diff --git a/test/smoke_install_snapshot.py b/test/smoke_install_snapshot.py new file mode 100644 index 0000000..4a8ca7d --- /dev/null +++ b/test/smoke_install_snapshot.py @@ -0,0 +1,130 @@ +"""Real three-process durable catch-up smoke, separate from deterministic schedules.""" +import pathlib +import socket +import struct +import subprocess +import sys +import tempfile +import time +sys.dont_write_bytecode = True +from smoke_kv_checkpoints import no_core, port, request, snapshot_boundary + + +def main(): + binary = str(pathlib.Path(sys.argv[1]).resolve()) + learner_mode = len(sys.argv) > 2 and sys.argv[2] == "learner" + with tempfile.TemporaryDirectory(prefix="miniraft-install-snapshot-") as temporary: + root = pathlib.Path(temporary) + ports = [] + while len(ports) < 3: + candidate = port() + if candidate not in ports: + ports.append(candidate) + logs = [] + directories = [] + for index in range(3): + directory = root / str(index + 1) + directory.mkdir() + directories.append(directory) + logs.append((directory / "output.log").open("w+")) + processes = [None] * 3 + + def start(index, use_role=True): + args = [binary, "--server", "--id", str(index + 1), "--durable"] + for node, endpoint in enumerate(ports): + args += ["--node", f"127.0.0.1:{endpoint}:{node + 1}"] + if learner_mode and use_role: + args += ["--learner", "3"] + if index < 2: + args += ["--snapshots", "1"] + processes[index] = subprocess.Popen(args, cwd=directories[index], stdout=logs[index], stderr=logs[index], preexec_fn=no_core) + + def call(index, key, value=None): + child = processes[index] + assert child is not None and child.poll() is None, f"node {index + 1} exited" + with socket.create_connection(("127.0.0.1", ports[index]), timeout=1) as sock: + sock.settimeout(2) + return request(sock, key, value, flags=2) # local stale read, never forwarded + + def wait_for(action, timeout): + end = time.monotonic() + timeout + while time.monotonic() < end: + try: + if action(): + return + except (OSError, AssertionError): + pass + time.sleep(0.05) + raise AssertionError("timed out waiting for cluster progress") + + try: + start(0) + start(1) # two voters form a majority; node 3 starts with no log later + values = {b"a": b"a" * 60000, b"b": b"b" * 60000, b"c": b"c" * 60000} + last = [0] + for key, value in values.items(): + def write(): + last[0], _ = call(0, key, value) + return last[0] > 0 + wait_for(write, 30) + + def compacted(): + for index in (0, 1): + image = directories[index] / f"state.snapshot.{index + 1}" + durable = directories[index] / f"state.durable.{index + 1}" + if not image.exists() or struct.unpack(" 0 and all(call(2, key)[1] == value for key, value in values.items()) + wait_for(caught_up, 25) + # Node 3 has no periodic snapshotting: its compacted image necessarily + # came from the InstallSnapshot RPC, not a local checkpoint timer. + assert not (directories[2] / "state.snapshot.3").exists() + wait_for(lambda: call(0, b"tail", b"after install")[0] > last[0], 15) + wait_for(lambda: call(2, b"tail")[1] == b"after install", 15) + processes[2].terminate() + processes[2].wait(timeout=5) + if learner_mode: + durable = directories[2] / "state.durable.3" + before = durable.read_bytes() + assert struct.unpack("= index: + break + time.sleep(0.02) + else: + raise AssertionError("checkpoint was not published") + assert snapshot_boundary(root / "state.durable.1") >= index, "compacted durable image was not published" + stop(process) + process, endpoint = launch(periodic=False) + with connect(process, endpoint) as sock: + _, value = request(sock, b"a") + assert value == b"persisted", value + stop(process) + process, _ = launch(["--node", f"127.0.0.1:{port()}:2"]) + assert process.wait(timeout=5) != 0, "changed membership must reject checkpoint" + before = (root / "state.durable.1").read_bytes() + damaged = bytearray(image.read_bytes()) + damaged[-1] ^= 1 + image.write_bytes(damaged) + process, _ = launch() + assert process.wait(timeout=5) != 0, "corrupt checkpoint must fail startup" + assert (root / "state.durable.1").read_bytes() == before + log.flush() + log.seek(0) + output = log.read() + assert "ERROR: AddressSanitizer" not in output and "runtime error:" not in output, output + print("KV checkpoint CLI: save, restart/read, configuration and corruption rejection passed") + except Exception: + log.flush() + log.seek(0) + print(log.read(), file=sys.stderr) + raise + finally: + if process is not None and process.poll() is None: + process.kill() + process.wait(timeout=5) + + +if __name__ == "__main__": + main() diff --git a/test/smoke_sql_snapshots.py b/test/smoke_sql_snapshots.py new file mode 100644 index 0000000..4f2d2b4 --- /dev/null +++ b/test/smoke_sql_snapshots.py @@ -0,0 +1,104 @@ +"""SQL CLI compaction and restart smoke; fault schedules live in C++ tests.""" +import pathlib +import re +import socket +import subprocess +import sys +import tempfile +import time +sys.dont_write_bytecode = True +from smoke_kv_checkpoints import no_core, port, snapshot_boundary + + +def main(): + binary = str(pathlib.Path(sys.argv[1]).resolve()) + with tempfile.TemporaryDirectory(prefix="miniraft-sql-snapshot-cli-") as temporary: + root = pathlib.Path(temporary) + endpoint = port() + node = f"127.0.0.1:{endpoint}:1" + args = [binary, "--server", "--id", "1", "--node", node, "--durable"] + log_path = root / "server.log" + process = None + with log_path.open("w+") as log: + def start(periodic): + extra = ["--snapshots", "1"] if periodic else [] + return subprocess.Popen([*args, *extra], cwd=root, stdout=log, stderr=log, preexec_fn=no_core) + + def stop(child): + child.terminate() + child.wait(timeout=5) + + def wait_ready(child): + deadline = time.monotonic() + 10 + while time.monotonic() < deadline: + assert child.poll() is None, "SQL server exited during startup" + try: + with socket.create_connection(("127.0.0.1", endpoint), timeout=0.1): + return + except OSError: + time.sleep(0.05) + raise AssertionError("SQL server did not listen") + + def run_client(line): + return subprocess.run([binary, "--client", "--node", node], cwd=root, + input=line + "\n", text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + timeout=15, preexec_fn=no_core) + + try: + for options in (["--server", "--snapshots", "1"], + ["--server", "--durable", "--snapshots", "0"], + ["--server", "--durable", "--snapshots"]): + rejected = subprocess.run([binary, *options], cwd=root, stdout=log, + stderr=log, timeout=5, preexec_fn=no_core) + assert rejected.returncode != 0, options + + process = start(periodic=True) + wait_ready(process) + write = "CREATE TABLE IF NOT EXISTS t (value TEXT); DELETE FROM t; INSERT INTO t VALUES ('persisted');" + write_index = None + deadline = time.monotonic() + 35 + while time.monotonic() < deadline: + assert process.poll() is None, "SQL server exited before accepting the write" + result = run_client(write) + match = re.search(r"commitIndex: (\d+)", result.stdout) + if match: + write_index = int(match.group(1)) + break + time.sleep(0.1) + assert write_index is not None, "SQL write did not commit" + + durable = root / "sql_state.durable.1" + deadline = time.monotonic() + 12 + while time.monotonic() < deadline: + if durable.exists() and snapshot_boundary(durable) >= write_index: + break + assert process.poll() is None, "SQL server exited during compaction" + time.sleep(0.05) + else: + raise AssertionError("SQL snapshot did not compact through the committed write") + + stop(process) + process = start(periodic=False) + wait_ready(process) + result = run_client("select value from t") + assert "value\npersisted\n" in result.stdout, result.stdout + stop(process) + + log.flush() + log.seek(0) + output = log.read() + assert "ERROR: AddressSanitizer" not in output and "runtime error:" not in output, output + print("SQL snapshot CLI: option checks, compaction and durable restart/read passed") + except Exception: + log.flush() + log.seek(0) + print(log.read(), file=sys.stderr) + raise + finally: + if process is not None and process.poll() is None: + process.kill() + process.wait(timeout=5) + + +if __name__ == "__main__": + main() diff --git a/test/test_app_payloads.cpp b/test/test_app_payloads.cpp new file mode 100644 index 0000000..7cc372e --- /dev/null +++ b/test/test_app_payloads.cpp @@ -0,0 +1,178 @@ +#include "../examples/kv.h" +#include "../examples/kv_messages.h" +#ifndef _WIN32 +#include +#include +#endif +#include +#include +#include +extern "C" { +#include +} +namespace { +template void Reject(F action) { + bool rejected = false; + try { action(); } catch (const TMessageError&) { rejected = true; } + assert_true(rejected); +} +void test_kv_prepare_rejects_inconsistent_lengths(void**) { + TKv kv; + auto command = NewHoldedMessage(); + command->KeySize = UINT16_MAX; + command->ValSize = UINT16_MAX; + Reject([&] { kv.Prepare(command); }); + assert_int_equal(kv.LastAppliedIndex, 0); +} +void test_kv_read_rejects_short_key(void**) { + TKv kv; + auto command = NewHoldedMessage(); + command->KeySize = UINT16_MAX; + Reject([&] { kv.Read(command, 0); }); +} +void test_kv_apply_rejects_before_state_mutation(void**) { + TKv kv; + auto entry = NewHoldedMessage(); + entry->KeySize = 1; + entry->ValSize = UINT16_MAX; + Reject([&] { kv.Write(entry, 1); }); + assert_int_equal(kv.LastAppliedIndex, 0); + auto list = NewHoldedMessage(); + assert_int_equal(kv.Read(list, 0)->Len, sizeof(TCommandResponse)); +} +void test_valid_kv_payload_roundtrip(void**) { + TKv kv; + auto command = NewHoldedMessage(sizeof(TWriteKv) + 4); + command->KeySize = 1; + command->ValSize = 3; + memcpy(command->TKvEntry::Data, "kabc", 4); + kv.Write(kv.Prepare(command), 1); + auto read = NewHoldedMessage(sizeof(TReadKv) + 1); + read->KeySize = 1; + read->Data[0] = 'k'; + auto reply = kv.Read(read, 1).Cast(); + assert_int_equal(reply->Len, sizeof(TCommandResponse) + 3); + assert_memory_equal(reply->Data, "abc", 3); + command->ValSize = 0; + kv.Write(kv.Prepare(command), 2); + assert_int_equal(kv.Read(read, 2)->Len, sizeof(TCommandResponse)); +} +void test_dummy_read_rejects_missing_index(void**) { + TDummyRsm rsm; + auto command = NewHoldedMessage(); + Reject([&] { rsm.Read(command, 0); }); +} +void test_dummy_read_copies_only_log_data(void**) { + TDummyRsm rsm; + auto entry = NewHoldedMessage(sizeof(TLogEntry) + 3); + memcpy(entry->Data, "abc", 3); + rsm.Write(entry, 1); + auto request = NewHoldedMessage(sizeof(TCommandRequest) + sizeof(int64_t)); + int64_t index = 1; + memcpy(request->Data, &index, sizeof(index)); + auto reply = rsm.Read(request, 1).Cast(); + assert_int_equal(reply->Len, sizeof(TCommandResponse) + 3); + assert_memory_equal(reply->Data, "abc", 3); +} +struct TEndpoint : INode { + void Send(TMessageHolder) override {} + void Drain() override {} +}; +void test_invalid_request_is_rejected_before_queue_or_forward(void**) { + for (bool write : {false, true}) { + auto state = std::make_shared(); + auto raft = std::make_shared(state, 1, TNodeDict{}); + if (!write) raft->Become(EState::LEADER); + auto kv = std::make_shared(); + TRequestProcessor processor(raft, kv, {}); + auto endpoint = std::make_shared(); + std::weak_ptr weak = endpoint; + auto command = NewHoldedMessage(); + command->Flags = write ? TCommandRequest::EWrite : TCommandRequest::EConsistent; + command->KeySize = UINT16_MAX; + Reject([&] { processor.OnCommandRequest(command, endpoint); }); + endpoint.reset(); + assert_true(weak.expired()); + assert_int_equal(state->LastLogIndex, 0); + } +} +void test_oversized_factory_argument_is_not_truncated(void**) { + if (sizeof(size_t) <= sizeof(uint32_t)) return; + bool rejected = false; + try { NewHoldedMessage(size_t(UINT32_MAX) + 1); } + catch (const std::invalid_argument&) { rejected = true; } + assert_true(rejected); +} +void test_read_application_error_does_not_stop_completion(void**) { + struct TFailRead : TKv { + TMessageHolder Read(TMessageHolder, uint64_t) override { + throw TMessageError("read output too large"); + } + }; + struct TSink : INode { + std::vector Cookies; + void Send(TMessageHolder message) override { + auto reply = message.Cast(); + assert_int_equal(reply->ErrorCode, 2); + Cookies.push_back(reply->Cookie); + } + void Drain() override {} + }; + auto state = std::make_shared(); + auto raft = std::make_shared(state, 1, TNodeDict{}); + raft->ProcessTimeout(ITimeSource::Time{}); + TRequestProcessor processor(raft, std::make_shared(), {}); + auto endpoint = std::make_shared(); + for (uint32_t flags : {0u, uint32_t(TCommandRequest::EConsistent)}) { + auto command = NewHoldedMessage(); + command->Flags = flags; + command->Cookie = flags + 7; + processor.OnCommandRequest(command, endpoint); + processor.ProcessCommitted(); + processor.ProcessWaiting(); + } + assert_int_equal(endpoint->Cookies.size(), 2); + assert_int_equal(endpoint->Cookies[0], 7); + assert_int_equal(endpoint->Cookies[1], 11); + assert_int_equal(raft->CurrentStateName(), EState::LEADER); +} +#ifndef _WIN32 +void test_invalid_committed_application_is_fatal(void**) { + const auto child = fork(); + assert_true(child >= 0); + if (child == 0) { + std::set_terminate([] { _exit(73); }); + auto state = std::make_shared(); + auto entry = NewHoldedMessage(); + entry->KeySize = UINT16_MAX; + state->Append(entry); + auto raft = std::make_shared(state, 1, TNodeDict{}); + const_cast(raft->GetVolatileState())->CommitIndex = 1; + TRequestProcessor processor(raft, std::make_shared(), {}); + try { processor.ProcessCommitted(); } catch (...) { _exit(74); } + _exit(75); + } + int status = 0; + assert_int_equal(waitpid(child, &status, 0), child); + assert_true(WIFEXITED(status)); + assert_int_equal(WEXITSTATUS(status), 73); +} +#endif +} +int main() { + const CMUnitTest tests[] = { + cmocka_unit_test(test_kv_prepare_rejects_inconsistent_lengths), + cmocka_unit_test(test_kv_read_rejects_short_key), + cmocka_unit_test(test_kv_apply_rejects_before_state_mutation), + cmocka_unit_test(test_valid_kv_payload_roundtrip), + cmocka_unit_test(test_dummy_read_rejects_missing_index), + cmocka_unit_test(test_dummy_read_copies_only_log_data), + cmocka_unit_test(test_invalid_request_is_rejected_before_queue_or_forward), + cmocka_unit_test(test_oversized_factory_argument_is_not_truncated), + cmocka_unit_test(test_read_application_error_does_not_stop_completion), +#ifndef _WIN32 + cmocka_unit_test(test_invalid_committed_application_is_fatal), +#endif + }; + return cmocka_run_group_tests(tests, nullptr, nullptr); +} diff --git a/test/test_compaction.cpp b/test/test_compaction.cpp new file mode 100644 index 0000000..aabb72c --- /dev/null +++ b/test/test_compaction.cpp @@ -0,0 +1,1079 @@ +#include +#include +#include +#include "../examples/kv.h" +#include "../examples/kv_messages.h" +#include +#include +#include +#include +#include +#include +#ifdef __linux__ +#include +#include +#endif +#include +#include +#include +extern "C" { +#include +} +namespace { +template void Reject(F action) { + bool rejected = false; + try { action(); } catch (const std::exception&) { rejected = true; } + assert_true(rejected); +} +struct TEndpoint : INode { + std::function)> SendTo; + explicit TEndpoint(std::function)> send) : SendTo(std::move(send)) {} + void Send(TMessageHolder message) override { SendTo(std::move(message)); } + void Drain() override {} +}; +TMessageHolder Command(const std::string& key, const std::string& value) { + auto message = NewHoldedMessage(sizeof(TWriteKv) + key.size() + value.size()); + message->Flags = TCommandRequest::EWrite; + message->KeySize = key.size(); message->ValSize = value.size(); + memcpy(message->TKvEntry::Data, key.data(), key.size()); + memcpy(message->TKvEntry::Data + key.size(), value.data(), value.size()); + return message; +} +std::string Read(TKv& kv, const std::string& key) { + auto message = NewHoldedMessage(sizeof(TReadKv) + key.size()); + message->KeySize = key.size(); memcpy(message->Data, key.data(), key.size()); + auto response = kv.Read(message, kv.LastAppliedIndex).Cast(); + return std::string(response->Data, response->Len - sizeof(TCommandResponse)); +} +struct TApplication : TKv { + size_t Loads = 0; + void LoadSnapshot(std::string_view bytes, uint64_t index) override { TKv::LoadSnapshot(bytes, index); ++Loads; } +}; +struct TCluster { + ITimeSource::Time Now{}; + std::vector> Nodes{3}; + std::vector> Apps{3}; + std::vector> Processors{3}; + std::deque> Packets; + std::set Isolated; + std::vector Learners; + explicit TCluster(std::vector learners = {}) : Learners(std::move(learners)) { + for (uint32_t id = 1; id <= 3; ++id) Build(id, std::make_shared()); + } + void Build(uint32_t id, std::shared_ptr state) { + TNodeDict peers; + for (uint32_t dst = 1; dst <= 3; ++dst) if (dst != id) + peers[dst] = std::make_shared([this](auto message) { Packets.push_back(std::move(message)); }); + Processors[id - 1].reset(); + Nodes[id - 1] = std::make_shared(state, id, peers, Learners); + Apps[id - 1] = std::make_shared(); + Processors[id - 1] = std::make_unique(Nodes[id - 1], Apps[id - 1], peers); + } + void Restart(uint32_t id) { auto state = Nodes[id - 1]->GetState(); Build(id, state); } + void Deliver(TMessageHolder message, bool duplicate = false) { + const auto dst = message.Cast()->Dst; + Nodes[dst - 1]->Process(Now, message); Processors[dst - 1]->ProcessCommitted(); + if (duplicate) { Nodes[dst - 1]->Process(Now, message); Processors[dst - 1]->ProcessCommitted(); } + } + void Drain(bool duplicate = false) { + size_t budget = 10000; + while (!Packets.empty()) { + assert_true(budget-- > 0); + auto message = Packets.front(); Packets.pop_front(); + auto header = message.Cast(); + if (!Isolated.contains(header->Src) && !Isolated.contains(header->Dst)) Deliver(message, duplicate); + } + } + void Elect(uint32_t id) { + Now += std::chrono::seconds(10); + Nodes[id - 1]->ProcessTimeout(Now); Drain(); + Nodes[id - 1]->ProcessTimeout(Now); Drain(); + assert_int_equal(Nodes[id - 1]->CurrentStateName(), EState::LEADER); + } + void Pump(uint32_t leader, int rounds = 40, bool duplicate = false) { + for (int i = 0; i < rounds; ++i) { + Now += std::chrono::milliseconds(100); + Nodes[leader - 1]->ProcessTimeout(Now); Drain(duplicate); + for (auto& processor : Processors) { processor->ProcessCommitted(); processor->ProcessWaiting(); } + } + } + void Write(uint32_t leader, const std::string& key, const std::string& value) { + Processors[leader - 1]->OnCommandRequest(Command(key, value), {}); Pump(leader); + } + void Prepare() { + Elect(1); Pump(1); Isolated = {3}; + for (const std::string key : {"a", "b", "c"}) Write(1, key, std::string(60000, key[0])); + Nodes[0]->Compact(); Nodes[1]->Compact(); + assert_true(Nodes[0]->GetState()->SnapshotIndex > 1); + Packets.clear(); Isolated.clear(); + } + TMessageHolder Take(uint32_t src, uint32_t dst, EMessageType type) { + for (auto it = Packets.begin(); it != Packets.end(); ++it) { + auto header = it->Cast(); + if (header->Src == src && header->Dst == dst && header->Type == uint32_t(type)) { + auto message = *it; Packets.erase(it); return message; + } + } + fail_msg("Missing packet %u -> %u type %u", src, dst, uint32_t(type)); return {}; + } + TMessageHolder SendChunk() { + Now += std::chrono::seconds(3); + Nodes[0]->ProcessTimeout(Now); + return Take(1, 3, EMessageType::INSTALL_SNAPSHOT_REQUEST).Cast(); + } + TMessageHolder AckChunk(TMessageHolder request) { + Deliver(request); + auto response = Take(3, 1, EMessageType::INSTALL_SNAPSHOT_RESPONSE).Cast(); + Deliver(response); return response; + } +}; +void test_compaction_lagging_follower_installs_snapshot_then_tail(void**) { + TCluster cluster; cluster.Prepare(); + const auto boundary = cluster.Nodes[0]->GetState()->SnapshotIndex; + cluster.Isolated = {3}; cluster.Write(1, "tail", "after snapshot"); cluster.Isolated.clear(); + cluster.Pump(1, 80, true); + assert_int_equal(cluster.Apps[2]->Loads, 1); + assert_int_equal(cluster.Nodes[2]->GetState()->SnapshotIndex, boundary); + for (auto& app : cluster.Apps) { + assert_true(Read(*app, "a") == std::string(60000, 'a')); + assert_true(Read(*app, "tail") == "after snapshot"); + assert_int_equal(app->LastAppliedIndex, cluster.Nodes[0]->GetLastIndex()); + } + assert_false(cluster.Nodes[0]->GetState()->Get(1)); +} +void test_simultaneous_startup_candidates_use_distinct_election_deadlines(void**) { + TCluster cluster; + for (auto& node : cluster.Nodes) node->ProcessTimeout(cluster.Now); + cluster.Drain(); // all voted for themselves in the first term + std::set deadlines; + uint32_t first = 1; + for (uint32_t id = 1; id <= 3; ++id) { + const auto due = cluster.Nodes[id - 1]->GetVolatileState()->ElectionDue; + deadlines.insert(due); + if (due < cluster.Nodes[first - 1]->GetVolatileState()->ElectionDue) first = id; + } + assert_int_equal(deadlines.size(), 3); + cluster.Now = *deadlines.begin(); + for (auto& node : cluster.Nodes) node->ProcessTimeout(cluster.Now); + cluster.Drain(); + cluster.Nodes[first - 1]->ProcessTimeout(cluster.Now); cluster.Drain(); + assert_int_equal(cluster.Nodes[first - 1]->CurrentStateName(), EState::LEADER); +} +void test_partial_transfer_restart_and_delayed_ack_are_safe(void**) { + TCluster cluster; cluster.Prepare(); + auto first = cluster.SendChunk(); + auto delayed = cluster.AckChunk(first); + assert_false(delayed->Installed); + assert_int_equal(cluster.Apps[2]->Loads, 0); + auto invalid = NewHoldedMessage(TMessageEx{.Src = 3, .Dst = 1, .Term = first->Term, .Seqno = first->Seqno}, + TInstallSnapshotResponse{.SnapshotIndex = first->SnapshotIndex, .NextOffset = 1, .Success = 1, .Installed = 1}); + cluster.Deliver(invalid); + assert_true(cluster.Nodes[0]->GetVolatileState()->MatchIndex.at(3) < first->SnapshotIndex); + cluster.Restart(3); // all partial bytes disappear, stable log/vote remain + auto second = cluster.SendChunk(); assert_true(second->Offset > 0); + auto failure = cluster.AckChunk(second); assert_false(failure->Success); + cluster.Deliver(delayed); // stale sequence cannot restore old progress + auto restarted = cluster.SendChunk(); assert_int_equal(restarted->Offset, 0); + cluster.AckChunk(restarted); cluster.Pump(1, 80, true); + assert_int_equal(cluster.Apps[2]->Loads, 1); + assert_true(Read(*cluster.Apps[2], "c") == std::string(60000, 'c')); +} +void test_snapshot_partial_ack_cannot_skip_unsent_bytes(void**) { + TCluster cluster; cluster.Prepare(); + auto chunk = cluster.SendChunk(); + const auto end = chunk->Offset + chunk->Len - sizeof(TInstallSnapshotRequest); + assert_true(end + 1 < chunk->TotalBytes); + auto invalid = NewHoldedMessage(TMessageEx{.Src = 3, .Dst = 1, .Term = chunk->Term, .Seqno = chunk->Seqno}, + TInstallSnapshotResponse{.SnapshotIndex = chunk->SnapshotIndex, .NextOffset = end + 1, .Success = 1}); + cluster.Deliver(invalid); + auto retry = cluster.SendChunk(); + assert_int_equal(retry->Offset, chunk->Offset); assert_int_equal(retry->Seqno, chunk->Seqno); + cluster.AckChunk(retry); auto next = cluster.SendChunk(); + assert_int_equal(next->Offset, end); + cluster.AckChunk(next); cluster.Pump(1, 80, true); + assert_int_equal(cluster.Apps[2]->Loads, 1); +} +void ExerciseDelayedSnapshotAcks(bool readTraffic) { + for (auto learners : {std::vector{}, std::vector{3}}) { + TCluster cluster(learners); cluster.Prepare(); + const auto boundary = cluster.Nodes[0]->GetState()->SnapshotIndex; + auto chunk = cluster.SendChunk(); + size_t chunks = 0; + while (true) { + assert_true(++chunks < 10); + cluster.Deliver(chunk); + auto ack = cluster.Take(3, 1, EMessageType::INSTALL_SNAPSHOT_RESPONSE).Cast(); + // Every ACK arrives after several retries. Drop retry packets while + // preserving the original successful reply, a valid lossy schedule. + for (int retry = 0; retry < 3; ++retry) { + if (readTraffic) cluster.Nodes[0]->ApproveRead(); + else { cluster.Now += std::chrono::seconds(3); cluster.Nodes[0]->ProcessTimeout(cluster.Now); } + auto retransmit = cluster.Take(1, 3, EMessageType::INSTALL_SNAPSHOT_REQUEST).Cast(); + assert_int_equal(retransmit->Offset, chunk->Offset); + cluster.Packets.clear(); + } + cluster.Deliver(ack); + if (ack->Installed) break; + cluster.Deliver(ack); // Also stale between consuming an ACK and the next send. + auto next = cluster.SendChunk(); + assert_int_equal(next->Offset, ack->NextOffset); + // A duplicate ACK for the previous chunk must not consume the new + // chunk's correlation or reset its progress. + cluster.Deliver(ack); + chunk = next; + } + assert_true(chunks > 1); + assert_int_equal(cluster.Nodes[0]->GetVolatileState()->MatchIndex.at(3), boundary); + assert_int_equal(cluster.Apps[2]->Loads, 1); + cluster.Write(1, "tail", "after delayed ACKs"); + cluster.Pump(1, 80); + assert_true(Read(*cluster.Apps[2], "tail") == "after delayed ACKs"); + if (!learners.empty()) assert_true(cluster.Nodes[0]->IsLearnerCaughtUp(3)); + } +} +void test_snapshot_timeout_retries_do_not_starve_delayed_acks(void**) { ExerciseDelayedSnapshotAcks(false); } +void test_snapshot_read_traffic_does_not_starve_delayed_acks(void**) { ExerciseDelayedSnapshotAcks(true); } +void test_compaction_during_each_chunk_cannot_restart_snapshot_catchup(void**) { + for (auto learners : {std::vector{}, std::vector{3}}) { + TCluster cluster(learners); cluster.Prepare(); + auto chunk = cluster.SendChunk(); + const auto boundary = chunk->SnapshotIndex; + std::weak_ptr original = cluster.Nodes[0]->GetState()->GetSnapshot(); + size_t chunks = 0; + while (true) { + assert_true(++chunks < 10); + cluster.Deliver(chunk); + auto ack = cluster.Take(3, 1, EMessageType::INSTALL_SNAPSHOT_RESPONSE).Cast(); + // Commit and compact again before every ACK arrives, including the + // final one. The majority remains healthy while the receiver lags. + cluster.Isolated = {3}; cluster.Packets.clear(); + cluster.Write(1, "latest", std::to_string(chunks)); + cluster.Nodes[0]->Compact(); + assert_true(cluster.Nodes[0]->GetState()->SnapshotIndex > boundary); + assert_false(original.expired()); // Owned by the active transfer. + cluster.Isolated.clear(); + auto retry = cluster.SendChunk(); + assert_int_equal(retry->SnapshotIndex, boundary); + assert_int_equal(retry->Offset, chunk->Offset); + cluster.Packets.clear(); cluster.Deliver(ack); + if (ack->Installed) break; + chunk = cluster.SendChunk(); + assert_int_equal(chunk->SnapshotIndex, boundary); + assert_int_equal(chunk->Offset, ack->NextOffset); + } + assert_true(chunks > 1); + assert_true(original.expired()); // Completion releases the pinned image. + assert_int_equal(cluster.Apps[2]->Loads, 1); + assert_int_equal(cluster.Nodes[2]->GetState()->SnapshotIndex, boundary); + // The older completed image does not claim the leader's newer prefix. + assert_int_equal(cluster.Nodes[0]->GetVolatileState()->MatchIndex.at(3), boundary); + assert_true(Read(*cluster.Apps[2], "latest").empty()); + cluster.Packets.clear(); + auto newer = cluster.SendChunk(); + assert_true(newer->SnapshotIndex > boundary); assert_int_equal(newer->Offset, 0); + cluster.AckChunk(newer); cluster.Pump(1, 80, true); + assert_true(Read(*cluster.Apps[2], "latest") == std::to_string(chunks)); + cluster.Write(1, "tail", "after compaction during transfer"); cluster.Pump(1, 80); + assert_true(Read(*cluster.Apps[2], "tail") == "after compaction during transfer"); + if (!learners.empty()) assert_true(cluster.Nodes[0]->IsLearnerCaughtUp(3)); + } +} +void test_compaction_is_deferred_until_one_pinned_generation_finishes(void**) { + TCluster cluster({3}); cluster.Prepare(); + auto firstChunk = cluster.SendChunk(); + const auto firstIndex = firstChunk->SnapshotIndex; + cluster.Deliver(firstChunk); // Lose the ACK; peer 3 keeps the first image pinned. + cluster.Packets.clear(); + auto* progress = const_cast(cluster.Nodes[0]->GetVolatileState()); + assert_true(progress->SnapshotRpcs.contains(3)); + + cluster.Isolated = {3}; + cluster.Write(1, "generation", "two"); + auto second = cluster.Nodes[0]->Compact(); + assert_true(second.Index > firstIndex); + cluster.Write(1, "generation", "three"); + assert_true(cluster.Apps[0]->LastAppliedIndex > second.Index); + + // Force a second peer to begin a transfer from the newer generation. + progress->NextIndex[2] = 1; + cluster.Packets.clear(); + cluster.Nodes[0]->ApproveRead(); + auto secondChunk = cluster.Take(1, 2, EMessageType::INSTALL_SNAPSHOT_REQUEST).Cast(); + assert_int_equal(secondChunk->SnapshotIndex, second.Index); + assert_int_equal(progress->SnapshotRpcs.at(3).Snapshot->Index, firstIndex); + assert_int_equal(progress->SnapshotRpcs.at(2).Snapshot->Index, second.Index); + + const auto storedBoundary = cluster.Nodes[0]->GetState()->SnapshotIndex; + const auto storedLastIndex = cluster.Nodes[0]->GetState()->LastLogIndex; + const auto appliedIndex = cluster.Apps[0]->LastAppliedIndex; + Reject([&] { cluster.Nodes[0]->Compact(); }); + assert_int_equal(cluster.Nodes[0]->GetState()->SnapshotIndex, storedBoundary); + assert_int_equal(cluster.Nodes[0]->GetState()->LastLogIndex, storedLastIndex); + assert_int_equal(cluster.Apps[0]->LastAppliedIndex, appliedIndex); + assert_true(progress->SnapshotRpcs.contains(2)); assert_true(progress->SnapshotRpcs.contains(3)); + + // Once peer 2 completes, compaction can advance while peer 3 still owns + // its older immutable image. + cluster.Deliver(secondChunk); + auto ack = cluster.Take(2, 1, EMessageType::INSTALL_SNAPSHOT_RESPONSE).Cast(); + cluster.Deliver(ack); + assert_true(ack->Installed); + assert_false(progress->SnapshotRpcs.contains(2)); assert_true(progress->SnapshotRpcs.contains(3)); + cluster.Packets.clear(); + auto latest = cluster.Nodes[0]->Compact(); + assert_true(latest.Index > storedBoundary); + assert_int_equal(cluster.Nodes[0]->GetState()->SnapshotIndex, latest.Index); +} +void test_leader_change_mid_snapshot_rejects_old_chunks(void**) { + TCluster cluster; cluster.Prepare(); + auto old = cluster.SendChunk(); cluster.AckChunk(old); + cluster.Packets.clear(); cluster.Isolated = {1}; + cluster.Elect(2); cluster.Pump(2, 80, true); + assert_true(cluster.Nodes[2]->GetState()->CurrentTerm > old->Term); + const auto loads = cluster.Apps[2]->Loads; + cluster.Deliver(old); + assert_int_equal(cluster.Apps[2]->Loads, loads); + cluster.Write(2, "new leader", "tail"); + assert_true(Read(*cluster.Apps[2], "new leader") == "tail"); +} +void test_partial_and_installed_snapshot_acks_do_not_confirm_strong_reads(void**) { + TCluster cluster; cluster.Prepare(); cluster.Isolated = {2}; + size_t replies = 0; + auto client = std::make_shared([&](auto message) { + assert_int_equal(message.template Cast()->ErrorCode, 0); ++replies; + }); + auto read = NewHoldedMessage(sizeof(TReadKv) + 1); + read->KeySize = 1; read->Data[0] = 'a'; read->Flags = TCommandRequest::EConsistent; + cluster.Processors[0]->OnCommandRequest(read, client); + auto chunk = cluster.Take(1, 3, EMessageType::INSTALL_SNAPSHOT_REQUEST).Cast(); + while (true) { + auto response = cluster.AckChunk(chunk); + cluster.Processors[0]->ProcessWaiting(); assert_int_equal(replies, 0); + if (response->Installed) break; + chunk = cluster.SendChunk(); + } + cluster.Pump(1); assert_int_equal(replies, 1); +} +void test_compacted_follower_hint_requires_boundary_probe_before_match(void**) { + TCluster cluster; cluster.Elect(1); cluster.Write(1, "key", "value"); + cluster.Nodes[1]->Compact(); + const auto boundary = cluster.Nodes[1]->GetState()->SnapshotIndex; + auto* progress = const_cast(cluster.Nodes[0]->GetVolatileState()); + progress->NextIndex[2] = 1; progress->MatchIndex[2] = 0; progress->RpcDue[2] = {}; + cluster.Now += std::chrono::seconds(3); cluster.Nodes[0]->ProcessTimeout(cluster.Now); + cluster.Deliver(cluster.Take(1, 2, EMessageType::APPEND_ENTRIES_REQUEST)); + auto hint = cluster.Take(2, 1, EMessageType::APPEND_ENTRIES_RESPONSE).Cast(); + assert_false(hint->Success); assert_int_equal(hint->MatchIndex, boundary); + cluster.Deliver(hint); + assert_int_equal(progress->MatchIndex[2], 0); assert_int_equal(progress->NextIndex[2], boundary + 1); + cluster.Pump(1); assert_int_equal(progress->MatchIndex[2], cluster.Nodes[0]->GetLastIndex()); +} + +TMessageHolder Chunk(const TSnapshot& snapshot, std::string_view bytes, uint64_t offset, uint32_t count) { + auto message = NewHoldedMessage(sizeof(TInstallSnapshotRequest) + count); + message->Src = 1; message->Dst = 2; message->Term = 3; message->Seqno = offset + 1; + message->SnapshotIndex = snapshot.Index; message->SnapshotTerm = snapshot.Term; + message->Offset = offset; message->TotalBytes = bytes.size(); message->Done = offset + count == bytes.size(); + message->LeaderId = 1; memcpy(message->Data, bytes.data() + offset, count); return message; +} +void test_joint_snapshot_is_rejected_even_if_boundary_is_already_covered(void**) { + TKv source; source.Write(source.Prepare(Command("a", "old")), 5); + TSnapshot fixed{5, 1, {1, 2}, source.SaveSnapshot()}; + std::vector> replies; + auto peer = std::make_shared([&](auto message) { replies.push_back(message.template Cast()); }); + auto state = std::make_shared(); + auto raft = std::make_shared(state, 2, TNodeDict{{1, peer}}); + auto app = std::make_shared(); TRequestProcessor processor(raft, app, {}); + auto fixedBytes = EncodeSnapshot(fixed); + raft->Process({}, Chunk(fixed, fixedBytes, 0, fixedBytes.size())); + assert_true(replies.back()->Installed); assert_int_equal(app->Loads, 1); + auto joint = fixed; joint.JointVoters = {2, 3}; joint.ConfigurationIndex = 4; joint.ConfigurationTerm = 1; + const auto bytes = EncodeSnapshot(joint); + const auto before = static_cast(*state); + const auto repliesBeforeUnsupported = replies.size(); + + // An unsupported snapshot format or an unclassifiable short first chunk + // must not persist a higher term before the runtime can reject it. + auto higherTermJoint = Chunk(joint, bytes, 0, static_cast(bytes.size())); + higherTermJoint->Term = state->CurrentTerm + 1; + raft->Process({}, higherTermJoint); + assert_true(static_cast(*state) == before); + assert_int_equal(replies.size(), repliesBeforeUnsupported); + auto wrongEmbeddedBoundary = fixed; wrongEmbeddedBoundary.Index = 4; + const auto wrongEmbeddedBoundaryBytes = EncodeSnapshot(wrongEmbeddedBoundary); + auto higherTermWrongBoundary = Chunk(fixed, wrongEmbeddedBoundaryBytes, + 0, static_cast(wrongEmbeddedBoundaryBytes.size())); + higherTermWrongBoundary->Term = state->CurrentTerm + 1; + raft->Process({}, higherTermWrongBoundary); + assert_true(static_cast(*state) == before); + assert_int_equal(replies.size(), repliesBeforeUnsupported); + auto wrongVotersBeforeTerm = fixed; wrongVotersBeforeTerm.Voters = {1, 3}; + const auto wrongVotersBeforeTermBytes = EncodeSnapshot(wrongVotersBeforeTerm); + auto higherTermWrongVoters = Chunk(wrongVotersBeforeTerm, wrongVotersBeforeTermBytes, + 0, static_cast(wrongVotersBeforeTermBytes.size())); + higherTermWrongVoters->Term = state->CurrentTerm + 1; + raft->Process({}, higherTermWrongVoters); + assert_true(static_cast(*state) == before); + assert_int_equal(replies.size(), repliesBeforeUnsupported); + auto higherTermShort = Chunk(joint, bytes, 0, 4); + higherTermShort->Term = state->CurrentTerm + 1; + raft->Process({}, higherTermShort); + assert_true(static_cast(*state) == before); + assert_int_equal(replies.size(), repliesBeforeUnsupported); + auto higherTermContinuation = Chunk(joint, bytes, 4, 4); + higherTermContinuation->Term = state->CurrentTerm + 1; + raft->Process({}, higherTermContinuation); + assert_true(static_cast(*state) == before); + assert_int_equal(replies.size(), repliesBeforeUnsupported); + + // At the current term, normal fail-closed rejection still returns a reply. + raft->Become(EState::CANDIDATE); + const auto electionDueBeforeUnsupported = raft->GetVolatileState()->ElectionDue; + raft->Process({}, Chunk(joint, bytes, 0, 4)); + assert_true(replies.back()->Success); assert_false(replies.back()->Installed); + assert_int_equal(raft->CurrentStateName(), EState::CANDIDATE); + assert_true(raft->GetVolatileState()->ElectionDue == electionDueBeforeUnsupported); + raft->Process({}, Chunk(joint, bytes, 4, 4)); + assert_false(replies.back()->Success); assert_false(replies.back()->Installed); + assert_int_equal(raft->CurrentStateName(), EState::CANDIDATE); + assert_true(raft->GetVolatileState()->ElectionDue == electionDueBeforeUnsupported); + raft->Process({}, Chunk(joint, bytes, 0, bytes.size())); + assert_false(replies.back()->Success); assert_false(replies.back()->Installed); + assert_int_equal(raft->CurrentStateName(), EState::CANDIDATE); + assert_true(raft->GetVolatileState()->ElectionDue == electionDueBeforeUnsupported); + assert_true(static_cast(*state) == before); assert_int_equal(app->Loads, 1); + + // Covered fast-path ACKs still require the embedded boundary and voter set + // to match the request and this node's fixed membership. + auto wrongVoters = fixed; wrongVoters.Voters = {1, 3}; + auto wrongVoterBytes = EncodeSnapshot(wrongVoters); + raft->Process({}, Chunk(wrongVoters, wrongVoterBytes, 0, wrongVoterBytes.size())); + assert_false(replies.back()->Success); assert_false(replies.back()->Installed); + assert_true(static_cast(*state) == before); assert_int_equal(app->Loads, 1); + + auto wrongBoundary = fixed; wrongBoundary.Index = 4; + auto wrongBoundaryBytes = EncodeSnapshot(wrongBoundary); + raft->Process({}, Chunk(fixed, wrongBoundaryBytes, 0, wrongBoundaryBytes.size())); + assert_false(replies.back()->Success); assert_false(replies.back()->Installed); + assert_true(static_cast(*state) == before); assert_int_equal(app->Loads, 1); + + // The metadata-only probe remains bounded and handles a split prefix. + const size_t metadataBytes = 36 + 4 * fixed.Voters.size(); + for (size_t offset = 0; offset < metadataBytes; offset += 4) { + const auto count = std::min(4, metadataBytes - offset); + raft->Process({}, Chunk(fixed, fixedBytes, offset, static_cast(count))); + if (offset + count < metadataBytes) { + assert_true(replies.back()->Success); assert_false(replies.back()->Installed); + assert_int_equal(replies.back()->NextOffset, offset + count); + } + } + assert_true(replies.back()->Success); assert_true(replies.back()->Installed); + assert_int_equal(replies.back()->NextOffset, fixedBytes.size()); + raft->Process({}, Chunk(joint, bytes, 0, 4)); + assert_true(replies.back()->Success); assert_false(replies.back()->Installed); + raft->Process({}, Chunk(joint, bytes, 4, 4)); + assert_false(replies.back()->Success); assert_false(replies.back()->Installed); + auto stable = fixed; stable.StableConfiguration = true; stable.ConfigurationIndex = 4; stable.ConfigurationTerm = 1; + auto stableBytes = EncodeSnapshot(stable); + raft->Process({}, Chunk(stable, stableBytes, 0, stableBytes.size())); + assert_true(replies.back()->Success); assert_true(replies.back()->Installed); + assert_true(static_cast(*state) == before); assert_int_equal(app->Loads, 1); + + // The stable-format covered probe validates the config identity and voter + // set from a split metadata prefix before crediting the already-held index. + const size_t stableMetadataBytes = 52 + 4 * stable.Voters.size(); + for (size_t offset = 0; offset < stableMetadataBytes; offset += 4) { + const auto count = std::min(4, stableMetadataBytes - offset); + raft->Process({}, Chunk(stable, stableBytes, offset, static_cast(count))); + if (offset + count < stableMetadataBytes) { + assert_true(replies.back()->Success); assert_false(replies.back()->Installed); + assert_int_equal(replies.back()->NextOffset, offset + count); + } + } + assert_true(replies.back()->Success); assert_true(replies.back()->Installed); + + auto stableWrongVoters = stable; stableWrongVoters.Voters = {1, 3}; + auto stableWrongVotersBytes = EncodeSnapshot(stableWrongVoters); + raft->Process({}, Chunk(stableWrongVoters, stableWrongVotersBytes, 0, stableWrongVotersBytes.size())); + assert_false(replies.back()->Success); assert_false(replies.back()->Installed); + auto invalidStableIdentity = stableBytes; + for (size_t i = 36; i < 44; ++i) invalidStableIdentity[i] = 0; + raft->Process({}, Chunk(stable, invalidStableIdentity, 0, invalidStableIdentity.size())); + assert_false(replies.back()->Success); assert_false(replies.back()->Installed); + joint.Index = 6; + const auto newer = EncodeSnapshot(joint); + raft->Process({}, Chunk(joint, newer, 0, newer.size())); + assert_false(replies.back()->Success); assert_false(replies.back()->Installed); + assert_true(static_cast(*state) == before); assert_int_equal(app->Loads, 1); + assert_true(Read(*app, "a") == "old"); +} +void test_unsupported_new_snapshot_does_not_accept_leader_contact(void**) { + TKv source; source.Write(source.Prepare(Command("a", "joint")), 5); + TSnapshot joint{5, 2, {1, 2}, source.SaveSnapshot()}; + joint.JointVoters = {2, 3}; joint.ConfigurationIndex = 4; joint.ConfigurationTerm = 2; + const auto bytes = EncodeSnapshot(joint); + std::vector> replies; + auto peer = std::make_shared([&](auto message) { + replies.push_back(message.template Cast()); + }); + auto state = std::make_shared(); + auto raft = std::make_shared(state, 2, TNodeDict{{1, peer}}); + auto app = std::make_shared(); TRequestProcessor processor(raft, app, {}); + state->CurrentTerm = 3; + raft->Become(EState::CANDIDATE); + const auto before = static_cast(*state); + const auto electionDue = raft->GetVolatileState()->ElectionDue; + + // This image is newer than the local commit boundary, so it exercises the + // full installation path rather than the covered-prefix fast path. + raft->Process({}, Chunk(joint, bytes, 0, 4)); + assert_true(replies.back()->Success); assert_false(replies.back()->Installed); + assert_int_equal(raft->CurrentStateName(), EState::CANDIDATE); + assert_true(raft->GetVolatileState()->ElectionDue == electionDue); + raft->Process({}, Chunk(joint, bytes, 4, 4)); + assert_false(replies.back()->Success); assert_false(replies.back()->Installed); + raft->Process({}, Chunk(joint, bytes, 0, static_cast(bytes.size()))); + assert_false(replies.back()->Success); assert_false(replies.back()->Installed); + assert_int_equal(raft->CurrentStateName(), EState::CANDIDATE); + assert_true(raft->GetVolatileState()->ElectionDue == electionDue); + assert_true(static_cast(*state) == before); + assert_int_equal(app->Loads, 0); assert_int_equal(app->LastAppliedIndex, 0); +} +void test_stable_configuration_snapshot_installs_and_recovers_with_fixed_membership(void**) { + TKv source; source.Write(source.Prepare(Command("a", "stable snapshot")), 5); + TSnapshot snapshot{5, 1, {1, 2}, source.SaveSnapshot()}; + snapshot.StableConfiguration = true; + snapshot.ConfigurationIndex = 4; snapshot.ConfigurationTerm = 1; + const auto bytes = EncodeSnapshot(snapshot); + std::vector> replies; + auto peer = std::make_shared([&](auto message) { replies.push_back(message.template Cast()); }); + auto state = std::make_shared(); + { + auto raft = std::make_shared(state, 2, TNodeDict{{1, peer}}); + auto app = std::make_shared(); TRequestProcessor processor(raft, app, {}); + const size_t metadataBytes = 52 + 4 * snapshot.Voters.size(); + raft->Process({}, Chunk(snapshot, bytes, 0, metadataBytes)); + assert_true(replies.back()->Success); assert_false(replies.back()->Installed); + assert_int_equal(replies.back()->NextOffset, metadataBytes); assert_int_equal(app->Loads, 0); + raft->Process({}, Chunk(snapshot, bytes, metadataBytes, + static_cast(bytes.size() - metadataBytes))); + assert_true(replies.back()->Success); assert_true(replies.back()->Installed); + assert_int_equal(replies.back()->NextOffset, bytes.size()); + assert_true(*state->GetSnapshot() == snapshot); + assert_int_equal(app->Loads, 1); assert_true(Read(*app, "a") == "stable snapshot"); + + // A covered stable image with a different configuration identity must + // not be acknowledged as progress. + auto conflictingCovered = snapshot; + conflictingCovered.ConfigurationIndex = 3; + const auto conflictingCoveredBytes = EncodeSnapshot(conflictingCovered); + const auto beforeConflictingCovered = static_cast(*state); + const auto repliesBeforeConflictingCovered = replies.size(); + auto higherTermConflictingCovered = Chunk(conflictingCovered, conflictingCoveredBytes, 0, + static_cast(conflictingCoveredBytes.size())); + higherTermConflictingCovered->Term = state->CurrentTerm + 1; + raft->Process({}, higherTermConflictingCovered); + assert_true(static_cast(*state) == beforeConflictingCovered); + assert_int_equal(replies.size(), repliesBeforeConflictingCovered); + assert_true(*state->GetSnapshot() == snapshot); assert_int_equal(app->Loads, 1); + + raft->Process({}, Chunk(conflictingCovered, conflictingCoveredBytes, 0, + static_cast(conflictingCoveredBytes.size()))); + assert_false(replies.back()->Success); assert_false(replies.back()->Installed); + assert_true(*state->GetSnapshot() == snapshot); assert_int_equal(app->Loads, 1); + + // A newer legacy image can be installed, but it must retain the + // receiver's already trusted stable configuration identity. + source.Write(source.Prepare(Command("a", "legacy refresh")), 6); + TSnapshot legacy{6, 1, {1, 2}, source.SaveSnapshot()}; + const auto legacyBytes = EncodeSnapshot(legacy); + raft->Process({}, Chunk(legacy, legacyBytes, 0, static_cast(legacyBytes.size()))); + assert_true(replies.back()->Success); assert_true(replies.back()->Installed); + auto preserved = legacy; + preserved.StableConfiguration = true; + preserved.ConfigurationIndex = snapshot.ConfigurationIndex; + preserved.ConfigurationTerm = snapshot.ConfigurationTerm; + assert_true(*state->GetSnapshot() == preserved); + assert_int_equal(app->Loads, 2); assert_true(Read(*app, "a") == "legacy refresh"); + + // Conflicting metadata at a newer boundary is rejected before either + // durable Raft state or the application can be replaced. + source.Write(source.Prepare(Command("a", "conflicting refresh")), 7); + TSnapshot conflicting{7, 1, {1, 2}, source.SaveSnapshot()}; + conflicting.StableConfiguration = true; + conflicting.ConfigurationIndex = 3; conflicting.ConfigurationTerm = 1; + const auto conflictingBytes = EncodeSnapshot(conflicting); + raft->Process({}, Chunk(conflicting, conflictingBytes, 0, + static_cast(conflictingBytes.size()))); + assert_false(replies.back()->Success); assert_false(replies.back()->Installed); + assert_true(*state->GetSnapshot() == preserved); + assert_int_equal(app->Loads, 2); assert_int_equal(app->LastAppliedIndex, 6); + assert_true(Read(*app, "a") == "legacy refresh"); + } + auto restarted = std::make_shared(state, 2, TNodeDict{{1, peer}}); + auto recovered = std::make_shared(); TRequestProcessor recovery(restarted, recovered, {}); + assert_int_equal(recovered->LastAppliedIndex, 6); + assert_true(Read(*recovered, "a") == "legacy refresh"); + const auto recoveredSnapshot = restarted->GetState()->GetSnapshot(); + assert_true(recoveredSnapshot->StableConfiguration); + assert_int_equal(recoveredSnapshot->ConfigurationIndex, 4); + assert_int_equal(recoveredSnapshot->ConfigurationTerm, 1); +} +void test_idle_partial_snapshot_expires_on_learner_despite_heartbeats(void**) { + auto state = std::make_shared(); + state->BindVoters({1}); + auto entry = NewHoldedMessage(); entry->Term = 1; state->Append(entry); + std::vector> replies; + auto peer = std::make_shared([&](auto message) { + if (message->Type == uint32_t(EMessageType::INSTALL_SNAPSHOT_RESPONSE)) + replies.push_back(message.template Cast()); + }); + auto raft = std::make_shared(state, 2, TNodeDict{{1, peer}}, std::vector{2}); + auto app = std::make_shared(); TRequestProcessor processor(raft, app, {}); + TKv source; source.Write(source.Prepare(Command("key", "value")), 1); + TSnapshot snapshot{1, 1, {1}, source.SaveSnapshot()}; + const auto bytes = EncodeSnapshot(snapshot); + const size_t metadataBytes = 36 + 4 * snapshot.Voters.size(); + + raft->Process({}, Chunk(snapshot, bytes, 0, metadataBytes)); + assert_true(replies.back()->Success); assert_false(replies.back()->Installed); + assert_int_equal(replies.back()->NextOffset, metadataBytes); assert_int_equal(app->Loads, 0); + + // Ordinary leader contact keeps Raft's election timer fresh, but must not + // keep an abandoned multi-chunk snapshot allocation alive. + const auto heartbeatAt = ITimeSource::Time{} + std::chrono::seconds(20); + auto heartbeat = NewHoldedMessage(TMessageEx{.Src = 1, .Dst = 2, .Term = 3}, + TAppendEntriesRequest{.PrevLogIndex = 1, .PrevLogTerm = 1, .LeaderCommit = 0, .LeaderId = 1}); + raft->Process(heartbeatAt, heartbeat); + // A matching retry is valid transfer activity and refreshes the deadline. + raft->Process(heartbeatAt, Chunk(snapshot, bytes, 0, metadataBytes)); + assert_true(replies.back()->Success); assert_int_equal(replies.back()->NextOffset, metadataBytes); + const auto nextChunkAt = heartbeatAt + std::chrono::seconds(10); + raft->ProcessTimeout(nextChunkAt); + raft->Process(nextChunkAt, Chunk(snapshot, bytes, metadataBytes, 8)); + assert_true(replies.back()->Success); assert_int_equal(replies.back()->NextOffset, metadataBytes + 8); + + // A continuation after expiry is rejected with offset zero so the leader + // restarts from the beginning instead of extending a discarded image. + const auto expiredAt = nextChunkAt + TTimeout::SnapshotTransferIdle + std::chrono::milliseconds(1); + raft->ProcessTimeout(expiredAt); + raft->Process(expiredAt, Chunk(snapshot, bytes, metadataBytes + 8, 8)); + assert_false(replies.back()->Success); assert_false(replies.back()->Installed); + assert_int_equal(replies.back()->NextOffset, 0); assert_int_equal(app->Loads, 0); + assert_int_equal(app->LastAppliedIndex, 0); assert_int_equal(state->SnapshotIndex, 0); + + raft->Process(expiredAt, Chunk(snapshot, bytes, 0, static_cast(bytes.size()))); + assert_true(replies.back()->Success); assert_true(replies.back()->Installed); + assert_int_equal(app->Loads, 1); assert_int_equal(app->LastAppliedIndex, 1); + assert_true(Read(*app, "key") == "value"); +} +void test_corrupt_snapshot_and_invalid_headers_cannot_mutate_application(void**) { + TKv source; source.Write(source.Prepare(Command("a", "new")), 5); + TSnapshot snapshot{5, 1, {1, 2}, source.SaveSnapshot()}; + const auto encoded = EncodeSnapshot(snapshot); + std::vector> replies; + auto peer = std::make_shared([&](auto message) { replies.push_back(message.template Cast()); }); + auto state = std::make_shared(); + auto raft = std::make_shared(state, 2, TNodeDict{{1, peer}}); + auto app = std::make_shared(); TRequestProcessor processor(raft, app, {}); + for (int invalid = 0; invalid < 6; ++invalid) { + auto request = Chunk(snapshot, encoded, 0, encoded.size()); request->Term = 99; + if (invalid == 0) request->Offset = UINT64_MAX; + if (invalid == 1) request->TotalBytes = TSnapshot::MaxBytes + 1; + if (invalid == 2) request->Done = 0; + if (invalid == 3) request->SnapshotTerm = 0; + if (invalid == 4) request->LeaderId = 9; + if (invalid == 5) request->SnapshotIndex = UINT64_MAX; + raft->Process({}, request); assert_int_equal(state->CurrentTerm, 1); + } + auto corrupt = Chunk(snapshot, encoded, 0, encoded.size()); corrupt->Data[encoded.size() - 1] ^= 1; + raft->Process({}, corrupt); assert_false(replies.back()->Success); + assert_int_equal(app->LastAppliedIndex, 0); assert_int_equal(state->SnapshotIndex, 0); + for (int invalid = 0; invalid < 2; ++invalid) { + auto wrong = snapshot; + if (invalid == 0) wrong.Voters = {1, 2, 3}; else wrong.Data = "invalid application image"; + const auto bytes = EncodeSnapshot(wrong); + raft->Process({}, Chunk(wrong, bytes, 0, bytes.size())); + assert_false(replies.back()->Success); assert_int_equal(state->SnapshotIndex, 0); + assert_int_equal(app->LastAppliedIndex, 0); + } + auto valid = Chunk(snapshot, encoded, 0, encoded.size()); raft->Process({}, valid); + assert_true(replies.back()->Installed); assert_int_equal(app->Loads, 1); + assert_true(Read(*app, "a") == "new"); + raft->Process({}, valid); assert_int_equal(app->Loads, 1); // duplicate completion is idempotent + snapshot.Index = 3; auto stale = EncodeSnapshot(snapshot); + raft->Process({}, Chunk(snapshot, stale, 0, stale.size())); + assert_int_equal(app->LastAppliedIndex, 5); assert_int_equal(app->Loads, 1); +} + +void test_older_snapshot_cannot_exceed_compacted_boundary_term(void**) { + std::vector> replies; + auto peer = std::make_shared([&](auto message) { replies.push_back(message.template Cast()); }); + auto state = std::make_shared(); state->CurrentTerm = 2; + TKv image; image.Write(image.Prepare(Command("key", "committed")), 5); + state->InstallSnapshot({5, 2, {1, 2}, image.SaveSnapshot()}); + auto raft = std::make_shared(state, 2, TNodeDict{{1, peer}}); + auto app = std::make_shared(); TRequestProcessor processor(raft, app, {}); + const auto before = static_cast(*state); + TSnapshot older{3, 3, {1, 2}, image.SaveSnapshot()}; + auto bytes = EncodeSnapshot(older); auto invalid = Chunk(older, bytes, 0, bytes.size()); invalid->Term = 9; + raft->Process({}, invalid); + assert_true(replies.empty()); assert_true(static_cast(*state) == before); + assert_int_equal(app->Loads, 1); assert_true(Read(*app, "key") == "committed"); + for (uint64_t term : {1, 2}) { + older.Term = term; bytes = EncodeSnapshot(older); + raft->Process({}, Chunk(older, bytes, 0, bytes.size())); + assert_true(replies.back()->Success); assert_true(replies.back()->Installed); + assert_int_equal(app->Loads, 1); assert_int_equal(state->SnapshotIndex, 5); + } +} +void test_newer_snapshot_cannot_regress_committed_term(void**) { + for (bool compacted : {false, true}) { + std::vector> replies; + auto peer = std::make_shared([&](auto message) { + if (message->Type == uint32_t(EMessageType::INSTALL_SNAPSHOT_RESPONSE)) + replies.push_back(message.template Cast()); + }); + auto state = std::make_shared(); + auto raft = std::make_shared(state, 2, TNodeDict{{1, peer}}); + auto app = std::make_shared(); TRequestProcessor processor(raft, app, {}); + state->CurrentTerm = 2; + for (uint64_t index = 1; index <= 3; ++index) { + auto entry = app->Prepare(Command("key", std::to_string(index))); + entry->Term = index == 1 ? 1 : 2; state->Append(entry); + } + auto heartbeat = NewHoldedMessage(TMessageEx{.Src = 1, .Dst = 2, .Term = 2}, + TAppendEntriesRequest{.PrevLogIndex = 3, .PrevLogTerm = 2, .LeaderCommit = 3, .LeaderId = 1}); + raft->Process({}, heartbeat); processor.ProcessCommitted(); + if (compacted) raft->Compact(); + assert_int_equal(app->LastAppliedIndex, 3); assert_true(Read(*app, "key") == "3"); + TKv source; source.Write(source.Prepare(Command("key", "future")), 4); + TSnapshot snapshot{4, 1, {1, 2}, source.SaveSnapshot()}; + auto bytes = EncodeSnapshot(snapshot); auto request = Chunk(snapshot, bytes, 0, bytes.size()); + request->Term = 9; // An invalid higher-term request must not change hard state. + const auto before = static_cast(*state); + raft->Process({}, request); + assert_true(static_cast(*state) == before); + assert_true(replies.empty()); assert_int_equal(app->LastAppliedIndex, 3); + assert_true(Read(*app, "key") == "3"); assert_int_equal(app->Loads, 0); + auto conflicting = snapshot; conflicting.Index = 3; + const auto conflictingBytes = EncodeSnapshot(conflicting); + auto covered = Chunk(conflicting, conflictingBytes, 0, conflictingBytes.size()); covered->Term = 9; + raft->Process({}, covered); + assert_true(static_cast(*state) == before); assert_true(replies.empty()); + snapshot.Term = 2; bytes = EncodeSnapshot(snapshot); + raft->Process({}, Chunk(snapshot, bytes, 0, bytes.size())); + assert_true(replies.back()->Installed); assert_int_equal(state->SnapshotIndex, 4); + assert_int_equal(app->LastAppliedIndex, 4); assert_true(Read(*app, "key") == "future"); + } +} + +void test_snapshot_is_rechecked_when_commit_advances_mid_transfer(void**) { + std::vector> replies; + auto peer = std::make_shared([&](auto message) { + if (message->Type == uint32_t(EMessageType::INSTALL_SNAPSHOT_RESPONSE)) + replies.push_back(message.template Cast()); + }); + auto state = std::make_shared(); auto raft = std::make_shared(state, 2, TNodeDict{{1, peer}}); + auto app = std::make_shared(); TRequestProcessor processor(raft, app, {}); + state->CurrentTerm = 2; + for (uint64_t index = 1; index <= 3; ++index) { + auto entry = app->Prepare(Command("key", std::to_string(index))); + entry->Term = index == 1 ? 1 : 2; state->Append(entry); + } + auto heartbeat = NewHoldedMessage(TMessageEx{.Src = 1, .Dst = 2, .Term = 3}, + TAppendEntriesRequest{.PrevLogIndex = 3, .PrevLogTerm = 2, .LeaderCommit = 1, .LeaderId = 1}); + raft->Process({}, heartbeat); processor.ProcessCommitted(); + TKv source; source.Write(source.Prepare(Command("key", "future")), 4); + TSnapshot snapshot{4, 1, {1, 2}, source.SaveSnapshot()}; + const auto bytes = EncodeSnapshot(snapshot); + raft->Process({}, Chunk(snapshot, bytes, 0, 8)); + assert_true(replies.back()->Success); assert_false(replies.back()->Installed); + heartbeat.Cast()->LeaderCommit = 3; + raft->Process({}, heartbeat); processor.ProcessCommitted(); + const auto before = static_cast(*state); const auto replyCount = replies.size(); + raft->Process({}, Chunk(snapshot, bytes, 8, bytes.size() - 8)); + assert_int_equal(replies.size(), replyCount); assert_true(static_cast(*state) == before); + assert_int_equal(app->LastAppliedIndex, 3); assert_true(Read(*app, "key") == "3"); + snapshot.Term = 2; const auto valid = EncodeSnapshot(snapshot); + raft->Process({}, Chunk(snapshot, valid, 0, valid.size())); + assert_true(replies.back()->Installed); assert_true(Read(*app, "key") == "future"); +} + +void CheckSnapshotStorageTerm(IState& state, std::function image = [] { return std::string{}; }) { + state.CurrentTerm = 2; state.Commit(); + state.InstallSnapshot({2, 2, {1}, "committed"}); + auto entry = NewHoldedMessage(); entry->Term = 2; state.Append(entry); + const auto before = static_cast(state); + const auto beforeImage = image(); + Reject([&] { state.InstallSnapshot({4, 1, {1}, "regressed"}); }); + assert_true(static_cast(state) == before); + assert_true(image() == beforeImage); + assert_true(state.GetSnapshot()->Data == "committed"); assert_int_equal(state.LogTerm(3), 2); + state.InstallSnapshot({4, 2, {1}, "valid"}); + assert_int_equal(state.SnapshotIndex, 4); assert_true(state.GetSnapshot()->Data == "valid"); +} +void test_memory_snapshot_storage_rejects_term_regression(void**) { + TState state; CheckSnapshotStorageTerm(state); +} +#ifdef __linux__ +void test_durable_snapshot_storage_rejects_term_regression(void**) { + auto dir = std::filesystem::temp_directory_path() / ("miniraft-term-boundary-" + + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count())); + assert_true(std::filesystem::create_directory(dir)); + auto name = (dir / "raft").string(); + { TDurableState state(name, 1); CheckSnapshotStorageTerm(state, [&] { + std::ifstream input(name + ".durable.1", std::ios::binary); + return std::string(std::istreambuf_iterator(input), {}); + }); } + { TDurableState state(name, 1); assert_int_equal(state.SnapshotIndex, 4); + assert_int_equal(state.SnapshotTerm, 2); assert_true(state.GetSnapshot()->Data == "valid"); } + std::error_code ignored; std::filesystem::remove_all(dir, ignored); +} +#endif + +void CheckSnapshotCannotConflictAtCommit(IState& state, std::function image = [] { return std::string{}; }) { + state.CurrentTerm = 2; + auto entry = NewHoldedMessage(); entry->Term = 2; state.Append(entry); + state.CommitIndex = 1; state.Commit(); + const auto before = static_cast(state); + const auto beforeImage = image(); + Reject([&] { state.InstallSnapshot({1, 1, {1}, "conflicting committed boundary"}); }); + assert_true(static_cast(state) == before); + assert_true(image() == beforeImage); +} +void test_memory_snapshot_cannot_replace_committed_index_with_conflicting_term(void**) { + TState state; CheckSnapshotCannotConflictAtCommit(state); +} +#ifdef __linux__ +void test_durable_snapshot_cannot_replace_committed_index_with_conflicting_term(void**) { + auto dir = std::filesystem::temp_directory_path() / ("miniraft-committed-snapshot-" + + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count())); + assert_true(std::filesystem::create_directory(dir)); + auto name = (dir / "raft").string(); + { TDurableState state(name, 1); CheckSnapshotCannotConflictAtCommit(state, [&] { + std::ifstream input(name + ".durable.1", std::ios::binary); + return std::string(std::istreambuf_iterator(input), {}); + }); } + { TDurableState state(name, 1); assert_int_equal(state.CommitIndex, 1); } + std::error_code ignored; std::filesystem::remove_all(dir, ignored); +} +#endif + +void test_remote_snapshot_retains_suffix_without_applying_it_before_commit(void**) { + auto state = std::make_shared(); + TKv prefix; + for (const std::string key : {"a", "b", "c", "d"}) { + auto entry = prefix.Prepare(Command(key, "value")); + state->Append(entry); + if (state->LastLogIndex <= 2) prefix.Write(entry, state->LastLogIndex); + } + auto peer = std::make_shared([](auto) {}); + auto raft = std::make_shared(state, 2, TNodeDict{{1, peer}}); + auto app = std::make_shared(); TRequestProcessor processor(raft, app, {}); + TSnapshot snapshot{2, 1, {1, 2}, prefix.SaveSnapshot()}; const auto bytes = EncodeSnapshot(snapshot); + raft->Process({}, Chunk(snapshot, bytes, 0, bytes.size())); processor.ProcessCommitted(); + assert_int_equal(state->LastLogIndex, 4); assert_int_equal(app->LastAppliedIndex, 2); + assert_true(Read(*app, "b") == "value"); assert_true(Read(*app, "c").empty()); + auto heartbeat = NewHoldedMessage(TMessageEx{.Src = 1, .Dst = 2, .Term = 3}, + TAppendEntriesRequest{.PrevLogIndex = 4, .PrevLogTerm = 1, .LeaderCommit = 4, .LeaderId = 1}); + raft->Process({}, heartbeat); processor.ProcessCommitted(); + assert_true(Read(*app, "d") == "value"); assert_int_equal(app->LastAppliedIndex, 4); + assert_int_equal(app->Loads, 1); +} +void test_snapshot_boundary_index_exhaustion_does_not_append_wrapped_noop(void**) { + auto state = std::make_shared(); + TKv image; + state->InstallSnapshot({uint64_t(INT64_MAX), 1, {1}, image.SaveSnapshot()}); + auto raft = std::make_shared(state, 1, TNodeDict{}); + auto app = std::make_shared(); TRequestProcessor processor(raft, app, {}); + Reject([&] { raft->ProcessTimeout({}); }); + assert_true(state->LastLogIndex == uint64_t(INT64_MAX)); + assert_int_equal(state->LastLogTerm, 1); +} +void CheckStorageIndexExhaustion(IState& state, std::function image = [] { return std::string{}; }) { + state.InstallSnapshot({uint64_t(INT64_MAX) - 1, 1, {1}, "boundary"}); + auto entry = NewHoldedMessage(); entry->Term = 1; state.Append(entry); + assert_true(state.LastLogIndex == uint64_t(INT64_MAX)); + const auto before = static_cast(state); const auto beforeImage = image(); + Reject([&] { state.Append(entry); }); + assert_true(static_cast(state) == before); assert_true(image() == beforeImage); + assert_int_equal(state.LogTerm(INT64_MAX), 1); assert_true(state.GetSnapshot()->Data == "boundary"); +} +void test_memory_storage_rejects_index_exhaustion(void**) { + TState state; CheckStorageIndexExhaustion(state); +} +#ifdef __linux__ +void test_durable_storage_rejects_index_exhaustion(void**) { + auto dir = std::filesystem::temp_directory_path() / ("miniraft-index-limit-" + + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count())); + assert_true(std::filesystem::create_directory(dir)); + auto name = (dir / "raft").string(); + { TDurableState state(name, 1); CheckStorageIndexExhaustion(state, [&] { + std::ifstream input(name + ".durable.1", std::ios::binary); + return std::string(std::istreambuf_iterator(input), {}); + }); } + { TDurableState state(name, 1); assert_true(state.LastLogIndex == uint64_t(INT64_MAX)); + assert_int_equal(state.LogTerm(INT64_MAX), 1); } + std::error_code ignored; std::filesystem::remove_all(dir, ignored); +} +#endif + +void ExerciseBoundary(IState& state) { + state.CurrentTerm = 3; state.VotedFor = 2; state.Commit(); + for (uint64_t term : {1, 2, 3}) { auto entry = NewHoldedMessage(); entry->Term = term; state.Append(entry); } + state.InstallSnapshot({2, 2, {1, 2}, "application"}); + assert_int_equal(state.LastLogIndex, 3); assert_int_equal(state.SnapshotIndex, 2); + assert_int_equal(state.LogTerm(1), 0); assert_int_equal(state.LogTerm(2), 2); assert_int_equal(state.Get(3)->Term, 3); + assert_false(state.Get(1)); assert_false(state.Get(2)); + state.RemoveLast(); + assert_int_equal(state.LastLogIndex, 2); assert_int_equal(state.LastLogTerm, 2); + auto entry = NewHoldedMessage(); entry->Term = 3; state.Append(entry); + assert_int_equal(state.LastLogIndex, 3); assert_int_equal(state.Get(3)->Term, 3); + state.InstallSnapshot({3, 2, {1, 2}, "different boundary"}); + assert_int_equal(state.LastLogIndex, 3); assert_int_equal(state.LastLogTerm, 2); assert_false(state.Get(3)); + Reject([&] { state.InstallSnapshot({2, 2, {1, 2}, "obsolete"}); }); + assert_int_equal(state.SnapshotIndex, 3); +} +void test_memory_snapshot_boundary_retains_matching_suffix_and_replaces_conflict(void**) { TState state; ExerciseBoundary(state); } +#ifdef __linux__ +struct TFiles { + std::filesystem::path Directory; + std::string Name; + TFiles() { + static unsigned next = 0; + Directory = std::filesystem::temp_directory_path() / ("miniraft-compaction-" + std::to_string(getpid()) + "-" + std::to_string(next++)); + assert_true(std::filesystem::create_directory(Directory)); Name = (Directory / "raft").string(); + } + ~TFiles() { std::error_code ignored; std::filesystem::remove_all(Directory, ignored); } +}; +struct TCountingDurable : TDurableState { + using TDurableState::TDurableState; + mutable unsigned Reads = 0; + TMessageHolder Get(int64_t index) const override { + ++Reads; return TDurableState::Get(index); + } +}; +void test_durable_term_lookup_avoids_payload_copies_across_compaction(void**) { + TFiles files; TCountingDurable state(files.Name, 1); + auto entry = NewHoldedMessage(sizeof(TLogEntry) + 1024 * 1024); + entry->Data[0] = 'x'; state.Append(entry); state.Append(entry); + state.InstallSnapshot({1, 1, {1}, "image"}); + for (int i = 0; i < 20; ++i) { + assert_int_equal(state.LogTerm(1), 1); assert_int_equal(state.LogTerm(2), 1); + assert_int_equal(state.LogTerm(3), 0); + } + assert_int_equal(state.Reads, 0); + auto read = state.Get(2); read->Data[0] = 'y'; read->Term = 2; + assert_int_equal(state.LogTerm(2), 1); assert_int_equal(state.Get(2)->Data[0], 'x'); +} +void test_durable_compacted_boundary_survives_restart(void**) { + TFiles files; + { TDurableState state(files.Name, 1); ExerciseBoundary(state); } + TDurableState state(files.Name, 1); + assert_int_equal(state.SnapshotIndex, 3); assert_int_equal(state.LogTerm(3), 2); + assert_int_equal(state.CurrentTerm, 3); assert_int_equal(state.VotedFor, 2); + assert_true(state.GetSnapshot()->Data == "different boundary"); + auto entry = NewHoldedMessage(); entry->Term = 3; state.Append(entry); + assert_int_equal(state.LastLogIndex, 4); assert_int_equal(state.Get(4)->Term, 3); +} +void test_snapshot_and_suffix_are_atomic_at_every_publication_stage(void**) { + using TStage = TDurableState::EStage; + for (bool priorSnapshot : {false, true}) for (bool retain : {false, true}) for (auto stage : {TStage::TempCreated, TStage::Written, TStage::FileSynced, + TStage::Renamed, TStage::DirectorySynced}) { + TFiles files; + { TDurableState state(files.Name, 1); state.CurrentTerm = 3; state.VotedFor = 2; + for (uint64_t term : {1, 2, 3}) { auto entry = NewHoldedMessage(); entry->Term = term; state.Append(entry); } + if (priorSnapshot) state.InstallSnapshot({1, 1, {1, 2}, "previous application"}); } + const auto child = fork(); assert_true(child >= 0); + if (child == 0) { + try { TDurableState state(files.Name, 1, [&](auto point) { if (point == stage) _exit(90); }); + state.InstallSnapshot({retain ? 2u : 4u, retain ? 2u : 3u, {1, 2}, "durable application"}); + } catch (...) { _exit(91); } _exit(92); + } + int status = 0; assert_int_equal(waitpid(child, &status, 0), child); + assert_true(WIFEXITED(status)); assert_int_equal(WEXITSTATUS(status), 90); + TDurableState state(files.Name, 1); + const bool updated = stage >= TStage::Renamed; + assert_int_equal(state.CurrentTerm, 3); assert_int_equal(state.VotedFor, 2); + assert_int_equal(state.SnapshotIndex, updated ? retain ? 2 : 4 : priorSnapshot ? 1 : 0); + assert_int_equal(state.LastLogIndex, updated && !retain ? 4 : 3); + if (updated) assert_true(state.GetSnapshot()->Data == "durable application"); + else if (priorSnapshot) assert_true(state.GetSnapshot()->Data == "previous application"); + if (retain || !updated) assert_int_equal(state.Get(3)->Term, 3); + } +} +void test_compaction_reclaims_durable_prefix_bytes(void**) { + TFiles files; TDurableState state(files.Name, 1); + for (int i = 0; i < 30; ++i) { + auto entry = NewHoldedMessage(sizeof(TLogEntry) + 4096); + memset(entry->Data, 0, 4096); state.Append(entry); + } + const auto before = std::filesystem::file_size(files.Name + ".durable.1"); + state.InstallSnapshot({29, 1, {1}, "small current application"}); + const auto after = std::filesystem::file_size(files.Name + ".durable.1"); + assert_true(after < before / 10); assert_int_equal(state.LastLogIndex, 30); + assert_true(bool(state.Get(30))); assert_false(state.Get(29)); +} +void test_durable_v4_semantic_corruption_rejects_without_rewriting(void**) { + for (int invalid = 0; invalid < 3; ++invalid) { + TFiles files; + { TDurableState state(files.Name, 1); ExerciseBoundary(state); } + const auto path = files.Name + ".durable.1"; + std::ifstream input(path, std::ios::binary); + std::string bytes(std::istreambuf_iterator(input), {}); input.close(); + assert_true(bytes.substr(0, 8) == "MRFTD004"); + if (invalid < 2) { + const auto offset = invalid == 0 ? 52 : 32; // snapshot length or retained entry count + for (size_t i = offset; i < size_t(offset + 8); ++i) bytes[i] = char(0xff); + } else bytes.back() ^= 1; // inner snapshot checksum with a valid outer checksum + const auto crc = NSnapshotCodec::Checksum(bytes.substr(0, 8) + bytes.substr(20)); + for (int i = 0; i < 4; ++i) bytes[16 + i] = char(crc >> (8 * i)); + { std::ofstream out(path, std::ios::binary); out.write(bytes.data(), bytes.size()); } + Reject([&] { TDurableState state(files.Name, 1); }); + std::ifstream after(path, std::ios::binary); + assert_true(std::string(std::istreambuf_iterator(after), {}) == bytes); + } +} +#endif +} +int main() { + const CMUnitTest tests[] = { + cmocka_unit_test(test_compaction_lagging_follower_installs_snapshot_then_tail), + cmocka_unit_test(test_simultaneous_startup_candidates_use_distinct_election_deadlines), + cmocka_unit_test(test_partial_transfer_restart_and_delayed_ack_are_safe), + cmocka_unit_test(test_snapshot_partial_ack_cannot_skip_unsent_bytes), + cmocka_unit_test(test_snapshot_timeout_retries_do_not_starve_delayed_acks), + cmocka_unit_test(test_snapshot_read_traffic_does_not_starve_delayed_acks), + cmocka_unit_test(test_compaction_during_each_chunk_cannot_restart_snapshot_catchup), + cmocka_unit_test(test_compaction_is_deferred_until_one_pinned_generation_finishes), + cmocka_unit_test(test_leader_change_mid_snapshot_rejects_old_chunks), + cmocka_unit_test(test_partial_and_installed_snapshot_acks_do_not_confirm_strong_reads), + cmocka_unit_test(test_compacted_follower_hint_requires_boundary_probe_before_match), + cmocka_unit_test(test_corrupt_snapshot_and_invalid_headers_cannot_mutate_application), + cmocka_unit_test(test_newer_snapshot_cannot_regress_committed_term), + cmocka_unit_test(test_older_snapshot_cannot_exceed_compacted_boundary_term), + cmocka_unit_test(test_snapshot_is_rechecked_when_commit_advances_mid_transfer), + cmocka_unit_test(test_memory_snapshot_storage_rejects_term_regression), + cmocka_unit_test(test_memory_snapshot_cannot_replace_committed_index_with_conflicting_term), + cmocka_unit_test(test_joint_snapshot_is_rejected_even_if_boundary_is_already_covered), + cmocka_unit_test(test_unsupported_new_snapshot_does_not_accept_leader_contact), + cmocka_unit_test(test_stable_configuration_snapshot_installs_and_recovers_with_fixed_membership), + cmocka_unit_test(test_idle_partial_snapshot_expires_on_learner_despite_heartbeats), + cmocka_unit_test(test_remote_snapshot_retains_suffix_without_applying_it_before_commit), + cmocka_unit_test(test_snapshot_boundary_index_exhaustion_does_not_append_wrapped_noop), + cmocka_unit_test(test_memory_storage_rejects_index_exhaustion), + cmocka_unit_test(test_memory_snapshot_boundary_retains_matching_suffix_and_replaces_conflict), +#ifdef __linux__ + cmocka_unit_test(test_durable_snapshot_storage_rejects_term_regression), + cmocka_unit_test(test_durable_snapshot_cannot_replace_committed_index_with_conflicting_term), + cmocka_unit_test(test_durable_storage_rejects_index_exhaustion), + cmocka_unit_test(test_durable_term_lookup_avoids_payload_copies_across_compaction), + cmocka_unit_test(test_durable_compacted_boundary_survives_restart), + cmocka_unit_test(test_snapshot_and_suffix_are_atomic_at_every_publication_stage), + cmocka_unit_test(test_compaction_reclaims_durable_prefix_bytes), + cmocka_unit_test(test_durable_v4_semantic_corruption_rejects_without_rewriting), +#endif + }; + return cmocka_run_group_tests(tests, nullptr, nullptr); +} diff --git a/test/test_configuration.cpp b/test/test_configuration.cpp new file mode 100644 index 0000000..fae2fc6 --- /dev/null +++ b/test/test_configuration.cpp @@ -0,0 +1,920 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +extern "C" { +#include +} +namespace { +template void Reject(F f) { + bool rejected = false; + try { f(); } catch (const std::exception&) { rejected = true; } + assert_true(rejected); +} +TConfigurationRecord Joint() { + return {EConfigurationPhase::Joint, 3, 2, 2, {1, 2, 3}, {3, 4, 5}}; +} +TMessageHolder Copy(TMessageHolder entry) { + auto copy = NewHoldedMessage(entry->Len); + memcpy(copy.Mes, entry.Mes, entry->Len); return copy; +} +void FixChecksum(TMessageHolder entry) { + const auto length = entry->Len - sizeof(TLogEntry); + const auto crc = NSnapshotCodec::Checksum(std::string_view(entry->Data, length - 4)); + for (int i = 0; i < 4; ++i) entry->Data[length - 4 + i] = char(crc >> (8 * i)); +} +struct TEndpoint : INode { + unsigned Sent = 0; + std::vector> Messages; + std::function)> Route; + void Send(TMessageHolder message) override { + ++Sent; + if (Route) Route(std::move(message)); + else Messages.push_back(std::move(message)); + } + void Drain() override {} +}; +struct TCountingRsm : TDummyRsm { + unsigned Writes = 0; + TMessageHolder Write(TMessageHolder entry, uint64_t index) override { + ++Writes; + return TDummyRsm::Write(std::move(entry), index); + } +}; +TMessageHolder Ordinary(uint64_t term) { + auto entry = NewHoldedMessage(); entry->Term = term; return entry; +} +void test_configuration_snapshot_projection_and_history_resume(void**) { + auto initial = TConfigurationHistory::Stable({1, 2, 3}, 0, 0, 0, 0); + Reject([&] { initial.MakeSnapshot("empty boundary"); }); + auto prefix = initial.Observe(1, Ordinary(1)); + const auto bootstrap = prefix.MakeSnapshot("bootstrap"); + assert_true(EncodeSnapshot(bootstrap).substr(0, 8) == "MRFTS001"); + assert_int_equal(bootstrap.Index, 1); assert_int_equal(bootstrap.Term, 1); + assert_false(bootstrap.HasConfigurationMetadata()); + auto joint = Joint(); joint.Term = 2; joint.PreviousIndex = joint.PreviousTerm = 0; + auto pending = prefix.Advance(2, EncodeConfigurationEntry(joint)).Observe(3, Ordinary(2)); + const auto pendingSnapshot = pending.MakeSnapshot(std::string("j\0x", 3)); + const auto jointBytes = EncodeSnapshot(pendingSnapshot); + assert_true(jointBytes.substr(0, 8) == "MRFTS002"); + auto restoredJoint = TConfigurationHistory::FromJointSnapshot(DecodeSnapshot(jointBytes)); + assert_int_equal(restoredJoint.ThroughIndex(), 3); assert_int_equal(restoredJoint.ConfigurationIndex(), 2); + assert_true(restoredJoint.OldVoters() == joint.OldVoters); assert_true(restoredJoint.NewVoters() == joint.NewVoters); + auto final = joint; final.Phase = EConfigurationPhase::Stable; final.Term = 3; + final.PreviousIndex = 2; final.PreviousTerm = 2; + auto stable = restoredJoint.Advance(4, EncodeConfigurationEntry(final)); + const auto stableSnapshot = stable.MakeSnapshot("stable"); + const auto stableBytes = EncodeSnapshot(stableSnapshot); + assert_true(stableBytes.substr(0, 8) == "MRFTS003"); + auto restoredStable = TConfigurationHistory::FromStableSnapshot(DecodeSnapshot(stableBytes)); + assert_int_equal(restoredStable.ConfigurationIndex(), 4); assert_int_equal(restoredStable.ConfigurationTerm(), 3); + assert_true(restoredStable.OldVoters() == joint.NewVoters); assert_false(restoredStable.IsJoint()); + TConfigurationRecord next{EConfigurationPhase::Joint, 3, 4, 3, joint.NewVoters, {3, 4, 6}}; + auto resumed = restoredStable.Advance(5, EncodeConfigurationEntry(next)); + assert_true(resumed.IsJoint()); assert_true(resumed.NewVoters() == next.NewVoters); + assert_true(pending.NewVoters() == joint.NewVoters); assert_false(stable.IsJoint()); + TState jointState; jointState.BindVoters(pendingSnapshot.Voters); + jointState.CurrentTerm = pendingSnapshot.Term; + jointState.InstallSnapshot(pendingSnapshot); + assert_true(*jointState.GetSnapshot() == pendingSnapshot); + TState stableState; stableState.BindVoters(stableSnapshot.Voters); + stableState.CurrentTerm = stableSnapshot.Term; + stableState.InstallSnapshot(stableSnapshot); + assert_true(*stableState.GetSnapshot() == stableSnapshot); + TState unbound; unbound.CurrentTerm = pendingSnapshot.Term; + Reject([&] { unbound.InstallSnapshot(pendingSnapshot); }); + unbound.Snapshot = std::make_shared(pendingSnapshot); + Reject([&] { unbound.BindVoters(pendingSnapshot.Voters); }); +} +void test_history_validates_joint_stable_chain_and_independent_rollback(void**) { + auto initial = TConfigurationHistory::Stable({1, 2, 3}, 0, 0, 0, 0); + auto prefix = initial.Observe(1, Ordinary(1)); + auto joint = Joint(); joint.Term = 2; joint.PreviousIndex = joint.PreviousTerm = 0; + auto pending = prefix.Advance(2, EncodeConfigurationEntry(joint)); + assert_true(pending.IsJoint()); + assert_true(pending.OldVoters() == joint.OldVoters); assert_true(pending.NewVoters() == joint.NewVoters); + assert_int_equal(pending.ConfigurationIndex(), 2); assert_int_equal(pending.ConfigurationTerm(), 2); + auto scanned = pending.Observe(3, Ordinary(2)); + auto final = joint; final.Phase = EConfigurationPhase::Stable; final.Term = 3; + final.PreviousIndex = 2; final.PreviousTerm = 2; + auto stable = scanned.Advance(4, EncodeConfigurationEntry(final)); + assert_false(stable.IsJoint()); assert_true(stable.OldVoters() == joint.NewVoters); + assert_int_equal(stable.ConfigurationIndex(), 4); assert_int_equal(stable.ThroughTerm(), 3); + // A replacement branch starts from the unchanged prior prefix. + auto replacement = joint; replacement.Term = 3; replacement.NewVoters = {3, 5, 6}; + auto alternate = prefix.Advance(2, EncodeConfigurationEntry(replacement)); + assert_true(alternate.NewVoters() == replacement.NewVoters); + assert_true(pending.NewVoters() == joint.NewVoters); + assert_false(prefix.IsJoint()); assert_int_equal(prefix.ThroughIndex(), 1); +} +void test_configuration_scan_compares_full_configuration_state(void**) { + auto committed = TConfigurationHistory::Stable({1, 2, 3}, 7, 3, 9, 4); + auto differentVoters = TConfigurationHistory::Stable({1, 2, 4}, 7, 3, 9, 4); + TConfigurationScan voterMismatch{committed, differentVoters}; + assert_true(voterMismatch.HasUncommittedConfiguration()); + + TSnapshot jointSnapshot{9, 4, {1, 2, 3}, "opaque"}; + jointSnapshot.JointVoters = {3, 4, 5}; + jointSnapshot.ConfigurationIndex = 7; jointSnapshot.ConfigurationTerm = 3; + auto joint = TConfigurationHistory::FromJointSnapshot(jointSnapshot); + TConfigurationScan phaseMismatch{committed, joint}; + assert_true(phaseMismatch.HasUncommittedConfiguration()); + + TConfigurationScan unchanged{committed, committed}; + assert_false(unchanged.HasUncommittedConfiguration()); +} +void test_history_from_joint_snapshot_requires_a_consistent_identity(void**) { + TSnapshot snapshot{8, 4, {1, 2, 3}, "opaque"}; + snapshot.JointVoters = {3, 4, 5}; snapshot.ConfigurationIndex = 5; snapshot.ConfigurationTerm = 3; + auto history = TConfigurationHistory::FromJointSnapshot(snapshot); + assert_true(history.IsJoint()); assert_int_equal(history.ConfigurationIndex(), 5); + assert_int_equal(history.ConfigurationTerm(), 3); assert_int_equal(history.ThroughIndex(), 8); + assert_int_equal(history.ThroughTerm(), 4); + auto final = Joint(); final.Phase = EConfigurationPhase::Stable; final.Term = 4; + final.PreviousIndex = 5; final.PreviousTerm = 3; + auto stable = history.Advance(9, EncodeConfigurationEntry(final)); + assert_false(stable.IsJoint()); assert_true(stable.OldVoters() == final.NewVoters); + auto fixed = snapshot; fixed.JointVoters.clear(); fixed.ConfigurationIndex = fixed.ConfigurationTerm = 0; + Reject([&] { TConfigurationHistory::FromJointSnapshot(fixed); }); + auto inconsistent = snapshot; inconsistent.ConfigurationIndex = inconsistent.Index; + Reject([&] { TConfigurationHistory::FromJointSnapshot(inconsistent); }); + Reject([&] { TConfigurationHistory::Stable({1}, 4, 2, 4, 3); }); + Reject([&] { TConfigurationHistory::Stable({1}, 5, 1, 4, 3); }); + Reject([&] { TConfigurationHistory::Stable({1}, 0, 0, 1, 0); }); +} +void test_history_rejects_skips_wrong_predecessors_and_term_regression(void**) { + auto base = TConfigurationHistory::Stable({1, 2, 3}, 0, 0, 0, 0); + auto joint = Joint(); joint.Term = 2; joint.PreviousIndex = joint.PreviousTerm = 0; + auto entry = EncodeConfigurationEntry(joint); + Reject([&] { base.Advance(2, entry); }); + Reject([&] { base.Observe(2, Ordinary(1)); }); + Reject([&] { base.Observe(1, entry); }); + Reject([&] { base.Advance(1, Ordinary(1)); }); + auto wrong = joint; wrong.OldVoters = {1, 2, 4}; + Reject([&] { base.Advance(1, EncodeConfigurationEntry(wrong)); }); + auto noOp = joint; noOp.NewVoters = noOp.OldVoters; + Reject([&] { base.Advance(1, EncodeConfigurationEntry(noOp)); }); + auto prefix = base.Observe(1, Ordinary(3)); + Reject([&] { prefix.Advance(2, entry); }); + Reject([&] { prefix.Observe(2, Ordinary(2)); }); + auto valid = joint; valid.Term = 3; + auto pending = prefix.Advance(2, EncodeConfigurationEntry(valid)); + wrong = valid; wrong.Phase = EConfigurationPhase::Stable; wrong.PreviousIndex = 2; wrong.PreviousTerm = 3; + wrong.NewVoters = {3, 4, 6}; + Reject([&] { pending.Advance(3, EncodeConfigurationEntry(wrong)); }); + wrong = valid; wrong.PreviousIndex = 2; wrong.PreviousTerm = 3; + Reject([&] { pending.Advance(3, EncodeConfigurationEntry(wrong)); }); + auto corrupt = EncodeConfigurationEntry(valid); corrupt->Data[0] ^= 1; + Reject([&] { prefix.Advance(2, corrupt); }); + auto exhausted = TConfigurationHistory::Stable({1}, 0, 0, uint64_t(INT64_MAX), 1); + Reject([&] { exhausted.Observe(UINT64_MAX, Ordinary(1)); }); + Reject([&] { exhausted.Advance(UINT64_MAX, entry); }); + assert_false(prefix.IsJoint()); assert_int_equal(prefix.ThroughIndex(), 1); + assert_true(pending.IsJoint()); assert_int_equal(pending.ThroughIndex(), 2); +} +void test_scan_requires_each_configuration_predecessor_to_be_committed(void**) { + TState state; state.CurrentTerm = 4; + state.Append(Ordinary(1)); + const TConfigurationRecord joint{EConfigurationPhase::Joint, 2, 0, 0, {1}, {1, 2}}; + state.Append(EncodeConfigurationEntry(joint)); + const TConfigurationRecord stable{EConfigurationPhase::Stable, 3, 2, 2, {1}, {1, 2}}; + state.Append(EncodeConfigurationEntry(stable)); + const TConfigurationRecord nextJoint{EConfigurationPhase::Joint, 4, 3, 3, {1, 2}, {2, 3}}; + state.Append(EncodeConfigurationEntry(nextJoint)); + + const auto baseline = TConfigurationHistory::Stable({1}, 0, 0, 0, 0); + // The first joint entry is committed, but its stable successor is not. + // The next joint transition must wait for that stable entry to commit. + Reject([&] { ScanConfigurationHistory(state, baseline, 2); }); + + const auto scan = ScanConfigurationHistory(state, baseline, 3); + assert_false(scan.Committed.IsJoint()); + assert_true(scan.Committed.OldVoters() == stable.NewVoters); + assert_true(scan.Latest.IsJoint()); + assert_true(scan.Latest.OldVoters() == nextJoint.OldVoters); + assert_true(scan.Latest.NewVoters() == nextJoint.NewVoters); + assert_true(scan.HasUncommittedConfiguration()); +} +struct TCountingState : TState { + mutable unsigned Gets = 0; + TMessageHolder Get(int64_t index) const override { ++Gets; return TState::Get(index); } +}; +struct TMismatchedMetadataState : TState { + uint64_t RetainedLogTerm(int64_t index) const override { return TState::RetainedLogTerm(index) + 1; } +}; +void test_scan_separates_commit_and_replaced_suffix(void**) { + TCountingState state; state.CurrentTerm = 3; + auto large = NewHoldedMessage(sizeof(TLogEntry) + 1024 * 1024); large->Term = 1; + TConfigurationRecord joint{EConfigurationPhase::Joint, 2, 0, 0, {1}, {1, 2}}; + auto final = joint; final.Phase = EConfigurationPhase::Stable; final.Term = 3; + final.PreviousIndex = 2; final.PreviousTerm = 2; + state.Append(large); state.Append(EncodeConfigurationEntry(joint)); + state.Append(Ordinary(2)); state.Append(EncodeConfigurationEntry(final)); + auto base = TConfigurationHistory::Stable({1}, 0, 0, 0, 0); + auto scan = ScanConfigurationHistory(state, base, 2); + assert_true(scan.Committed.IsJoint()); assert_int_equal(scan.Committed.ThroughIndex(), 2); + assert_false(scan.Latest.IsJoint()); assert_true(scan.Latest.OldVoters() == final.NewVoters); + assert_true(scan.HasUncommittedConfiguration()); assert_int_equal(state.Gets, 2); + const auto committedImage = scan.Committed.MakeSnapshot("committed image"); + assert_true(committedImage.IsJoint()); assert_int_equal(committedImage.Index, 2); + assert_int_equal(committedImage.ConfigurationIndex, 2); + scan = ScanConfigurationHistory(state, base, 4); + assert_false(scan.HasUncommittedConfiguration()); assert_int_equal(scan.Committed.ThroughIndex(), 4); + state.RemoveLast(); + scan = ScanConfigurationHistory(state, base, 2); + assert_true(scan.Latest.IsJoint()); assert_false(scan.HasUncommittedConfiguration()); + state.Append(EncodeConfigurationEntry(final)); + Reject([&] { ScanConfigurationHistory(state, base, 1); }); + Reject([&] { base.ObserveMetadata(1, 1, TLogEntry::EConfiguration); }); + Reject([&] { base.ObserveMetadata(2, 1, 0); }); +} +void test_scan_from_joint_snapshot_and_rejects_mismatched_baselines(void**) { + TSnapshot snapshot{2, 2, {1, 2}, "image"}; + snapshot.JointVoters = {2, 3}; snapshot.ConfigurationIndex = 2; snapshot.ConfigurationTerm = 2; + // The original binding is retained while the snapshot describes an active + // configuration reached by an earlier committed transition. + TState state; state.BindVoters({7, 8}); state.CurrentTerm = 2; + state.InstallSnapshot(snapshot); state.CurrentTerm = 3; state.Append(Ordinary(2)); + TConfigurationRecord final{EConfigurationPhase::Stable, 3, 2, 2, {1, 2}, {2, 3}}; + state.Append(EncodeConfigurationEntry(final)); + auto base = TConfigurationHistory::FromJointSnapshot(snapshot); + auto scan = ScanConfigurationHistory(state, base, 2); + assert_true(scan.Committed.IsJoint()); assert_false(scan.Latest.IsJoint()); + assert_true(scan.Latest.OldVoters() == final.NewVoters); assert_true(scan.HasUncommittedConfiguration()); + assert_false(ScanConfigurationHistory(state, base, 4).HasUncommittedConfiguration()); + Reject([&] { ScanConfigurationHistory(state, base, 1); }); + Reject([&] { ScanConfigurationHistory(state, base, 5); }); + Reject([&] { ScanConfigurationHistory(state, TConfigurationHistory::Stable({1, 2}, 0, 0, 2, 2), 2); }); + auto wrong = snapshot; wrong.JointVoters = {2, 4}; + Reject([&] { ScanConfigurationHistory(state, TConfigurationHistory::FromJointSnapshot(wrong), 2); }); + state.BoundVoters = {1, 2, 4}; + assert_true(ScanConfigurationHistory(state, base, 2).Committed.IsJoint()); + state.BoundVoters = {7, 8}; + state.LastLogTerm = 2; + Reject([&] { ScanConfigurationHistory(state, base, 2); }); +} +void test_scan_from_stable_configuration_snapshot(void**) { + TSnapshot snapshot{5, 2, {1, 2}, "image"}; snapshot.StableConfiguration = true; + snapshot.ConfigurationIndex = 4; snapshot.ConfigurationTerm = 2; + auto decoded = DecodeSnapshot(EncodeSnapshot(snapshot)); + auto baseline = TConfigurationHistory::FromStableSnapshot(decoded); + assert_false(baseline.IsJoint()); assert_int_equal(baseline.ConfigurationIndex(), 4); + TState state; state.BindVoters({7, 8}); + state.CurrentTerm = 2; state.InstallSnapshot(decoded); + state.CurrentTerm = 3; + TConfigurationRecord joint{EConfigurationPhase::Joint, 3, 4, 2, {1, 2}, {2, 3}}; + state.Append(EncodeConfigurationEntry(joint)); + auto scan = ScanConfigurationHistory(state, baseline, 5); + assert_false(scan.Committed.IsJoint()); assert_true(scan.Latest.IsJoint()); + assert_true(scan.HasUncommittedConfiguration()); + auto wrong = TConfigurationHistory::Stable({1, 2}, 0, 0, 5, 2); + Reject([&] { ScanConfigurationHistory(state, wrong, 5); }); + wrong = TConfigurationHistory::Stable({1, 2}, 4, 1, 5, 2); + Reject([&] { ScanConfigurationHistory(state, wrong, 5); }); + Reject([&] { TConfigurationHistory::FromJointSnapshot(decoded); }); + Reject([&] { TConfigurationHistory::FromStableSnapshot(TSnapshot{5, 2, {1, 2}, "legacy"}); }); + Reject([&] { state.InstallSnapshot(snapshot); }); + auto wrongPredecessor = joint; wrongPredecessor.PreviousIndex = 3; + state.RemoveLast(); state.Append(EncodeConfigurationEntry(wrongPredecessor)); + Reject([&] { ScanConfigurationHistory(state, baseline, 5); }); +} +void test_scan_fails_closed_on_invalid_suffix(void**) { + TState state; state.CurrentTerm = 3; + auto base = TConfigurationHistory::Stable({1}, 0, 0, 0, 0); + state.Append(Ordinary(2)); + TConfigurationRecord wrong{EConfigurationPhase::Joint, 3, 1, 2, {1}, {1, 2}}; + state.Append(EncodeConfigurationEntry(wrong)); + const auto fields = static_cast(state); + Reject([&] { ScanConfigurationHistory(state, base, 1); }); + assert_true(static_cast(state) == fields); assert_int_equal(base.ThroughIndex(), 0); + state.RemoveLast(); state.Append(Ordinary(1)); + Reject([&] { ScanConfigurationHistory(state, base, 1); }); + state.RemoveLast(); state.Append(Ordinary(3)); + state.BoundVoters = {2}; + Reject([&] { ScanConfigurationHistory(state, base, 1); }); + state.BoundVoters.clear(); state.CurrentTerm = 2; + Reject([&] { ScanConfigurationHistory(state, base, 1); }); + TState corrupt; corrupt.CurrentTerm = 3; + TConfigurationRecord joint{EConfigurationPhase::Joint, 2, 0, 0, {1}, {1, 2}}; + auto damaged = EncodeConfigurationEntry(joint); damaged->Data[0] ^= 1; + corrupt.Append(damaged); + Reject([&] { ScanConfigurationHistory(corrupt, base, 0); }); + TMismatchedMetadataState mismatch; mismatch.CurrentTerm = 3; + mismatch.Append(EncodeConfigurationEntry(joint)); + Reject([&] { ScanConfigurationHistory(mismatch, base, 0); }); +} +void test_configuration_roundtrips_both_phases_and_bounded_maximum(void**) { + auto record = Joint(); + for (auto phase : {EConfigurationPhase::Joint, EConfigurationPhase::Stable}) { + record.Phase = phase; + auto entry = EncodeConfigurationEntry(record); + assert_int_equal(entry->Flags, TLogEntry::EConfiguration); + assert_true(DecodeConfigurationEntry(entry) == record); + } + record = {EConfigurationPhase::Joint, 1, 0, 0, {1}, {1, 2}}; + assert_true(DecodeConfigurationEntry(EncodeConfigurationEntry(record)) == record); + record.OldVoters.clear(); record.NewVoters.clear(); + for (uint32_t id = 1; id <= TSnapshot::MaxVoters; ++id) { + record.OldVoters.push_back(id); record.NewVoters.push_back(id + TSnapshot::MaxVoters); + } + auto max = EncodeConfigurationEntry(record); + assert_int_equal(max->Len, sizeof(TLogEntry) + 48 + 8 * TSnapshot::MaxVoters); + assert_true(DecodeConfigurationEntry(max) == record); +} +void test_configuration_rejects_invalid_fields_and_valid_crc_corruption(void**) { + for (int mode = 0; mode < 12; ++mode) { + auto record = Joint(); + if (mode == 0) record.Phase = EConfigurationPhase(0); + if (mode == 1) record.Term = 0; + if (mode == 2) record.PreviousIndex = UINT64_MAX; + if (mode == 3) record.PreviousIndex = 0; + if (mode == 4) record.PreviousTerm = 0; + if (mode == 5) record.PreviousTerm = 4; + if (mode == 6) record.OldVoters.clear(); + if (mode == 7) record.NewVoters = {0, 1}; + if (mode == 8) record.NewVoters = {1, 1}; + if (mode == 9) record.NewVoters = {3, 2}; + if (mode == 10) { record.Phase = EConfigurationPhase::Stable; record.PreviousIndex = record.PreviousTerm = 0; } + if (mode == 11) record.OldVoters.resize(TSnapshot::MaxVoters + 1, 1); + Reject([&] { EncodeConfigurationEntry(record); }); + } + auto original = EncodeConfigurationEntry(Joint()); + const auto payload = original->Len - sizeof(TLogEntry); + for (size_t i = 0; i < payload; ++i) { + auto bad = Copy(original); bad->Data[i] ^= 1; + Reject([&] { DecodeConfigurationEntry(bad); }); + bad = Copy(original); bad->Len = sizeof(TLogEntry) + i; + Reject([&] { DecodeConfigurationEntry(bad); }); + } + for (size_t offset : {size_t(8), size_t(12), size_t(20), size_t(28), size_t(36), size_t(40), size_t(44), size_t(56)}) { + auto bad = Copy(original); + const auto width = offset == 12 || offset == 20 || offset == 28 ? 8 : 4; + for (size_t i = offset; i < offset + width; ++i) bad->Data[i] = offset >= 44 ? 0 : char(0xff); + FixChecksum(bad); Reject([&] { DecodeConfigurationEntry(bad); }); + } + auto bad = Copy(original); bad->Term++; Reject([&] { DecodeConfigurationEntry(bad); }); + bad = Copy(original); bad->Flags |= TLogEntry::EStub; Reject([&] { DecodeConfigurationEntry(bad); }); + bad = Copy(original); bad->Len += 1; Reject([&] { DecodeConfigurationEntry(bad); }); + bad = Copy(original); bad.InitPayload(1); Reject([&] { DecodeConfigurationEntry(bad); }); +} +void test_configuration_rpc_and_proposal_are_rejected_before_mutation(void**) { + auto peer = std::make_shared(); auto state = std::make_shared(); + TRaft follower(state, 2, {{1, peer}}); + auto record = Joint(); record.Term = 2; record.PreviousIndex = record.PreviousTerm = 0; + auto config = EncodeConfigurationEntry(record); + auto request = NewHoldedMessage(TMessageEx{.Src = 1, .Dst = 2, .Term = 2}, + TAppendEntriesRequest{.LeaderId = 1, .Nentries = 1}); + request.InitPayload(1); request.Payload[0] = config; + const auto before = static_cast(*state); + follower.Process({}, request); + assert_true(static_cast(*state) == before); assert_int_equal(peer->Sent, 0); + config->Flags |= TLogEntry::EStub; follower.Process({}, request); + assert_true(static_cast(*state) == before); assert_int_equal(peer->Sent, 0); + auto normal = NewHoldedMessage(); normal->Term = 2; request.Payload[0] = normal; + follower.Process({}, request); assert_int_equal(state->CurrentTerm, 2); assert_int_equal(state->LastLogIndex, 1); + assert_int_equal(peer->Sent, 1); + auto leaderState = std::make_shared(); TRaft leader(leaderState, 1, {}); + leader.ProcessTimeout({}); assert_int_equal(leader.CurrentStateName(), EState::LEADER); + const auto leaderBefore = static_cast(*leaderState); + auto proposed = EncodeConfigurationEntry(record); + Reject([&] { leader.Append(proposed); }); + assert_true(static_cast(*leaderState) == leaderBefore); assert_int_equal(proposed->Term, 2); + Reject([&] { leader.ProposeJointConfiguration({1, 2}); }); // Volatile stores cannot authorize a live change. + assert_true(static_cast(*leaderState) == leaderBefore); +} +void test_joint_membership_addition_and_finalization_replicate_deterministically(void**) { +#ifdef __linux__ + auto dir = std::filesystem::temp_directory_path() / ("miniraft-live-config-" + + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count())); + assert_true(std::filesystem::create_directory(dir)); + const auto name = (dir / "raft").string(); + { + auto toLearner = std::make_shared(); + auto toLeader = std::make_shared(); + auto leaderState = std::make_shared(name, 1); + auto learnerState = std::make_shared(name, 2); + TRaft leader(leaderState, 1, {{2, toLearner}}, {2}); + TRaft learner(learnerState, 2, {{1, toLeader}}, {2}); + const auto now = ITimeSource::Time{}; + toLearner->Route = [&](auto message) { learner.Process(now, std::move(message)); }; + toLeader->Route = [&](auto message) { leader.Process(now, std::move(message)); }; + + // A well-formed but wrong predecessor must not make the higher term + // durable or earn an AppendEntries response. + TConfigurationRecord invalid{EConfigurationPhase::Joint, 2, 0, 0, {2}, {1, 2}}; + auto invalidRequest = NewHoldedMessage(TMessageEx{.Src = 1, .Dst = 2, .Term = 2}, + TAppendEntriesRequest{.LeaderId = 1, .Nentries = 1}); + invalidRequest.InitPayload(1); invalidRequest.Payload[0] = EncodeConfigurationEntry(invalid); + const auto beforeInvalidRequest = static_cast(*learnerState); + learner.Process(now, invalidRequest); + assert_true(static_cast(*learnerState) == beforeInvalidRequest); + assert_int_equal(toLeader->Sent, 0); + + leader.ProcessTimeout(now); + assert_true(leader.CurrentStateName() == EState::LEADER); + leader.ProcessTimeout(now + std::chrono::seconds(10)); + assert_true(leader.IsLearnerCommitAcknowledged(2)); + + const auto beforeRejectedProposal = static_cast(*leaderState); + Reject([&] { leader.ProposeJointConfiguration({2}); }); // Cannot remove the active leader. + assert_true(static_cast(*leaderState) == beforeRejectedProposal); + const auto jointIndex = leader.ProposeJointConfiguration({1, 2}); + assert_int_equal(jointIndex, 2); + assert_true(leader.IsVoter(2)); + assert_true(leader.GetVoterIds() == (std::vector{1, 2})); + + leader.ProcessTimeout(now + std::chrono::seconds(20)); + assert_true(learner.IsVoter(2)); + assert_int_equal(leader.GetVolatileState()->CommitIndex, jointIndex); + assert_int_equal(learnerState->CommitIndex, 1); // The commit watermark needs another RPC. + leader.ProcessTimeout(now + std::chrono::seconds(30)); + assert_int_equal(learnerState->CommitIndex, jointIndex); + + const auto stableIndex = leader.FinalizeConfiguration(); + assert_int_equal(stableIndex, 3); + leader.ProcessTimeout(now + std::chrono::seconds(40)); + assert_int_equal(leader.GetVolatileState()->CommitIndex, stableIndex); + assert_true(leader.GetVoterIds() == (std::vector{1, 2})); + leader.ProcessTimeout(now + std::chrono::seconds(50)); + assert_int_equal(learnerState->CommitIndex, stableIndex); + assert_true(leader.GetLearnerIds().empty()); + assert_true(learner.IsVoter(2)); + assert_true(learner.GetVoterIds() == (std::vector{1, 2})); + + // Removing a different voter still requires both old and new + // majorities until the stable successor commits. + const auto removalJoint = leader.ProposeJointConfiguration({1}); + assert_int_equal(removalJoint, 4); + assert_true(leader.IsVoter(2)); + leader.ProcessTimeout(now + std::chrono::seconds(60)); + assert_int_equal(leader.GetVolatileState()->CommitIndex, removalJoint); + leader.ProcessTimeout(now + std::chrono::seconds(70)); + assert_int_equal(learnerState->CommitIndex, removalJoint); + const auto removalStable = leader.FinalizeConfiguration(); + assert_int_equal(removalStable, 5); + leader.ProcessTimeout(now + std::chrono::seconds(80)); + assert_int_equal(leader.GetVolatileState()->CommitIndex, removalStable); + assert_false(leader.IsVoter(2)); + leader.ProcessTimeout(now + std::chrono::seconds(90)); + assert_int_equal(learnerState->CommitIndex, removalStable); + assert_false(leader.IsVoter(2)); + assert_false(learner.IsVoter(2)); + assert_true(leader.GetVoterIds() == (std::vector{1})); + assert_true(learner.GetVoterIds() == (std::vector{1})); + } + std::error_code ignored; std::filesystem::remove_all(dir, ignored); +#endif +} +void test_uncommitted_joint_suffix_replacement_restores_old_voters(void**) { +#ifdef __linux__ + auto dir = std::filesystem::temp_directory_path() / ("miniraft-config-rollback-" + + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count())); + assert_true(std::filesystem::create_directory(dir)); + const auto name = (dir / "raft").string(); + { + auto peer = std::make_shared(); + auto state = std::make_shared(name, 2); + state->BindVoters({1}); + state->CurrentTerm = 2; + auto joint = TConfigurationRecord{EConfigurationPhase::Joint, 2, 0, 0, {1}, {1, 2}}; + state->Append(EncodeConfigurationEntry(joint)); + state->Commit(); + TRaft follower(state, 2, {{1, peer}}, {2}); + assert_true(follower.IsVoter(2)); + + auto ordinary = Ordinary(3); + auto replacement = NewHoldedMessage(TMessageEx{.Src = 1, .Dst = 2, .Term = 3}, + TAppendEntriesRequest{.LeaderId = 1, .Nentries = 1}); + replacement.InitPayload(1); replacement.Payload[0] = ordinary; + follower.Process(ITimeSource::Time{}, replacement); + assert_int_equal(state->CurrentTerm, 3); + assert_int_equal(state->LastLogIndex, 1); + assert_int_equal(state->RetainedLogFlags(1), 0); + assert_false(follower.IsVoter(2)); + assert_true(follower.GetVoterIds() == (std::vector{1})); + assert_int_equal(peer->Sent, 1); + } + std::error_code ignored; std::filesystem::remove_all(dir, ignored); +#endif +} +void test_durable_restart_recovers_stable_membership_roles(void**) { +#ifdef __linux__ + auto dir = std::filesystem::temp_directory_path() / ("miniraft-config-restart-" + + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count())); + assert_true(std::filesystem::create_directory(dir)); + const auto name = (dir / "raft").string(); + { + auto state = std::make_shared(name, 2); + state->BindVoters({1}); // Original binding stays fixed through both transitions. + state->CurrentTerm = 3; + auto addJoint = TConfigurationRecord{EConfigurationPhase::Joint, 2, 0, 0, {1}, {1, 2}}; + state->Append(EncodeConfigurationEntry(addJoint)); + state->CommitIndex = 1; state->Commit(); + auto addStable = addJoint; addStable.Phase = EConfigurationPhase::Stable; + addStable.PreviousIndex = 1; addStable.PreviousTerm = 2; + state->Append(EncodeConfigurationEntry(addStable)); + state->CommitIndex = 2; state->Commit(); + auto removeJoint = TConfigurationRecord{EConfigurationPhase::Joint, 3, 2, 2, {1, 2}, {1}}; + state->Append(EncodeConfigurationEntry(removeJoint)); + state->CommitIndex = 3; state->Commit(); + auto removeStable = removeJoint; removeStable.Phase = EConfigurationPhase::Stable; + removeStable.PreviousIndex = 3; removeStable.PreviousTerm = 3; + state->Append(EncodeConfigurationEntry(removeStable)); + state->CommitIndex = 4; state->Commit(); + } + { + auto state = std::make_shared(name, 2); + auto peer = std::make_shared(); + TRaft raft(state, 2, {{1, peer}}, {2}); // Reconstruct from the original binding. + assert_true(state->GetBoundVoters() == (std::vector{1})); + assert_false(raft.IsVoter(2)); + assert_true(raft.GetVoterIds() == (std::vector{1})); + assert_true(raft.GetLearnerIds() == (std::vector{2})); + assert_int_equal(raft.GetVolatileState()->CommitIndex, 4); + } + std::error_code ignored; std::filesystem::remove_all(dir, ignored); +#endif +} +void test_configuration_application_advances_without_calling_state_machine(void**) { + auto state = std::make_shared(); auto record = Joint(); record.Term = 1; + record.PreviousIndex = record.PreviousTerm = 0; + state->Append(EncodeConfigurationEntry(record)); + Reject([&] { TRaft invalid(state, 1, {}); }); + assert_true(state->GetBoundVoters().empty()); + auto live = std::make_shared(); auto raft = std::make_shared(live, 1, TNodeDict{}); + auto app = std::make_shared(); TRequestProcessor processor(raft, app, {}); + raft->ProcessTimeout({}); + processor.ProcessCommitted(); + assert_int_equal(app->LastAppliedIndex, 1); // Leader's initial stub. + record.Term = live->CurrentTerm; + live->Append(EncodeConfigurationEntry(record)); + raft->ProcessTimeout(ITimeSource::Time{} + std::chrono::seconds(1)); + processor.ProcessCommitted(); + assert_int_equal(app->LastAppliedIndex, 2); + assert_int_equal(app->Writes, 0); + auto compacted = std::make_shared(); + { TRaft bootstrap(compacted, 1, {}); } + compacted->Append(NewHoldedMessage()); + compacted->InstallSnapshot({1, 1, {1}, "fixed image"}); + compacted->Append(EncodeConfigurationEntry(record)); + const auto fields = static_cast(*compacted); + Reject([&] { TRaft invalid(compacted, 1, {}); }); + assert_true(static_cast(*compacted) == fields); + assert_true(compacted->GetSnapshot()->Data == "fixed image"); +} +#ifdef __linux__ +void test_durable_restart_rejects_reserved_entry_without_rewriting(void**) { + auto dir = std::filesystem::temp_directory_path() / ("miniraft-config-" + + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count())); + assert_true(std::filesystem::create_directory(dir)); + const auto name = (dir / "raft").string(); const auto path = name + ".durable.1"; + auto record = Joint(); record.Term = 1; record.PreviousIndex = record.PreviousTerm = 0; + { auto state = std::make_shared(name, 1); + { TRaft bootstrap(state, 1, {}); } + state->Append(EncodeConfigurationEntry(record)); } + std::ifstream input(path, std::ios::binary); std::string before(std::istreambuf_iterator(input), {}); input.close(); + { auto state = std::make_shared(name, 1); + const auto fields = static_cast(*state); + Reject([&] { TRaft invalid(state, 1, {}); }); + assert_true(static_cast(*state) == fields); } + std::ifstream after(path, std::ios::binary); + assert_true(std::string(std::istreambuf_iterator(after), {}) == before); + std::error_code ignored; std::filesystem::remove_all(dir, ignored); +} +void test_legacy_restart_uses_safe_flag_lookup_and_preserves_files(void**) { + auto dir = std::filesystem::temp_directory_path() / ("miniraft-legacy-config-" + + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count())); + assert_true(std::filesystem::create_directory(dir)); + const auto name = (dir / "raft").string(); + auto record = Joint(); record.Term = 1; record.PreviousIndex = record.PreviousTerm = 0; + { TDiskState state(name, 2); state.Append(EncodeConfigurationEntry(record)); } + std::vector before; + for (auto suffix : {".entries.2", ".index.2", ".state.2"}) { + std::ifstream input(name + suffix, std::ios::binary); + before.emplace_back(std::istreambuf_iterator(input), std::istreambuf_iterator()); + } + { auto state = std::make_shared(name, 2); + const auto fields = static_cast(*state); + Reject([&] { TRaft invalid(state, 2, {}); }); + assert_true(static_cast(*state) == fields); } + size_t i = 0; + for (auto suffix : {".entries.2", ".index.2", ".state.2"}) { + std::ifstream input(name + suffix, std::ios::binary); + assert_true(std::string(std::istreambuf_iterator(input), {}) == before[i++]); + } + std::error_code ignored; std::filesystem::remove_all(dir, ignored); +} +void test_offline_history_scan_after_durable_snapshot_and_restart(void**) { + auto dir = std::filesystem::temp_directory_path() / ("miniraft-history-" + + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count())); + assert_true(std::filesystem::create_directory(dir)); + const auto name = (dir / "raft").string(); const auto path = name + ".durable.1"; + TConfigurationRecord joint{EConfigurationPhase::Joint, 2, 0, 0, {1}, {1, 2}}; + auto final = joint; final.Phase = EConfigurationPhase::Stable; final.Term = 3; + final.PreviousIndex = 2; final.PreviousTerm = 2; + { + auto state = std::make_shared(name, 1); + { TRaft bootstrap(state, 1, {}); } + state->Append(Ordinary(1)); state->InstallSnapshot({1, 1, {1}, "fixed image"}); + state->CurrentTerm = 3; state->Commit(); + state->Append(EncodeConfigurationEntry(joint)); state->Append(Ordinary(2)); + state->Append(EncodeConfigurationEntry(final)); + } + std::ifstream input(path, std::ios::binary); std::string before(std::istreambuf_iterator(input), {}); input.close(); + { + auto state = std::make_shared(name, 1); + // This fixture explicitly knows that index 1 precedes any config entry. + auto history = TConfigurationHistory::Stable({1}, 0, 0, state->SnapshotIndex, state->SnapshotTerm); + const auto scan = ScanConfigurationHistory(*state, history, 2); + assert_true(scan.Committed.IsJoint()); assert_false(scan.Latest.IsJoint()); + assert_true(scan.Latest.OldVoters() == final.NewVoters); + assert_true(scan.HasUncommittedConfiguration()); + history = history.Advance(2, state->Get(2)); + assert_true(history.IsJoint()); + history = history.Observe(3, state->Get(3)); + history = history.Advance(4, state->Get(4)); + assert_false(history.IsJoint()); assert_true(history.OldVoters() == final.NewVoters); + assert_int_equal(history.ConfigurationIndex(), 4); + Reject([&] { TRaft disabled(state, 1, {}); }); + assert_true(state->GetSnapshot()->Data == "fixed image"); + } + std::ifstream after(path, std::ios::binary); + assert_true(std::string(std::istreambuf_iterator(after), {}) == before); + std::error_code ignored; std::filesystem::remove_all(dir, ignored); +} +void test_raft_recovers_joint_quorum_for_candidate_election(void**) { + auto dir = std::filesystem::temp_directory_path() / ("miniraft-joint-quorum-" + + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count())); + assert_true(std::filesystem::create_directory(dir)); + const auto name = (dir / "raft").string(); + { + auto state = std::make_shared(name, 4); + state->BindVoters({1, 2, 3}); + state->CurrentTerm = 2; + state->Commit(); + TConfigurationRecord joint{EConfigurationPhase::Joint, 2, 0, 0, {1, 2, 3}, {3, 4, 5}}; + state->Append(EncodeConfigurationEntry(joint)); + state->Commit(); + + TNodeDict peers; + std::unordered_map> endpoints; + for (uint32_t id : {1, 2, 3, 5}) { + auto endpoint = std::make_shared(); + endpoints.emplace(id, endpoint); + peers.emplace(id, endpoint); + } + auto raft = std::make_shared(state, 4, peers, std::vector{4, 5}); + assert_true(raft->IsVoter(4)); + assert_true(raft->IsVoter(5)); + assert_int_equal(raft->GetNservers(), 5); + + auto now = ITimeSource::Time{}; + raft->ProcessTimeout(now); // self vote alone cannot satisfy either joint majority + assert_true(raft->CurrentStateName() == EState::CANDIDATE); + auto vote = [&](uint32_t source) { + raft->Process(now, NewHoldedMessage(TMessageEx{.Src = source, .Dst = 4, .Term = 3}, + TRequestVoteResponse{.VoteGranted = 1})); + raft->ProcessTimeout(now + std::chrono::milliseconds(1)); + }; + + vote(1); // This leaves both voter sets below their strict majority. + assert_true(raft->CurrentStateName() == EState::CANDIDATE); + vote(3); // 3 plus self satisfies new, while 1 plus 3 satisfies old. + assert_true(raft->CurrentStateName() == EState::LEADER); + + raft->ProcessTimeout(now + std::chrono::milliseconds(2)); + auto ack = [&](uint32_t source, uint64_t expectedIndex, int responseMillis) { + auto request = endpoints.at(source)->Messages.back().Cast(); + const auto matchIndex = request->PrevLogIndex + request->Nentries; + assert_int_equal(matchIndex, expectedIndex); + raft->Process(now + std::chrono::milliseconds(responseMillis), + NewHoldedMessage(TMessageEx{.Src = source, .Dst = 4, .Term = 3, .Seqno = request->Seqno}, + TAppendEntriesResponse{.MatchIndex = matchIndex, .Success = 1})); + }; + ack(3, 1, 3); + ack(5, 1, 3); + ack(1, 1, 3); + assert_int_equal(raft->GetVolatileState()->CommitIndex, 0); + + raft->ProcessTimeout(now + std::chrono::milliseconds(4)); + ack(3, 2, 5); + assert_int_equal(raft->GetVolatileState()->CommitIndex, 0); + ack(5, 2, 5); + assert_int_equal(raft->GetVolatileState()->CommitIndex, 0); + assert_int_equal(raft->CommitSeqno(), 0); + ack(1, 2, 5); + assert_int_equal(raft->GetVolatileState()->CommitIndex, 2); + assert_true(raft->CommitSeqno() > 0); + } + std::error_code ignored; std::filesystem::remove_all(dir, ignored); +} +void test_raft_keeps_joint_quorum_until_stable_successor_commits(void**) { + auto dir = std::filesystem::temp_directory_path() / ("miniraft-joint-finalize-" + + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count())); + assert_true(std::filesystem::create_directory(dir)); + auto state = std::make_shared((dir / "raft").string(), 4); + state->BindVoters({1, 2, 3}); + state->CurrentTerm = 3; + state->Commit(); + TConfigurationRecord joint{EConfigurationPhase::Joint, 2, 0, 0, {1, 2, 3}, {3, 4, 5}}; + state->Append(EncodeConfigurationEntry(joint)); + auto stable = joint; + stable.Phase = EConfigurationPhase::Stable; + stable.Term = 3; + stable.PreviousIndex = 1; + stable.PreviousTerm = 2; + state->Append(EncodeConfigurationEntry(stable)); + state->CommitIndex = 1; + state->Commit(); + + TNodeDict peers; + for (uint32_t id : {1, 2, 3, 5}) peers.emplace(id, std::make_shared()); + { + TRaft inJoint(state, 4, peers, std::vector{4, 5}); + assert_true(inJoint.IsVoter(1)); + assert_true(inJoint.IsVoter(5)); + assert_int_equal(inJoint.GetNservers(), 5); + } + + state->CommitIndex = 2; + state->Commit(); + { + TRaft afterStableCommit(state, 4, peers, std::vector{4, 5}); + assert_false(afterStableCommit.IsVoter(1)); + assert_false(afterStableCommit.IsVoter(2)); + assert_true(afterStableCommit.IsVoter(3)); + assert_true(afterStableCommit.IsVoter(5)); + assert_int_equal(afterStableCommit.GetNservers(), 3); + } + state.reset(); + std::error_code ignored; std::filesystem::remove_all(dir, ignored); +} +void test_raft_rejects_stable_snapshot_that_changes_active_voters(void**) { + auto state = std::make_shared(); + state->BindVoters({1, 2, 3}); + state->CurrentTerm = 5; + TSnapshot snapshot{5, 4, {2, 3, 4}, "stable image"}; + snapshot.StableConfiguration = true; + snapshot.ConfigurationIndex = 4; + snapshot.ConfigurationTerm = 3; + state->InstallSnapshot(snapshot); + + TNodeDict peers{ + {3, std::make_shared()}, + {4, std::make_shared()}, + }; + const auto fields = static_cast(*state); + Reject([&] { TRaft raft(state, 2, peers); }); + assert_true(static_cast(*state) == fields); + assert_true(state->GetBoundVoters() == std::vector({1, 2, 3})); +} + +void test_persisted_commit_boundary_drives_committed_configuration_scan(void**) { + auto dir = std::filesystem::temp_directory_path() / ("miniraft-committed-history-" + + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count())); + assert_true(std::filesystem::create_directory(dir)); + const auto name = (dir / "raft").string(); + TConfigurationRecord joint{EConfigurationPhase::Joint, 2, 0, 0, {1}, {1, 2}}; + auto final = joint; final.Phase = EConfigurationPhase::Stable; final.Term = 3; + final.PreviousIndex = 2; final.PreviousTerm = 2; + { + auto state = std::make_shared(name, 1); + { TRaft bootstrap(state, 1, {}); } + state->CurrentTerm = 3; state->Commit(); + state->Append(Ordinary(1)); + state->Append(EncodeConfigurationEntry(joint)); + state->Append(Ordinary(2)); + state->Append(EncodeConfigurationEntry(final)); + state->CommitIndex = 3; state->Commit(); + } + { + TDurableState state(name, 1); + auto baseline = TConfigurationHistory::Stable({1}, 0, 0, 0, 0); + const auto scan = ScanCommittedConfigurationHistory(state, baseline); + assert_int_equal(state.CommitIndex, 3); + assert_true(scan.Committed.IsJoint()); + assert_int_equal(scan.Committed.ConfigurationIndex(), 2); + assert_false(scan.Latest.IsJoint()); + assert_true(scan.Latest.OldVoters() == final.NewVoters); + assert_true(scan.HasUncommittedConfiguration()); + Reject([&] { + auto wrong = TConfigurationHistory::Stable({2}, 0, 0, 0, 0); + ScanCommittedConfigurationHistory(state, wrong); + }); + } + TState volatileState; volatileState.BindVoters({1}); + Reject([&] { + auto baseline = TConfigurationHistory::Stable({1}, 0, 0, 0, 0); + ScanCommittedConfigurationHistory(volatileState, baseline); + }); + std::error_code ignored; std::filesystem::remove_all(dir, ignored); +} + +void test_configuration_snapshots_preserve_original_binding_across_transitions(void**) { + auto dir = std::filesystem::temp_directory_path() / ("miniraft-config-snapshot-binding-" + + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count())); + assert_true(std::filesystem::create_directory(dir)); + const auto name = (dir / "raft").string(); + const std::vector original{1, 2, 3}; + const std::vector firstActive{2, 3, 4}; + const std::vector secondActive{2, 4, 5}; + const std::vector thirdActive{4, 5, 6}; + + TSnapshot jointSnapshot{5, 4, firstActive, "joint image"}; + jointSnapshot.JointVoters = secondActive; + jointSnapshot.ConfigurationIndex = 4; jointSnapshot.ConfigurationTerm = 3; + TConfigurationRecord stable{EConfigurationPhase::Stable, 5, 4, 3, firstActive, secondActive}; + + { + TDurableState state(name, 2); + state.BindVoters(original); + state.CurrentTerm = 4; state.Commit(); + state.InstallSnapshot(jointSnapshot); + assert_true(state.GetBoundVoters() == original); + assert_true(*state.GetSnapshot() == jointSnapshot); + } + + TSnapshot stableSnapshot{6, 5, secondActive, "stable image"}; + stableSnapshot.StableConfiguration = true; + stableSnapshot.ConfigurationIndex = 6; stableSnapshot.ConfigurationTerm = 5; + TConfigurationRecord nextJoint{EConfigurationPhase::Joint, 6, 6, 5, secondActive, thirdActive}; + { + TDurableState state(name, 2); + assert_true(state.GetBoundVoters() == original); + assert_true(*state.GetSnapshot() == jointSnapshot); + auto history = TConfigurationHistory::FromJointSnapshot(*state.GetSnapshot()); + state.CurrentTerm = 5; state.Commit(); + state.Append(EncodeConfigurationEntry(stable)); + state.CommitIndex = state.LastLogIndex; state.Commit(); + auto scan = ScanCommittedConfigurationHistory(state, history); + assert_false(scan.Committed.IsJoint()); + assert_true(scan.Committed.OldVoters() == secondActive); + + state.InstallSnapshot(stableSnapshot); + state.CurrentTerm = 6; state.Commit(); + state.Append(EncodeConfigurationEntry(nextJoint)); + state.CommitIndex = state.LastLogIndex; state.Commit(); + } + + { + auto state = std::make_shared(name, 2); + assert_true(state->GetBoundVoters() == original); + assert_true(*state->GetSnapshot() == stableSnapshot); + auto baseline = TConfigurationHistory::FromStableSnapshot(*state->GetSnapshot()); + const auto scan = ScanCommittedConfigurationHistory(*state, baseline); + assert_true(scan.Committed.IsJoint()); + assert_true(scan.Committed.OldVoters() == secondActive); + assert_true(scan.Committed.NewVoters() == thirdActive); + assert_false(scan.HasUncommittedConfiguration()); + + const auto fields = static_cast(*state); + Reject([&] { TRaft disabled(state, 2, {{4, std::make_shared()}, + {5, std::make_shared()}}); }); + assert_true(static_cast(*state) == fields); + assert_true(state->GetBoundVoters() == original); + } + std::error_code ignored; std::filesystem::remove_all(dir, ignored); +} +#endif +} +int main() { + const CMUnitTest tests[] = { + cmocka_unit_test(test_configuration_snapshot_projection_and_history_resume), + cmocka_unit_test(test_history_validates_joint_stable_chain_and_independent_rollback), + cmocka_unit_test(test_configuration_scan_compares_full_configuration_state), + cmocka_unit_test(test_history_from_joint_snapshot_requires_a_consistent_identity), + cmocka_unit_test(test_history_rejects_skips_wrong_predecessors_and_term_regression), + cmocka_unit_test(test_scan_requires_each_configuration_predecessor_to_be_committed), + cmocka_unit_test(test_scan_separates_commit_and_replaced_suffix), + cmocka_unit_test(test_scan_from_joint_snapshot_and_rejects_mismatched_baselines), + cmocka_unit_test(test_scan_from_stable_configuration_snapshot), + cmocka_unit_test(test_scan_fails_closed_on_invalid_suffix), + cmocka_unit_test(test_configuration_roundtrips_both_phases_and_bounded_maximum), + cmocka_unit_test(test_configuration_rejects_invalid_fields_and_valid_crc_corruption), + cmocka_unit_test(test_configuration_rpc_and_proposal_are_rejected_before_mutation), + cmocka_unit_test(test_joint_membership_addition_and_finalization_replicate_deterministically), + cmocka_unit_test(test_uncommitted_joint_suffix_replacement_restores_old_voters), + cmocka_unit_test(test_configuration_application_advances_without_calling_state_machine), +#ifdef __linux__ + cmocka_unit_test(test_durable_restart_recovers_stable_membership_roles), + cmocka_unit_test(test_durable_restart_rejects_reserved_entry_without_rewriting), + cmocka_unit_test(test_legacy_restart_uses_safe_flag_lookup_and_preserves_files), + cmocka_unit_test(test_offline_history_scan_after_durable_snapshot_and_restart), + cmocka_unit_test(test_raft_recovers_joint_quorum_for_candidate_election), + cmocka_unit_test(test_raft_keeps_joint_quorum_until_stable_successor_commits), + cmocka_unit_test(test_raft_rejects_stable_snapshot_that_changes_active_voters), + cmocka_unit_test(test_persisted_commit_boundary_drives_committed_configuration_scan), + cmocka_unit_test(test_configuration_snapshots_preserve_original_binding_across_transitions), +#endif + }; + return cmocka_run_group_tests(tests, nullptr, nullptr); +} diff --git a/test/test_correctness.cpp b/test/test_correctness.cpp new file mode 100644 index 0000000..a80e7ae --- /dev/null +++ b/test/test_correctness.cpp @@ -0,0 +1,933 @@ +#include +#include +#include +#include +#include +#include +#include +extern "C" { +#include +} + +namespace { +struct TEndpoint : INode { + std::function)> Deliver; + explicit TEndpoint(std::function)> f) : Deliver(std::move(f)) {} + void Send(TMessageHolder m) override { Deliver(std::move(m)); } + void Drain() override {} +}; + +// Virtual time and a manually scheduled network: no sockets, sleeps or wall clock. +struct TCluster { + ITimeSource::Time Now{}; + std::vector> Nodes; + std::deque> Packets; + std::set Isolated; + explicit TCluster(uint32_t count) { + for (uint32_t id = 1; id <= count; ++id) { + TNodeDict peers; + for (uint32_t dst = 1; dst <= count; ++dst) { + if (dst != id) peers[dst] = std::make_shared([this](auto m) { + Packets.push_back(std::move(m)); + }); + } + Nodes.push_back(std::make_shared(std::make_shared(), id, peers)); + } + } + void Restart(uint32_t id) { + auto state = Nodes.at(id - 1)->GetState(); + TNodeDict peers; + for (uint32_t dst = 1; dst <= Nodes.size(); ++dst) { + if (dst != id) peers[dst] = std::make_shared([this](auto m) { + Packets.push_back(std::move(m)); + }); + } + // Retain stable term/vote/log; discard all Raft volatile state. + Nodes.at(id - 1) = std::make_shared(state, id, peers); + } + void Drain(bool reverse = false, bool duplicate = false) { + size_t budget = 10000; + while (!Packets.empty()) { + assert_true(budget-- > 0); + auto m = reverse ? Packets.back() : Packets.front(); + if (reverse) Packets.pop_back(); else Packets.pop_front(); + auto ex = m.Cast(); + if (Isolated.count(ex->Src) || Isolated.count(ex->Dst)) continue; + Nodes.at(ex->Dst - 1)->Process(Now, m); + if (duplicate) Nodes.at(ex->Dst - 1)->Process(Now, m); + } + } + void Elect(uint32_t id) { + Now += std::chrono::seconds(10); + auto& node = Nodes.at(id - 1); + node->ProcessTimeout(Now); + Drain(); + node->ProcessTimeout(Now); + Drain(); + assert_int_equal(node->CurrentStateName(), EState::LEADER); + } + void Replicate(uint32_t id, bool reverse = false, bool duplicate = false) { + for (int round = 0; round < 12; ++round) { + Now += std::chrono::milliseconds(500); + Nodes.at(id - 1)->ProcessTimeout(Now); + Drain(reverse, duplicate); + } + } + void AssertCommittedPrefix() { + for (auto& a : Nodes) for (auto& b : Nodes) { + auto end = std::min(a->GetVolatileState()->CommitIndex, b->GetVolatileState()->CommitIndex); + for (uint64_t i = 1; i <= end; ++i) { + auto x = a->GetState()->Get(i), y = b->GetState()->Get(i); + assert_int_equal(x->Len, y->Len); + assert_memory_equal(x.Mes, y.Mes, x->Len); + } + } + } + TMessageHolder DeliverOne(uint32_t src, uint32_t dst) { + for (auto it = Packets.begin(); it != Packets.end(); ++it) { + auto ex = it->Cast(); + if (ex->Src == src && ex->Dst == dst) { + auto message = *it; + Packets.erase(it); + Nodes.at(dst - 1)->Process(Now, message); + return message; + } + } + fail_msg("Expected a queued packet from %u to %u", src, dst); + return {}; + } + void RoundTrip(uint32_t leader, uint32_t peer, bool duplicate = false) { + DeliverOne(leader, peer); + auto response = DeliverOne(peer, leader); + if (duplicate) Nodes.at(leader - 1)->Process(Now, response); + } +}; + +struct TReadClient { + struct TReply { uint32_t Cookie; uint32_t Error; }; + std::vector Replies; + std::shared_ptr Rsm = std::make_shared(); + TRequestProcessor Processor; + std::shared_ptr Endpoint; + explicit TReadClient(std::shared_ptr raft) + : Processor(raft, Rsm, {}) + , Endpoint(std::make_shared([this](auto message) { + auto response = message.template Cast(); + Replies.push_back({response->Cookie, response->ErrorCode}); + })) {} + void Read(uint32_t cookie, uint32_t flags = TCommandRequest::EConsistent) { + auto command = NewHoldedMessage(sizeof(TCommandRequest) + sizeof(int64_t)); + int64_t index = 0; + memcpy(command->Data, &index, sizeof(index)); + command->Cookie = cookie; + command->Flags = flags; + Processor.OnCommandRequest(command, Endpoint); + } + void Poll() { Processor.ProcessCommitted(); Processor.ProcessWaiting(); } +}; + +struct TEchoRsm : IRsm { + std::vector Applied; + std::weak_ptr LastReply; + TMessageHolder Read(TMessageHolder, uint64_t index) override { + return NewHoldedMessage(TCommandResponse{.Index = index}); + } + TMessageHolder Prepare(TMessageHolder command) override { + const auto size = command->Len - sizeof(TCommandRequest); + auto entry = NewHoldedMessage(sizeof(TLogEntry) + size); + memcpy(entry->Data, command->Data, size); + return entry; + } + TMessageHolder Write(TMessageHolder entry, uint64_t index) override { + const auto size = entry->Len - sizeof(TLogEntry); + Applied.emplace_back(entry->Data, size); + LastAppliedIndex = index; + auto reply = NewHoldedMessage(sizeof(TCommandResponse) + size); + reply->Index = index; + memcpy(reply->Data, entry->Data, size); + LastReply = reply.RawData; + return reply; + } +}; +struct TWriteClient { + struct TReply { uint32_t Cookie, Error; std::string Data; }; + std::vector Replies; + std::shared_ptr Rsm = std::make_shared(); + TRequestProcessor Processor; + std::shared_ptr Endpoint; + explicit TWriteClient(std::shared_ptr raft) + : Processor(raft, Rsm, {}) + , Endpoint(std::make_shared([this](auto message) { + auto response = message.template Cast(); + Replies.push_back({response->Cookie, response->ErrorCode, + std::string(response->Data, response->Len - sizeof(TCommandResponse))}); + })) {} + void Write(uint32_t cookie, const std::string& value) { + auto command = NewHoldedMessage(sizeof(TCommandRequest) + value.size()); + command->Cookie = cookie; + command->Flags = TCommandRequest::EWrite; + memcpy(command->Data, value.data(), value.size()); + Processor.OnCommandRequest(command, Endpoint); + } + void Read(uint32_t cookie, uint32_t flags = 0) { + auto command = NewHoldedMessage(); + command->Cookie = cookie; + command->Flags = flags; + Processor.OnCommandRequest(command, Endpoint); + } + void Poll() { Processor.ProcessCommitted(); Processor.ProcessWaiting(); } +}; +TMessageHolder LogValue(const std::string& value) { + auto entry = NewHoldedMessage(sizeof(TLogEntry) + value.size()); + memcpy(entry->Data, value.data(), value.size()); + return entry; +} + +void test_nonleader_proposal_rejection_preserves_store_and_input(void**) { + for (int phase = 0; phase < 3; ++phase) { + TCluster cluster(3); + auto raft = cluster.Nodes[0]; + if (phase == 1) { + raft->ProcessTimeout(cluster.Now); + assert_int_equal(raft->CurrentStateName(), EState::CANDIDATE); + } else if (phase == 2) { + cluster.Elect(1); cluster.Replicate(1); + auto vote = NewHoldedMessage( + TMessageEx{.Src = 2, .Dst = 1, .Term = raft->GetState()->CurrentTerm + 1}, + TRequestVoteRequest{.LastLogIndex = raft->GetLastIndex(), + .LastLogTerm = raft->GetState()->LastLogTerm, .CandidateId = 2}); + raft->Process(cluster.Now, vote); + assert_int_equal(raft->CurrentStateName(), EState::FOLLOWER); + } + const auto before = static_cast(*raft->GetState()); + auto entry = LogValue("must not be proposed"); entry->Term = 77; + bool rejected = false; + try { raft->Append(entry); } catch (const std::logic_error&) { rejected = true; } + assert_true(rejected); + assert_true(static_cast(*raft->GetState()) == before); + assert_int_equal(entry->Term, 77); + } +} +void test_mutating_proposal_after_append_cannot_change_committed_command(void**) { + TCluster cluster(3); cluster.Elect(1); cluster.Replicate(1); + TWriteClient leader(cluster.Nodes[0]), follower(cluster.Nodes[1]), other(cluster.Nodes[2]); + auto entry = LogValue("valid"); + auto index = cluster.Nodes[0]->Append(entry); + memcpy(entry->Data, "rogue", 5); + cluster.Replicate(1, true, true); + leader.Poll(); follower.Poll(); other.Poll(); + for (auto* client : {&leader, &follower, &other}) { + assert_int_equal(client->Rsm->Applied.size(), 1); + assert_string_equal(client->Rsm->Applied[0].c_str(), "valid"); + } + assert_true(cluster.Nodes[0]->GetVolatileState()->CommitIndex >= index); + cluster.AssertCommittedPrefix(); +} + +void test_follower_proposal_cannot_diverge_same_term_committed_command(void**) { + TCluster cluster(3); cluster.Elect(1); cluster.Replicate(1); + TWriteClient leader(cluster.Nodes[0]), follower(cluster.Nodes[1]), other(cluster.Nodes[2]); + bool rejected = false; + try { cluster.Nodes[1]->Append(LogValue("rogue")); } + catch (const std::logic_error&) { rejected = true; } + leader.Write(77, "valid"); // Same index/term as the unauthorized follower entry on old code. + cluster.Replicate(1, true, true); + leader.Poll(); follower.Poll(); other.Poll(); + for (auto* client : {&leader, &follower, &other}) { + assert_int_equal(client->Rsm->Applied.size(), 1); + assert_string_equal(client->Rsm->Applied[0].c_str(), "valid"); + } + cluster.AssertCommittedPrefix(); + assert_true(rejected); + assert_int_equal(leader.Replies.size(), 1); + assert_int_equal(leader.Replies[0].Error, 0); +} + +void test_forwarded_write_and_strong_read_complete_through_real_raft(void**) { + TCluster cluster(3); + cluster.Elect(1); + cluster.Replicate(1); + std::deque> requests; + std::deque> responses; + auto peer = std::make_shared([&](auto message) { + requests.push_back(message.template Cast()); + }); + auto returnPath = std::make_shared([&](auto message) { + responses.push_back(message.template Cast()); + }); + auto leaderRsm = std::make_shared(); + auto originRsm = std::make_shared(); + TRequestProcessor leader(cluster.Nodes[0], leaderRsm, {}); + TRequestProcessor origin(cluster.Nodes[1], originRsm, TNodeDict{{1, peer}}); + std::vector cookies; + auto client = std::make_shared([&](auto message) { + auto response = message.template Cast(); + assert_int_equal(response->ErrorCode, 0); + cookies.push_back(response->Cookie); + }); + for (uint32_t cookie : {42, 77}) { + auto command = NewHoldedMessage(sizeof(TCommandRequest) + 5); + command->Cookie = cookie; + command->Flags = cookie == 42 ? TCommandRequest::EWrite : TCommandRequest::EConsistent; + memcpy(command->Data, "value", 5); + origin.OnCommandRequest(command, client); + assert_int_equal(requests.size(), 1); + leader.OnCommandRequest(requests.front(), returnPath); + requests.pop_front(); + cluster.Replicate(1, true, true); + leader.ProcessCommitted(); + leader.ProcessWaiting(); + assert_int_equal(responses.size(), 1); + origin.OnCommandResponse(responses.front(), peer); + origin.OnCommandResponse(responses.front(), peer); + responses.pop_front(); + assert_int_equal(cookies.back(), cookie); + } + assert_int_equal(cookies.size(), 2); + assert_int_equal(leaderRsm->Applied.size(), 1); + assert_string_equal(leaderRsm->Applied[0].c_str(), "value"); + assert_true(originRsm->Applied.empty()); +} + +void test_replaced_write_never_receives_another_commands_reply(void**) { + TCluster cluster(3); + cluster.Elect(1); + cluster.Replicate(1); + cluster.Nodes[0]->Append(LogValue("uncommitted-padding")); + TWriteClient client(cluster.Nodes[0]); + client.Write(11, "lost-command"); // index 3, old term + cluster.Isolated = {1}; + cluster.Elect(2); // new-term no-op at index 2 + cluster.Nodes[1]->Append(LogValue("replacement-command")); // index 3, new term + cluster.Replicate(2); + cluster.Isolated.clear(); + cluster.Replicate(2); + client.Poll(); + assert_int_equal(client.Replies.size(), 1); + assert_int_equal(client.Replies[0].Cookie, 11); + assert_int_equal(client.Replies[0].Error, 1); + assert_true(client.Replies[0].Data.empty()); + assert_int_equal(client.Rsm->Applied.size(), 1); + assert_string_equal(client.Rsm->Applied[0].c_str(), "replacement-command"); + client.Poll(); + assert_int_equal(client.Replies.size(), 1); +} + +void test_stepdown_rejects_write_without_waiting_for_application(void**) { + TCluster cluster(3); + cluster.Elect(1); + cluster.Replicate(1); + TWriteClient client(cluster.Nodes[0]); + client.Write(11, "pending"); + cluster.Elect(2); + client.Processor.CheckStateChange(); + assert_int_equal(client.Replies.size(), 1); + assert_int_equal(client.Replies[0].Error, 1); +} + +void test_committed_but_unanswered_write_has_unknown_outcome_on_stepdown(void**) { + TCluster cluster(3); + cluster.Elect(1); + cluster.Replicate(1); + TWriteClient client(cluster.Nodes[0]); + client.Write(11, "committed"); + cluster.Replicate(1); + client.Processor.ProcessCommitted(); // application reply cached, not sent + assert_false(client.Rsm->LastReply.expired()); + cluster.Elect(2); + client.Poll(); + assert_int_equal(client.Replies.size(), 1); + assert_int_equal(client.Replies[0].Error, 1); + assert_int_equal(client.Rsm->Applied.size(), 1); + assert_string_equal(client.Rsm->Applied[0].c_str(), "committed"); + assert_true(client.Rsm->LastReply.expired()); + client.Poll(); + assert_int_equal(client.Rsm->Applied.size(), 1); + assert_int_equal(client.Replies.size(), 1); +} + +void test_old_write_is_rejected_after_unobserved_reelection(void**) { + TCluster cluster(3); + cluster.Elect(1); + cluster.Replicate(1); + TWriteClient client(cluster.Nodes[0]); + client.Write(11, "lost"); + cluster.Elect(2); + cluster.Elect(1); + cluster.Replicate(1); + client.Write(22, "new"); + assert_int_equal(client.Replies.size(), 1); + assert_int_equal(client.Replies[0].Cookie, 11); + assert_int_equal(client.Replies[0].Error, 1); + cluster.Replicate(1); + client.Poll(); + assert_int_equal(client.Replies.size(), 2); + assert_int_equal(client.Replies[1].Cookie, 22); + assert_int_equal(client.Replies[1].Error, 0); + assert_string_equal(client.Replies[1].Data.c_str(), "new"); +} + +void test_default_read_of_lost_tail_is_rejected_on_stepdown(void**) { + TCluster cluster(3); + cluster.Elect(1); + cluster.Replicate(1); + cluster.Nodes[0]->Append(LogValue("lost-1")); + cluster.Nodes[0]->Append(LogValue("lost-2")); + TWriteClient client(cluster.Nodes[0]); + client.Read(11); + cluster.Elect(2); + client.Processor.CheckStateChange(); + assert_int_equal(client.Replies.size(), 1); + assert_int_equal(client.Replies[0].Error, 1); +} + +void test_request_waiting_for_leader_runs_when_local_node_is_elected(void**) { + for (uint32_t count : {1, 3}) { + TCluster cluster(count); + TWriteClient client(cluster.Nodes[0]); + client.Write(0, "waiting-for-election"); + cluster.Elect(1); + client.Processor.CheckStateChange(); + assert_int_equal(cluster.Nodes[0]->GetLastIndex(), 2); + cluster.Replicate(1); + client.Poll(); + assert_int_equal(client.Replies.size(), 1); + assert_int_equal(client.Replies[0].Error, 0); + assert_string_equal(client.Replies[0].Data.c_str(), "waiting-for-election"); + } +} + +void test_write_replaced_by_noop_is_rejected_without_application_reply(void**) { + TCluster cluster(3); + cluster.Elect(1); + cluster.Replicate(1); + TWriteClient client(cluster.Nodes[0]); + client.Write(11, "lost"); + cluster.Elect(2); + cluster.Replicate(2); + client.Poll(); + assert_int_equal(client.Replies.size(), 1); + assert_int_equal(client.Replies[0].Cookie, 11); + assert_int_equal(client.Replies[0].Error, 1); + assert_true(client.Rsm->Applied.empty()); +} + +void test_stale_read_does_not_wait_for_uncommitted_tail(void**) { + TCluster cluster(3); + cluster.Elect(1); + cluster.Replicate(1); + cluster.Nodes[0]->Append(LogValue("uncommitted")); + TWriteClient client(cluster.Nodes[0]); + client.Read(11, TCommandRequest::EStale); + client.Poll(); + assert_int_equal(client.Replies.size(), 1); + assert_int_equal(client.Replies[0].Error, 0); + assert_true(client.Rsm->Applied.empty()); +} + +void test_disconnect_removes_local_write_and_read_waiters(void**) { + TCluster cluster(3); + cluster.Elect(1); + cluster.Replicate(1); + TWriteClient client(cluster.Nodes[0]); + client.Write(11, "still-may-commit"); + client.Read(22); + client.Read(33, TCommandRequest::EConsistent); + std::vector otherReplies; + auto other = std::make_shared([&](auto message) { + auto reply = message.template Cast(); + assert_int_equal(reply->ErrorCode, 0); + otherReplies.push_back(reply->Cookie); + }); + auto command = NewHoldedMessage(sizeof(TCommandRequest) + 5); + command->Cookie = 44; + command->Flags = TCommandRequest::EWrite; + memcpy(command->Data, "other", 5); + client.Processor.OnCommandRequest(command, other); + std::weak_ptr endpoint = client.Endpoint; + client.Processor.CleanUp(client.Endpoint); + client.Endpoint.reset(); + assert_true(endpoint.expired()); + cluster.Replicate(1); + client.Poll(); + assert_true(client.Replies.empty()); + assert_int_equal(client.Rsm->Applied.size(), 2); + assert_int_equal(otherReplies.size(), 1); + assert_int_equal(otherReplies[0], 44); + assert_true(client.Rsm->LastReply.expired()); +} + +void test_disconnect_removes_requests_waiting_for_leader(void**) { + TCluster cluster(3); + TWriteClient client(cluster.Nodes[0]); + client.Write(0, "disconnected-before-election"); + std::weak_ptr endpoint = client.Endpoint; + client.Processor.CleanUp(client.Endpoint); + client.Endpoint.reset(); + assert_true(endpoint.expired()); + cluster.Elect(1); + client.Processor.CheckStateChange(); + assert_int_equal(cluster.Nodes[0]->GetLastIndex(), 1); +} + +void test_disconnect_releases_cached_write_reply(void**) { + TCluster cluster(3); + cluster.Elect(1); + cluster.Replicate(1); + TWriteClient client(cluster.Nodes[0]); + client.Write(11, "committed"); + cluster.Replicate(1); + client.Processor.ProcessCommitted(); + assert_false(client.Rsm->LastReply.expired()); + client.Processor.CleanUp(client.Endpoint); + assert_true(client.Rsm->LastReply.expired()); + client.Poll(); + assert_true(client.Replies.empty()); +} + +void test_follower_application_does_not_retain_unused_replies(void**) { + TCluster cluster(3); + cluster.Elect(1); + cluster.Replicate(1); + TWriteClient follower(cluster.Nodes[1]); + cluster.Nodes[0]->Append(LogValue("background-command")); + cluster.Replicate(1); + follower.Processor.ProcessCommitted(); + assert_int_equal(follower.Rsm->Applied.size(), 1); + assert_true(follower.Rsm->LastReply.expired()); +} + +void test_strong_reads_require_fresh_majority_in_all_cluster_sizes(void**) { + for (uint32_t count = 1; count <= 8; ++count) { + TCluster cluster(count); + cluster.Elect(1); + cluster.Replicate(1); + TReadClient client(cluster.Nodes[0]); + client.Read(11); + client.Poll(); + assert_int_equal(client.Replies.size(), count == 1 ? 1 : 0); + for (uint32_t peer = 2; peer <= count; ++peer) { + cluster.RoundTrip(1, peer, true); + client.Poll(); + assert_int_equal(client.Replies.size(), peer >= count / 2 + 1 ? 1 : 0); + } + assert_int_equal(client.Replies[0].Cookie, 11); + assert_int_equal(client.Replies[0].Error, 0); + } +} + +void test_stale_leader_with_one_follower_cannot_serve_strong_read(void**) { + TCluster cluster(5); + cluster.Elect(1); + cluster.Replicate(1); + // The majority elects a new leader and commits a later entry while the + // old leader remains connected to one old-term follower. + cluster.Isolated = {1, 2}; + cluster.Elect(3); + auto latest = cluster.Nodes[2]->Append(NewHoldedMessage()); + cluster.Replicate(3); + assert_true(cluster.Nodes[2]->GetVolatileState()->CommitIndex >= latest); + cluster.Isolated.clear(); + TReadClient client(cluster.Nodes[0]); + client.Read(11); + cluster.RoundTrip(1, 2, true); + client.Poll(); + assert_true(client.Replies.empty()); + // A current-term leader's follower reveals the higher term. Pending reads + // must fail, rather than serving the old leader's state. + cluster.RoundTrip(1, 3); + client.Poll(); + assert_int_equal(client.Replies.size(), 1); + assert_int_equal(client.Replies[0].Error, 1); +} + +void test_new_read_cannot_reuse_previous_read_quorum(void**) { + TCluster cluster(5); + cluster.Elect(1); + cluster.Replicate(1); + TReadClient client(cluster.Nodes[0]); + client.Read(11); + cluster.RoundTrip(1, 2); + cluster.RoundTrip(1, 3); + client.Poll(); + assert_int_equal(client.Replies.size(), 1); + client.Read(22); + cluster.RoundTrip(1, 2, true); + client.Poll(); + assert_int_equal(client.Replies.size(), 1); + cluster.RoundTrip(1, 3); + client.Poll(); + assert_int_equal(client.Replies.size(), 2); + assert_int_equal(client.Replies[1].Cookie, 22); +} + +void test_ready_read_is_cancelled_on_leadership_change(void**) { + TCluster cluster(3); + cluster.Elect(1); + cluster.Replicate(1); + TReadClient client(cluster.Nodes[0]); + client.Read(11); + cluster.RoundTrip(1, 2); + cluster.Elect(2); // Lose leadership before the application processes reads. + client.Poll(); + assert_int_equal(client.Replies.size(), 1); + assert_int_equal(client.Replies[0].Error, 1); +} + +void test_consistent_flag_takes_precedence_over_stale(void**) { + TCluster cluster(3); + cluster.Elect(1); + cluster.Replicate(1); + TReadClient client(cluster.Nodes[0]); + client.Read(11, TCommandRequest::EConsistent | TCommandRequest::EStale); + client.Poll(); + assert_true(client.Replies.empty()); + cluster.RoundTrip(1, 2); + client.Poll(); + assert_int_equal(client.Replies.size(), 1); + assert_int_equal(client.Replies[0].Error, 0); +} + +void test_delayed_pre_read_replies_cannot_confirm_new_read(void**) { + TCluster cluster(5); + cluster.Elect(1); + cluster.Replicate(1); + cluster.Nodes[0]->ProcessTimeout(cluster.Now += TTimeout::Election); + for (uint32_t peer = 2; peer <= 5; ++peer) cluster.DeliverOne(1, peer); + TReadClient client(cluster.Nodes[0]); + client.Read(11); + for (uint32_t peer = 2; peer <= 5; ++peer) cluster.DeliverOne(peer, 1); + client.Poll(); + assert_true(client.Replies.empty()); + cluster.RoundTrip(1, 2, true); + client.Poll(); + assert_true(client.Replies.empty()); + cluster.RoundTrip(1, 3); + client.Poll(); + assert_int_equal(client.Replies.size(), 1); +} + +void test_new_leader_waits_for_current_term_commit_and_application(void**) { + TCluster cluster(3); + for (auto& node : cluster.Nodes) { + node->GetState()->Append(NewHoldedMessage()); + const_cast(node->GetVolatileState())->CommitIndex = 1; + } + cluster.Elect(1); + // First RPCs probe the old prefix; the new-term no-op is not committed yet. + assert_int_equal(cluster.Nodes[0]->GetVolatileState()->CommitIndex, 1); + assert_int_equal(cluster.Nodes[0]->CommitSeqno(), 0); + TReadClient client(cluster.Nodes[0]); + client.Read(11); + client.Poll(); + assert_true(client.Replies.empty()); + cluster.RoundTrip(1, 2); + assert_int_equal(cluster.Nodes[0]->GetVolatileState()->CommitIndex, 2); + // Fresh confirmation alone is insufficient until the local state machine + // has applied the read's target prefix. + client.Processor.ProcessWaiting(); + assert_true(client.Replies.empty()); + client.Poll(); + assert_int_equal(client.Replies.size(), 1); + assert_int_equal(client.Replies[0].Error, 0); +} + +void test_old_term_read_is_rejected_after_reelection(void**) { + TCluster cluster(3); + cluster.Elect(1); + cluster.Replicate(1); + TReadClient client(cluster.Nodes[0]); + client.Read(11); + cluster.Elect(2); + cluster.Elect(1); + cluster.Replicate(1); + // No application poll observed the intermediate follower state. The term + // tag must still reject the old operation before accepting another read. + client.Read(22); + assert_int_equal(client.Replies.size(), 1); + assert_int_equal(client.Replies[0].Cookie, 11); + assert_int_equal(client.Replies[0].Error, 1); + cluster.RoundTrip(1, 2); + client.Poll(); + assert_int_equal(client.Replies.size(), 2); + assert_int_equal(client.Replies[1].Cookie, 22); + assert_int_equal(client.Replies[1].Error, 0); +} + +void test_majority_all_cluster_sizes(void**) { + for (int count = 1; count <= 8; ++count) { + TState state; + state.Append(NewHoldedMessage()); + for (int copies = 1; copies <= count; ++copies) { + TVolatileState v; + for (int peer = 1; peer < copies; ++peer) v.MatchIndex[peer + 1] = 1; + v.CommitAdvance(count, state); + assert_int_equal(v.CommitIndex, copies >= count / 2 + 1 ? 1 : 0); + } + } +} + +void CheckRequests(uint32_t flags, bool partial) { + auto state = std::make_shared(); + auto raft = std::make_shared(state, 1, TNodeDict{}); + if (flags & TCommandRequest::EConsistent) raft->ProcessTimeout(ITimeSource::Time{}); + else raft->Become(EState::LEADER); + auto rsm = std::make_shared(); + TRequestProcessor processor(raft, rsm, {}); + std::vector replies; + auto client = std::make_shared([&](auto m) { + replies.push_back(m.template Cast()->Cookie); + }); + for (uint32_t cookie : {11, 22}) { + auto command = NewHoldedMessage(sizeof(TCommandRequest) + sizeof(int64_t)); + int64_t readIndex = 0; + memcpy(command->Data, &readIndex, sizeof(readIndex)); + command->Flags = flags; + command->Cookie = cookie; + processor.OnCommandRequest(command, client); + } + if (partial) { + // Only the first write is committed; a later request must not block it. + const_cast(raft->GetVolatileState())->CommitIndex = 1; + processor.ProcessCommitted(); + processor.ProcessWaiting(); + assert_int_equal(replies.size(), 1); + assert_int_equal(replies[0], 11); + } + raft->ProcessTimeout(ITimeSource::Time{}); + processor.ProcessCommitted(); + processor.ProcessWaiting(); + assert_int_equal(replies.size(), 2); + assert_int_equal(replies[0], 11); + assert_int_equal(replies[1], 22); + processor.ProcessWaiting(); + assert_int_equal(replies.size(), 2); +} +void test_read_fifo(void**) { CheckRequests(0, false); } +void test_strong_read_fifo(void**) { CheckRequests(TCommandRequest::EConsistent, false); } +void test_write_fifo(void**) { CheckRequests(TCommandRequest::EWrite, false); } +void test_partial_write_completion(void**) { CheckRequests(TCommandRequest::EWrite, true); } + +void test_loss_reorder_duplicate_and_recovery(void**) { + TCluster cluster(3); + cluster.Elect(1); + auto index = cluster.Nodes[0]->Append(NewHoldedMessage()); + cluster.Nodes[0]->ProcessTimeout(cluster.Now += std::chrono::seconds(1)); + cluster.Packets.clear(); // Drop a complete replication round. + assert_true(cluster.Nodes[0]->GetVolatileState()->CommitIndex < index); + cluster.Replicate(1, true, true); + for (auto& node : cluster.Nodes) assert_int_equal(node->GetVolatileState()->CommitIndex, index); + cluster.AssertCommittedPrefix(); +} + +void test_partition_leader_replacement_and_heal(void**) { + TCluster cluster(3); + cluster.Elect(1); + auto committed = cluster.Nodes[0]->Append(NewHoldedMessage()); + cluster.Replicate(1); + cluster.Nodes[0]->ProcessTimeout(cluster.Now += std::chrono::seconds(1)); + auto delayed = std::move(cluster.Packets); + cluster.Packets.clear(); + cluster.Isolated.insert(1); + auto lost = NewHoldedMessage(); + lost->Flags = 8; // Distinguish the minority's uncommitted entry. + auto minority = cluster.Nodes[0]->Append(lost); + cluster.Replicate(1); + assert_int_equal(cluster.Nodes[0]->GetVolatileState()->CommitIndex, committed); + cluster.Elect(2); + auto replacement = cluster.Nodes[1]->Append(NewHoldedMessage()); + cluster.Replicate(2); + assert_true(cluster.Nodes[1]->GetVolatileState()->CommitIndex >= replacement); + cluster.AssertCommittedPrefix(); + cluster.Isolated.clear(); + // Old-term traffic arrives after the new leader has committed its entry. + cluster.Packets = std::move(delayed); + cluster.Drain(true, true); + cluster.Replicate(2, true, true); + assert_int_equal(cluster.Nodes[0]->CurrentStateName(), EState::FOLLOWER); + for (auto& node : cluster.Nodes) { + assert_int_equal(node->GetVolatileState()->CommitIndex, replacement); + assert_int_equal(node->GetState()->Get(minority)->Flags, TLogEntry::EStub); + } + cluster.AssertCommittedPrefix(); +} + +void test_even_cluster_partition_requires_majority(void**) { + TCluster cluster(4); + cluster.Elect(1); + cluster.Replicate(1); + auto previous = cluster.Nodes[0]->GetVolatileState()->CommitIndex; + cluster.Isolated = {3, 4}; + auto index = cluster.Nodes[0]->Append(NewHoldedMessage()); + cluster.Replicate(1); + assert_int_equal(cluster.Nodes[0]->GetVolatileState()->CommitIndex, previous); + assert_int_equal(cluster.Nodes[1]->GetVolatileState()->CommitIndex, previous); + cluster.Isolated.clear(); + cluster.Replicate(1, true, true); + for (auto& node : cluster.Nodes) assert_int_equal(node->GetVolatileState()->CommitIndex, index); + cluster.AssertCommittedPrefix(); +} +std::shared_ptr RecoverApplication(const TEchoRsm& old) { + auto recovered = std::make_shared(); + recovered->Applied = old.Applied; + recovered->LastAppliedIndex = old.LastAppliedIndex; + return recovered; +} +void test_recovered_application_progress_is_not_rewound(void**) { + TCluster cluster(1); + cluster.Elect(1); + cluster.Nodes[0]->Append(LogValue("once")); + cluster.Replicate(1); + auto original = std::make_shared(); + { TRequestProcessor processor(cluster.Nodes[0], original, {}); processor.ProcessCommitted(); } + auto recovered = RecoverApplication(*original); + cluster.Restart(1); + TRequestProcessor processor(cluster.Nodes[0], recovered, {}); + processor.ProcessCommitted(); // timer before the first post-restart election + assert_int_equal(recovered->LastAppliedIndex, original->LastAppliedIndex); + cluster.Elect(1); + processor.ProcessCommitted(); + processor.ProcessCommitted(); + assert_true(recovered->Applied == std::vector{"once"}); +} +void test_application_progress_never_decreases_on_lower_commit_input(void**) { + TCluster cluster(1); + cluster.Elect(1); + auto recovered = std::make_shared(); + recovered->LastAppliedIndex = 1; + TRequestProcessor processor(cluster.Nodes[0], recovered, {}); + // Defensive check even if an embedder violates commit-index monotonicity. + const_cast(cluster.Nodes[0]->GetVolatileState())->CommitIndex = 0; + processor.ProcessCommitted(); + assert_int_equal(recovered->LastAppliedIndex, 1); +} +void test_recovered_applied_prefix_is_protected_and_suffix_can_change(void**) { + TCluster cluster(2); + auto state = cluster.Nodes[0]->GetState(); + state->CurrentTerm = 2; + for (const std::string value : {"applied", "uncommitted"}) { + auto entry = LogValue(value); + entry->Term = 2; + state->Append(entry); + } + auto recovered = std::make_shared(); + recovered->Applied = {"applied"}; + recovered->LastAppliedIndex = 1; + TRequestProcessor processor(cluster.Nodes[0], recovered, {}); + auto request = [&](uint64_t prev, uint64_t prevTerm, const std::string& value) { + auto message = NewHoldedMessage(TMessageEx{.Src = 2, .Dst = 1, .Term = 3}, + TAppendEntriesRequest{.PrevLogIndex = prev, .PrevLogTerm = prevTerm, + .LeaderCommit = 2, .LeaderId = 2, .Nentries = 1}); + message.InitPayload(1); + auto entry = LogValue(value); + entry->Term = 3; + message.Payload[0] = entry; + return message; + }; + cluster.Nodes[0]->Process(cluster.Now, request(0, 0, "overwrite-applied")); + assert_int_equal(state->LogTerm(1), 2); + assert_int_equal(cluster.Nodes[0]->GetVolatileState()->CommitIndex, 1); + cluster.Nodes[0]->Process(cluster.Now, request(1, 2, "replacement")); + processor.ProcessCommitted(); + assert_true(recovered->Applied == (std::vector{"applied", "replacement"})); + assert_int_equal(recovered->LastAppliedIndex, 2); +} +void test_application_ahead_of_recovered_log_is_rejected(void**) { + TCluster cluster(1); + for (const auto index : {uint64_t(1), UINT64_MAX}) { + auto recovered = std::make_shared(); + recovered->LastAppliedIndex = index; + bool rejected = false; + try { TRequestProcessor processor(cluster.Nodes[0], recovered, {}); } + catch (const std::invalid_argument&) { rejected = true; } + assert_true(rejected); + assert_true(recovered->LastAppliedIndex == index); + assert_int_equal(cluster.Nodes[0]->GetVolatileState()->CommitIndex, 0); + assert_int_equal(cluster.Nodes[0]->GetLastIndex(), 0); + } +} +void test_cluster_restart_preserves_applied_history_and_rebuilds_volatile_application(void**) { + TCluster cluster(3); + cluster.Elect(1); + cluster.Nodes[0]->Append(LogValue("first")); + cluster.Replicate(1); + std::vector> recovered; + for (auto& node : cluster.Nodes) { + auto rsm = std::make_shared(); + TRequestProcessor processor(node, rsm, {}); + processor.ProcessCommitted(); + recovered.push_back(RecoverApplication(*rsm)); + } + cluster.Nodes[0]->Append(LogValue("old-uncommitted-tail")); + // Keep delayed old-term traffic across all node restarts. + cluster.Nodes[0]->ProcessTimeout(cluster.Now + std::chrono::seconds(1)); + auto delayed = std::move(cluster.Packets); + cluster.Packets.clear(); + for (uint32_t id = 1; id <= 3; ++id) cluster.Restart(id); + // Node 3 instead uses a volatile application and must rebuild from index zero. + recovered[2] = std::make_shared(); + std::vector> processors; + for (size_t i = 0; i < recovered.size(); ++i) { + processors.push_back(std::make_unique(cluster.Nodes[i], recovered[i], TNodeDict{})); + processors.back()->ProcessCommitted(); + } + cluster.Elect(2); + // Recovered old-term commitment cannot replace a fresh current-term read quorum. + assert_int_equal(cluster.Nodes[1]->CommitSeqno(), 0); + cluster.Nodes[1]->Append(LogValue("second")); + cluster.Replicate(2, true, true); + cluster.Packets = std::move(delayed); + cluster.Drain(true, true); + cluster.Replicate(2, true, true); + for (auto& processor : processors) { processor->ProcessCommitted(); processor->ProcessCommitted(); } + for (const auto& rsm : recovered) { + assert_true(rsm->Applied == (std::vector{"first", "second"})); + assert_int_equal(rsm->LastAppliedIndex, cluster.Nodes[1]->GetVolatileState()->CommitIndex); + } + cluster.AssertCommittedPrefix(); +} + +} + +int main() { + const CMUnitTest tests[] = { + cmocka_unit_test(test_nonleader_proposal_rejection_preserves_store_and_input), + cmocka_unit_test(test_mutating_proposal_after_append_cannot_change_committed_command), + cmocka_unit_test(test_follower_proposal_cannot_diverge_same_term_committed_command), + cmocka_unit_test(test_recovered_application_progress_is_not_rewound), + cmocka_unit_test(test_application_progress_never_decreases_on_lower_commit_input), + cmocka_unit_test(test_recovered_applied_prefix_is_protected_and_suffix_can_change), + cmocka_unit_test(test_application_ahead_of_recovered_log_is_rejected), + cmocka_unit_test(test_cluster_restart_preserves_applied_history_and_rebuilds_volatile_application), + cmocka_unit_test(test_majority_all_cluster_sizes), + cmocka_unit_test(test_read_fifo), + cmocka_unit_test(test_strong_read_fifo), + cmocka_unit_test(test_write_fifo), + cmocka_unit_test(test_partial_write_completion), + cmocka_unit_test(test_loss_reorder_duplicate_and_recovery), + cmocka_unit_test(test_partition_leader_replacement_and_heal), + cmocka_unit_test(test_even_cluster_partition_requires_majority), + cmocka_unit_test(test_strong_reads_require_fresh_majority_in_all_cluster_sizes), + cmocka_unit_test(test_stale_leader_with_one_follower_cannot_serve_strong_read), + cmocka_unit_test(test_new_read_cannot_reuse_previous_read_quorum), + cmocka_unit_test(test_ready_read_is_cancelled_on_leadership_change), + cmocka_unit_test(test_consistent_flag_takes_precedence_over_stale), + cmocka_unit_test(test_delayed_pre_read_replies_cannot_confirm_new_read), + cmocka_unit_test(test_new_leader_waits_for_current_term_commit_and_application), + cmocka_unit_test(test_old_term_read_is_rejected_after_reelection), + cmocka_unit_test(test_replaced_write_never_receives_another_commands_reply), + cmocka_unit_test(test_stepdown_rejects_write_without_waiting_for_application), + cmocka_unit_test(test_committed_but_unanswered_write_has_unknown_outcome_on_stepdown), + cmocka_unit_test(test_old_write_is_rejected_after_unobserved_reelection), + cmocka_unit_test(test_default_read_of_lost_tail_is_rejected_on_stepdown), + cmocka_unit_test(test_request_waiting_for_leader_runs_when_local_node_is_elected), + cmocka_unit_test(test_disconnect_removes_local_write_and_read_waiters), + cmocka_unit_test(test_disconnect_removes_requests_waiting_for_leader), + cmocka_unit_test(test_disconnect_releases_cached_write_reply), + cmocka_unit_test(test_follower_application_does_not_retain_unused_replies), + cmocka_unit_test(test_write_replaced_by_noop_is_rejected_without_application_reply), + cmocka_unit_test(test_stale_read_does_not_wait_for_uncommitted_tail), + cmocka_unit_test(test_forwarded_write_and_strong_read_complete_through_real_raft), + }; + return cmocka_run_group_tests(tests, nullptr, nullptr); +} diff --git a/test/test_durable.cpp b/test/test_durable.cpp new file mode 100644 index 0000000..86e6477 --- /dev/null +++ b/test/test_durable.cpp @@ -0,0 +1,432 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +extern "C" { +#include +} + +namespace { +namespace fs = std::filesystem; +using TStage = TDurableState::EStage; +struct TFiles { + fs::path Directory; + std::string Name; + TFiles() { + static unsigned next = 0; + Directory = fs::temp_directory_path() / ("miniraft-durable-" + + std::to_string(getpid()) + "-" + std::to_string(next++)); + assert_true(fs::create_directory(Directory)); + Name = (Directory / "raft").string(); + } + ~TFiles() { std::error_code ignored; fs::remove_all(Directory, ignored); } + fs::path Image() const { return Name + ".durable.1"; } + void Seed(); +}; +void Put(std::string& bytes, uint64_t value, unsigned width) { + for (unsigned i = 0; i < width; ++i) bytes.push_back(char(value >> (8 * i))); +} +uint32_t Crc32(std::string_view bytes) { + uint32_t crc = UINT32_MAX; + for (unsigned char byte : bytes) { + crc ^= byte; + for (int bit = 0; bit < 8; ++bit) crc = (crc >> 1) ^ (0xedb88320u & (0u - (crc & 1u))); + } + return ~crc; +} +std::string LegacyImage(std::string_view magic, bool hasSnapshot, bool configured) { + std::string payload; + Put(payload, 3, 8); // current term + Put(payload, 1, 4); // vote + Put(payload, 1, 8); // retained entry count + if (configured) { + Put(payload, 2, 4); Put(payload, 1, 4); Put(payload, 2, 4); + } + if (hasSnapshot) { + const auto image = EncodeSnapshot(TSnapshot{1, 2, {1, 2}, "legacy snapshot"}); + Put(payload, image.size(), 8); payload.append(image); + } + Put(payload, 3, 8); Put(payload, 0, 8); Put(payload, 4, 4); payload.append("tail"); + std::string checksumInput; + if (magic != "MRFTD001") checksumInput.append(magic); + checksumInput.append(payload); + std::string bytes(magic); + Put(bytes, payload.size(), 8); Put(bytes, Crc32(checksumInput), 4); bytes.append(payload); + return bytes; +} +TMessageHolder Entry(uint64_t term, const std::string& data) { + auto entry = NewHoldedMessage(sizeof(TLogEntry) + data.size()); + entry->Term = term; + memcpy(entry->Data, data.data(), data.size()); + return entry; +} +std::string Data(TDurableState& state, int64_t index); + +void test_v1_v2_v3_images_recover_conservatively_and_upgrade_on_publish(void**) { + for (const auto [magic, hasSnapshot, configured] : { + std::tuple{"MRFTD001", false, false}, + {"MRFTD002", true, false}, + {"MRFTD003", true, true}, + }) { + TFiles files; + const auto legacy = LegacyImage(magic, hasSnapshot, configured); + { std::ofstream output(files.Image(), std::ios::binary); output.write(legacy.data(), legacy.size()); } + { + TDurableState state(files.Name, 1); + assert_int_equal(state.CurrentTerm, 3); + assert_int_equal(state.SnapshotIndex, hasSnapshot ? 1 : 0); + assert_int_equal(state.CommitIndex, state.SnapshotIndex); + assert_int_equal(state.LastLogIndex, hasSnapshot ? 2 : 1); + if (!configured) state.BindVotersFromTrustedBaseline({1, 2}); + assert_true((state.GetBoundVoters() == std::vector{1, 2})); + assert_true(state.SupportsCommitIndexPersistence()); + if (hasSnapshot) assert_true(state.GetSnapshot()->Data == "legacy snapshot"); + assert_string_equal(Data(state, state.LastLogIndex).c_str(), "tail"); + state.Commit(); + } + std::ifstream upgraded(files.Image(), std::ios::binary); + std::string published(8, '\0'); upgraded.read(published.data(), published.size()); + assert_true(published == "MRFTD004"); + } +} +void TFiles::Seed() { + TDurableState state(Name, 1); + state.CurrentTerm = 3; + state.VotedFor = 2; + state.Append(Entry(1, "first")); + state.Append(Entry(2, "second")); +} +std::string Data(TDurableState& state, int64_t index) { + auto entry = state.Get(index); + return std::string(entry->Data, entry->Len - sizeof(TLogEntry)); +} +enum class EOperation { Append, Truncate, Vote, Replace, CommitIndex }; +void Operate(TDurableState& state, EOperation operation) { + if (operation == EOperation::Truncate) state.RemoveLast(); + else if (operation == EOperation::CommitIndex) { + state.CommitIndex = 1; + state.Commit(); + } + else if (operation == EOperation::Vote) { + state.CurrentTerm = 4; + state.VotedFor = 3; + state.Commit(); + } else state.Append(Entry(3, "replacement")); +} +void Check(TDurableState& state, EOperation operation, bool updated) { + assert_int_equal(state.CurrentTerm, updated && operation == EOperation::Vote ? 4 : 3); + assert_int_equal(state.VotedFor, updated && operation == EOperation::Vote ? 3 : 2); + const auto oldCount = operation == EOperation::Replace ? 1 : 2; + const auto count = !updated ? oldCount : operation == EOperation::Truncate ? 1 : + operation == EOperation::Vote || operation == EOperation::CommitIndex ? 2 : oldCount + 1; + assert_int_equal(state.LastLogIndex, count); + assert_int_equal(state.CommitIndex, updated && operation == EOperation::CommitIndex ? 1 : 0); + assert_string_equal(Data(state, 1).c_str(), "first"); + if (count >= 2) { + const bool replaced = updated && operation == EOperation::Replace; + assert_string_equal(Data(state, 2).c_str(), replaced ? "replacement" : "second"); + assert_int_equal(state.Get(2)->Term, replaced ? 3 : 2); + } + if (count == 3) assert_string_equal(Data(state, 3).c_str(), "replacement"); + assert_int_equal(state.LastLogTerm, state.Get(count)->Term); +} +void Wait(pid_t child, int expected) { + int status; + assert_int_equal(waitpid(child, &status, 0), child); + assert_true(WIFEXITED(status)); + assert_int_equal(WEXITSTATUS(status), expected); +} + +void test_crash_at_every_publication_boundary(void**) { + for (auto operation : {EOperation::Append, EOperation::Truncate, EOperation::Vote, EOperation::Replace, EOperation::CommitIndex}) { + for (auto stage : {TStage::TempCreated, TStage::Written, TStage::FileSynced, + TStage::Renamed, TStage::DirectorySynced}) { + TFiles files; + files.Seed(); + const auto child = fork(); + assert_true(child >= 0); + if (child == 0) { + try { + bool armed = false; + TDurableState state(files.Name, 1, [&](auto point) { + if (armed && point == stage) _exit(90); + }); + if (operation == EOperation::Replace) state.RemoveLast(); + armed = true; + Operate(state, operation); + } catch (...) { _exit(2); } + _exit(3); + } + Wait(child, 90); + TDurableState recovered(files.Name, 1); + Check(recovered, operation, stage >= TStage::Renamed); + } + } +} + +void test_acknowledged_updates_survive_immediate_exit(void**) { + for (auto operation : {EOperation::Append, EOperation::Truncate, EOperation::Vote, EOperation::Replace, EOperation::CommitIndex}) { + TFiles files; + files.Seed(); + const auto child = fork(); + assert_true(child >= 0); + if (child == 0) { + try { + TDurableState state(files.Name, 1); + if (operation == EOperation::Replace) state.RemoveLast(); + Operate(state, operation); + _exit(0); // No destructors or stream flushing after acknowledgement. + } catch (...) { _exit(2); } + } + Wait(child, 0); + TDurableState recovered(files.Name, 1); + Check(recovered, operation, true); + } +} + +struct TEndpoint : INode { + std::vector> Sent; + std::vector* Events = nullptr; + void Send(TMessageHolder message) override { + Sent.push_back(std::move(message)); + if (Events) Events->push_back("ack"); + } + void Drain() override {} +}; + +void test_follower_persists_commit_before_ack_and_recovers_it(void**) { + TFiles files; + std::vector events; + auto peer = std::make_shared(); + peer->Events = &events; + { + auto state = std::make_shared(files.Name, 1, [&](auto stage) { + if (stage == TStage::DirectorySynced) events.push_back("durable"); + }); + TRaft raft(state, 1, {{2, peer}}); + events.clear(); // Ignore initial image and voter-binding publications. + auto request = NewHoldedMessage(TMessageEx{.Src = 2, .Dst = 1, .Term = 1}, + TAppendEntriesRequest{.PrevLogIndex = 0, .PrevLogTerm = 0, .LeaderCommit = 1, + .LeaderId = 2, .Nentries = 1}); + request.InitPayload(1); + request.Payload[0] = Entry(1, "committed"); + raft.Process(ITimeSource::Time{}, request); + assert_int_equal(state->CommitIndex, 1); + assert_int_equal(peer->Sent.size(), 1); + assert_true(peer->Sent.back().Cast()->Success); + assert_true((events == std::vector{"durable", "durable", "ack"})); + } + TDurableState recovered(files.Name, 1); + assert_int_equal(recovered.CommitIndex, 1); + assert_int_equal(recovered.LastLogIndex, 1); + bool protectedEntry = false; + try { recovered.RemoveLast(); } + catch (const std::logic_error&) { protectedEntry = true; } + assert_true(protectedEntry); + assert_int_equal(recovered.LastLogIndex, 1); +} + +void test_publication_order(void**) { + TFiles files; + files.Seed(); + std::vector stages; + TDurableState state(files.Name, 1, [&](auto stage) { stages.push_back(stage); }); + state.CurrentTerm = 4; + state.Commit(); + const std::vector expected{TStage::TempCreated, TStage::Written, TStage::FileSynced, + TStage::Renamed, TStage::DirectorySynced}; + assert_true(stages == expected); +} + +void test_live_publication_failure_is_fatal(void**) { + TFiles files; + files.Seed(); + const auto child = fork(); + assert_true(child >= 0); + if (child == 0) { + std::set_terminate([] { _exit(73); }); + try { + TDurableState state(files.Name, 1, [](auto stage) { + if (stage == TStage::FileSynced) throw std::runtime_error("injected failure"); + }); + state.CurrentTerm = 4; + state.Commit(); + } catch (...) { _exit(74); } + _exit(75); + } + Wait(child, 73); + TDurableState recovered(files.Name, 1); + assert_int_equal(recovered.CurrentTerm, 3); + assert_int_equal(recovered.VotedFor, 2); +} + +void test_write_error_is_fatal_and_preserves_image(void**) { + TFiles files; + files.Seed(); + const auto child = fork(); + assert_true(child >= 0); + if (child == 0) { + std::set_terminate([] { _exit(73); }); + try { + TDurableState state(files.Name, 1); + struct rlimit limit; + if (getrlimit(RLIMIT_FSIZE, &limit) != 0) _exit(76); + limit.rlim_cur = 0; + if (setrlimit(RLIMIT_FSIZE, &limit) != 0) _exit(76); + std::signal(SIGXFSZ, SIG_IGN); + state.Append(Entry(3, "must not be acknowledged")); + } catch (...) { _exit(74); } + _exit(75); + } + Wait(child, 73); + TDurableState recovered(files.Name, 1); + Check(recovered, EOperation::Append, false); +} + +void test_interrupted_initialization_is_not_reset(void**) { + for (auto stage : {TStage::TempCreated, TStage::Written, TStage::FileSynced, + TStage::Renamed, TStage::DirectorySynced}) { + TFiles files; + const auto child = fork(); + assert_true(child >= 0); + if (child == 0) { + try { + TDurableState state(files.Name, 1, [&](auto point) { + if (point == stage) _exit(90); + }); + } catch (...) { _exit(2); } + _exit(3); + } + Wait(child, 90); + bool rejected = false; + try { + TDurableState recovered(files.Name, 1); + assert_int_equal(recovered.CurrentTerm, 1); + assert_int_equal(recovered.VotedFor, 0); + assert_int_equal(recovered.LastLogIndex, 0); + } catch (const std::runtime_error&) { rejected = true; } + assert_int_equal(rejected, stage < TStage::Renamed); + } +} + +void Reject(const TFiles& files) { + bool rejected = false; + try { TDurableState state(files.Name, 1); } + catch (const std::runtime_error&) { rejected = true; } + assert_true(rejected); +} +void test_corrupt_images_are_rejected(void**) { + for (const auto offset : {0, 8, 16, 20, 30, 45}) { + TFiles files; + files.Seed(); + std::fstream file(files.Image(), std::ios::in | std::ios::out | std::ios::binary); + file.seekg(offset); + char byte; + file.read(&byte, 1); + byte ^= 1; + file.seekp(offset); + file.write(&byte, 1); + file.close(); + Reject(files); + } + for (const auto size : {0, 7, 19, 21, 40}) { + TFiles files; + files.Seed(); + fs::resize_file(files.Image(), size); + Reject(files); + } +} +void test_single_writer_lock(void**) { + TFiles files; + files.Seed(); + { + TDurableState writer(files.Name, 1); + Reject(files); + } + TDurableState afterClose(files.Name, 1); + assert_int_equal(afterClose.CurrentTerm, 3); +} +void test_missing_image_is_not_recreated(void**) { + TFiles files; + files.Seed(); + fs::remove(files.Image()); + Reject(files); + assert_false(fs::exists(files.Image())); +} +void test_legacy_store_is_not_silently_migrated(void**) { + TFiles files; + { TDiskState legacy(files.Name, 1); } + Reject(files); + assert_false(fs::exists(files.Image())); +} +void test_get_and_append_do_not_share_mutable_entries(void**) { + TFiles files; + { + TDurableState state(files.Name, 1); + auto entry = Entry(1, "data"); + state.Append(entry); + entry->Term = 100; + auto result = state.Get(1); + assert_int_equal(result->Term, 1); + result->Term = 99; + assert_int_equal(state.Get(1)->Term, 1); + state.Commit(); + } + TDurableState state(files.Name, 1); + assert_int_equal(state.Get(1)->Term, 1); +} +void test_raft_election_and_application_replay_after_restart(void**) { + TFiles files; + uint64_t term = 0, index = 0; + for (int restart = 0; restart < 2; ++restart) { + auto state = std::make_shared(files.Name, 1); + auto raft = std::make_shared(state, 1, TNodeDict{}); + raft->ProcessTimeout(ITimeSource::Time{}); + assert_int_equal(raft->CurrentStateName(), EState::LEADER); + assert_true(state->CurrentTerm > term); + term = state->CurrentTerm; + if (restart == 0) { + index = raft->Append(Entry(term, "application-command")); + raft->ProcessTimeout(ITimeSource::Time{}); + } + assert_true(raft->GetVolatileState()->CommitIndex >= index); + assert_string_equal(Data(*state, index).c_str(), "application-command"); + auto rsm = std::make_shared(); + TRequestProcessor processor(raft, rsm, {}); + processor.ProcessCommitted(); + assert_int_equal(rsm->LastAppliedIndex, state->LastLogIndex); + } +} +} + +int main() { + const CMUnitTest tests[] = { + cmocka_unit_test(test_crash_at_every_publication_boundary), + cmocka_unit_test(test_acknowledged_updates_survive_immediate_exit), + cmocka_unit_test(test_follower_persists_commit_before_ack_and_recovers_it), + cmocka_unit_test(test_publication_order), + cmocka_unit_test(test_v1_v2_v3_images_recover_conservatively_and_upgrade_on_publish), + cmocka_unit_test(test_live_publication_failure_is_fatal), + cmocka_unit_test(test_write_error_is_fatal_and_preserves_image), + cmocka_unit_test(test_interrupted_initialization_is_not_reset), + cmocka_unit_test(test_corrupt_images_are_rejected), + cmocka_unit_test(test_single_writer_lock), + cmocka_unit_test(test_missing_image_is_not_recreated), + cmocka_unit_test(test_legacy_store_is_not_silently_migrated), + cmocka_unit_test(test_get_and_append_do_not_share_mutable_entries), + cmocka_unit_test(test_raft_election_and_application_replay_after_restart), + }; + return cmocka_run_group_tests(tests, nullptr, nullptr); +} diff --git a/test/test_forwarding.cpp b/test/test_forwarding.cpp new file mode 100644 index 0000000..c5d4193 --- /dev/null +++ b/test/test_forwarding.cpp @@ -0,0 +1,264 @@ +#include +#include +#include +#include +#include +#include +extern "C" { +#include +} + +namespace { +struct TEndpoint : INode { + std::function)> Callback; + explicit TEndpoint(std::function)> cb) : Callback(std::move(cb)) {} + void Send(TMessageHolder message) override { Callback(std::move(message)); } + void Drain() override {} +}; +struct TClient { + struct TReply { uint32_t Cookie, Error; std::string Data; }; + std::vector Replies; + std::shared_ptr Endpoint = std::make_shared([this](auto message) { + auto response = message.template Cast(); + Replies.push_back({response->Cookie, response->ErrorCode, + std::string(response->Data, response->Len - sizeof(TCommandResponse))}); + }); +}; +struct TFollower { + std::vector> Sent; + std::shared_ptr Leader = std::make_shared([this](auto message) { + if (message->Type == uint32_t(EMessageType::COMMAND_REQUEST)) + Sent.push_back(message.template Cast()); + }); + std::shared_ptr Other = std::make_shared([this](auto message) { + if (message->Type == uint32_t(EMessageType::COMMAND_REQUEST)) + Sent.push_back(message.template Cast()); + }); + TNodeDict Peers{{1, Leader}, {3, Other}}; + std::shared_ptr Raft = std::make_shared(std::make_shared(), 2, Peers); + std::shared_ptr Rsm = std::make_shared(); + TRequestProcessor Processor{Raft, Rsm, Peers}; + explicit TFollower(bool known = true) { if (known) LearnLeader(1, 3); } + void LearnLeader(uint32_t id, uint64_t term) { + auto message = NewHoldedMessage(TMessageEx{.Src = id, .Dst = 2, .Term = term}, + TAppendEntriesRequest{.LeaderId = id}); + Raft->Process(ITimeSource::Time{}, message); + } + TMessageHolder Request(TClient& client, uint32_t cookie, uint32_t extra = 0) { + auto request = NewHoldedMessage(sizeof(TCommandRequest) + 5); + request->Cookie = cookie; + request->Flags = TCommandRequest::EWrite | extra; + memcpy(request->Data, "value", 5); + Processor.OnCommandRequest(request, client.Endpoint); + return request; + } + TMessageHolder Response(size_t sent, uint32_t error = 0) { + auto response = NewHoldedMessage(sizeof(TCommandResponse) + 6); + response->Cookie = Sent.at(sent)->Cookie; + response->ErrorCode = error; + memcpy(response->Data, "answer", 6); + return response; + } + void Reply(size_t sent, uint32_t error = 0) { + Processor.OnCommandResponse(Response(sent, error), Leader); + } +}; + +void test_malformed_forwarded_reply_does_not_consume_route(void**) { + TFollower follower; TClient client; + follower.Request(client, 42); + auto response = follower.Response(0); + response->Len += 8; + bool rejected = false; + try { follower.Processor.OnCommandResponse(response, follower.Leader); } + catch (const TMessageError&) { rejected = true; } + assert_true(rejected); assert_true(client.Replies.empty()); + auto wrong = follower.Response(0); + wrong->Type = uint32_t(EMessageType::LOG_ENTRY); + rejected = false; + try { follower.Processor.OnCommandResponse(wrong, follower.Leader); } + catch (const TMessageError&) { rejected = true; } + assert_true(rejected); assert_true(client.Replies.empty()); + follower.Reply(0); + assert_int_equal(client.Replies.size(), 1); + assert_int_equal(client.Replies[0].Cookie, 42); + assert_string_equal(client.Replies[0].Data.c_str(), "answer"); +} + +void test_nonzero_client_cookies_are_forwarded(void**) { + for (uint32_t cookie : {42u, UINT32_MAX}) { + TFollower follower; + TClient client; + follower.Request(client, cookie); + assert_int_equal(follower.Sent.size(), 1); + follower.Reply(0); + assert_int_equal(client.Replies.size(), 1); + assert_int_equal(client.Replies[0].Cookie, cookie); + assert_int_equal(client.Replies[0].Error, 0); + } +} +void test_zero_client_cookie_is_restored(void**) { + TFollower follower; + TClient client; + follower.Request(client, 0); + assert_int_equal(follower.Sent.size(), 1); + follower.Reply(0); + assert_int_equal(client.Replies.size(), 1); + assert_int_equal(client.Replies[0].Cookie, 0); + assert_string_equal(client.Replies[0].Data.c_str(), "answer"); +} +void test_forwarding_copies_request_and_marks_one_hop(void**) { + TFollower follower; + TClient client; + auto original = follower.Request(client, 0); + assert_int_equal(original->Cookie, 0); + assert_int_equal(original->Flags, TCommandRequest::EWrite); + assert_true(follower.Sent[0]->Flags & TCommandRequest::EForwarded); + assert_true(follower.Sent[0].RawData != original.RawData); +} +void test_out_of_order_and_duplicate_replies_preserve_client_routes(void**) { + TFollower follower; + TClient first, second; + follower.Request(first, 0); + follower.Request(second, 0); + assert_int_not_equal(follower.Sent[0]->Cookie, follower.Sent[1]->Cookie); + follower.Reply(1); + follower.Reply(1); + assert_true(first.Replies.empty()); + assert_int_equal(second.Replies.size(), 1); + assert_int_equal(second.Replies[0].Cookie, 0); + follower.Reply(0); + assert_int_equal(first.Replies.size(), 1); + assert_int_equal(first.Replies[0].Cookie, 0); +} +void test_forwarded_request_is_rejected_on_term_change(void**) { + TFollower follower; + TClient client; + follower.Request(client, 0); + follower.LearnLeader(3, 4); + follower.Processor.CheckStateChange(); + assert_int_equal(client.Replies.size(), 1); + assert_int_equal(client.Replies[0].Cookie, 0); + assert_int_equal(client.Replies[0].Error, 1); + follower.Reply(0); + assert_int_equal(client.Replies.size(), 1); +} +void test_candidate_transition_rejects_forwarded_requests(void**) { + TFollower follower; + TClient client; + follower.Request(client, 0); + follower.Raft->ProcessTimeout(ITimeSource::Time{} + std::chrono::seconds(20)); + assert_int_equal(follower.Raft->CurrentStateName(), EState::CANDIDATE); + follower.Processor.CheckStateChange(); + assert_int_equal(client.Replies.size(), 1); + assert_int_equal(client.Replies[0].Error, 1); +} +void test_late_reply_prunes_stale_route_before_application_poll(void**) { + TFollower follower; + TClient client; + follower.Request(client, 0); + follower.LearnLeader(3, 4); + follower.Reply(0); // No CheckStateChange/ProcessWaiting in between. + assert_int_equal(client.Replies.size(), 1); + assert_int_equal(client.Replies[0].Error, 1); + assert_int_equal(client.Replies[0].Cookie, 0); +} +void test_client_disconnect_releases_only_its_forwarded_route(void**) { + TFollower follower; + TClient first, second; + follower.Request(first, 0); + follower.Request(second, 0); + std::weak_ptr endpoint = first.Endpoint; + follower.Processor.CleanUp(first.Endpoint); + first.Endpoint.reset(); + assert_true(endpoint.expired()); + follower.Reply(0); + assert_true(first.Replies.empty()); + follower.Reply(1); + assert_int_equal(second.Replies.size(), 1); + assert_int_equal(second.Replies[0].Error, 0); +} +void test_already_forwarded_request_is_not_relayed_again(void**) { + TFollower follower; + TClient client; + follower.Request(client, 0, TCommandRequest::EForwarded); + assert_true(follower.Sent.empty()); + assert_int_equal(client.Replies.size(), 1); + assert_int_equal(client.Replies[0].Error, 1); +} +void test_nonzero_cookie_waits_for_leader_discovery(void**) { + TFollower follower(false); + TClient client; + follower.Request(client, 77); + assert_true(client.Replies.empty()); + assert_true(follower.Sent.empty()); + follower.LearnLeader(1, 3); + follower.Processor.CheckStateChange(); + assert_int_equal(follower.Sent.size(), 1); + follower.Reply(0); + assert_int_equal(client.Replies[0].Cookie, 77); +} +void test_response_requires_original_peer_and_preserves_wire_message(void**) { + TFollower follower; + TClient client; + follower.Request(client, 0); + auto response = follower.Response(0, 77); + const auto cookie = response->Cookie; + follower.Processor.OnCommandResponse(response); + follower.Processor.OnCommandResponse(response, follower.Other); + assert_true(client.Replies.empty()); + follower.Processor.OnCommandResponse(response, follower.Leader); + assert_int_equal(client.Replies.size(), 1); + assert_int_equal(client.Replies[0].Cookie, 0); + assert_int_equal(client.Replies[0].Error, 77); + assert_int_equal(response->Cookie, cookie); +} +void test_peer_disconnect_cancels_once_and_does_not_reuse_cookies(void**) { + TFollower follower; + TClient first, second; + follower.Request(first, 42); + follower.Request(second, 43); + follower.Processor.OnPeerDisconnected(follower.Other); + assert_true(first.Replies.empty()); + assert_true(second.Replies.empty()); + follower.Processor.OnPeerDisconnected(follower.Leader); + follower.Processor.OnPeerDisconnected(follower.Leader); + assert_int_equal(first.Replies.size(), 1); + assert_int_equal(second.Replies.size(), 1); + assert_int_equal(first.Replies[0].Cookie, 42); + assert_int_equal(second.Replies[0].Cookie, 43); + assert_int_equal(first.Replies[0].Error, 1); + assert_int_equal(second.Replies[0].Error, 1); + std::weak_ptr endpoint = first.Endpoint; + first.Endpoint.reset(); + assert_true(endpoint.expired()); + TClient afterReconnect; + follower.Request(afterReconnect, 44); + assert_true(follower.Sent[2]->Cookie > follower.Sent[1]->Cookie); + follower.Reply(0); // Late old-connection reply cannot redeem the new route. + assert_true(afterReconnect.Replies.empty()); + follower.Reply(2); + assert_int_equal(afterReconnect.Replies.size(), 1); + assert_int_equal(afterReconnect.Replies[0].Cookie, 44); + assert_int_equal(afterReconnect.Replies[0].Error, 0); +} +} + +int main() { + const CMUnitTest tests[] = { + cmocka_unit_test(test_malformed_forwarded_reply_does_not_consume_route), + cmocka_unit_test(test_nonzero_client_cookies_are_forwarded), + cmocka_unit_test(test_zero_client_cookie_is_restored), + cmocka_unit_test(test_forwarding_copies_request_and_marks_one_hop), + cmocka_unit_test(test_out_of_order_and_duplicate_replies_preserve_client_routes), + cmocka_unit_test(test_forwarded_request_is_rejected_on_term_change), + cmocka_unit_test(test_candidate_transition_rejects_forwarded_requests), + cmocka_unit_test(test_late_reply_prunes_stale_route_before_application_poll), + cmocka_unit_test(test_client_disconnect_releases_only_its_forwarded_route), + cmocka_unit_test(test_already_forwarded_request_is_not_relayed_again), + cmocka_unit_test(test_nonzero_cookie_waits_for_leader_discovery), + cmocka_unit_test(test_response_requires_original_peer_and_preserves_wire_message), + cmocka_unit_test(test_peer_disconnect_cancels_once_and_does_not_reuse_cookies), + }; + return cmocka_run_group_tests(tests, nullptr, nullptr); +} diff --git a/test/test_learners.cpp b/test/test_learners.cpp new file mode 100644 index 0000000..260ee88 --- /dev/null +++ b/test/test_learners.cpp @@ -0,0 +1,808 @@ +#include +#include +#include +#include +#include "../examples/kv.h" +#include "../examples/kv_messages.h" +#include +#include +#include +#include +#include +#ifdef __linux__ +#include +#include +#endif +#include +#include +#include +extern "C" { +#include +} +namespace { +template void Reject(F action) { + bool rejected = false; + try { action(); } catch (const std::exception&) { rejected = true; } + assert_true(rejected); +} +struct TEndpoint : INode { + std::function)> Callback; + explicit TEndpoint(std::function)> callback = {}) : Callback(std::move(callback)) {} + void Send(TMessageHolder message) override { if (Callback) Callback(std::move(message)); } + void Drain() override {} +}; +TNodeDict Peers(uint32_t self, uint32_t count = 4) { + TNodeDict peers; + for (uint32_t id = 1; id <= count; ++id) if (id != self) peers[id] = std::make_shared(); + return peers; +} +TMessageHolder Command(const std::string& key, const std::string& value) { + auto command = NewHoldedMessage(sizeof(TWriteKv) + key.size() + value.size()); + command->Flags = TCommandRequest::EWrite; command->KeySize = key.size(); command->ValSize = value.size(); + memcpy(command->TKvEntry::Data, key.data(), key.size()); + memcpy(command->TKvEntry::Data + key.size(), value.data(), value.size()); + return command; +} +std::string Read(TKv& kv, const std::string& key) { + auto command = NewHoldedMessage(sizeof(TReadKv) + key.size()); + command->KeySize = key.size(); memcpy(command->Data, key.data(), key.size()); + auto result = kv.Read(command, kv.LastAppliedIndex).Cast(); + return std::string(result->Data, result->Len - sizeof(TCommandResponse)); +} +struct TApp : TKv { + size_t Loads = 0; + void LoadSnapshot(std::string_view bytes, uint64_t index) override { TKv::LoadSnapshot(bytes, index); ++Loads; } +}; +struct TCluster { + ITimeSource::Time Now{}; + std::vector> Nodes{4}; + std::vector> Apps{4}; + std::vector> Processors{4}; + std::deque> Packets; + std::vector VoteTargets; + std::set Isolated; + bool StageVoterLearner = false; + explicit TCluster(bool stageVoterLearner = false) : StageVoterLearner(stageVoterLearner) { + for (uint32_t id = 1; id <= 4; ++id) Build(id, std::make_shared()); + } + void Build(uint32_t id, std::shared_ptr state) { + auto peers = Peers(id); + if (StageVoterLearner && id != 4) peers.erase(4); + for (auto& [_, peer] : peers) peer = std::make_shared([this](auto message) { + if (message->Type == uint32_t(EMessageType::REQUEST_VOTE_REQUEST)) VoteTargets.push_back(message.template Cast()->Dst); + Packets.push_back(std::move(message)); + }); + Processors[id - 1].reset(); + Nodes[id - 1] = std::make_shared(state, id, peers, + StageVoterLearner && id != 4 ? std::vector{} : std::vector{4}); + Apps[id - 1] = std::make_shared(); + Processors[id - 1] = std::make_unique(Nodes[id - 1], Apps[id - 1], peers); + } + void Drain(bool duplicate = false) { + size_t budget = 10000; + while (!Packets.empty()) { + assert_true(budget-- > 0); + auto message = Packets.front(); Packets.pop_front(); auto header = message.Cast(); + if (Isolated.contains(header->Src) || Isolated.contains(header->Dst)) continue; + Nodes[header->Dst - 1]->Process(Now, message); + if (duplicate) Nodes[header->Dst - 1]->Process(Now, message); + Processors[header->Dst - 1]->ProcessCommitted(); + } + } + void Elect(uint32_t id) { + Now += std::chrono::seconds(10); Nodes[id - 1]->ProcessTimeout(Now); Drain(); + Nodes[id - 1]->ProcessTimeout(Now); Drain(); + assert_int_equal(Nodes[id - 1]->CurrentStateName(), EState::LEADER); + } + void Pump(uint32_t leader = 1, int rounds = 40, bool duplicate = false) { + for (int i = 0; i < rounds; ++i) { + Now += std::chrono::milliseconds(100); Nodes[leader - 1]->ProcessTimeout(Now); Drain(duplicate); + for (auto& processor : Processors) { processor->ProcessCommitted(); processor->ProcessWaiting(); } + } + } + void Write(const std::string& key, const std::string& value) { Processors[0]->OnCommandRequest(Command(key, value), {}); Pump(); } +}; +void test_invalid_learner_role_configurations_fail_before_binding(void**) { + for (auto learners : {std::vector{0}, std::vector{4, 4}, + std::vector{5}, std::vector{1, 2, 3, 4}}) { + auto state = std::make_shared(); + Reject([&] { TRaft raft(state, 1, Peers(1), learners); }); + assert_true(state->GetBoundVoters().empty()); assert_int_equal(state->CurrentTerm, 1); + } +} +void test_learner_never_campaigns_votes_or_proposes(void**) { + std::vector> replies; + auto peers = Peers(4); + peers[1] = std::make_shared([&](auto message) { replies.push_back(message.template Cast()); }); + auto state = std::make_shared(); TRaft learner(state, 4, peers, {4}); + for (int i = 0; i < 5; ++i) learner.ProcessTimeout(ITimeSource::Time{} + std::chrono::hours(i + 1)); + assert_int_equal(state->CurrentTerm, 1); assert_int_equal(state->VotedFor, 0); + Reject([&] { learner.Become(EState::CANDIDATE); }); Reject([&] { learner.Become(EState::LEADER); }); + Reject([&] { learner.Append(NewHoldedMessage()); }); + auto request = NewHoldedMessage(TMessageEx{.Src = 1, .Dst = 4, .Term = 5}, + TRequestVoteRequest{.LastLogIndex = 0, .LastLogTerm = 0, .CandidateId = 1}); + learner.Process({}, request); + assert_int_equal(replies.size(), 1); assert_false(replies[0]->VoteGranted); + assert_int_equal(state->CurrentTerm, 5); assert_int_equal(state->VotedFor, 0); + assert_int_equal(learner.CurrentStateName(), EState::FOLLOWER); +} +void test_learner_messages_cannot_change_voter_term_or_leadership(void**) { + TCluster cluster; cluster.Elect(1); cluster.Pump(); + const auto term = cluster.Nodes[0]->GetState()->CurrentTerm; + std::vector> messages; + const TMessageEx header{.Src = 4, .Dst = 1, .Term = 99}; + messages.push_back(NewHoldedMessage(header, TRequestVoteRequest{.LastLogIndex = 0, .LastLogTerm = 0, .CandidateId = 4})); + messages.push_back(NewHoldedMessage(header, TRequestVoteResponse{.VoteGranted = 1})); + messages.push_back(NewHoldedMessage(header, TAppendEntriesRequest{.LeaderId = 4})); + messages.push_back(NewHoldedMessage(header, TAppendEntriesResponse{.MatchIndex = 0, .Success = 0})); + auto install = NewHoldedMessage(sizeof(TInstallSnapshotRequest) + 1); + install->Src = 4; install->Dst = 1; install->Term = 99; install->SnapshotIndex = 1; install->SnapshotTerm = 1; + install->TotalBytes = 44; install->LeaderId = 4; install->Data[0] = 'x'; messages.push_back(install); + messages.push_back(NewHoldedMessage(header, TInstallSnapshotResponse{.SnapshotIndex = 1})); + for (auto message : messages) { + cluster.Nodes[0]->Process(cluster.Now, message); + assert_int_equal(cluster.Nodes[0]->GetState()->CurrentTerm, term); + assert_int_equal(cluster.Nodes[0]->CurrentStateName(), EState::LEADER); + } +} +void test_voter_commit_quorums_ignore_arbitrary_learner_matches(void**) { + TState state; state.Append(NewHoldedMessage()); + for (uint32_t count = 1; count <= 8; ++count) for (uint32_t acknowledgements = 0; acknowledgements < count; ++acknowledgements) { + std::vector voters; + for (uint32_t id = 1; id <= count; ++id) voters.push_back(id); + TVolatileState progress; + for (uint32_t id = 2; id <= acknowledgements + 1; ++id) progress.MatchIndex[id] = 1; + for (uint32_t id = count + 1; id <= count + 5; ++id) progress.MatchIndex[id] = 1; + progress.CommitAdvance(voters, 1, state); + assert_int_equal(progress.CommitIndex, acknowledgements + 1 > count / 2 ? 1 : 0); + } +} +void test_single_voter_commits_and_reads_without_learner_acknowledgements(void**) { + auto state = std::make_shared(); + auto raft = std::make_shared(state, 1, Peers(1), std::vector{2, 3, 4}); + auto app = std::make_shared(); TRequestProcessor processor(raft, app, {}); + raft->ProcessTimeout({}); assert_int_equal(raft->CurrentStateName(), EState::LEADER); + assert_int_equal(raft->GetNservers(), 1); assert_int_equal(raft->GetNpeers(), 3); + processor.OnCommandRequest(Command("a", "committed alone"), {}); + raft->ProcessTimeout({}); processor.ProcessCommitted(); + assert_true(Read(*app, "a") == "committed alone"); + size_t replies = 0; + auto client = std::make_shared([&](auto message) { + assert_int_equal(message.template Cast()->ErrorCode, 0); ++replies; + }); + auto read = NewHoldedMessage(sizeof(TReadKv) + 1); + read->KeySize = 1; read->Data[0] = 'a'; read->Flags = TCommandRequest::EConsistent; + processor.OnCommandRequest(read, client); processor.ProcessWaiting(); assert_int_equal(replies, 1); +} +void test_learner_grants_cannot_elect_candidate_and_no_votes_are_solicited(void**) { + TCluster cluster; cluster.Isolated = {2, 3}; + cluster.Nodes[0]->ProcessTimeout(cluster.Now); cluster.Drain(); + auto grant = NewHoldedMessage(TMessageEx{.Src = 4, .Dst = 1, .Term = cluster.Nodes[0]->GetState()->CurrentTerm}, + TRequestVoteResponse{.VoteGranted = 1}); + cluster.Nodes[0]->Process(cluster.Now, grant); cluster.Nodes[0]->ProcessTimeout(cluster.Now); + assert_int_equal(cluster.Nodes[0]->CurrentStateName(), EState::CANDIDATE); + assert_int_equal(cluster.Nodes[0]->GetMinVotes(), 2); + for (auto id : cluster.VoteTargets) assert_true(id != 4); + cluster.Isolated = {3}; cluster.Elect(1); +} +void test_learner_replication_cannot_commit_or_satisfy_strong_read(void**) { + TCluster cluster; cluster.Elect(1); cluster.Pump(); + const auto before = cluster.Nodes[0]->GetVolatileState()->CommitIndex; + cluster.Isolated = {2, 3}; cluster.Write("x", "new"); + assert_true(cluster.Nodes[0]->GetVolatileState()->MatchIndex.at(4) > before); + assert_int_equal(cluster.Nodes[0]->GetVolatileState()->CommitIndex, before); + assert_true(Read(*cluster.Apps[0], "x").empty()); assert_true(Read(*cluster.Apps[3], "x").empty()); + size_t replies = 0; + auto client = std::make_shared([&](auto message) { + auto reply = message.template Cast(); + assert_int_equal(reply->Cookie, 77); assert_int_equal(reply->ErrorCode, 0); ++replies; + }); + auto read = NewHoldedMessage(sizeof(TReadKv) + 1); + read->Flags = TCommandRequest::EConsistent; read->Cookie = 77; read->KeySize = 1; read->Data[0] = 'x'; + cluster.Processors[0]->OnCommandRequest(read, client); cluster.Pump(); assert_int_equal(replies, 0); + cluster.Isolated = {3}; cluster.Pump(1, 80); + assert_int_equal(replies, 1); assert_true(Read(*cluster.Apps[3], "x") == "new"); +} +void test_learner_commit_ack_waits_for_commit_propagation(void**) { + TCluster cluster; cluster.Elect(1); cluster.Pump(); + auto leader = cluster.Nodes[0]; + const auto index = leader->Append(cluster.Apps[0]->Prepare(Command("pending", "value"))); + + // The learner can have the complete log while the voters cannot yet commit + // this entry. Its ACK only proves the previous commit boundary. + cluster.Isolated = {2, 3}; cluster.Pump(1, 40); + assert_int_equal(leader->GetVolatileState()->CommitIndex, index - 1); + assert_true(leader->GetVolatileState()->MatchIndex.at(4) >= index); + assert_true(leader->IsLearnerCaughtUp(4)); + assert_true(leader->IsLearnerCommitAcknowledged(4)); + + // Now let the voter majority commit while the learner is partitioned. The + // log watermark remains true, but the correlated commit acknowledgement is + // stale until the leader sends the new boundary and receives a reply. + cluster.Isolated = {4}; cluster.Pump(1, 40); + assert_int_equal(leader->GetVolatileState()->CommitIndex, index); + assert_true(leader->IsLearnerCaughtUp(4)); + assert_false(leader->IsLearnerCommitAcknowledged(4)); + + cluster.Isolated.clear(); cluster.Pump(1, 40); + assert_true(leader->IsLearnerCommitAcknowledged(4)); + assert_int_equal(cluster.Apps[3]->LastAppliedIndex, index); +} +void test_learner_snapshot_catchup_status_and_restart_preserve_role(void**) { + TCluster cluster; cluster.Isolated = {4}; cluster.Elect(1); + for (const std::string key : {"a", "b", "c"}) cluster.Write(key, std::string(60000, key[0])); + auto snapshot = cluster.Nodes[0]->Compact(); + assert_true(snapshot.Voters == (std::vector{1, 2, 3})); + assert_false(cluster.Nodes[0]->IsLearnerCaughtUp(4)); cluster.Isolated.clear(); cluster.Pump(1, 100, true); + assert_int_equal(cluster.Apps[3]->Loads, 1); assert_true(cluster.Nodes[0]->IsLearnerCaughtUp(4)); + assert_true(cluster.Nodes[0]->IsLearnerCommitAcknowledged(4)); + assert_false(cluster.Nodes[0]->IsLearnerCaughtUp(2)); assert_false(cluster.Nodes[0]->IsLearnerCaughtUp(99)); + assert_true(Read(*cluster.Apps[3], "c") == std::string(60000, 'c')); + cluster.Nodes[0]->Append(cluster.Apps[0]->Prepare(Command("tail", "later"))); + assert_false(cluster.Nodes[0]->IsLearnerCaughtUp(4)); cluster.Pump(); + assert_true(cluster.Nodes[0]->IsLearnerCaughtUp(4)); + auto state = cluster.Nodes[3]->GetState(); cluster.Build(4, state); + cluster.Nodes[3]->ProcessTimeout(cluster.Now + std::chrono::hours(1)); cluster.Pump(); + assert_true(Read(*cluster.Apps[3], "tail") == "later"); + assert_false(cluster.Nodes[3]->IsVoter(4)); assert_int_equal(state->VotedFor, 0); + assert_int_equal(cluster.Nodes[3]->CurrentStateName(), EState::FOLLOWER); +} +void test_restart_cannot_promote_learner_or_change_voters_but_can_add_transport(void**) { + auto state = std::make_shared(); TRaft learner(state, 4, Peers(4), {4}); + const auto fields = static_cast(*state); + Reject([&] { TRaft promoted(state, 4, Peers(4)); }); + assert_true(static_cast(*state) == fields); + TRaft expanded(state, 4, Peers(4, 5), {4, 5}); + assert_true(expanded.GetVoterIds() == (std::vector{1, 2, 3})); + assert_true(expanded.GetLearnerIds() == (std::vector{4, 5})); + Reject([&] { state->BindVoters({1, 2, 3, 4}); }); + Reject([&] { state->InstallSnapshot({1, 1, {1, 2, 3, 4}, "changed voters"}); }); +} +void test_nonfresh_unbound_store_requires_snapshot_proof_for_learners(void**) { + auto state = std::make_shared(); + state->CurrentTerm = 2; state->VotedFor = 1; state->Append(NewHoldedMessage()); + Reject([&] { TRaft learnerAware(state, 1, Peers(1), {4}); }); + assert_true(state->GetBoundVoters().empty()); + TKv image; state->InstallSnapshot({1, 1, {1, 2, 3}, image.SaveSnapshot()}); + TRaft adopted(state, 1, Peers(1), {4}); + assert_true(state->GetBoundVoters() == (std::vector{1, 2, 3})); +} +void test_online_learner_staging_snapshot_catchup_and_quorum_isolation(void**) { + TCluster cluster(true); cluster.Elect(1); + for (const std::string key : {"a", "b", "c"}) cluster.Write(key, std::string(60000, key[0])); + auto leader = cluster.Nodes[0]; const auto fixed = leader->GetVoterIds(); + const auto snapshot = leader->Compact(); assert_true(snapshot.Index > 1); + auto endpoint = std::make_shared([&](auto message) { cluster.Packets.push_back(std::move(message)); }); + const auto fields = static_cast(*leader->GetState()); + Reject([&] { leader->AddLearnerTarget(2, endpoint); }); + leader->AddLearnerTarget(4, endpoint); + assert_true(static_cast(*leader->GetState()) == fields); + assert_true(leader->GetState()->GetBoundVoters() == fixed); + assert_true(leader->GetVoterIds() == fixed); assert_int_equal(leader->GetNservers(), 3); + assert_int_equal(leader->GetNpeers(), 3); assert_false(leader->IsLearnerCaughtUp(4)); + assert_true(leader->GetLearnerIds() == std::vector({4})); + cluster.Pump(1, 120, true); + assert_true(leader->IsLearnerCaughtUp(4)); assert_int_equal(cluster.Apps[3]->Loads, 1); + assert_true(Read(*cluster.Apps[3], "c") == std::string(60000, 'c')); + Reject([&] { TRaft promoted(leader->GetState(), 1, Peers(1)); }); + assert_true(leader->GetState()->GetBoundVoters() == fixed); + const auto before = leader->GetVolatileState()->CommitIndex; + cluster.Isolated = {2, 3}; cluster.Write("uncommitted", "value"); + assert_true(leader->GetVolatileState()->MatchIndex.at(4) > before); + assert_int_equal(leader->GetVolatileState()->CommitIndex, before); + assert_true(Read(*cluster.Apps[0], "uncommitted").empty()); + size_t replies = 0; + auto client = std::make_shared([&](auto message) { + assert_int_equal(message.template Cast()->ErrorCode, 0); ++replies; + }); + auto read = NewHoldedMessage(sizeof(TReadKv) + 1); + read->Flags = TCommandRequest::EConsistent; read->KeySize = 1; read->Data[0] = 'a'; + cluster.Processors[0]->OnCommandRequest(read, client); cluster.Pump(); + assert_int_equal(replies, 0); + cluster.Isolated = {3}; cluster.Pump(1, 80); + assert_int_equal(replies, 1); + assert_true(Read(*cluster.Apps[0], "uncommitted") == "value"); + assert_true(leader->IsLearnerCaughtUp(4)); +} +void test_online_learner_removal_and_stale_reply_after_readdition(void**) { + TCluster cluster(true); cluster.Elect(1); cluster.Pump(); + auto leader = cluster.Nodes[0]; + auto endpoint = std::make_shared([&](auto message) { cluster.Packets.push_back(std::move(message)); }); + leader->AddLearnerTarget(4, endpoint); cluster.Pump(); assert_true(leader->IsLearnerCaughtUp(4)); + leader->ApproveRead(); + TMessageHolder request; + for (auto it = cluster.Packets.begin(); it != cluster.Packets.end(); ++it) { + auto header = it->Cast(); + if (header->Src == 1 && header->Dst == 4 && header->Type == uint32_t(EMessageType::APPEND_ENTRIES_REQUEST)) { + request = *it; cluster.Packets.erase(it); break; + } + } + assert_true(bool(request)); + cluster.Nodes[3]->Process(cluster.Now, request); + TMessageHolder delayed; + for (auto it = cluster.Packets.begin(); it != cluster.Packets.end(); ++it) { + auto header = it->Cast(); + if (header->Src == 4 && header->Dst == 1 && header->Type == uint32_t(EMessageType::APPEND_ENTRIES_RESPONSE)) { + delayed = *it; cluster.Packets.erase(it); break; + } + } + assert_true(bool(delayed)); cluster.Packets.clear(); + assert_true(leader->RemoveLearnerTarget(4) == endpoint); + assert_true(leader->GetLearnerIds().empty()); assert_int_equal(leader->GetNpeers(), 2); + assert_false(leader->IsLearnerCaughtUp(4)); + leader->Process(cluster.Now, delayed); // no longer a configured peer + leader->AddLearnerTarget(4, endpoint); + cluster.Now += std::chrono::seconds(3); leader->ProcessTimeout(cluster.Now); + leader->Process(cluster.Now, delayed); // old RPC identity cannot confirm the new target + assert_false(leader->IsLearnerCaughtUp(4)); + assert_int_equal(leader->GetVolatileState()->MatchIndex.at(4), 0); + cluster.Drain(); cluster.Pump(); assert_true(leader->IsLearnerCaughtUp(4)); + assert_true(leader->GetVoterIds() == std::vector({1, 2, 3})); +} +void test_online_learner_removal_during_snapshot_and_delayed_ack(void**) { + TCluster cluster(true); cluster.Elect(1); + for (const std::string key : {"a", "b", "c"}) cluster.Write(key, std::string(60000, key[0])); + auto leader = cluster.Nodes[0]; leader->Compact(); + auto endpoint = std::make_shared([&](auto message) { cluster.Packets.push_back(std::move(message)); }); + const auto nextSnapshot = [&]() -> TMessageHolder { + for (int attempt = 0; attempt < 40; ++attempt) { + cluster.Now += std::chrono::seconds(3); leader->ProcessTimeout(cluster.Now); + size_t budget = 1000; + while (!cluster.Packets.empty()) { + assert_true(budget-- > 0); + auto message = cluster.Packets.front(); cluster.Packets.pop_front(); + auto header = message.Cast(); + if (header->Src == 1 && header->Dst == 4 && header->Type == uint32_t(EMessageType::INSTALL_SNAPSHOT_REQUEST)) + return message; + cluster.Nodes[header->Dst - 1]->Process(cluster.Now, message); + cluster.Processors[header->Dst - 1]->ProcessCommitted(); + } + } + fail_msg("Learner did not enter snapshot transfer"); return {}; + }; + leader->AddLearnerTarget(4, endpoint); + auto first = nextSnapshot(); const auto firstSeq = first.Cast()->Seqno; + cluster.Nodes[3]->Process(cluster.Now, first); + TMessageHolder delayed; + for (auto it = cluster.Packets.begin(); it != cluster.Packets.end(); ++it) { + auto header = it->Cast(); + if (header->Src == 4 && header->Dst == 1 && header->Type == uint32_t(EMessageType::INSTALL_SNAPSHOT_RESPONSE)) { + delayed = *it; cluster.Packets.erase(it); break; + } + } + assert_true(bool(delayed)); assert_false(delayed.Cast()->Installed); + assert_true(leader->GetVolatileState()->SnapshotRpcs.contains(4)); + assert_true(leader->RemoveLearnerTarget(4) == endpoint); + assert_false(leader->GetVolatileState()->SnapshotRpcs.contains(4)); cluster.Packets.clear(); + leader->AddLearnerTarget(4, endpoint); + auto replacement = nextSnapshot(); const auto replacementSeq = replacement.Cast()->Seqno; + assert_true(replacementSeq != firstSeq); + leader->Process(cluster.Now, delayed); + const auto& pending = leader->GetVolatileState()->SnapshotRpcs.at(4); + assert_int_equal(pending.Seqno, replacementSeq); assert_int_equal(pending.Offset, 0); + cluster.Nodes[3]->Process(cluster.Now, replacement); cluster.Drain(); cluster.Pump(1, 120, true); + assert_true(leader->IsLearnerCaughtUp(4)); assert_int_equal(cluster.Apps[3]->Loads, 1); + assert_true(Read(*cluster.Apps[3], "c") == std::string(60000, 'c')); +} +void test_online_learner_requires_reconciliation_after_leader_change(void**) { + TCluster cluster(true); cluster.Elect(1); + auto endpoint = std::make_shared([&](auto message) { cluster.Packets.push_back(std::move(message)); }); + cluster.Nodes[0]->AddLearnerTarget(4, endpoint); cluster.Pump(); + assert_true(cluster.Nodes[0]->IsLearnerCaughtUp(4)); + cluster.Isolated = {1}; cluster.Elect(2); cluster.Pump(2); + assert_true(cluster.Nodes[1]->GetLearnerIds().empty()); + assert_false(cluster.Nodes[1]->IsLearnerCaughtUp(4)); + cluster.Nodes[1]->AddLearnerTarget(4, endpoint); cluster.Pump(2); + assert_true(cluster.Nodes[1]->IsLearnerCaughtUp(4)); + cluster.Processors[1]->OnCommandRequest(Command("after", "failover"), {}); cluster.Pump(2, 80); + assert_true(Read(*cluster.Apps[3], "after") == "failover"); + assert_true(cluster.Nodes[1]->GetVoterIds() == std::vector({1, 2, 3})); + cluster.Isolated.clear(); cluster.Pump(2, 80); + assert_int_equal(cluster.Nodes[0]->CurrentStateName(), EState::FOLLOWER); + assert_true(cluster.Nodes[0]->GetLearnerIds() == std::vector({4})); +} +void test_online_learner_staging_rejects_invalid_calls(void**) { + auto state = std::make_shared(); auto raft = std::make_shared(state, 1, Peers(1, 3)); + auto endpoint = std::make_shared(); + Reject([&] { raft->AddLearnerTarget(4, endpoint); }); + Reject([&] { raft->RemoveLearnerTarget(2); }); + raft->ProcessTimeout({}); // candidate; no remote votes + Reject([&] { raft->AddLearnerTarget(4, endpoint); }); + auto single = std::make_shared(std::make_shared(), 1, TNodeDict{}); + single->ProcessTimeout({}); assert_int_equal(single->CurrentStateName(), EState::LEADER); + for (uint32_t id : {0u, 1u}) Reject([&] { single->AddLearnerTarget(id, endpoint); }); + Reject([&] { single->AddLearnerTarget(2, {}); }); + single->AddLearnerTarget(2, endpoint); + Reject([&] { single->AddLearnerTarget(2, endpoint); }); + Reject([&] { single->RemoveLearnerTarget(1); }); + Reject([&] { single->RemoveLearnerTarget(3); }); + assert_true(single->GetVoterIds() == std::vector({1})); + assert_int_equal(single->GetNservers(), 1); assert_int_equal(single->GetNpeers(), 1); + auto unbound = std::make_shared(); unbound->CurrentTerm = 2; + unbound->Append(NewHoldedMessage()); + TRaft oldStore(unbound, 1, {}); oldStore.ProcessTimeout({}); + assert_int_equal(oldStore.CurrentStateName(), EState::LEADER); + assert_true(unbound->GetBoundVoters().empty()); + Reject([&] { oldStore.AddLearnerTarget(2, endpoint); }); + assert_true(oldStore.GetLearnerIds().empty()); assert_int_equal(oldStore.GetNpeers(), 0); +} +void test_recovered_hard_state_must_match_fixed_voters(void**) { + for (int mode = 0; mode < 2; ++mode) { + auto state = std::make_shared(); + state->CurrentTerm = mode == 0 ? 2 : 0; state->VotedFor = mode == 0 ? 4 : 0; + const auto before = static_cast(*state); + Reject([&] { TRaft invalid(state, 1, Peers(1, 3)); }); + assert_true(static_cast(*state) == before); assert_true(state->GetBoundVoters().empty()); + } + for (uint32_t vote : {0u, 1u, 2u}) { + auto state = std::make_shared(); state->CurrentTerm = 2; state->VotedFor = vote; + TRaft valid(state, 1, Peers(1, 3)); assert_int_equal(state->VotedFor, vote); + } +} +#ifdef __linux__ +struct TFiles { + std::filesystem::path Directory; std::string Name; + TFiles() { + static unsigned next = 0; + do { + Directory = std::filesystem::temp_directory_path() / + ("miniraft-learners-" + std::to_string(getpid()) + "-" + std::to_string(next++)); + } while (std::filesystem::exists(Directory)); + assert_true(std::filesystem::create_directory(Directory)); Name = (Directory / "raft").string(); + } + ~TFiles() { std::error_code ignored; std::filesystem::remove_all(Directory, ignored); } +}; +void test_legacy_recovered_vote_outside_voters_preserves_storage(void**) { + TFiles files; + { TDiskState state(files.Name, 1); state.CurrentTerm = 2; state.VotedFor = 4; state.Commit(); } + std::vector before; + for (auto suffix : {".entries.1", ".index.1", ".state.1"}) { + std::ifstream input(files.Name + suffix, std::ios::binary); + before.emplace_back(std::istreambuf_iterator(input), std::istreambuf_iterator()); + } + { auto state = std::make_shared(files.Name, 1); + const auto fields = static_cast(*state); + Reject([&] { TRaft invalid(state, 1, Peers(1, 3)); }); + assert_true(static_cast(*state) == fields); } + size_t i = 0; + for (auto suffix : {".entries.1", ".index.1", ".state.1"}) { + std::ifstream input(files.Name + suffix, std::ios::binary); + assert_true(std::string(std::istreambuf_iterator(input), {}) == before[i++]); + } +} +void test_durable_voter_binding_is_atomic_before_first_vote(void**) { + using TStage = TDurableState::EStage; + for (auto stage : {TStage::TempCreated, TStage::Written, TStage::FileSynced, TStage::Renamed, TStage::DirectorySynced}) { + TFiles files; + auto child = fork(); assert_true(child >= 0); + if (!child) { + bool armed = false; + try { auto state = std::make_shared(files.Name, 4, [&](auto point) { if (armed && point == stage) _exit(90); }); + armed = true; + TRaft raft(state, 4, Peers(4), {4}); + } catch (...) { _exit(91); } _exit(92); + } + int status = 0; assert_int_equal(waitpid(child, &status, 0), child); + assert_true(WIFEXITED(status)); assert_int_equal(WEXITSTATUS(status), 90); + TDurableState state(files.Name, 4); + if (stage >= TStage::Renamed) assert_true(state.GetBoundVoters() == (std::vector{1, 2, 3})); + else assert_true(state.GetBoundVoters().empty()); + assert_int_equal(state.CurrentTerm, 1); assert_int_equal(state.VotedFor, 0); assert_int_equal(state.LastLogIndex, 0); + } +} +void test_trusted_legacy_voter_baseline_migrates_durable_state(void**) { + { + TFiles files; + { + TDurableState fresh(files.Name, 1); + Reject([&] { fresh.BindVotersFromTrustedBaseline({1, 2, 3}); }); + } + auto recovered = std::make_shared(files.Name, 1); + assert_int_equal(recovered->CurrentTerm, 1); + assert_int_equal(recovered->LastLogIndex, 0); + Reject([&] { TRaft missingBaseline(recovered, 1, Peers(1, 3)); }); + recovered->BindVotersFromTrustedBaseline({1, 2, 3}); + assert_true(recovered->GetBoundVoters() == (std::vector{1, 2, 3})); + TRaft raft(recovered, 1, Peers(1, 3)); + assert_true(raft.GetVoterIds() == (std::vector{1, 2, 3})); + } + { + TFiles files; TStateFields before; + { + TDurableState legacy(files.Name, 1); + legacy.CurrentTerm = 3; legacy.VotedFor = 1; + auto entry = NewHoldedMessage(); entry->Term = 2; + legacy.Append(entry); before = legacy; + } + const auto path = files.Name + ".durable.1"; + std::ifstream originalInput(path, std::ios::binary); + const std::string original(std::istreambuf_iterator(originalInput), {}); + assert_true(original.substr(0, 8) == "MRFTD004"); + + { + auto state = std::make_shared(files.Name, 1); + assert_true(state->GetBoundVoters().empty()); + Reject([&] { state->BindVoters({1, 2, 3}); }); + Reject([&] { state->BindVotersFromTrustedBaseline({2, 3}); }); + assert_true(static_cast(*state) == before); + assert_true(state->GetBoundVoters().empty()); + std::ifstream unchanged(path, std::ios::binary); + assert_true(std::string(std::istreambuf_iterator(unchanged), {}) == original); + + state->BindVotersFromTrustedBaseline({1, 2, 3}); + assert_true(static_cast(*state) == before); + assert_true(state->GetBoundVoters() == (std::vector{1, 2, 3})); + assert_int_equal(state->RetainedLogTerm(1), 2); + } + { std::ifstream migrated(path, std::ios::binary); std::string magic(8, '\0'); + migrated.read(magic.data(), magic.size()); assert_true(magic == "MRFTD004"); } + auto recovered = std::make_shared(files.Name, 1); + assert_true(static_cast(*recovered) == before); + assert_true(recovered->GetBoundVoters() == (std::vector{1, 2, 3})); + TRaft raft(recovered, 1, Peers(1, 3)); + assert_true(raft.GetVoterIds() == (std::vector{1, 2, 3})); + } + { + TFiles files; TStateFields before; std::string application; + { + TDurableState legacy(files.Name, 1); TKv image; + legacy.CurrentTerm = 3; legacy.VotedFor = 1; + auto entry = image.Prepare(Command("migrated", "value")); entry->Term = 2; + legacy.Append(entry); image.Write(entry, 1); + application = image.SaveSnapshot(); + legacy.InstallSnapshot({1, 2, {1, 2, 3}, application}); + auto tail = NewHoldedMessage(); tail->Term = 3; legacy.Append(tail); + before = legacy; + } + const auto path = files.Name + ".durable.1"; + std::ifstream originalInput(path, std::ios::binary); + const std::string original(std::istreambuf_iterator(originalInput), {}); + assert_true(original.substr(0, 8) == "MRFTD004"); + + { + auto state = std::make_shared(files.Name, 1); + Reject([&] { state->BindVotersFromTrustedBaseline({1, 2}); }); + assert_true(static_cast(*state) == before); + assert_true(state->GetBoundVoters().empty()); + std::ifstream unchanged(path, std::ios::binary); + assert_true(std::string(std::istreambuf_iterator(unchanged), {}) == original); + state->BindVotersFromTrustedBaseline({1, 2, 3}); + assert_true(static_cast(*state) == before); + assert_true(state->GetSnapshot()->Data == application); + assert_true(state->GetBoundVoters() == (std::vector{1, 2, 3})); + } + { std::ifstream migrated(path, std::ios::binary); std::string magic(8, '\0'); + migrated.read(magic.data(), magic.size()); assert_true(magic == "MRFTD004"); } + auto recovered = std::make_shared(files.Name, 1); + assert_true(static_cast(*recovered) == before); + assert_true(recovered->GetBoundVoters() == (std::vector{1, 2, 3})); + auto raft = std::make_shared(recovered, 1, Peers(1, 3)); + auto app = std::make_shared(); TRequestProcessor processor(raft, app, {}); + assert_true(Read(*app, "migrated") == "value"); + assert_true(bool(recovered->Get(2))); + } +} +void test_trusted_legacy_voter_baseline_refuses_configuration_logs(void**) { + TFiles files; + { + TDurableState legacy(files.Name, 1); + legacy.CurrentTerm = 2; + auto entry = NewHoldedMessage(); entry->Term = 2; entry->Flags = TLogEntry::EConfiguration; + legacy.Append(entry); + } + const auto path = files.Name + ".durable.1"; + std::ifstream originalInput(path, std::ios::binary); + const std::string original(std::istreambuf_iterator(originalInput), {}); + auto state = std::make_shared(files.Name, 1); + Reject([&] { state->BindVotersFromTrustedBaseline({1, 2, 3}); }); + assert_true(state->GetBoundVoters().empty()); assert_int_equal(state->LastLogIndex, 1); + std::ifstream unchanged(path, std::ios::binary); + assert_true(std::string(std::istreambuf_iterator(unchanged), {}) == original); +} +void test_trusted_legacy_voter_baseline_must_precede_raft_startup(void**) { + TFiles files; + { + TDurableState legacy(files.Name, 1); + legacy.CurrentTerm = 2; legacy.VotedFor = 1; + legacy.Commit(); + } + const auto path = files.Name + ".durable.1"; + std::ifstream originalInput(path, std::ios::binary); + const std::string original(std::istreambuf_iterator(originalInput), {}); + auto state = std::make_shared(files.Name, 1); + Reject([&] { TRaft missingBaseline(state, 1, Peers(1, 3)); }); + assert_true(state->GetBoundVoters().empty()); + assert_int_equal(state->VotedFor, 1); + std::ifstream unchanged(path, std::ios::binary); + assert_true(std::string(std::istreambuf_iterator(unchanged), {}) == original); + + state->BindVotersFromTrustedBaseline({1, 2, 3}); + auto raft = std::make_shared(state, 1, Peers(1, 3)); + assert_true(raft->GetVoterIds() == (std::vector{1, 2, 3})); + assert_true(state->GetBoundVoters() == (std::vector{1, 2, 3})); + std::ifstream migratedInput(path, std::ios::binary); + const std::string migrated(std::istreambuf_iterator(migratedInput), {}); + Reject([&] { state->BindVotersFromTrustedBaseline({1, 2, 3}); }); + std::ifstream after(path, std::ios::binary); + assert_true(std::string(std::istreambuf_iterator(after), {}) == migrated); +} +void test_trusted_legacy_voter_binding_is_atomic_at_each_publish_stage(void**) { + using TStage = TDurableState::EStage; + const std::vector stages = {TStage::TempCreated, TStage::Written, TStage::FileSynced, + TStage::Renamed, TStage::DirectorySynced}; + for (bool withSnapshot : {false, true}) { + for (auto stage : stages) { + TFiles files; + { + TDurableState legacy(files.Name, 1); legacy.CurrentTerm = 3; legacy.VotedFor = 1; + if (withSnapshot) { + TKv image; auto entry = image.Prepare(Command("atomic", "snapshot")); + entry->Term = 2; legacy.Append(entry); image.Write(entry, 1); + legacy.InstallSnapshot({1, 2, {1, 2, 3}, image.SaveSnapshot()}); + auto tail = NewHoldedMessage(); tail->Term = 3; legacy.Append(tail); + } else { + auto entry = NewHoldedMessage(); entry->Term = 2; legacy.Append(entry); + } + } + auto child = fork(); assert_true(child >= 0); + if (!child) { + try { + TDurableState state(files.Name, 1, [&](auto point) { + if (point == stage) _exit(90); + }); + state.BindVotersFromTrustedBaseline({1, 2, 3}); + } catch (...) { _exit(91); } + _exit(92); + } + int status = 0; assert_int_equal(waitpid(child, &status, 0), child); + assert_true(WIFEXITED(status)); assert_int_equal(WEXITSTATUS(status), 90); + + TDurableState recovered(files.Name, 1); + if (stage < TStage::Renamed) { + assert_true(recovered.GetBoundVoters().empty()); + assert_int_equal(recovered.LastLogIndex, withSnapshot ? 2 : 1); + } else { + assert_true(recovered.GetBoundVoters() == (std::vector{1, 2, 3})); + assert_int_equal(recovered.LastLogIndex, withSnapshot ? 2 : 1); + } + if (withSnapshot) { + assert_true(bool(recovered.GetSnapshot())); + assert_int_equal(recovered.SnapshotIndex, 1); + assert_int_equal(recovered.SnapshotTerm, 2); + } + } + } +} +void test_durable_learner_snapshot_suffix_and_role_survive_restart(void**) { + TFiles files; + { + auto state = std::make_shared(files.Name, 4); TRaft raft(state, 4, Peers(4), {4}); + TKv image; auto entry = image.Prepare(Command("a", "snapshot")); state->Append(entry); image.Write(entry, 1); + state->InstallSnapshot({1, 1, {1, 2, 3}, image.SaveSnapshot()}); + state->Append(image.Prepare(Command("b", "tail"))); + } + auto state = std::make_shared(files.Name, 4); + Reject([&] { TRaft promoted(state, 4, Peers(4)); }); + auto raft = std::make_shared(state, 4, Peers(4), std::vector{4}); + auto app = std::make_shared(); TRequestProcessor processor(raft, app, {}); + assert_true(Read(*app, "a") == "snapshot"); assert_true(Read(*app, "b").empty()); + auto heartbeat = NewHoldedMessage(TMessageEx{.Src = 1, .Dst = 4, .Term = 1}, + TAppendEntriesRequest{.PrevLogIndex = 2, .PrevLogTerm = 1, .LeaderCommit = 2, .LeaderId = 1}); + raft->Process({}, heartbeat); processor.ProcessCommitted(); + assert_true(Read(*app, "b") == "tail"); assert_int_equal(state->VotedFor, 0); + assert_true(state->GetBoundVoters() == (std::vector{1, 2, 3})); +} +void test_durable_v4_corrupt_commit_or_roles_fail_without_rewriting(void**) { + for (size_t offset : {size_t(40), size_t(52), size_t(56), size_t(64)}) { + TFiles files; + { auto state = std::make_shared(files.Name, 4); TRaft raft(state, 4, Peers(4), {4}); } + const auto path = files.Name + ".durable.4"; + std::ifstream input(path, std::ios::binary); std::string bytes(std::istreambuf_iterator(input), {}); input.close(); + assert_true(bytes.substr(0, 8) == "MRFTD004"); + const auto width = offset == 40 || offset == 64 ? 8 : 4; + for (size_t i = offset; i < offset + width; ++i) bytes[i] = offset == 52 || offset == 56 ? 0 : char(0xff); + const auto crc = NSnapshotCodec::Checksum(bytes.substr(0, 8) + bytes.substr(20)); + for (int i = 0; i < 4; ++i) bytes[16 + i] = char(crc >> (8 * i)); + { std::ofstream out(path, std::ios::binary); out.write(bytes.data(), bytes.size()); } + Reject([&] { TDurableState invalid(files.Name, 4); }); + std::ifstream after(path, std::ios::binary); assert_true(std::string(std::istreambuf_iterator(after), {}) == bytes); + } +} +void test_durable_snapshot_adoption_preserves_vote_log_and_application(void**) { + TFiles files; TStateFields before; std::string application; + { + TDurableState state(files.Name, 1); TKv app; + state.CurrentTerm = 3; state.VotedFor = 1; + auto entry = app.Prepare(Command("a", "snapshot")); state.Append(entry); app.Write(entry, 1); + application = app.SaveSnapshot(); state.InstallSnapshot({1, 1, {1, 2, 3}, application}); + state.Append(app.Prepare(Command("b", "tail"))); before = state; + } + const auto path = files.Name + ".durable.1"; + { std::ifstream input(path, std::ios::binary); std::string magic(8, '\0'); input.read(magic.data(), 8); assert_true(magic == "MRFTD004"); } + { + auto state = std::make_shared(files.Name, 1); + auto raft = std::make_shared(state, 1, Peers(1), std::vector{4}); + assert_true(static_cast(*state) == before); + assert_true(state->GetSnapshot()->Data == application); assert_true(bool(state->Get(2))); + auto app = std::make_shared(); TRequestProcessor processor(raft, app, {}); + assert_true(Read(*app, "a") == "snapshot"); assert_true(Read(*app, "b").empty()); + Reject([&] { TRaft promoted(state, 1, Peers(1)); }); + } + TDurableState state(files.Name, 1); + assert_true(static_cast(state) == before); + assert_true(state.GetSnapshot()->Data == application); + assert_true(state.GetBoundVoters() == (std::vector{1, 2, 3})); +} +void test_legacy_disk_backend_cannot_enable_learner_roles(void**) { + TFiles files; auto state = std::make_shared(files.Name, 4); + Reject([&] { TRaft raft(state, 4, Peers(4), {4}); }); + assert_int_equal(state->CurrentTerm, 1); assert_int_equal(state->VotedFor, 0); +} +void test_legacy_leader_cannot_stage_online_learner(void**) { + TFiles files; auto state = std::make_shared(files.Name, 1); + TRaft raft(state, 1, {}); raft.ProcessTimeout({}); + assert_int_equal(raft.CurrentStateName(), EState::LEADER); + const auto fields = static_cast(*state); + Reject([&] { raft.AddLearnerTarget(2, std::make_shared()); }); + assert_true(static_cast(*state) == fields); + assert_true(raft.GetLearnerIds().empty()); assert_int_equal(raft.GetNpeers(), 0); +} +void test_online_learner_staging_does_not_change_durable_binding(void**) { + TFiles files; const auto path = files.Name + ".durable.1"; + { + auto state = std::make_shared(files.Name, 1); TRaft raft(state, 1, {}); + raft.ProcessTimeout({}); assert_int_equal(raft.CurrentStateName(), EState::LEADER); + std::ifstream input(path, std::ios::binary); std::string before(std::istreambuf_iterator(input), {}); input.close(); + auto endpoint = std::make_shared(); raft.AddLearnerTarget(2, endpoint); + assert_true(state->GetBoundVoters() == std::vector({1})); + std::ifstream afterAdd(path, std::ios::binary); + assert_true(std::string(std::istreambuf_iterator(afterAdd), {}) == before); + assert_true(raft.RemoveLearnerTarget(2) == endpoint); + std::ifstream afterRemove(path, std::ios::binary); + assert_true(std::string(std::istreambuf_iterator(afterRemove), {}) == before); + raft.AddLearnerTarget(2, endpoint); // Leave staged at process shutdown. + std::ifstream afterReadd(path, std::ios::binary); + assert_true(std::string(std::istreambuf_iterator(afterReadd), {}) == before); + } + auto recovered = std::make_shared(files.Name, 1); + TRaft raft(recovered, 1, {}); + assert_true(raft.GetLearnerIds().empty()); assert_true(recovered->GetBoundVoters() == std::vector({1})); +} +#endif +} +int main() { + const CMUnitTest tests[] = { + cmocka_unit_test(test_invalid_learner_role_configurations_fail_before_binding), + cmocka_unit_test(test_learner_never_campaigns_votes_or_proposes), + cmocka_unit_test(test_learner_messages_cannot_change_voter_term_or_leadership), + cmocka_unit_test(test_voter_commit_quorums_ignore_arbitrary_learner_matches), + cmocka_unit_test(test_single_voter_commits_and_reads_without_learner_acknowledgements), + cmocka_unit_test(test_learner_grants_cannot_elect_candidate_and_no_votes_are_solicited), + cmocka_unit_test(test_learner_replication_cannot_commit_or_satisfy_strong_read), + cmocka_unit_test(test_learner_commit_ack_waits_for_commit_propagation), + cmocka_unit_test(test_learner_snapshot_catchup_status_and_restart_preserve_role), + cmocka_unit_test(test_restart_cannot_promote_learner_or_change_voters_but_can_add_transport), + cmocka_unit_test(test_nonfresh_unbound_store_requires_snapshot_proof_for_learners), + cmocka_unit_test(test_online_learner_staging_snapshot_catchup_and_quorum_isolation), + cmocka_unit_test(test_online_learner_removal_and_stale_reply_after_readdition), + cmocka_unit_test(test_online_learner_removal_during_snapshot_and_delayed_ack), + cmocka_unit_test(test_online_learner_requires_reconciliation_after_leader_change), + cmocka_unit_test(test_online_learner_staging_rejects_invalid_calls), + cmocka_unit_test(test_recovered_hard_state_must_match_fixed_voters), +#ifdef __linux__ + cmocka_unit_test(test_legacy_recovered_vote_outside_voters_preserves_storage), + cmocka_unit_test(test_durable_voter_binding_is_atomic_before_first_vote), + cmocka_unit_test(test_durable_learner_snapshot_suffix_and_role_survive_restart), + cmocka_unit_test(test_durable_v4_corrupt_commit_or_roles_fail_without_rewriting), + cmocka_unit_test(test_durable_snapshot_adoption_preserves_vote_log_and_application), + cmocka_unit_test(test_trusted_legacy_voter_baseline_migrates_durable_state), + cmocka_unit_test(test_trusted_legacy_voter_baseline_refuses_configuration_logs), + cmocka_unit_test(test_trusted_legacy_voter_baseline_must_precede_raft_startup), + cmocka_unit_test(test_trusted_legacy_voter_binding_is_atomic_at_each_publish_stage), + cmocka_unit_test(test_legacy_disk_backend_cannot_enable_learner_roles), + cmocka_unit_test(test_legacy_leader_cannot_stage_online_learner), + cmocka_unit_test(test_online_learner_staging_does_not_change_durable_binding), +#endif + }; + return cmocka_run_group_tests(tests, nullptr, nullptr); +} diff --git a/test/test_message_safety.cpp b/test/test_message_safety.cpp new file mode 100644 index 0000000..9b0d05b --- /dev/null +++ b/test/test_message_safety.cpp @@ -0,0 +1,173 @@ +#include +#include +#include +#include +#include +extern "C" { +#include +} + +namespace { +void test_typed_allocations_reject_undersized_storage(void**) { + for (uint32_t size : {0u, 7u, 8u, uint32_t(sizeof(TRequestVoteRequest) - 1)}) { + for (bool explicitType : {false, true}) { + bool rejected = false; + try { + if (explicitType) NewHoldedMessage(uint32_t(EMessageType::REQUEST_VOTE_REQUEST), size); + else NewHoldedMessage(size); + } catch (const std::invalid_argument&) { rejected = true; } + assert_true(rejected); + } + } +} +void test_short_wrong_type_and_empty_casts_are_rejected(void**) { + auto shortFrame = NewHoldedMessage(uint32_t(EMessageType::REQUEST_VOTE_REQUEST), sizeof(TMessage)); + auto wrongType = NewHoldedMessage(uint32_t(EMessageType::LOG_ENTRY), sizeof(TRequestVoteRequest)); + for (auto message : {shortFrame, wrongType, TMessageHolder{}}) { + assert_false(message.IsEx()); + assert_false(message.Maybe()); + bool rejected = false; + try { message.Cast(); } catch (const std::bad_cast&) { rejected = true; } + assert_true(rejected); + } +} +struct TEndpoint : INode { + std::vector> Sent; + void Send(TMessageHolder message) override { + if (message->Type == uint32_t(EMessageType::APPEND_ENTRIES_REQUEST)) + Sent.push_back(message.Cast()); + } + void Drain() override {} +}; +struct TCountingState : TState { + mutable unsigned Reads = 0; + TMessageHolder Get(int64_t index) const override { + ++Reads; return TState::Get(index); + } +}; +void test_memory_term_lookup_avoids_copying_full_entry(void**) { + TCountingState state; + auto entry = NewHoldedMessage(sizeof(TLogEntry) + 1024 * 1024); + entry->Term = 1; entry->Data[0] = 'x'; state.Append(entry); + for (int i = 0; i < 20; ++i) assert_int_equal(state.LogTerm(1), 1); + assert_int_equal(state.Reads, 0); + auto copy = state.Get(1); copy->Term = 9; copy->Data[0] = 'y'; + assert_int_equal(state.LogTerm(1), 1); + assert_int_equal(state.Get(1)->Data[0], 'x'); +} +void test_memory_log_does_not_alias_proposed_or_read_entries(void**) { + auto entry = NewHoldedMessage(sizeof(TLogEntry) + 1); + entry->Data[0] = 'a'; + TState state; + state.Append(entry); + entry->Term = 9; entry->Data[0] = 'b'; + auto read = state.Get(1); + assert_int_equal(read->Term, 1); assert_int_equal(read->Data[0], 'a'); + read->Term = 7; read->Data[0] = 'c'; + assert_int_equal(state.Get(1)->Term, 1); assert_int_equal(state.Get(1)->Data[0], 'a'); + assert_int_equal(state.LastLogTerm, 1); +} +void test_memory_log_constructor_owns_initial_entries(void**) { + auto entry = NewHoldedMessage(sizeof(TLogEntry) + 1); + entry->Data[0] = 'x'; + TState state(1, 0, {entry}); + entry->Data[0] = 'y'; entry->Term = 2; + assert_int_equal(state.Get(1)->Data[0], 'x'); assert_int_equal(state.LogTerm(1), 1); +} +void test_memory_state_copies_do_not_share_mutable_log_entries(void**) { + TState source; source.Append(NewHoldedMessage()); + TState copied(source), assigned; assigned = source; + source.Log[0]->Term = 4; + assert_int_equal(copied.LogTerm(1), 1); + assert_int_equal(assigned.LogTerm(1), 1); + copied.Log[0]->Term = 5; + assert_int_equal(assigned.LogTerm(1), 1); +} +void test_raft_batches_respect_budget_and_continue_replication(void**) { + auto state = std::make_shared(); + state->CurrentTerm = 2; + auto entry = NewHoldedMessage(1024 * 1024); + entry->Term = 2; + for (int i = 0; i < 100; ++i) state->Append(entry); + auto peer = std::make_shared(); + TRaft raft(state, 1, {{2, peer}}); + raft.Become(EState::LEADER); + auto* progress = const_cast(raft.GetVolatileState()); + progress->BatchSize[2] = 1024; + raft.ProcessTimeout(ITimeSource::Time{}); + assert_int_equal(peer->Sent.size(), 1); + auto first = peer->Sent[0]; + ValidateMessageTree(first); + assert_true(first->Nentries > 0); + assert_true(first->Nentries < 100); + assert_int_equal(first->Nentries, first.PayloadSize); + auto reply = NewHoldedMessage(TMessageEx{.Src = 2, .Dst = 1, .Term = 2, .Seqno = first->Seqno}, + TAppendEntriesResponse{.MatchIndex = first->Nentries, .Success = 1}); + raft.Process(ITimeSource::Time{}, reply); + raft.ProcessTimeout(ITimeSource::Time{}); + assert_int_equal(peer->Sent.size(), 2); + auto second = peer->Sent[1]; + ValidateMessageTree(second); + assert_int_equal(second->PrevLogIndex, first->Nentries); + assert_int_equal(first->Nentries + second->Nentries, 100); + assert_true(second->Nentries > 0); +} +void test_invalid_proposal_does_not_mutate_persistent_state(void**) { + auto state = std::make_shared(); + TRaft raft(state, 1, {}); + raft.Become(EState::LEADER); + for (uint32_t length : {0u, uint32_t(sizeof(TLogEntry) - 1), TMessageLimits{}.MaxFrameBytes + 1}) { + auto entry = NewHoldedMessage(); + entry->Len = length; + entry->Term = 99; + bool rejected = false; + try { raft.Append(entry); } catch (const TMessageError&) { rejected = true; } + assert_true(rejected); + assert_int_equal(state->LastLogIndex, 0); + assert_int_equal(entry->Term, 99); + } +} +void test_declared_length_cannot_exceed_actual_proposal_buffer(void**) { + auto state = std::make_shared(); + TRaft raft(state, 1, {}); raft.Become(EState::LEADER); + auto entry = NewHoldedMessage(); + entry->Len = sizeof(TLogEntry) + 8; + bool rejected = false; + try { raft.Append(entry); } catch (const TMessageError&) { rejected = true; } + assert_true(rejected); + assert_int_equal(state->LastLogIndex, 0); +} +void test_invalid_holder_extents_are_rejected_before_batch_access(void**) { + auto shortEntry = NewHoldedMessage(); + shortEntry->Len += 8; + assert_false(shortEntry.Maybe()); + bool rejected = false; + try { ValidateMessageTree(shortEntry); } catch (const TMessageError&) { rejected = true; } + assert_true(rejected); + auto append = NewHoldedMessage(TAppendEntriesRequest{.Nentries = 2}); + append.InitPayload(1); append.Payload[0] = NewHoldedMessage(); + append.PayloadSize = 2; + rejected = false; + try { ValidateMessageTree(append); } catch (const TMessageError&) { rejected = true; } + assert_true(rejected); + append.PayloadSize = 1; append->Nentries = 1; append.Payload[0] = shortEntry; + rejected = false; + try { ValidateMessageTree(append); } catch (const TMessageError&) { rejected = true; } + assert_true(rejected); +} +} +int main() { + const CMUnitTest tests[] = { + cmocka_unit_test(test_memory_term_lookup_avoids_copying_full_entry), + cmocka_unit_test(test_memory_log_does_not_alias_proposed_or_read_entries), + cmocka_unit_test(test_memory_log_constructor_owns_initial_entries), + cmocka_unit_test(test_memory_state_copies_do_not_share_mutable_log_entries), + cmocka_unit_test(test_typed_allocations_reject_undersized_storage), + cmocka_unit_test(test_short_wrong_type_and_empty_casts_are_rejected), + cmocka_unit_test(test_raft_batches_respect_budget_and_continue_replication), + cmocka_unit_test(test_invalid_proposal_does_not_mutate_persistent_state), + cmocka_unit_test(test_declared_length_cannot_exceed_actual_proposal_buffer), + cmocka_unit_test(test_invalid_holder_extents_are_rejected_before_batch_access), + }; + return cmocka_run_group_tests(tests, nullptr, nullptr); +} diff --git a/test/test_network_frames.cpp b/test/test_network_frames.cpp new file mode 100644 index 0000000..558f0c7 --- /dev/null +++ b/test/test_network_frames.cpp @@ -0,0 +1,265 @@ +#include +#include +#include +#include +#include +extern "C" { +#include +} + +namespace { +// Immediate deterministic I/O: no sockets, sleeps, pollers or wall clock. +struct TStream { + std::vector Bytes; + size_t Position = 0; + size_t Chunk = SIZE_MAX; + size_t MaxRequested = 0; + unsigned Reads = 0; + std::vector Output; + NNet::TValueTask ReadSome(void* out, size_t length) { + ++Reads; + MaxRequested = std::max(MaxRequested, length); + const auto count = std::min({length, Chunk, Bytes.size() - Position}); + if (count) memcpy(out, Bytes.data() + Position, count); + Position += count; + co_return count; + } + NNet::TValueTask WriteSome(const void* data, size_t length) { + const auto count = std::min(length, Chunk); + auto bytes = static_cast(data); + Output.insert(Output.end(), bytes, bytes + count); + co_return count; + } + void Add(uint32_t type, uint32_t length) { + const char* a = reinterpret_cast(&type); + const char* b = reinterpret_cast(&length); + Bytes.insert(Bytes.end(), a, a + sizeof(type)); + Bytes.insert(Bytes.end(), b, b + sizeof(length)); + } + template void Add(TMessageHolder message) { + auto data = reinterpret_cast(message.Mes); + Bytes.insert(Bytes.end(), data, data + message->Len); + } +}; +TMessageHolder Read(TStream& stream) { + auto task = TMessageReader(stream).Read(); + assert_true(task.done()); + return task.await_resume(); +} +bool Rejected(TStream& stream) { + try { Read(stream); } catch (const std::runtime_error&) { return true; } + return false; +} + +void test_fragmented_header_and_body_are_read_exactly(void**) { + TStream stream; + stream.Chunk = 1; + auto original = NewHoldedMessage(TRequestVoteRequest{.LastLogIndex = 7, .LastLogTerm = 2, .CandidateId = 1}); + stream.Add(original); + bool complete = false; + try { + auto received = Read(stream).Cast(); + assert_int_equal(received->LastLogIndex, 7); + assert_int_equal(received->LastLogTerm, 2); + assert_int_equal(stream.Position, original->Len); + complete = true; + } catch (const std::runtime_error&) {} + assert_true(complete); +} +void test_short_fixed_frame_is_rejected(void**) { + TStream stream; + stream.Add(uint32_t(EMessageType::REQUEST_VOTE_REQUEST), sizeof(TMessage)); + assert_true(Rejected(stream)); +} +void test_unknown_types_and_short_snapshot_headers_are_rejected(void**) { + for (auto type : {0u, 6u, 7u, 10u, UINT32_MAX}) { + TStream stream; + stream.Add(type, sizeof(TMessage)); + assert_true(Rejected(stream)); + } +} +void test_append_entries_rejects_non_log_child(void**) { + TStream stream; + stream.Add(NewHoldedMessage(TAppendEntriesRequest{.Nentries = 1})); + stream.Add(NewHoldedMessage()); + assert_true(Rejected(stream)); +} +void test_append_entries_rejects_nested_append(void**) { + TStream stream; + stream.Add(NewHoldedMessage(TAppendEntriesRequest{.Nentries = 1})); + stream.Add(NewHoldedMessage()); + assert_true(Rejected(stream)); +} +void test_oversized_frame_is_rejected_before_body_read(void**) { + TStream stream; + stream.Add(uint32_t(EMessageType::COMMAND_REQUEST), 16 * 1024 * 1024 + 1); + assert_true(Rejected(stream)); + assert_true(stream.MaxRequested <= sizeof(TMessage)); +} +void test_all_truncated_headers_are_rejected(void**) { + for (size_t size = 0; size < sizeof(TMessage); ++size) { + TStream stream; + stream.Add(uint32_t(EMessageType::COMMAND_REQUEST), sizeof(TCommandRequest)); + stream.Bytes.resize(size); + stream.Chunk = 1; + assert_true(Rejected(stream)); + } +} +void test_invalid_lengths_are_rejected_before_body_read(void**) { + for (auto length : {0u, 1u, 7u, 8u, 15u, UINT32_MAX}) { + TStream stream; + stream.Add(uint32_t(EMessageType::COMMAND_REQUEST), length); + assert_true(Rejected(stream)); + assert_int_equal(stream.Reads, 1); + } + TStream stream; + stream.Add(uint32_t(EMessageType::REQUEST_VOTE_REQUEST), sizeof(TRequestVoteRequest) + 1); + assert_true(Rejected(stream)); + assert_int_equal(stream.Reads, 1); +} +void test_all_truncated_bodies_are_rejected(void**) { + auto message = NewHoldedMessage(); + for (size_t size = sizeof(TMessage); size < message->Len; ++size) { + TStream stream; + stream.Add(message); + stream.Bytes.resize(size); + stream.Chunk = 3; + assert_true(Rejected(stream)); + } +} +void test_count_and_minimum_batch_budget_are_checked_before_children(void**) { + for (auto count : {4097u, UINT32_MAX}) { + TStream stream; + stream.Add(NewHoldedMessage(TAppendEntriesRequest{.Nentries = count})); + assert_true(Rejected(stream)); + assert_int_equal(stream.Reads, 2); + } + TStream stream; + stream.Add(NewHoldedMessage(TAppendEntriesRequest{.Nentries = 1})); + TMessageLimits limits; + limits.MaxBatchBytes = sizeof(TAppendEntriesRequest) + sizeof(TLogEntry) - 1; + auto task = TMessageReader(stream, limits).Read(); + bool rejected = false; + try { task.await_resume(); } catch (const TMessageError&) { rejected = true; } + assert_true(rejected); + assert_int_equal(stream.Reads, 2); +} +void test_aggregate_budget_is_shared_between_entries(void**) { + for (bool enough : {false, true}) { + TStream stream; + stream.Add(NewHoldedMessage(TAppendEntriesRequest{.Nentries = 2})); + stream.Add(NewHoldedMessage(sizeof(TLogEntry) + 1)); + stream.Add(NewHoldedMessage(sizeof(TLogEntry) + 1)); + TMessageLimits limits; + limits.MaxBatchBytes = stream.Bytes.size() - (enough ? 0 : 1); + auto task = TMessageReader(stream, limits).Read(); + bool rejected = false; + try { + auto message = task.await_resume(); + assert_int_equal(message.PayloadSize, 2); + } catch (const TMessageError&) { rejected = true; } + assert_int_equal(rejected, !enough); + if (!enough) assert_int_equal(stream.Position, + sizeof(TAppendEntriesRequest) + sizeof(TLogEntry) + 1 + sizeof(TMessage)); + } +} +void test_failed_reader_cannot_resume_at_another_frame(void**) { + TStream stream; + stream.Add(10, sizeof(TMessage)); + stream.Add(NewHoldedMessage()); + TMessageReader reader(stream); + for (int attempt = 0; attempt < 2; ++attempt) { + auto task = reader.Read(); + bool rejected = false; + try { task.await_resume(); } catch (const TMessageError&) { rejected = true; } + assert_true(rejected); + assert_int_equal(stream.Position, sizeof(TMessage)); + } +} +void test_writer_validates_entire_batch_before_first_byte(void**) { + for (int invalid = 0; invalid < 7; ++invalid) { + auto root = NewHoldedMessage(TAppendEntriesRequest{.Nentries = 1}); + root.InitPayload(1); + root.Payload[0] = NewHoldedMessage(); + TMessageLimits limits; + if (invalid == 0) root->Nentries = 2; + if (invalid == 1) root.Payload[0] = NewHoldedMessage(); + if (invalid == 2) root.Payload[0] = NewHoldedMessage(); + if (invalid == 3) limits.MaxBatchBytes = sizeof(TAppendEntriesRequest) + sizeof(TLogEntry) - 1; + if (invalid == 4) root.Payload.reset(); + if (invalid == 5) root.Payload[0] = {}; + if (invalid == 6) root.Payload[0].InitPayload(1); + TStream stream; + auto task = TMessageWriter(stream, limits).Write(root); + bool rejected = false; + try { task.await_resume(); } catch (const TMessageError&) { rejected = true; } + assert_true(rejected); + assert_true(stream.Output.empty()); + } +} +void test_fragmented_roundtrip_keeps_consecutive_frame_boundaries(void**) { + auto root = NewHoldedMessage(TAppendEntriesRequest{.Nentries = 2}); + root.InitPayload(2); + root.Payload[0] = NewHoldedMessage(); + root.Payload[1] = NewHoldedMessage(sizeof(TLogEntry) + 3); + memcpy(root.Payload[1].Cast()->Data, "abc", 3); + TStream stream; + stream.Chunk = 1; + auto batch = TMessageWriter(stream).Write(root); + batch.await_resume(); + auto command = TMessageWriter(stream).Write(NewHoldedMessage()); + command.await_resume(); + stream.Bytes = stream.Output; + auto received = Read(stream); + assert_int_equal(received.PayloadSize, 2); + auto entry = received.Payload[1].Cast(); + assert_memory_equal(entry->Data, "abc", 3); + auto next = Read(stream).Cast(); + assert_int_equal(next->Len, sizeof(TCommandRequest)); + assert_int_equal(stream.Position, stream.Bytes.size()); +} +void test_fragmented_snapshot_request_and_response_roundtrip(void**) { + for (uint32_t length : {uint32_t(sizeof(TInstallSnapshotRequest)), + uint32_t(sizeof(TInstallSnapshotRequest) + TInstallSnapshotRequest::MaxChunkBytes + 1)}) { + TStream invalid; invalid.Add(uint32_t(EMessageType::INSTALL_SNAPSHOT_REQUEST), length); + assert_true(Rejected(invalid)); assert_int_equal(invalid.Position, sizeof(TMessage)); + } + auto request = NewHoldedMessage(sizeof(TInstallSnapshotRequest) + 1024); + request->Src = 1; request->Dst = 2; request->Term = 3; request->Seqno = 7; + request->SnapshotIndex = 100; request->SnapshotTerm = 2; + request->TotalBytes = 1024; request->Offset = 0; request->Done = 1; request->LeaderId = 1; + memset(request->Data, 's', 1024); + auto response = NewHoldedMessage(TMessageEx{.Src = 2, .Dst = 1, .Term = 3, .Seqno = 7}, + TInstallSnapshotResponse{.SnapshotIndex = 100, .NextOffset = 1024, .Success = 1, .Installed = 1}); + TStream stream; stream.Chunk = 3; + auto send = TMessageWriter(stream).Write(request); send.await_resume(); + auto reply = TMessageWriter(stream).Write(response); reply.await_resume(); + stream.Bytes = stream.Output; + auto received = Read(stream); assert_true(received.IsEx()); + auto snapshot = received.Cast(); + assert_int_equal(snapshot->SnapshotIndex, 100); assert_memory_equal(snapshot->Data, request->Data, 1024); + auto ack = Read(stream).Cast(); assert_true(ack->Installed); + assert_int_equal(ack->NextOffset, 1024); assert_int_equal(stream.Position, stream.Bytes.size()); +} +} + +int main() { + const CMUnitTest tests[] = { + cmocka_unit_test(test_fragmented_header_and_body_are_read_exactly), + cmocka_unit_test(test_short_fixed_frame_is_rejected), + cmocka_unit_test(test_unknown_types_and_short_snapshot_headers_are_rejected), + cmocka_unit_test(test_append_entries_rejects_non_log_child), + cmocka_unit_test(test_append_entries_rejects_nested_append), + cmocka_unit_test(test_oversized_frame_is_rejected_before_body_read), + cmocka_unit_test(test_all_truncated_headers_are_rejected), + cmocka_unit_test(test_invalid_lengths_are_rejected_before_body_read), + cmocka_unit_test(test_all_truncated_bodies_are_rejected), + cmocka_unit_test(test_count_and_minimum_batch_budget_are_checked_before_children), + cmocka_unit_test(test_aggregate_budget_is_shared_between_entries), + cmocka_unit_test(test_failed_reader_cannot_resume_at_another_frame), + cmocka_unit_test(test_writer_validates_entire_batch_before_first_byte), + cmocka_unit_test(test_fragmented_roundtrip_keeps_consecutive_frame_boundaries), + cmocka_unit_test(test_fragmented_snapshot_request_and_response_roundtrip), + }; + return cmocka_run_group_tests(tests, nullptr, nullptr); +} diff --git a/test/test_node_lifecycle.cpp b/test/test_node_lifecycle.cpp new file mode 100644 index 0000000..7d0405d --- /dev/null +++ b/test/test_node_lifecycle.cpp @@ -0,0 +1,91 @@ +#include +#include +#include +#include +extern "C" { +#include +} + +namespace { +struct TPoller : NNet::TPollerBase { + void Tick() { ProcessTimers(); } + void TickOne() { + assert_false(Timers_.empty()); + auto timer = Timers_.top(); Timers_.pop(); + LastFiredTimer_ = timer.Id; + if (timer.Handle) timer.Handle.resume(); + } +}; +// Model the dependency's connect awaiter: a pending timer retains the coroutine +// handle, and destroying the awaiter does not unregister that timer. The timer +// is already due, so the schedule needs neither sockets nor wall-clock waits. +struct TSocket { + TPoller* Owner = nullptr; + bool FailWrite = false; + bool DuplicateDeadline = false; + auto Connect(NNet::TTime deadline = NNet::TTime::max()) { + struct TAwaiter { + TPoller* Owner; + bool DuplicateDeadline; + bool await_ready() { return false; } + void await_suspend(std::coroutine_handle<> h) { + Owner->AddTimer({}, h); // Controllable stand-in for I/O readiness. + if (DuplicateDeadline) Owner->AddTimer({}, h); + } + void await_resume() {} + }; + return TAwaiter{Owner, DuplicateDeadline && deadline != NNet::TTime::max()}; + } + TPoller* Poller() { return Owner; } + NNet::TValueTask WriteSome(const void*, size_t length) { + if (FailWrite) throw std::runtime_error("injected write failure"); + co_return length; + } +}; +void test_disconnect_during_pending_connect_does_not_destroy_timer_target(void**) { + TPoller poller; + TNode node([&](const auto&) { return TSocket{&poller}; }, "peer", + NNet::TAddress("127.0.0.1", 1), std::make_shared()); + node.Drain(); assert_false(node.IsConnected()); + node.Disconnect(); // Must preserve the active reconnect until it completes. + poller.Tick(); + assert_true(node.IsConnected()); + node.Disconnect(); assert_false(node.IsConnected()); +} +void test_old_reader_failure_cannot_disconnect_writer_reconnect(void**) { + TPoller poller; + TNode node([&](const auto&) { return TSocket{&poller}; }, "peer", + NNet::TAddress("127.0.0.1", 1), std::make_shared()); + node.Drain(); poller.Tick(); assert_true(node.IsConnected()); + const auto old = node.GetConnectionGeneration(); + node.Sock().FailWrite = true; + node.Send(NewHoldedMessage()); node.Drain(); + assert_false(node.IsConnected()); assert_true(node.GetConnectionGeneration() != old); + node.Disconnect(old); // Old reader unwinds while the new connect is pending. + poller.Tick(); assert_true(node.IsConnected()); + node.Disconnect(old); // It is stale even after the replacement connected. + assert_true(node.IsConnected()); + node.Send(NewHoldedMessage()); node.Drain(); + node.Disconnect(node.GetConnectionGeneration()); assert_false(node.IsConnected()); +} +void test_connect_cannot_leave_a_second_deadline_wakeup_after_completion(void**) { + TPoller poller; + // The dependency's finite-deadline connect registers the same coroutine for + // both I/O and a timer. Model a second wakeup surviving the first, as seen + // in CI when timeout/I/O completion races with the retry sleep. + TNode node([&](const auto&) { return TSocket{&poller, false, true}; }, "peer", + NNet::TAddress("127.0.0.1", 1), std::make_shared()); + node.Drain(); poller.TickOne(); assert_true(node.IsConnected()); + node.Disconnect(); + poller.Tick(); // A leftover deadline must not resume the destroyed connector. + assert_false(node.IsConnected()); +} +} +int main() { + const CMUnitTest tests[] = { + cmocka_unit_test(test_disconnect_during_pending_connect_does_not_destroy_timer_target), + cmocka_unit_test(test_old_reader_failure_cannot_disconnect_writer_reconnect), + cmocka_unit_test(test_connect_cannot_leave_a_second_deadline_wakeup_after_completion), + }; + return cmocka_run_group_tests(tests, nullptr, nullptr); +} diff --git a/test/test_persist_recovery.cpp b/test/test_persist_recovery.cpp new file mode 100644 index 0000000..449018e --- /dev/null +++ b/test/test_persist_recovery.cpp @@ -0,0 +1,316 @@ +#include +#include +#include +#include +#include +#include +#ifndef _WIN32 +#include +#include +#endif +#include +#include +#include +extern "C" { +#include +} + +namespace { +namespace fs = std::filesystem; +struct TFiles { + fs::path Directory; + std::string Name; + TFiles() { + static unsigned counter = 0; + Directory = fs::temp_directory_path() / + ("miniraft-recovery-" + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count()) + + "-" + std::to_string(counter++)); + assert_true(fs::create_directory(Directory)); + Name = (Directory / "raft").string(); + } + ~TFiles() { std::error_code ignored; fs::remove_all(Directory, ignored); } + fs::path Path(const std::string& kind) const { return Name + "." + kind + ".1"; } + void Seed() { + TDiskState state(Name, 1); + state.CurrentTerm = 7; + state.VotedFor = 2; + for (uint64_t term : {2, 5}) { + auto entry = NewHoldedMessage(); + entry->Term = term; + state.Append(entry); + } + } + template void Overwrite(const std::string& kind, uint64_t offset, T value) { + std::fstream file(Path(kind), std::ios::in | std::ios::out | std::ios::binary); + file.seekp(offset); + file.write(reinterpret_cast(&value), sizeof(value)); + file.flush(); + assert_true(bool(file)); + } + void RejectRecovery() { + auto contents = [&] { + std::map result; + for (auto& file : fs::directory_iterator(Directory)) { + std::ifstream input(file.path(), std::ios::binary); + result[file.path().filename().string()] = + std::string(std::istreambuf_iterator(input), {}); + } + return result; + }; + const auto before = contents(); + bool rejected = false; + try { TDiskState state(Name, 1); } + catch (const std::runtime_error&) { rejected = true; } + assert_true(rejected); + assert_true(before == contents()); + } +}; + +void test_truncated_metadata_is_not_reset(void**) { + for (uint64_t size : {0, 1, 8, 16, 19}) { + TFiles files; + files.Seed(); + fs::resize_file(files.Path("state"), size); + files.RejectRecovery(); + assert_int_equal(fs::file_size(files.Path("state")), size); + } +} +void test_oversized_metadata_is_rejected(void**) { + TFiles files; + files.Seed(); + fs::resize_file(files.Path("state"), 21); + files.RejectRecovery(); + assert_int_equal(fs::file_size(files.Path("state")), 21); +} +void test_missing_storage_is_not_recreated(void**) { + for (const std::string kind : {"state", "index", "entries"}) { + TFiles files; + files.Seed(); + fs::remove(files.Path(kind)); + files.RejectRecovery(); + assert_false(fs::exists(files.Path(kind))); + } +} +void test_short_index_is_rejected(void**) { + TFiles files; + files.Seed(); + fs::resize_file(files.Path("index"), sizeof(uint64_t)); + files.RejectRecovery(); +} +void test_short_entry_is_rejected(void**) { + TFiles files; + files.Seed(); + fs::resize_file(files.Path("entries"), sizeof(TLogEntry) + 4); + files.RejectRecovery(); +} +void test_invalid_entry_header_is_rejected(void**) { + for (uint32_t len : {0u, 7u, 8u, uint32_t(sizeof(TLogEntry) - 1), UINT32_MAX}) { + TFiles files; + files.Seed(); + files.Overwrite("entries", sizeof(uint32_t), len); + files.RejectRecovery(); + } + TFiles files; + files.Seed(); + files.Overwrite("entries", 0, uint32_t(EMessageType::COMMAND_REQUEST)); + files.RejectRecovery(); +} +void test_invalid_offsets_are_rejected(void**) { + for (uint64_t offset : {uint64_t(0), uint64_t(1), UINT64_MAX}) { + TFiles files; + files.Seed(); + files.Overwrite("index", sizeof(uint64_t), offset); + files.RejectRecovery(); + } +} +void test_invalid_term_is_rejected(void**) { + TFiles files; + files.Seed(); + files.Overwrite("state", sizeof(uint64_t), uint64_t(0)); + files.RejectRecovery(); +} +void test_invalid_log_count_is_rejected(void**) { + TFiles files; + files.Seed(); + files.Overwrite("state", 0, UINT64_MAX); + files.RejectRecovery(); +} +void test_truncate_updates_last_term_and_restarts(void**) { + TFiles files; + files.Seed(); + { + TDiskState state(files.Name, 1); + state.RemoveLast(); + assert_int_equal(state.LastLogIndex, 1); + assert_int_equal(state.LastLogTerm, 2); + } + { + TDiskState state(files.Name, 1); + assert_int_equal(state.CurrentTerm, 7); + assert_int_equal(state.VotedFor, 2); + assert_int_equal(state.LastLogTerm, 2); + state.RemoveLast(); + assert_int_equal(state.LastLogIndex, 0); + assert_int_equal(state.LastLogTerm, 0); + } + { + // Logical truncation leaves a suffix on disk. Reopen it as an empty + // log, then ensure the replacement really starts at byte zero. + TDiskState state(files.Name, 1); + assert_int_equal(state.LastLogIndex, 0); + assert_int_equal(state.LastLogTerm, 0); + auto entry = NewHoldedMessage(); + entry->Term = 7; + state.Append(entry); + assert_int_equal(state.Get(1)->Term, 7); + } + TDiskState recovered(files.Name, 1); + assert_int_equal(recovered.LastLogIndex, 1); + assert_int_equal(recovered.LastLogTerm, 7); + assert_int_equal(recovered.Get(1)->Term, 7); + assert_int_equal(recovered.VotedFor, 2); + // Also replace an empty log without reopening: the shared stream position + // is still at the end of the removed entry, rather than at byte zero. + recovered.RemoveLast(); + auto replacement = NewHoldedMessage(); + replacement->Term = 6; + recovered.Append(replacement); + assert_int_equal(recovered.Get(1)->Term, 6); + assert_int_equal(recovered.LastLogTerm, 6); +} +void test_zero_log_entry_terms_are_rejected_without_mutation(void**) { + for (uint64_t index : {0, 1}) { + TFiles files; + files.Seed(); + files.Overwrite("entries", index * sizeof(TLogEntry) + sizeof(TMessage), uint64_t(0)); + files.RejectRecovery(); + } +} +void test_log_terms_newer_than_current_term_are_rejected(void**) { + for (uint64_t term : {uint64_t(8), UINT64_MAX}) { + TFiles files; + files.Seed(); + files.Overwrite("entries", sizeof(TLogEntry) + sizeof(TMessage), term); + files.RejectRecovery(); + } + TFiles files; + files.Seed(); + files.Overwrite("state", sizeof(uint64_t), uint64_t(4)); + files.RejectRecovery(); +} +void test_decreasing_log_terms_are_rejected(void**) { + for (uint64_t index : {0, 1}) { + TFiles files; + files.Seed(); + files.Overwrite("entries", index * sizeof(TLogEntry) + sizeof(TMessage), uint64_t(index == 0 ? 6 : 1)); + files.RejectRecovery(); + } +} +void test_equal_and_skipped_terms_and_full_width_current_term_recover(void**) { + TFiles files; + { + TDiskState state(files.Name, 1); + state.CurrentTerm = UINT64_MAX; + state.VotedFor = 2; + for (uint64_t term : {uint64_t(1), uint64_t(1), uint64_t(5), UINT64_MAX, UINT64_MAX}) { + auto entry = NewHoldedMessage(); + entry->Term = term; + state.Append(entry); + } + } + TDiskState recovered(files.Name, 1); + assert_true(recovered.CurrentTerm == UINT64_MAX); + assert_true(recovered.LastLogTerm == UINT64_MAX); + assert_int_equal(recovered.LastLogIndex, 5); + assert_int_equal(recovered.VotedFor, 2); + assert_int_equal(recovered.Get(1)->Term, 1); + assert_int_equal(recovered.Get(2)->Term, 1); + assert_int_equal(recovered.Get(3)->Term, 5); +} +void test_invalid_terms_in_logically_removed_suffix_are_ignored(void**) { + TFiles files; + files.Seed(); + { + TDiskState state(files.Name, 1); + state.RemoveLast(); + } + files.Overwrite("entries", sizeof(TLogEntry) + sizeof(TMessage), uint64_t(0)); + TDiskState recovered(files.Name, 1); + assert_int_equal(recovered.LastLogIndex, 1); + assert_int_equal(recovered.LastLogTerm, 2); + auto entry = NewHoldedMessage(); + entry->Term = 7; + recovered.Append(entry); + assert_int_equal(recovered.LastLogIndex, 2); + assert_int_equal(recovered.Get(2)->Term, 7); +} + +#ifndef _WIN32 +void test_live_storage_failure_is_fatal(void**) { + TFiles files; + files.Seed(); + const auto child = fork(); + assert_true(child >= 0); + if (child == 0) { + std::set_terminate([] { _exit(73); }); + try { + TDiskState state(files.Name, 1); + fs::resize_file(files.Path("entries"), 0); + state.Get(1); + } catch (...) { _exit(74); } + _exit(75); + } + int status = 0; + assert_int_equal(waitpid(child, &status, 0), child); + assert_true(WIFEXITED(status)); + assert_int_equal(WEXITSTATUS(status), 73); +} +void test_live_invalid_log_term_is_fatal(void**) { + for (uint64_t term : {uint64_t(0), uint64_t(8)}) { + TFiles files; + files.Seed(); + const auto child = fork(); + assert_true(child >= 0); + if (child == 0) { + std::set_terminate([] { _exit(73); }); + try { + TDiskState state(files.Name, 1); + files.Overwrite("entries", sizeof(TMessage), term); + state.Get(1); + } catch (...) { _exit(74); } + _exit(75); + } + int status = 0; + assert_int_equal(waitpid(child, &status, 0), child); + assert_true(WIFEXITED(status)); + assert_int_equal(WEXITSTATUS(status), 73); + } +} + +#endif +} + +int main() { + const CMUnitTest tests[] = { + cmocka_unit_test(test_truncated_metadata_is_not_reset), + cmocka_unit_test(test_oversized_metadata_is_rejected), + cmocka_unit_test(test_missing_storage_is_not_recreated), + cmocka_unit_test(test_short_index_is_rejected), + cmocka_unit_test(test_short_entry_is_rejected), + cmocka_unit_test(test_invalid_entry_header_is_rejected), + cmocka_unit_test(test_invalid_offsets_are_rejected), + cmocka_unit_test(test_invalid_term_is_rejected), + cmocka_unit_test(test_invalid_log_count_is_rejected), + cmocka_unit_test(test_truncate_updates_last_term_and_restarts), + cmocka_unit_test(test_zero_log_entry_terms_are_rejected_without_mutation), + cmocka_unit_test(test_log_terms_newer_than_current_term_are_rejected), + cmocka_unit_test(test_decreasing_log_terms_are_rejected), + cmocka_unit_test(test_equal_and_skipped_terms_and_full_width_current_term_recover), + cmocka_unit_test(test_invalid_terms_in_logically_removed_suffix_are_ignored), +#ifndef _WIN32 + cmocka_unit_test(test_live_storage_failure_is_fatal), + cmocka_unit_test(test_live_invalid_log_term_is_fatal), +#endif + }; + return cmocka_run_group_tests(tests, nullptr, nullptr); +} diff --git a/test/test_quorum.cpp b/test/test_quorum.cpp new file mode 100644 index 0000000..52ffe97 --- /dev/null +++ b/test/test_quorum.cpp @@ -0,0 +1,101 @@ +#include +#include +#include +#include +#include +#include +extern "C" { +#include +} +namespace { +std::vector Members(unsigned mask) { + std::vector out; + for (uint32_t id = 1; id <= 5; ++id) if (mask & (1u << (id - 1))) out.push_back(id); + return out; +} +bool Majority(unsigned members, unsigned votes) { + return std::popcount(members & votes) > std::popcount(members) / 2; +} +void test_stable_quorum_matches_strict_majority_order_statistic(void**) { + for (uint32_t count = 1; count <= 8; ++count) { + std::vector voters; + for (uint32_t id = 1; id <= count; ++id) voters.push_back(id); + TQuorum quorum(voters); + for (uint64_t seed = 0; seed < 64; ++seed) { + auto value = [&](uint32_t id) { return (seed * 7 + id * id * 3) % 17; }; + std::vector expected; + for (auto id : voters) expected.push_back(value(id)); + std::sort(expected.begin(), expected.end()); + assert_int_equal(quorum.Value(value), expected[(count - 1) / 2]); + } + } +} +void test_all_small_joint_election_quorums(void**) { + for (unsigned old = 1; old < 32; ++old) for (unsigned next = 1; next < 32; ++next) { + TQuorum quorum(Members(old), Members(next)); + for (unsigned mask = 0; mask < 32; ++mask) { + std::unordered_set votes{99}; + for (auto id : Members(mask)) votes.insert(id); + assert_int_equal(quorum.HasQuorum(votes), Majority(old, mask) && Majority(next, mask)); + for (uint32_t self = 1; self <= 6; ++self) { + const auto withSelf = self <= 5 ? mask | (1u << (self - 1)) : mask; + assert_int_equal(quorum.HasQuorum(votes, self), Majority(old, withSelf) && Majority(next, withSelf)); + } + } + } +} +void test_joint_commit_and_read_watermarks_need_both_majorities(void**) { + for (unsigned old = 1; old < 32; ++old) for (unsigned next = 1; next < 32; ++next) { + TQuorum quorum(Members(old), Members(next)); + for (unsigned pattern = 0; pattern < 32; ++pattern) { + const auto value = [&](uint32_t id) -> uint64_t { + return pattern & (1u << (id - 1)) ? 11 - id : id % 3; + }; + uint64_t expected = 0; + for (uint64_t threshold = 1; threshold <= 10; ++threshold) { + unsigned matched = 0; + for (uint32_t id = 1; id <= 5; ++id) if (value(id) >= threshold) matched |= 1u << (id - 1); + if (Majority(old, matched) && Majority(next, matched)) expected = threshold; + } + assert_int_equal(quorum.Value(value), expected); + } + } +} +void test_invalid_configurations_are_rejected(void**) { + for (auto bad : {std::vector{}, std::vector{0}, + std::vector{1, 1}, std::vector{2, 1}}) { + bool rejected = false; + try { TQuorum invalid(bad); } catch (const std::invalid_argument&) { rejected = true; } + assert_true(rejected); rejected = false; + try { TQuorum invalid({1}, bad); } catch (const std::invalid_argument&) { rejected = true; } + assert_true(rejected); + } +} +void test_joint_commit_respects_both_majorities_and_current_term(void**) { + TState state; state.CurrentTerm = 2; + auto old = NewHoldedMessage(); old->Term = 1; state.Append(old); + auto current = NewHoldedMessage(); current->Term = 2; state.Append(current); + TQuorum quorum({1, 2, 3}, {3, 4, 5}); + TVolatileState progress; + progress.MatchIndex[2] = 2; progress.MatchIndex[99] = 2; + progress.CommitAdvance(quorum, 1, state); assert_int_equal(progress.CommitIndex, 0); + progress.MatchIndex[3] = 1; progress.MatchIndex[4] = 1; + progress.CommitAdvance(quorum, 1, state); assert_int_equal(progress.CommitIndex, 0); + progress.MatchIndex[3] = 2; + progress.CommitAdvance(quorum, 1, state); assert_int_equal(progress.CommitIndex, 0); + progress.MatchIndex[4] = 2; + progress.CommitAdvance(quorum, 1, state); assert_int_equal(progress.CommitIndex, 2); + progress.MatchIndex.clear(); + progress.CommitAdvance(quorum, 1, state); assert_int_equal(progress.CommitIndex, 2); +} +} +int main() { + const CMUnitTest tests[] = { + cmocka_unit_test(test_stable_quorum_matches_strict_majority_order_statistic), + cmocka_unit_test(test_all_small_joint_election_quorums), + cmocka_unit_test(test_joint_commit_and_read_watermarks_need_both_majorities), + cmocka_unit_test(test_invalid_configurations_are_rejected), + cmocka_unit_test(test_joint_commit_respects_both_majorities_and_current_term), + }; + return cmocka_run_group_tests(tests, nullptr, nullptr); +} diff --git a/test/test_raft.cpp b/test/test_raft.cpp index eb66e0b..0756b42 100644 --- a/test/test_raft.cpp +++ b/test/test_raft.cpp @@ -272,14 +272,14 @@ void test_follower_append_entries_7a(void**) { }; auto ts = std::make_shared(); auto raft = MakeRaft(onSend, 3, TState{ - /*.CurrentTerm =*/ 1, + /*.CurrentTerm =*/ 6, /*.VotedFor =*/ 2, /*.Log =*/ MakeLog({1,1,1,4,4,5,5,6,6}) }); auto mes = NewHoldedMessage(TMessageEx { .Src = 2, .Dst = 1, - .Term = 1, + .Term = 6, }, TAppendEntriesRequest { .PrevLogIndex = 9, .PrevLogTerm = 6, @@ -305,14 +305,14 @@ void test_follower_append_entries_7b(void**) { }; auto ts = std::make_shared(); auto raft = MakeRaft(onSend, 3, TState{ - /*.CurrentTerm =*/ 1, + /*.CurrentTerm =*/ 6, /*.VotedFor =*/ 2, /*.Log =*/ MakeLog({1,1,1,4}) }); auto mes = NewHoldedMessage(TMessageEx { .Src = 2, .Dst = 1, - .Term = 1, + .Term = 6, }, TAppendEntriesRequest { .PrevLogIndex = 4, .PrevLogTerm = 4, @@ -338,14 +338,14 @@ void test_follower_append_entries_7c(void**) { }; auto ts = std::make_shared(); auto raft = MakeRaft(onSend, 3, TState{ - /*.CurrentTerm =*/ 1, + /*.CurrentTerm =*/ 6, /*.VotedFor =*/ 2, /*.Log =*/ MakeLog({1,1,1,4,4,5,5,6,6,6,6}) }); auto mes = NewHoldedMessage(TMessageEx { .Src = 2, .Dst = 1, - .Term = 1, + .Term = 6, }, TAppendEntriesRequest { .PrevLogIndex = 9, .PrevLogTerm = 6, @@ -371,7 +371,7 @@ void test_follower_append_entries_7f(void**) { }; auto ts = std::make_shared(); auto raft = MakeRaft(onSend, 3, TState{ - /*.CurrentTerm =*/ 1, + /*.CurrentTerm =*/ 3, /*.VotedFor =*/ 2, /*.Log =*/ MakeLog({1,1,1,2,2,2,3,3,3,3,3}) }); @@ -425,9 +425,9 @@ void test_follower_append_entries_empty_to_empty_log(void**) { } void test_candidate_initiate_election(void**) { - std::vector> messages; + std::vector> messages; auto onSend = [&](auto message) { - messages.emplace_back(message.template Cast()); + messages.emplace_back(message.template Cast()); }; auto ts = std::make_shared(); auto raft = MakeRaft(onSend, 3); @@ -465,8 +465,8 @@ void test_candidate_vote_request_small_term(void**) { .Dst = 1, .Term = 0, }, TRequestVoteRequest { - .LastLogIndex = 1, - .LastLogTerm = 1, + .LastLogIndex = 0, + .LastLogTerm = 0, .CandidateId = 2, }); raft->Process(ts->Now(), std::move(req)); @@ -557,7 +557,7 @@ void test_candidate_vote_after_start(void**) { // request with higher term => follower req = NewHoldedMessage(TMessageEx { - .Src = 2, + .Src = 3, .Dst = 1, .Term = 3, }, TRequestVoteRequest { @@ -675,6 +675,8 @@ void test_disk_state(void**) { assert_int_equal(state.LastLogIndex, 0); assert_int_equal(state.CurrentTerm, 1); assert_int_equal(state.VotedFor, 0); + state.CurrentTerm = 6; + state.Commit(); auto log = MakeLog({1,1,1,4,4,5,5,6,6}); for (auto entry : log) { state.Append(entry); @@ -730,13 +732,15 @@ void test_disk_state_restore(void**) { assert_int_equal(state->LastLogIndex, 0); assert_int_equal(state->CurrentTerm, 1); assert_int_equal(state->VotedFor, 0); + // Raft persists a new current term before appending entries from that term. + state->CurrentTerm = 10; + state->Commit(); auto log = MakeLog({1,1,1,4,4,5,5,6,6}); for (auto entry : log) { state->Append(entry); } assert_int_equal(state->LastLogIndex, 9); state->VotedFor = 2; - state->CurrentTerm = 10; state->Commit(); state.reset(); diff --git a/test/test_read_write.cpp b/test/test_read_write.cpp index b600ed6..c40e921 100644 --- a/test/test_read_write.cpp +++ b/test/test_read_write.cpp @@ -4,6 +4,8 @@ #include #include #include +#include +#include #include #include @@ -153,11 +155,65 @@ void test_read_write_payload(void**) { h1.destroy(); h2.destroy(); } +void test_server_stages_and_removes_runtime_learner(void**) { + NNet::TLoop loop; + auto timeSource = std::make_shared(); + auto raft = std::make_shared(std::make_shared(), 1, TNodeDict{}); + raft->Become(EState::LEADER); + NNet::TSocket listener(NNet::TAddress{"127.0.0.1", 0}, loop.Poller()); + TRaftServer server( + loop.Poller(), std::move(listener), raft, std::make_shared(), TNodeDict{}, timeSource); + auto makeEndpoint = [&] { + return std::make_shared>( + [&](const NNet::TAddress& addr) { return NNet::TSocket(addr, loop.Poller()); }, + "learner", NNet::TAddress{"127.0.0.1", 65534}, timeSource); + }; + auto endpoint = makeEndpoint(); + + server.AddLearnerTarget(2, endpoint); + assert_true(raft->GetLearnerIds() == std::vector({2})); + assert_int_equal(raft->GetNpeers(), 1); + assert_int_equal(raft->GetNservers(), 1); + + bool duplicateRejected = false; + try { + server.AddLearnerTarget(2, makeEndpoint()); + } catch (const std::invalid_argument&) { + duplicateRejected = true; + } + assert_true(duplicateRejected); + assert_true(raft->GetLearnerIds() == std::vector({2})); + + auto removed = server.RemoveLearnerTarget(2); + assert_true(removed == endpoint); + assert_true(raft->GetLearnerIds().empty()); + assert_int_equal(raft->GetNpeers(), 0); + + raft->Become(EState::FOLLOWER); + bool nonleaderRejected = false; + try { + server.AddLearnerTarget(3, makeEndpoint()); + } catch (const std::logic_error&) { + nonleaderRejected = true; + } + assert_true(nonleaderRejected); + assert_true(raft->GetLearnerIds().empty()); + assert_int_equal(raft->GetNpeers(), 0); + + raft->Become(EState::LEADER); + auto retryEndpoint = makeEndpoint(); + server.AddLearnerTarget(3, retryEndpoint); + assert_true(raft->GetLearnerIds() == std::vector({3})); + assert_true(server.RemoveLearnerTarget(3) == retryEndpoint); + assert_true(raft->GetLearnerIds().empty()); +} + int main() { const struct CMUnitTest tests[] = { cmocka_unit_test(test_message_send_recv), cmocka_unit_test(test_read_write), cmocka_unit_test(test_read_write_payload), + cmocka_unit_test(test_server_stages_and_removes_runtime_learner), }; return cmocka_run_group_tests(tests, NULL, NULL); }; diff --git a/test/test_rpc_validation.cpp b/test/test_rpc_validation.cpp new file mode 100644 index 0000000..8163d51 --- /dev/null +++ b/test/test_rpc_validation.cpp @@ -0,0 +1,283 @@ +#include +#include +#include +#include +#include +extern "C" { +#include +} + +namespace { +struct TEndpoint : INode { + std::vector> Sent; + void Send(TMessageHolder message) override { Sent.push_back(message); } + void Drain() override {} +}; +struct TFixture { + std::shared_ptr State = std::make_shared(); + std::shared_ptr Peer = std::make_shared(); + std::shared_ptr Other = std::make_shared(); + TRaft Raft{State, 1, {{2, Peer}, {3, Other}}}; + ITimeSource::Time Now{}; + TFixture() { State->CurrentTerm = 3; State->VotedFor = 1; } + void Append(uint64_t term) { + auto entry = NewHoldedMessage(); + entry->Term = term; + State->Append(entry); + } + TMessageHolder Send() { + Raft.Become(EState::LEADER); + Raft.ProcessTimeout(Now); + return Peer->Sent.back().Cast(); + } + TMessageHolder Ack(uint64_t sequence, uint64_t index, bool success = true) { + return NewHoldedMessage(TMessageEx{.Src = 2, .Dst = 1, .Term = 3, .Seqno = sequence}, + TAppendEntriesResponse{.MatchIndex = index, .Success = uint32_t(success)}); + } + TMessageHolder Request(uint64_t prev, uint64_t prevTerm, + uint64_t commit, std::initializer_list terms = {}) { + auto request = NewHoldedMessage(TMessageEx{.Src = 2, .Dst = 1, .Term = 4}, + TAppendEntriesRequest{.PrevLogIndex = prev, .PrevLogTerm = prevTerm, + .LeaderCommit = commit, .LeaderId = 2, .Nentries = uint32_t(terms.size())}); + request.InitPayload(terms.size()); + uint32_t i = 0; + for (auto term : terms) { + auto entry = NewHoldedMessage(); + entry->Term = term; + request.Payload[i++] = entry; + } + return request; + } +}; +void test_unknown_sender_cannot_change_term(void**) { + TFixture fixture; + auto response = NewHoldedMessage(TMessageEx{.Src = 99, .Dst = 1, .Term = 100}, + TRequestVoteResponse{.VoteGranted = 1}); + fixture.Raft.Process(fixture.Now, response); + assert_int_equal(fixture.State->CurrentTerm, 3); + assert_int_equal(fixture.State->VotedFor, 1); +} +void test_wrong_destination_cannot_change_term(void**) { + TFixture fixture; + auto response = NewHoldedMessage(TMessageEx{.Src = 2, .Dst = 3, .Term = 100}, + TRequestVoteResponse{.VoteGranted = 1}); + fixture.Raft.Process(fixture.Now, response); + assert_int_equal(fixture.State->CurrentTerm, 3); +} +void test_candidate_identity_must_match_sender(void**) { + TFixture fixture; + auto request = NewHoldedMessage(TMessageEx{.Src = 2, .Dst = 1, .Term = 4}, + TRequestVoteRequest{.LastLogIndex = 0, .LastLogTerm = 0, .CandidateId = 3}); + fixture.Raft.Process(fixture.Now, request); + assert_int_equal(fixture.State->CurrentTerm, 3); + assert_int_equal(fixture.State->VotedFor, 1); + assert_true(fixture.Peer->Sent.empty()); +} +void test_out_of_range_ack_cannot_wrap_next_index(void**) { + TFixture fixture; + fixture.Append(3); + auto request = fixture.Send(); + fixture.Raft.Process(fixture.Now, fixture.Ack(request->Seqno, UINT64_MAX)); + assert_int_equal(fixture.Raft.GetVolatileState()->NextIndex.at(2), 1); + assert_int_equal(fixture.Raft.GetVolatileState()->CommitIndex, 0); + fixture.Raft.Process(fixture.Now, fixture.Ack(request->Seqno, 1)); + assert_int_equal(fixture.Raft.GetVolatileState()->NextIndex.at(2), 2); +} +void test_unknown_sequence_cannot_commit(void**) { + TFixture fixture; + fixture.Append(3); + auto request = fixture.Send(); + fixture.Raft.Process(fixture.Now, fixture.Ack(request->Seqno + 100, 1)); + assert_int_equal(fixture.Raft.GetVolatileState()->CommitIndex, 0); + fixture.Raft.Process(fixture.Now, fixture.Ack(request->Seqno, 1)); + assert_int_equal(fixture.Raft.GetVolatileState()->CommitIndex, 1); +} +void test_ack_cannot_confirm_an_entry_not_in_its_rpc(void**) { + TFixture fixture; + fixture.Append(1); + fixture.Append(3); + auto request = fixture.Send(); + assert_int_equal(request->Nentries, 1); + fixture.Raft.Process(fixture.Now, fixture.Ack(request->Seqno, 2)); + assert_int_equal(fixture.Raft.GetVolatileState()->CommitIndex, 0); + assert_int_equal(fixture.Raft.GetVolatileState()->NextIndex.at(2), 1); +} +void test_old_failure_cannot_regress_new_success(void**) { + TFixture fixture; + fixture.Append(3); + auto first = fixture.Send(); + fixture.Now += TTimeout::Rpc + std::chrono::milliseconds(1); + auto second = fixture.Send(); + fixture.Raft.Process(fixture.Now, fixture.Ack(second->Seqno, 1)); + fixture.Raft.Process(fixture.Now, fixture.Ack(first->Seqno, 0, false)); + assert_int_equal(fixture.Raft.GetVolatileState()->NextIndex.at(2), 2); + assert_int_equal(fixture.Raft.GetVolatileState()->BatchSize.at(2), 1024); +} +void test_duplicate_failure_does_not_repeat_backoff(void**) { + TFixture fixture; + fixture.Append(3); + auto request = fixture.Send(); + auto response = fixture.Ack(request->Seqno, 0, false); + fixture.Raft.Process(fixture.Now, response); + auto backoff = fixture.Raft.GetVolatileState()->BackOff.at(2); + fixture.Raft.Process(fixture.Now, response); + assert_int_equal(fixture.Raft.GetVolatileState()->BackOff.at(2), backoff); +} +void test_follower_commits_only_prefix_matched_by_rpc(void**) { + TFixture fixture; + fixture.Append(1); + fixture.Append(2); + fixture.Append(2); + auto request = fixture.Request(1, 1, 9); + fixture.Raft.Process(fixture.Now, request); + assert_int_equal(fixture.Raft.GetVolatileState()->CommitIndex, 1); + assert_int_equal(fixture.State->LastLogIndex, 3); +} +void test_commit_index_preserves_full_width(void**) { + TVolatileState state; + const uint64_t index = (uint64_t(1) << 32) + 123; + state.SetCommitIndex(index); + assert_int_equal(state.CommitIndex, index); +} +void test_invalid_batch_terms_do_not_partially_mutate_log(void**) { + TFixture fixture; + fixture.Append(1); + auto request = fixture.Request(1, 1, 0, {3, 2}); + fixture.Raft.Process(fixture.Now, request); + assert_int_equal(fixture.State->CurrentTerm, 3); + assert_int_equal(fixture.State->LastLogIndex, 1); +} +void test_committed_prefix_cannot_be_replaced(void**) { + TFixture fixture; + fixture.Append(1); + fixture.Append(2); + const_cast(fixture.Raft.GetVolatileState())->CommitIndex = 2; + fixture.Raft.Process(fixture.Now, fixture.Request(1, 1, 2, {3})); + assert_int_equal(fixture.State->LogTerm(2), 2); + assert_int_equal(fixture.State->CurrentTerm, 3); + assert_int_equal(fixture.Raft.GetVolatileState()->CommitIndex, 2); +} +void test_impossible_vote_log_metadata_is_ignored(void**) { + for (int invalid = 0; invalid < 3; ++invalid) { + TFixture fixture; + auto request = NewHoldedMessage(TMessageEx{.Src = 2, .Dst = 1, .Term = 4}, + TRequestVoteRequest{.LastLogIndex = 1, .LastLogTerm = 3, .CandidateId = 2}); + if (invalid == 0) request->LastLogIndex = UINT64_MAX; + if (invalid == 1) request->LastLogTerm = 5; + if (invalid == 2) request->LastLogIndex = 0; + fixture.Raft.Process(fixture.Now, request); + assert_int_equal(fixture.State->CurrentTerm, 3); + assert_int_equal(fixture.State->VotedFor, 1); + } +} +void test_invalid_peer_configuration_is_rejected(void**) { + auto state = std::make_shared(); + auto peer = std::make_shared(); + for (int invalid = 0; invalid < 4; ++invalid) { + bool rejected = false; + try { + if (invalid == 0) { TRaft raft(state, 0, {}); } + if (invalid == 1) { TRaft raft(state, 1, {{0, peer}}); } + if (invalid == 2) { TRaft raft(state, 1, {{1, peer}}); } + if (invalid == 3) { TRaft raft(state, 1, {{2, {}}}); } + } catch (const std::invalid_argument&) { rejected = true; } + assert_true(rejected); + } +} +void test_invalid_requests_and_boolean_fields_are_ignored(void**) { + for (int invalid = 0; invalid < 8; ++invalid) { + TFixture fixture; + fixture.Append(1); + auto request = fixture.Request(1, 1, 0, {3}); + if (invalid == 0) request->Src = 99; + if (invalid == 1) request->LeaderId = 3; + if (invalid == 2) request->PrevLogIndex = UINT64_MAX; + if (invalid == 3) request->LeaderCommit = UINT64_MAX; + if (invalid == 4) request.Payload[0].Cast()->Term = 0; + if (invalid == 5) request.Payload[0].Cast()->Term = 5; + if (invalid == 6) request->Nentries = 2; + if (invalid == 7) request->PrevLogIndex = uint64_t(INT64_MAX); + fixture.Raft.Process(fixture.Now, request); + assert_int_equal(fixture.State->CurrentTerm, 3); + assert_int_equal(fixture.State->LastLogIndex, 1); + assert_true(fixture.Peer->Sent.empty()); + } + TFixture fixture; + auto vote = NewHoldedMessage(TMessageEx{.Src = 2, .Dst = 1, .Term = 100}, + TRequestVoteResponse{.VoteGranted = 2}); + fixture.Raft.Process(fixture.Now, vote); + assert_int_equal(fixture.State->CurrentTerm, 3); + auto append = fixture.Ack(1, 0); + append->Term = 100; + append->Success = 2; + fixture.Raft.Process(fixture.Now, append); + assert_int_equal(fixture.State->CurrentTerm, 3); +} +void test_old_evicted_response_is_ignored_and_latest_progresses(void**) { + TFixture fixture; + fixture.Append(3); + auto first = fixture.Send(); + for (int i = 0; i < 300; ++i) { + fixture.Now += TTimeout::Rpc + std::chrono::milliseconds(1); + fixture.Send(); + } + auto latest = fixture.Peer->Sent.back().Cast(); + fixture.Raft.Process(fixture.Now, fixture.Ack(first->Seqno, 1)); + assert_int_equal(fixture.Raft.GetVolatileState()->CommitIndex, 0); + fixture.Raft.Process(fixture.Now, fixture.Ack(latest->Seqno, 1)); + assert_int_equal(fixture.Raft.GetVolatileState()->CommitIndex, 1); +} +void test_term_exhaustion_does_not_wrap_or_change_vote(void**) { + TFixture fixture; + fixture.State->CurrentTerm = UINT64_MAX; + bool rejected = false; + try { fixture.Raft.ProcessTimeout(fixture.Now); } + catch (const std::overflow_error&) { rejected = true; } + assert_true(rejected); + assert_int_equal(fixture.State->CurrentTerm, UINT64_MAX); + assert_int_equal(fixture.State->VotedFor, 1); + assert_int_equal(fixture.Raft.CurrentStateName(), EState::FOLLOWER); +} +void test_lower_inflight_probe_cannot_regress_matched_progress(void**) { + TFixture fixture; + fixture.Append(3); + fixture.Append(3); + auto* progress = const_cast(fixture.Raft.GetVolatileState()); + progress->NextIndex[2] = 3; // newly elected leader initially probes its tail + auto first = fixture.Send(); + fixture.Now += TTimeout::Rpc + std::chrono::milliseconds(1); + auto higher = fixture.Send(); + fixture.Raft.Process(fixture.Now, fixture.Ack(first->Seqno, 0, false)); + fixture.Now += TTimeout::Rpc + std::chrono::milliseconds(1); + auto lower = fixture.Send(); + assert_int_equal(higher->PrevLogIndex, 2); + assert_int_equal(lower->PrevLogIndex, 1); + fixture.Raft.Process(fixture.Now, fixture.Ack(higher->Seqno, 2)); + fixture.Raft.Process(fixture.Now, fixture.Ack(lower->Seqno, 1)); + assert_int_equal(fixture.Raft.GetVolatileState()->MatchIndex.at(2), 2); + assert_int_equal(fixture.Raft.GetVolatileState()->NextIndex.at(2), 3); +} +} +int main() { + const CMUnitTest tests[] = { + cmocka_unit_test(test_unknown_sender_cannot_change_term), + cmocka_unit_test(test_wrong_destination_cannot_change_term), + cmocka_unit_test(test_candidate_identity_must_match_sender), + cmocka_unit_test(test_out_of_range_ack_cannot_wrap_next_index), + cmocka_unit_test(test_unknown_sequence_cannot_commit), + cmocka_unit_test(test_ack_cannot_confirm_an_entry_not_in_its_rpc), + cmocka_unit_test(test_old_failure_cannot_regress_new_success), + cmocka_unit_test(test_duplicate_failure_does_not_repeat_backoff), + cmocka_unit_test(test_follower_commits_only_prefix_matched_by_rpc), + cmocka_unit_test(test_commit_index_preserves_full_width), + cmocka_unit_test(test_invalid_batch_terms_do_not_partially_mutate_log), + cmocka_unit_test(test_committed_prefix_cannot_be_replaced), + cmocka_unit_test(test_impossible_vote_log_metadata_is_ignored), + cmocka_unit_test(test_invalid_peer_configuration_is_rejected), + cmocka_unit_test(test_invalid_requests_and_boolean_fields_are_ignored), + cmocka_unit_test(test_old_evicted_response_is_ignored_and_latest_progresses), + cmocka_unit_test(test_term_exhaustion_does_not_wrap_or_change_vote), + cmocka_unit_test(test_lower_inflight_probe_cannot_regress_matched_progress), + }; + return cmocka_run_group_tests(tests, nullptr, nullptr); +} diff --git a/test/test_snapshots.cpp b/test/test_snapshots.cpp new file mode 100644 index 0000000..9dd993b --- /dev/null +++ b/test/test_snapshots.cpp @@ -0,0 +1,772 @@ +#include +#include +#include +#include +#include "../examples/kv.h" +#include "../examples/kv_messages.h" +#include +#include +#include +#include +#ifdef __linux__ +#include +#include +#endif +#include +#include +#include +extern "C" { +#include +} + +namespace { +template void Reject(F action) { + bool rejected = false; + try { action(); } catch (const std::exception&) { rejected = true; } + assert_true(rejected); +} +struct TSink : INode { void Send(TMessageHolder) override {} void Drain() override {} }; +TMessageHolder Command(const std::string& key, const std::string& value) { + auto command = NewHoldedMessage(sizeof(TWriteKv) + key.size() + value.size()); + command->Flags = TCommandRequest::EWrite; + command->KeySize = key.size(); + command->ValSize = value.size(); + memcpy(command->TKvEntry::Data, key.data(), key.size()); + memcpy(command->TKvEntry::Data + key.size(), value.data(), value.size()); + return command; +} +std::string Read(TKv& kv, const std::string& key) { + auto command = NewHoldedMessage(sizeof(TReadKv) + key.size()); + command->KeySize = key.size(); + memcpy(command->Data, key.data(), key.size()); + auto reply = kv.Read(command, kv.LastAppliedIndex).Cast(); + return std::string(reply->Data, reply->Len - sizeof(TCommandResponse)); +} +void Apply(TKv& kv, const std::string& key, const std::string& value, uint64_t index) { + kv.Write(kv.Prepare(Command(key, value)), index); +} +TSnapshot Sample() { return {2, 1, {1, 2, 3}, std::string("a\0b", 3)}; } +TSnapshot JointSample() { + auto snapshot = Sample(); + snapshot.JointVoters = {3, 4, 5}; + snapshot.ConfigurationIndex = 1; snapshot.ConfigurationTerm = 1; + return snapshot; +} +void FixChecksum(std::string& bytes) { + bytes.resize(bytes.size() - 4); + NSnapshotCodec::Put(bytes, NSnapshotCodec::Checksum(bytes), 4); +} +struct TFiles { + std::filesystem::path Directory; + std::string Name; + TFiles() { + static unsigned next = 0; + Directory = std::filesystem::temp_directory_path() / ("miniraft-snapshot-" + + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count()) + "-" + std::to_string(next++)); + assert_true(std::filesystem::create_directory(Directory)); + Name = (Directory / "raft").string(); + } + ~TFiles() { std::error_code ignored; std::filesystem::remove_all(Directory, ignored); } + std::string Image() const { return Name + ".snapshot.1"; } +}; +void test_snapshot_codec_roundtrip_and_all_byte_corruption(void**) { + const auto original = Sample(); + const auto encoded = EncodeSnapshot(original); + assert_true(DecodeSnapshot(encoded) == original); + for (size_t i = 0; i < encoded.size(); ++i) { + auto damaged = encoded; + damaged[i] ^= 1; + Reject([&] { DecodeSnapshot(damaged); }); + Reject([&] { DecodeSnapshot(std::string_view(encoded).substr(0, i)); }); + } +} +void test_joint_snapshot_codec_roundtrip_and_corruption(void**) { + auto fixed = EncodeSnapshot(Sample()); + // Golden v1 image, including the original CRC and embedded NUL byte. + const std::string hex = "4d5246545330303102000000000000000100000000000000030000000300000000000000010000000200000003000000610062e0c541c6"; + std::string golden; + for (size_t i = 0; i < hex.size(); i += 2) golden.push_back(char(std::stoul(hex.substr(i, 2), nullptr, 16))); + assert_true(fixed == golden); + assert_true(DecodeSnapshot(golden) == Sample()); + assert_false(DecodeSnapshot(fixed).IsJoint()); + const auto original = JointSample(); + const auto encoded = EncodeSnapshot(original); + assert_true(encoded.substr(0, 8) == "MRFTS002"); + assert_true(DecodeSnapshot(encoded) == original); + for (size_t i = 0; i < encoded.size(); ++i) { + auto damaged = encoded; damaged[i] ^= 1; + Reject([&] { DecodeSnapshot(damaged); }); + Reject([&] { DecodeSnapshot(std::string_view(encoded).substr(0, i)); }); + } +} +void test_stable_configuration_snapshot_codec_and_runtime_guards(void**) { + auto snapshot = Sample(); snapshot.StableConfiguration = true; + snapshot.ConfigurationIndex = 1; snapshot.ConfigurationTerm = 1; + const auto encoded = EncodeSnapshot(snapshot); + assert_true(encoded.substr(0, 8) == "MRFTS003"); + assert_true(DecodeSnapshot(encoded) == snapshot); assert_false(snapshot.IsJoint()); + for (size_t i = 0; i < encoded.size(); ++i) { + auto damaged = encoded; damaged[i] ^= 1; + Reject([&] { DecodeSnapshot(damaged); }); + Reject([&] { DecodeSnapshot(std::string_view(encoded).substr(0, i)); }); + } + for (int mode = 0; mode < 6; ++mode) { + auto bad = snapshot; + if (mode == 0) bad.ConfigurationIndex = 0; + if (mode == 1) bad.ConfigurationIndex = 3; + if (mode == 2) bad.ConfigurationTerm = 0; + if (mode == 3) bad.ConfigurationTerm = 2; + if (mode == 4) { bad.Term = 2; bad.ConfigurationIndex = 2; bad.ConfigurationTerm = 1; } + if (mode == 5) bad.JointVoters = {3, 4}; + Reject([&] { EncodeSnapshot(bad); }); + } + for (size_t offset : {size_t(36), size_t(44)}) { + auto damaged = encoded; + for (size_t i = offset; i < offset + 8; ++i) damaged[i] = char(0xff); + FixChecksum(damaged); Reject([&] { DecodeSnapshot(damaged); }); + } + auto memory = std::make_shared(); + Reject([&] { memory->InstallSnapshot(snapshot); }); + memory->Snapshot = std::make_shared(snapshot); + Reject([&] { TRaft unsupported(memory, 1, {{2, std::make_shared()}, {3, std::make_shared()}}); }); + assert_true(memory->GetBoundVoters().empty()); + auto bound = std::make_shared(); + bound->BindVoters(snapshot.Voters); + bound->InstallSnapshot(snapshot); + TRaft recovered(bound, 1, {{2, std::make_shared()}, {3, std::make_shared()}}); + assert_int_equal(recovered.GetVolatileState()->CommitIndex, snapshot.Index); + auto changedBinding = std::make_shared(); + changedBinding->BindVoters({1, 2, 4}); + changedBinding->CurrentTerm = snapshot.Term; + changedBinding->Snapshot = std::make_shared(snapshot); + changedBinding->SnapshotIndex = changedBinding->LastLogIndex = snapshot.Index; + changedBinding->SnapshotTerm = changedBinding->LastLogTerm = snapshot.Term; + Reject([&] { TRaft invalid(changedBinding, 1, {{2, std::make_shared()}, {3, std::make_shared()}}); }); + assert_true(changedBinding->GetBoundVoters() == std::vector({1, 2, 4})); +#ifdef __linux__ + TFiles files; + { TSnapshotStore store(files.Name, 1); store.Save(snapshot); assert_true(store.Load().value() == snapshot); } + { TDurableState state(files.Name, 1); Reject([&] { state.InstallSnapshot(snapshot); }); } +#endif +} +void test_joint_snapshot_metadata_rejects_invalid_boundaries_and_voters(void**) { + for (int mode = 0; mode < 10; ++mode) { + auto bad = JointSample(); + if (mode == 0) bad.JointVoters.clear(); + if (mode == 1) bad.JointVoters = {0, 1}; + if (mode == 2) bad.JointVoters = {1, 1}; + if (mode == 3) bad.JointVoters = {2, 1}; + if (mode == 4) bad.JointVoters.resize(TSnapshot::MaxVoters + 1, 1); + if (mode == 5) bad.ConfigurationIndex = 0; + if (mode == 6) bad.ConfigurationIndex = bad.Index + 1; + if (mode == 7) bad.ConfigurationTerm = 0; + if (mode == 8) bad.ConfigurationTerm = bad.Term + 1; + if (mode == 9) { bad.Term = 2; bad.ConfigurationIndex = bad.Index; bad.ConfigurationTerm = 1; } + Reject([&] { EncodeSnapshot(bad); }); + } + for (size_t offset : {size_t(36), size_t(44), size_t(52)}) { + auto damaged = EncodeSnapshot(JointSample()); + const auto width = offset == 52 ? 4 : 8; + for (size_t i = offset; i < offset + width; ++i) damaged[i] = char(0xff); + FixChecksum(damaged); Reject([&] { DecodeSnapshot(damaged); }); + } + auto empty = EncodeSnapshot(JointSample()); + for (size_t i = 36; i < 56; ++i) empty[i] = 0; + FixChecksum(empty); Reject([&] { DecodeSnapshot(empty); }); + auto mismatch = EncodeSnapshot(JointSample()); + mismatch[16] = 2; // snapshot term + mismatch[36] = 2; // configuration index equals the snapshot index + FixChecksum(mismatch); Reject([&] { DecodeSnapshot(mismatch); }); +} +void test_joint_snapshot_cannot_be_installed_or_restored_by_fixed_runtime(void**) { + auto snapshot = JointSample(); + auto state = std::make_shared(); + state->Append(NewHoldedMessage()); state->Append(NewHoldedMessage()); + const auto before = static_cast(*state); + Reject([&] { state->InstallSnapshot(snapshot); }); + assert_true(static_cast(*state) == before); + TRaft raft(state, 1, {{2, std::make_shared()}, {3, std::make_shared()}}); + TKv app; + Reject([&] { RestoreSnapshot(snapshot, raft, app); }); + assert_int_equal(app.LastAppliedIndex, 0); + auto recovered = std::make_shared(); + recovered->Snapshot = std::make_shared(snapshot); + Reject([&] { TRaft unsupported(recovered, 1, {{2, std::make_shared()}, {3, std::make_shared()}}); }); + assert_true(recovered->GetBoundVoters().empty()); +} +void test_startup_rejects_inconsistent_snapshot_boundaries_before_binding(void**) { + for (int mode = 0; mode < 11; ++mode) { + auto state = std::make_shared(); state->CurrentTerm = 2; + state->SnapshotIndex = state->LastLogIndex = 5; + state->SnapshotTerm = state->LastLogTerm = 2; + TSnapshot image{5, 2, {1, 2}, "opaque"}; + if (mode == 1) image.Index = 4; + if (mode == 2) image.Term = 1; + if (mode == 3) state->LastLogTerm = 1; + if (mode == 4) state->CurrentTerm = 1; + if (mode == 5) image.Voters = {1, 1}; + if (mode == 6 || mode == 7) { + state->SnapshotIndex = state->LastLogIndex = 0; + state->SnapshotTerm = mode == 6 ? 2 : 0; state->LastLogTerm = 0; + } + if (mode == 8) image.Voters = {1, 3}; + if (mode != 0 && mode != 6) state->Snapshot = std::make_shared(image); + if (mode == 9 || mode == 10) { + auto entry = NewHoldedMessage(); entry->Term = mode == 9 ? 1 : 3; state->Append(entry); + } + const auto before = static_cast(*state); + Reject([&] { TRaft invalid(state, 1, {{2, std::make_shared()}}); }); + assert_true(static_cast(*state) == before); + assert_true(state->GetBoundVoters().empty()); + } + auto valid = std::make_shared(); valid->CurrentTerm = 2; + valid->SnapshotIndex = valid->LastLogIndex = 5; valid->SnapshotTerm = valid->LastLogTerm = 2; + valid->Snapshot = std::make_shared(TSnapshot{5, 2, {1, 2}, "opaque"}); + TRaft raft(valid, 1, {{2, std::make_shared()}}); + assert_int_equal(raft.GetVolatileState()->CommitIndex, 5); + assert_true(valid->GetBoundVoters() == std::vector({1, 2})); + auto entry = NewHoldedMessage(); entry->Term = 2; valid->Append(entry); + TRaft withSuffix(valid, 1, {{2, std::make_shared()}}); + assert_int_equal(withSuffix.GetVolatileState()->CommitIndex, 5); +} +void test_startup_rejects_invalid_compacted_suffix_terms(void**) { + for (int mode = 0; mode < 4; ++mode) { + auto state = std::make_shared(); state->CurrentTerm = 3; + state->SnapshotIndex = state->LastLogIndex = 5; state->SnapshotTerm = state->LastLogTerm = 2; + state->Snapshot = std::make_shared(TSnapshot{5, 2, {1, 2}, "opaque"}); + auto first = NewHoldedMessage(); first->Term = mode == 0 ? 1 : mode == 2 ? 0 : mode == 3 ? 4 : 2; + auto last = NewHoldedMessage(); last->Term = mode == 1 ? 3 : 2; + state->Append(first); state->Append(last); + if (mode == 1) state->LastLogTerm = 2; // in range but differs from the actual last entry + const auto before = static_cast(*state); + Reject([&] { TRaft invalid(state, 1, {{2, std::make_shared()}}); }); + assert_true(static_cast(*state) == before); assert_true(state->GetBoundVoters().empty()); + } + auto valid = std::make_shared(); valid->CurrentTerm = 3; + valid->SnapshotIndex = valid->LastLogIndex = 5; valid->SnapshotTerm = valid->LastLogTerm = 2; + valid->Snapshot = std::make_shared(TSnapshot{5, 2, {1, 2}, "opaque"}); + for (uint64_t term : {2, 2, 3}) { auto entry = NewHoldedMessage(); entry->Term = term; valid->Append(entry); } + TRaft raft(valid, 1, {{2, std::make_shared()}}); + assert_int_equal(raft.GetVolatileState()->CommitIndex, 5); +} +void test_snapshot_metadata_and_declared_lengths_are_bounded(void**) { + auto snapshot = Sample(); + for (uint64_t index : {uint64_t(0), UINT64_MAX}) { + auto bad = snapshot; bad.Index = index; + Reject([&] { EncodeSnapshot(bad); }); + } + for (auto voters : {std::vector{}, std::vector{0, 1}, + std::vector{1, 1}, std::vector{2, 1}}) { + auto bad = snapshot; bad.Voters = voters; + Reject([&] { EncodeSnapshot(bad); }); + } + auto bad = snapshot; bad.Term = 0; + Reject([&] { EncodeSnapshot(bad); }); + for (size_t offset : {size_t(0), size_t(8), size_t(16), size_t(24), size_t(28)}) { + auto damaged = EncodeSnapshot(snapshot); + for (size_t i = offset; i < offset + (offset == 24 ? 4 : 8); ++i) damaged[i] = char(0xff); + FixChecksum(damaged); // Exercise semantic validation, not just CRC rejection. + if (offset == 16) { + // Terms are full-width; UINT64_MAX is valid in the codec. + assert_true(DecodeSnapshot(damaged).Term == UINT64_MAX); + } else Reject([&] { DecodeSnapshot(damaged); }); + } +} +void test_kv_snapshot_is_canonical_and_binary_safe(void**) { + TKv a, b; + const std::string binary("a\0z", 3); + Apply(a, "z", "last", 1); Apply(a, "a", binary, 2); Apply(a, "gone", "remove", 3); Apply(a, "gone", {}, 4); + Apply(b, "a", binary, 1); Apply(b, "z", "last", 2); + assert_true(a.SaveSnapshot() == b.SaveSnapshot()); + TKv restored; + restored.LoadSnapshot(a.SaveSnapshot(), 4); + assert_int_equal(restored.LastAppliedIndex, 4); + assert_true(Read(restored, "a") == binary); + assert_true(Read(restored, "z") == "last"); + assert_true(Read(restored, "gone").empty()); +} +void test_kv_invalid_snapshot_preserves_existing_state_and_progress(void**) { + TKv source, target; + Apply(source, "a", "one", 1); Apply(source, "b", "two", 2); + Apply(target, "old", "preserved", 1); + const auto bytes = source.SaveSnapshot(); + for (size_t size = 0; size < bytes.size(); ++size) { + Reject([&] { target.LoadSnapshot(std::string_view(bytes).substr(0, size), 2); }); + assert_true(Read(target, "old") == "preserved"); + assert_int_equal(target.LastAppliedIndex, 1); + } + auto trailing = bytes + "x"; + Reject([&] { target.LoadSnapshot(trailing, 2); }); + auto duplicate = bytes; + duplicate[36] = 'a'; // second record's key; duplicate canonical key must fail + Reject([&] { target.LoadSnapshot(duplicate, 2); }); + std::string oversized = "MRKV0001"; + NSnapshotCodec::Put(oversized, 1, 8); + NSnapshotCodec::Put(oversized, 1, 4); + NSnapshotCodec::Put(oversized, uint64_t(UINT16_MAX) + 1, 4); + NSnapshotCodec::Append(oversized, "a"); + NSnapshotCodec::Append(oversized, std::string(size_t(UINT16_MAX) + 1, 'x')); + Reject([&] { target.LoadSnapshot(oversized, 2); }); + assert_true(Read(target, "old") == "preserved"); + assert_int_equal(target.LastAppliedIndex, 1); +} +void test_capture_requires_committed_applied_progress(void**) { + auto state = std::make_shared(); + TRaft raft(state, 1, {}); + TKv kv; + Reject([&] { CaptureSnapshot(raft, kv); }); + auto entry = kv.Prepare(Command("a", "one")); + state->Append(entry); + kv.Write(entry, 1); + Reject([&] { CaptureSnapshot(raft, kv); }); + const_cast(raft.GetVolatileState())->CommitIndex = 1; + const auto snapshot = CaptureSnapshot(raft, kv); + assert_int_equal(snapshot.Index, 1); + assert_int_equal(snapshot.Term, 1); + assert_true(snapshot.Voters == std::vector{1}); +} +void test_restore_rejects_mismatched_log_configuration_and_used_application(void**) { + auto state = std::make_shared(); + TKv source; + auto entry = source.Prepare(Command("a", "one")); + state->Append(entry); source.Write(entry, 1); + TRaft raft(state, 1, {{2, std::make_shared()}, {3, std::make_shared()}}); + const_cast(raft.GetVolatileState())->CommitIndex = 1; + auto snapshot = CaptureSnapshot(raft, source); + for (int mode = 0; mode < 4; ++mode) { + auto bad = snapshot; + if (mode == 0) bad.Index = 2; + if (mode == 1) bad.Term = 2; + if (mode == 2) bad.Voters = {1, 2}; + if (mode == 3) bad.Data = "invalid KV data"; + TKv restored; + Reject([&] { RestoreSnapshot(bad, raft, restored); }); + assert_int_equal(restored.LastAppliedIndex, 0); + assert_true(Read(restored, "a").empty()); + } + TKv restored; + RestoreSnapshot(snapshot, raft, restored); + Reject([&] { RestoreSnapshot(snapshot, raft, restored); }); + assert_true(Read(restored, "a") == "one"); +} +void test_restore_rejects_checkpoint_with_conflicting_stable_identity(void**) { + const std::vector voters{1, 2, 3}; + auto state = std::make_shared(); + state->BindVoters(voters); + TSnapshot authoritative{2, 1, voters, "authoritative"}; + authoritative.StableConfiguration = true; + authoritative.ConfigurationIndex = 1; + authoritative.ConfigurationTerm = 1; + state->InstallSnapshot(authoritative); + state->CurrentTerm = 2; + auto entry = NewHoldedMessage(); entry->Term = 2; + state->Append(entry); + TRaft raft(state, 1, {{2, std::make_shared()}, {3, std::make_shared()}}); + const_cast(raft.GetVolatileState())->CommitIndex = 3; + + TKv source; Apply(source, "a", "checkpoint value", 3); + TSnapshot checkpoint{3, 2, voters, source.SaveSnapshot()}; + checkpoint.StableConfiguration = true; + for (int mode = 0; mode < 2; ++mode) { + auto conflicting = checkpoint; + if (mode == 0) conflicting.ConfigurationIndex = 2; + else conflicting.ConfigurationTerm = 2; + TKv rejected; + Reject([&] { RestoreSnapshot(conflicting, raft, rejected); }); + assert_int_equal(rejected.LastAppliedIndex, 0); + assert_true(Read(rejected, "a").empty()); + } + + // Legacy checkpoints contain no identity; the local trusted identity is + // still enough to accept their application image at a matching boundary. + checkpoint.StableConfiguration = false; + checkpoint.ConfigurationIndex = checkpoint.ConfigurationTerm = 0; + TKv legacy; + RestoreSnapshot(checkpoint, raft, legacy); + assert_int_equal(legacy.LastAppliedIndex, 3); + assert_true(Read(legacy, "a") == "checkpoint value"); +} +void test_startup_restores_stale_application_from_durable_snapshot_boundary(void**) { + auto state = std::make_shared(); + TKv source; Apply(source, "snapshot", "committed", 2); + TSnapshot snapshot{2, 1, {1}, source.SaveSnapshot()}; + state->InstallSnapshot(snapshot); + auto raft = std::make_shared(state, 1, TNodeDict{}); + + auto stale = std::make_shared(); Apply(*stale, "stale", "must be replaced", 1); + assert_int_equal(stale->LastAppliedIndex, 1); + TRequestProcessor processor(raft, stale, {}); + assert_int_equal(stale->LastAppliedIndex, 2); + assert_true(Read(*stale, "snapshot") == "committed"); + assert_true(Read(*stale, "stale").empty()); + assert_int_equal(raft->GetVolatileState()->CommitIndex, 2); +} +void CheckStableSnapshotIdentityContinuity(IState& state) { + const std::vector voters{1, 2, 3}; + state.BindVoters(voters); + TSnapshot stable{2, 1, voters, "stable image"}; + stable.StableConfiguration = true; + stable.ConfigurationIndex = 1; + stable.ConfigurationTerm = 1; + state.InstallSnapshot(stable); + + TSnapshot legacy{3, 1, voters, "legacy refresh"}; + state.InstallSnapshot(legacy); + auto preserved = legacy; + preserved.StableConfiguration = true; + preserved.ConfigurationIndex = stable.ConfigurationIndex; + preserved.ConfigurationTerm = stable.ConfigurationTerm; + assert_true(*state.GetSnapshot() == preserved); + + auto conflicting = TSnapshot{4, 1, voters, "conflicting stable image"}; + conflicting.StableConfiguration = true; + conflicting.ConfigurationIndex = 2; + conflicting.ConfigurationTerm = 1; + const auto before = static_cast(state); + Reject([&] { state.InstallSnapshot(conflicting); }); + assert_true(static_cast(state) == before); + assert_true(*state.GetSnapshot() == preserved); +} +TSnapshot CheckConfigurationSnapshotTransitionContinuity(IState& state) { + const std::vector original{1, 2, 3}; + const std::vector active{2, 3, 4}; + const std::vector next{3, 4, 5}; + state.BindVoters(original); + + TSnapshot stable{2, 1, active, "stable before transition"}; + stable.StableConfiguration = true; + stable.ConfigurationIndex = 1; stable.ConfigurationTerm = 1; + state.InstallSnapshot(stable); + + // Neither a legacy refresh nor a stable image with the same identity may + // silently restore the original or any other voter list. + TSnapshot legacyWithOriginal{3, 1, original, "legacy stale membership"}; + Reject([&] { state.InstallSnapshot(legacyWithOriginal); }); + auto stableWithConflict = stable; stableWithConflict.Index = 3; + stableWithConflict.Voters = original; + Reject([&] { state.InstallSnapshot(stableWithConflict); }); + assert_true(*state.GetSnapshot() == stable); + + TSnapshot legacy{3, 1, active, "legacy refresh"}; + state.InstallSnapshot(legacy); + stable.Index = legacy.Index; stable.Data = legacy.Data; + assert_true(*state.GetSnapshot() == stable); + + state.CurrentTerm = 2; state.Commit(); + TSnapshot wrongJoint{4, 2, original, "wrong old set"}; + wrongJoint.JointVoters = next; + wrongJoint.ConfigurationIndex = 4; wrongJoint.ConfigurationTerm = 2; + Reject([&] { state.InstallSnapshot(wrongJoint); }); + TSnapshot staleJoint{4, 2, active, "configuration predates stable snapshot"}; + staleJoint.JointVoters = next; + staleJoint.ConfigurationIndex = 3; staleJoint.ConfigurationTerm = 1; + Reject([&] { state.InstallSnapshot(staleJoint); }); + assert_true(*state.GetSnapshot() == stable); + + TSnapshot joint{4, 2, active, "joint transition"}; + joint.JointVoters = next; + joint.ConfigurationIndex = 4; joint.ConfigurationTerm = 2; + state.InstallSnapshot(joint); + auto conflictingJoint = joint; conflictingJoint.Index = 5; + conflictingJoint.JointVoters = {3, 5, 6}; + Reject([&] { state.InstallSnapshot(conflictingJoint); }); + + TSnapshot legacyDowngrade{5, 2, active, "cannot erase joint phase"}; + Reject([&] { state.InstallSnapshot(legacyDowngrade); }); + TSnapshot wrongStable{5, 2, active, "wrong stable set"}; + wrongStable.StableConfiguration = true; + wrongStable.ConfigurationIndex = 5; wrongStable.ConfigurationTerm = 2; + Reject([&] { state.InstallSnapshot(wrongStable); }); + TSnapshot regressedStable{5, 2, next, "configuration term predates joint snapshot"}; + regressedStable.StableConfiguration = true; + regressedStable.ConfigurationIndex = 5; regressedStable.ConfigurationTerm = 1; + Reject([&] { state.InstallSnapshot(regressedStable); }); + + TSnapshot final{5, 2, next, "stable after transition"}; + final.StableConfiguration = true; + final.ConfigurationIndex = 5; final.ConfigurationTerm = 2; + state.InstallSnapshot(final); + + auto conflictingFinal = final; conflictingFinal.Index = 6; + conflictingFinal.Voters = {3, 4, 6}; + Reject([&] { state.InstallSnapshot(conflictingFinal); }); + assert_true(*state.GetSnapshot() == final); + assert_true(state.GetBoundVoters() == original); + return final; +} +void test_memory_snapshot_storage_preserves_stable_configuration_identity(void**) { + TState state; + CheckStableSnapshotIdentityContinuity(state); +} +void test_memory_snapshot_storage_checks_configuration_transition_continuity(void**) { + TState state; + CheckConfigurationSnapshotTransitionContinuity(state); +} +#ifdef __linux__ +void test_durable_snapshot_storage_preserves_stable_configuration_identity(void**) { + TFiles files; + { + TDurableState state(files.Name, 1); + CheckStableSnapshotIdentityContinuity(state); + } + TDurableState recovered(files.Name, 1); + auto snapshot = recovered.GetSnapshot(); + assert_true(snapshot && snapshot->StableConfiguration); + assert_int_equal(snapshot->Index, 3); + assert_int_equal(snapshot->ConfigurationIndex, 1); + assert_int_equal(snapshot->ConfigurationTerm, 1); + assert_true(snapshot->Data == "legacy refresh"); +} +void test_durable_snapshot_storage_checks_configuration_transition_continuity(void**) { + TFiles files; + const std::vector original{1, 2, 3}; + TSnapshot final; + { + TDurableState state(files.Name, 1); + final = CheckConfigurationSnapshotTransitionContinuity(state); + } + TDurableState recovered(files.Name, 1); + assert_true(recovered.GetBoundVoters() == original); + assert_true(recovered.GetSnapshot() && *recovered.GetSnapshot() == final); +} +void test_stable_configuration_snapshot_durable_runtime_recovery_and_compaction(void**) { + TFiles files; + TKv source; Apply(source, "a", "one", 1); Apply(source, "b", "two", 2); + TSnapshot snapshot{2, 1, {1, 2, 3}, source.SaveSnapshot()}; + snapshot.StableConfiguration = true; + snapshot.ConfigurationIndex = 1; snapshot.ConfigurationTerm = 1; + { + TDurableState state(files.Name, 1); + state.BindVoters(snapshot.Voters); + state.InstallSnapshot(snapshot); + } + TSnapshot compacted; + { + auto state = std::make_shared(files.Name, 1); + assert_true(state->GetBoundVoters() == snapshot.Voters); + assert_true(*state->GetSnapshot() == snapshot); + auto raft = std::make_shared(state, 1, + TNodeDict{{2, std::make_shared()}, {3, std::make_shared()}}); + auto app = std::make_shared(); + TRequestProcessor processor(raft, app, {}); + assert_int_equal(app->LastAppliedIndex, 2); + assert_true(Read(*app, "a") == "one"); assert_true(Read(*app, "b") == "two"); + + auto entry = app->Prepare(Command("tail", "after recovery")); entry->Term = 1; + state->Append(entry); app->Write(entry, 3); + const_cast(raft->GetVolatileState())->CommitIndex = 3; + compacted = raft->Compact(); + assert_true(compacted.StableConfiguration); + assert_int_equal(compacted.ConfigurationIndex, snapshot.ConfigurationIndex); + assert_int_equal(compacted.ConfigurationTerm, snapshot.ConfigurationTerm); + assert_int_equal(compacted.Index, 3); + } + auto state = std::make_shared(files.Name, 1); + assert_true(*state->GetSnapshot() == compacted); + auto raft = std::make_shared(state, 1, + TNodeDict{{2, std::make_shared()}, {3, std::make_shared()}}); + auto app = std::make_shared(); TRequestProcessor processor(raft, app, {}); + assert_int_equal(app->LastAppliedIndex, compacted.Index); + assert_true(Read(*app, "a") == "one"); assert_true(Read(*app, "b") == "two"); + assert_true(Read(*app, "tail") == "after recovery"); +} +void test_unbound_durable_configuration_snapshot_recovery_fails_closed(void**) { + TFiles files; const auto path = files.Name + ".durable.1"; + { + TDurableState state(files.Name, 1); + state.InstallSnapshot(Sample()); + } + std::ifstream input(path, std::ios::binary); + std::string bytes(std::istreambuf_iterator(input), {}); input.close(); + assert_true(bytes.substr(0, 8) == "MRFTD004"); + auto snapshot = Sample(); snapshot.StableConfiguration = true; + snapshot.ConfigurationIndex = 1; snapshot.ConfigurationTerm = 1; + const auto image = EncodeSnapshot(snapshot); + constexpr size_t imageLengthOffset = 52; + constexpr size_t imageOffset = 60; + uint64_t oldLength = 0; + for (int i = 0; i < 8; ++i) oldLength |= uint64_t(uint8_t(bytes[imageLengthOffset + i])) << (8 * i); + assert_true(oldLength == bytes.size() - imageOffset); + bytes.replace(imageOffset, size_t(oldLength), image); + for (int i = 0; i < 8; ++i) bytes[imageLengthOffset + i] = char(uint64_t(image.size()) >> (8 * i)); + for (int i = 0; i < 8; ++i) bytes[8 + i] = char(uint64_t(bytes.size() - 20) >> (8 * i)); + const auto crc = NSnapshotCodec::Checksum(bytes.substr(0, 8) + bytes.substr(20)); + for (int i = 0; i < 4; ++i) bytes[16 + i] = char(crc >> (8 * i)); + { std::ofstream output(path, std::ios::binary | std::ios::trunc); output.write(bytes.data(), bytes.size()); } + Reject([&] { TDurableState state(files.Name, 1); }); + std::ifstream after(path, std::ios::binary); + assert_true(std::string(std::istreambuf_iterator(after), {}) == bytes); +} +void test_joint_snapshot_export_and_durable_roundtrip_but_runtime_rejects(void**) { + TFiles files; + { TSnapshotStore exportStore(files.Name, 1); exportStore.Save(JointSample()); + assert_true(exportStore.Load().value() == JointSample()); } + const auto path = files.Name + ".durable.1"; + { + TDurableState state(files.Name, 1); + state.BindVoters(JointSample().Voters); + state.InstallSnapshot(JointSample()); + } + std::string bytes; + { + TDurableState state(files.Name, 1); + assert_true(state.GetBoundVoters() == JointSample().Voters); + assert_true(*state.GetSnapshot() == JointSample()); + const auto before = static_cast(state); + // Legacy snapshots still cannot change the original voter binding. + TSnapshot mismatched{3, 1, {1, 2, 4}, "legacy changed voters"}; + std::ifstream input(path, std::ios::binary); + bytes.assign(std::istreambuf_iterator(input), {}); + Reject([&] { state.InstallSnapshot(mismatched); }); + assert_true(static_cast(state) == before); + std::ifstream unchanged(path, std::ios::binary); + assert_true(std::string(std::istreambuf_iterator(unchanged), {}) == bytes); + } + Reject([&] { TRaft raft(std::make_shared(files.Name, 1), 1, + {{2, std::make_shared()}, {3, std::make_shared()}}); }); + std::ifstream after(path, std::ios::binary); + assert_true(std::string(std::istreambuf_iterator(after), {}) == bytes); +} +void test_snapshot_store_rejects_oversized_files_and_temp_symlinks(void**) { + TFiles files; + TSnapshotStore store(files.Name, 1); + store.Save(Sample()); + const auto saved = store.Load(); + std::filesystem::create_symlink(files.Image(), files.Image() + ".tmp"); + Reject([&] { store.Save(Sample()); }); + assert_true(store.Load() == saved); + std::filesystem::resize_file(files.Image(), TSnapshot::MaxBytes + 1); + Reject([&] { store.Load(); }); + assert_true(std::filesystem::file_size(files.Image()) == TSnapshot::MaxBytes + 1); +} +void test_snapshot_store_roundtrip_lock_and_corrupt_recovery(void**) { + TFiles files; + { + TSnapshotStore store(files.Name, 1); + assert_false(store.Load().has_value()); + Reject([&] { TSnapshotStore second(files.Name, 1); }); + store.Save(Sample()); + assert_true(store.Load() == std::optional{Sample()}); + } + std::ifstream input(files.Image(), std::ios::binary); + std::string bytes(std::istreambuf_iterator(input), {}); + input.close(); + bytes.back() ^= 1; + { std::ofstream out(files.Image(), std::ios::binary); out.write(bytes.data(), bytes.size()); } + TSnapshotStore store(files.Name, 1); + Reject([&] { store.Load(); }); + std::ifstream after(files.Image(), std::ios::binary); + assert_true(std::string(std::istreambuf_iterator(after), {}) == bytes); +} +void test_snapshot_crash_at_each_publication_boundary(void**) { + using TStage = TSnapshotStore::EStage; + for (bool initial : {false, true}) for (auto stage : {TStage::TempCreated, TStage::Written, + TStage::FileSynced, TStage::Renamed, TStage::DirectorySynced}) { + TFiles files; + if (!initial) { TSnapshotStore store(files.Name, 1); store.Save(Sample()); } + auto updated = Sample(); updated.Index = 3; updated.Data = "new committed state"; + const auto child = fork(); + assert_true(child >= 0); + if (child == 0) { + try { + TSnapshotStore store(files.Name, 1, [&](auto point) { if (point == stage) _exit(90); }); + store.Save(updated); + } catch (...) { _exit(91); } + _exit(92); + } + int status = 0; + assert_int_equal(waitpid(child, &status, 0), child); + assert_true(WIFEXITED(status)); assert_int_equal(WEXITSTATUS(status), 90); + TSnapshotStore store(files.Name, 1); + auto selected = store.Load(); + if (stage >= TStage::Renamed) assert_true(selected == std::optional{updated}); + else if (initial) assert_false(selected.has_value()); + else assert_true(selected == std::optional{Sample()}); + } +} +void test_snapshot_publication_error_preserves_old_checkpoint(void**) { + TFiles files; + { TSnapshotStore store(files.Name, 1); store.Save(Sample()); } + TSnapshotStore store(files.Name, 1, [](auto stage) { + if (stage == TSnapshotStore::EStage::Written) throw std::runtime_error("injected checkpoint failure"); + }); + auto updated = Sample(); updated.Index = 3; + Reject([&] { store.Save(updated); }); + assert_true(store.Load() == std::optional{Sample()}); +} +void test_durable_kv_restart_restores_checkpoint_and_replays_only_tail(void**) { + struct TCountingKv : TKv { + size_t Applied = 0; + TMessageHolder Write(TMessageHolder entry, uint64_t index) override { + ++Applied; return TKv::Write(entry, index); + } + }; + TFiles files; + { + auto state = std::make_shared(files.Name, 1); + auto raft = std::make_shared(state, 1, TNodeDict{}); + auto kv = std::make_shared(); + TRequestProcessor processor(raft, kv, {}); + auto now = ITimeSource::Time{}; + raft->ProcessTimeout(now); + auto write = [&](const std::string& key, const std::string& value) { + processor.OnCommandRequest(Command(key, value), {}); + now += std::chrono::milliseconds(500); + raft->ProcessTimeout(now); processor.ProcessCommitted(); + }; + write("a", "one"); + { TSnapshotStore store(files.Name, 1); store.Save(CaptureSnapshot(*raft, *kv)); } + write("b", "tail"); + assert_int_equal(state->LastLogIndex, 3); + } + auto state = std::make_shared(files.Name, 1); + auto raft = std::make_shared(state, 1, TNodeDict{}); + auto kv = std::make_shared(); + TSnapshotStore store(files.Name, 1); + auto snapshot = store.Load(); assert_true(snapshot.has_value()); + RestoreSnapshot(*snapshot, *raft, *kv); + assert_true(Read(*kv, "a") == "one"); assert_true(Read(*kv, "b").empty()); + TRequestProcessor processor(raft, kv, {}); + processor.ProcessCommitted(); assert_int_equal(kv->Applied, 1); + assert_true(Read(*kv, "b") == "tail"); + raft->ProcessTimeout(ITimeSource::Time{}); processor.ProcessCommitted(); + assert_int_equal(kv->Applied, 1); + assert_true(Read(*kv, "a") == "one"); assert_true(Read(*kv, "b") == "tail"); + assert_int_equal(kv->LastAppliedIndex, 4); + assert_int_equal(state->LastLogIndex, 4); // no compaction: every entry is retained +} +#endif +} +int main() { + const CMUnitTest tests[] = { + cmocka_unit_test(test_snapshot_codec_roundtrip_and_all_byte_corruption), + cmocka_unit_test(test_joint_snapshot_codec_roundtrip_and_corruption), + cmocka_unit_test(test_stable_configuration_snapshot_codec_and_runtime_guards), + cmocka_unit_test(test_joint_snapshot_metadata_rejects_invalid_boundaries_and_voters), + cmocka_unit_test(test_joint_snapshot_cannot_be_installed_or_restored_by_fixed_runtime), + cmocka_unit_test(test_startup_rejects_inconsistent_snapshot_boundaries_before_binding), + cmocka_unit_test(test_startup_rejects_invalid_compacted_suffix_terms), + cmocka_unit_test(test_snapshot_metadata_and_declared_lengths_are_bounded), + cmocka_unit_test(test_kv_snapshot_is_canonical_and_binary_safe), + cmocka_unit_test(test_kv_invalid_snapshot_preserves_existing_state_and_progress), + cmocka_unit_test(test_capture_requires_committed_applied_progress), + cmocka_unit_test(test_restore_rejects_mismatched_log_configuration_and_used_application), + cmocka_unit_test(test_restore_rejects_checkpoint_with_conflicting_stable_identity), + cmocka_unit_test(test_startup_restores_stale_application_from_durable_snapshot_boundary), + cmocka_unit_test(test_memory_snapshot_storage_preserves_stable_configuration_identity), + cmocka_unit_test(test_memory_snapshot_storage_checks_configuration_transition_continuity), +#ifdef __linux__ + cmocka_unit_test(test_durable_snapshot_storage_preserves_stable_configuration_identity), + cmocka_unit_test(test_durable_snapshot_storage_checks_configuration_transition_continuity), + cmocka_unit_test(test_stable_configuration_snapshot_durable_runtime_recovery_and_compaction), + cmocka_unit_test(test_unbound_durable_configuration_snapshot_recovery_fails_closed), + cmocka_unit_test(test_joint_snapshot_export_and_durable_roundtrip_but_runtime_rejects), + cmocka_unit_test(test_snapshot_store_rejects_oversized_files_and_temp_symlinks), + cmocka_unit_test(test_snapshot_store_roundtrip_lock_and_corrupt_recovery), + cmocka_unit_test(test_snapshot_crash_at_each_publication_boundary), + cmocka_unit_test(test_snapshot_publication_error_preserves_old_checkpoint), + cmocka_unit_test(test_durable_kv_restart_restores_checkpoint_and_replays_only_tail), +#endif + }; + return cmocka_run_group_tests(tests, nullptr, nullptr); +} diff --git a/test/test_sql_payloads.cpp b/test/test_sql_payloads.cpp new file mode 100644 index 0000000..1644afd --- /dev/null +++ b/test/test_sql_payloads.cpp @@ -0,0 +1,569 @@ +#include "../examples/sql_rsm.h" +#include +#include +#include +#include +#include +#include +#include +extern "C" { +#include +} +namespace { +TMessageHolder Query(const std::string& text) { + auto message = NewHoldedMessage(sizeof(TReadSql) + text.size()); + message->QuerySize = text.size(); + memcpy(message->Query, text.data(), text.size()); + return message; +} +std::string ReadText(TSql& sql, const std::string& query) { + auto reply = sql.Read(Query(query), sql.LastAppliedIndex).Cast(); + return std::string(reply->Data, reply->Len - sizeof(TCommandResponse)); +} +std::string Apply(TSql& sql, const std::string& query, uint64_t index) { + auto command = Query(query); + command->Flags = TCommandRequest::EWrite; + auto reply = sql.Write(sql.Prepare(command), index).Cast(); + return std::string(reply->Data, reply->Len - sizeof(TCommandResponse)); +} +void Seed(TSql& sql) { Apply(sql, "CREATE TABLE t (value TEXT); INSERT INTO t VALUES ('original');", 1); } +template void Reject(F action) { + bool rejected = false; + try { action(); } catch (const TMessageError&) { rejected = true; } + assert_true(rejected); +} +void test_sql_prepare_rejects_short_query(void**) { + TSql sql(":memory:", 1); + auto command = NewHoldedMessage(); + command->QuerySize = UINT32_MAX; + Reject([&] { sql.Prepare(command); }); +} +void test_sql_read_rejects_short_query(void**) { + TSql sql(":memory:", 1); + auto command = NewHoldedMessage(); + command->QuerySize = UINT32_MAX; + Reject([&] { sql.Read(command, 0); }); +} +void test_sql_apply_rejects_before_progress(void**) { + TSql sql(":memory:", 1); + auto entry = NewHoldedMessage(); + entry->QuerySize = UINT32_MAX; + Reject([&] { sql.Write(entry, 1); }); + assert_int_equal(sql.LastAppliedIndex, 0); +} +void test_valid_sql_payload_roundtrip(void**) { + TSql sql(":memory:", 1); + const std::string query = "CREATE TABLE test (value TEXT); INSERT INTO test VALUES ('ok');"; + auto command = NewHoldedMessage(sizeof(TWriteSql) + query.size()); + command->QuerySize = query.size(); + memcpy(command->Query, query.data(), query.size()); + sql.Write(sql.Prepare(command), 1); + assert_int_equal(sql.LastAppliedIndex, 1); + const std::string select = "SELECT value FROM test"; + auto read = NewHoldedMessage(sizeof(TReadSql) + select.size()); + read->QuerySize = select.size(); + memcpy(read->Query, select.data(), select.size()); + auto result = sql.Read(read, 1).Cast(); + std::string text(result->Data, result->Len - sizeof(TCommandResponse)); + assert_true(text.find("ok") != std::string::npos); +} +void test_sql_validation_runs_before_request_admission(void**) { + auto state = std::make_shared(); + auto raft = std::make_shared(state, 1, TNodeDict{}); + auto sql = std::make_shared(":memory:", 1); + TRequestProcessor processor(raft, sql, {}); + for (bool write : {false, true}) { + raft->Become(write ? EState::FOLLOWER : EState::LEADER); + auto command = NewHoldedMessage(); + command->Flags = write ? TCommandRequest::EWrite : TCommandRequest::EConsistent; + command->QuerySize = UINT32_MAX; + Reject([&] { processor.OnCommandRequest(command, {}); }); + assert_int_equal(state->LastLogIndex, 0); + } +} +void test_sql_read_cannot_insert_update_or_delete(void**) { + for (const std::string query : {"INSERT INTO t VALUES ('bad')", "UPDATE t SET value='bad'", "DELETE FROM t"}) { + TSql sql(":memory:", 1); + Seed(sql); + Reject([&] { sql.Read(Query(query), 1); }); + assert_true(ReadText(sql, "SELECT value FROM t").find("original") != std::string::npos); + assert_true(ReadText(sql, "SELECT count(*) FROM t").find("1") != std::string::npos); + assert_int_equal(sql.LastAppliedIndex, 1); + } +} +void test_sql_read_rejects_multi_statement_mutation_before_execution(void**) { + TSql sql(":memory:", 1); + Seed(sql); + Reject([&] { sql.Read(Query("SELECT value FROM t; UPDATE t SET value='bad';"), 1); }); + assert_true(ReadText(sql, "SELECT value FROM t").find("original") != std::string::npos); +} +void test_sql_read_cannot_change_schema(void**) { + for (const std::string query : {"DROP TABLE t", "CREATE TABLE rogue (x)", "ALTER TABLE t ADD COLUMN x"}) { + TSql sql(":memory:", 1); + Seed(sql); + Reject([&] { sql.Read(Query(query), 1); }); + assert_true(ReadText(sql, "SELECT value FROM t").find("original") != std::string::npos); + } +} +void test_sql_read_rejects_connection_control_and_unsafe_functions(void**) { + for (const std::string query : {"PRAGMA query_only=OFF", "BEGIN", "COMMIT", "SAVEPOINT x", + "ATTACH ':memory:' AS other", "VACUUM", "SELECT load_extension('nonexistent')", + "SELECT readfile('nonexistent')"}) { + TSql sql(":memory:", 1); + Reject([&] { sql.Read(Query(query), 0); }); + assert_int_equal(sql.LastAppliedIndex, 0); + } +} +void test_sql_read_supports_comments_ctes_and_safe_functions(void**) { + TSql sql(":memory:", 1); + Seed(sql); + assert_true(ReadText(sql, "-- leading comment\n SELECT upper(value), 'a;b' FROM t; -- tail").find("ORIGINAL") != std::string::npos); + assert_true(ReadText(sql, "WITH RECURSIVE n(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM n WHERE x<3) SELECT sum(x) FROM n").find("6") != std::string::npos); +} +void test_sql_read_rejects_nul_empty_and_second_select(void**) { + TSql sql(":memory:", 1); + const std::string embedded("SELECT 1\0;SELECT 2", 18); + for (const auto& query : {std::string{}, std::string("-- only comment"), std::string("SELECT 1; SELECT 2"), embedded}) + Reject([&] { sql.Read(Query(query), 0); }); +} +void test_sql_read_policy_is_restored_after_rejection(void**) { + TSql sql(":memory:", 1); + Seed(sql); + Reject([&] { sql.Read(Query("DELETE FROM t"), 1); }); + Apply(sql, "INSERT INTO t VALUES ('second')", 2); + assert_int_equal(sql.LastAppliedIndex, 2); + assert_true(ReadText(sql, "SELECT count(*) FROM t").find("2") != std::string::npos); +} +struct TDatabase { + std::filesystem::path Directory; + std::string Path; + TDatabase() { + static unsigned counter = 0; + Directory = std::filesystem::temp_directory_path() / + ("miniraft-sql-" + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count()) + + "-" + std::to_string(counter++)); + assert_true(std::filesystem::create_directory(Directory)); + Path = (Directory / "db").string(); + } + ~TDatabase() { std::error_code ignored; std::filesystem::remove_all(Directory, ignored); } + void External(const std::string& query) { + sqlite3* db = nullptr; + assert_int_equal(sqlite3_open((Path + ".1").c_str(), &db), SQLITE_OK); + int rc = sqlite3_exec(db, query.c_str(), nullptr, nullptr, nullptr); + sqlite3_close(db); + assert_int_equal(rc, SQLITE_OK); + } +}; +void test_sql_snapshot_roundtrip_and_rejection_are_atomic(void**) { + TDatabase sourceFile, targetFile; + std::string image; + { + TSql source(sourceFile.Path, 1); + Seed(source); + assert_true(Apply(source, "INSERT INTO t VALUES ('second')", 2).empty()); + image = source.SaveSnapshot(); + assert_true(image.substr(0, 16) == std::string("SQLite format 3\0", 16)); + source.ValidateSnapshot(image, 2); + Reject([&] { source.ValidateSnapshot(image, 3); }); + auto corrupt = image; corrupt[0] ^= 1; + Reject([&] { source.ValidateSnapshot(corrupt, 2); }); + } + { + TSql target(targetFile.Path, 1); + Seed(target); + auto corrupt = image; corrupt[0] ^= 1; + Reject([&] { target.LoadSnapshot(corrupt, 2); }); + Reject([&] { target.LoadSnapshot(image, 3); }); + assert_int_equal(target.LastAppliedIndex, 1); + assert_true(ReadText(target, "SELECT value FROM t") == "value\noriginal\n"); + + target.LoadSnapshot(image, 2); + assert_int_equal(target.LastAppliedIndex, 2); + assert_true(ReadText(target, "SELECT value FROM t ORDER BY rowid") == "value\noriginal\nsecond\n"); + } + { + TSql restored(targetFile.Path, 1); + assert_int_equal(restored.LastAppliedIndex, 2); + assert_true(ReadText(restored, "SELECT count(*) FROM t") == "count(*)\n2\n"); + } +} +void test_sql_snapshot_installation_busy_failure_preserves_database(void**) { + TDatabase sourceFile, targetFile; + TSql source(sourceFile.Path, 1); Seed(source); + assert_true(Apply(source, "INSERT INTO t VALUES ('snapshot')", 2).empty()); + const auto image = source.SaveSnapshot(); + TSql target(targetFile.Path, 1); Seed(target); + + sqlite3* other = nullptr; + assert_int_equal(sqlite3_open((targetFile.Path + ".1").c_str(), &other), SQLITE_OK); + assert_int_equal(sqlite3_exec(other, "BEGIN IMMEDIATE", nullptr, nullptr, nullptr), SQLITE_OK); + Reject([&] { target.LoadSnapshot(image, 2); }); + assert_int_equal(target.LastAppliedIndex, 1); + assert_true(ReadText(target, "SELECT value FROM t") == "value\noriginal\n"); + assert_int_equal(sqlite3_exec(other, "ROLLBACK", nullptr, nullptr, nullptr), SQLITE_OK); + assert_int_equal(sqlite3_close(other), SQLITE_OK); + + target.LoadSnapshot(image, 2); + assert_int_equal(target.LastAppliedIndex, 2); + assert_true(ReadText(target, "SELECT value FROM t") == "value\noriginal\nsnapshot\n"); +} +void test_sql_snapshot_includes_committed_wal_pages(void**) { + TDatabase file; + TSql source(file.Path, 1); + Seed(source); + + sqlite3* wal = nullptr; + assert_int_equal(sqlite3_open((file.Path + ".1").c_str(), &wal), SQLITE_OK); + assert_int_equal(sqlite3_exec(wal, "PRAGMA journal_mode=WAL", nullptr, nullptr, nullptr), SQLITE_OK); + assert_int_equal(sqlite3_close(wal), SQLITE_OK); + assert_true(Apply(source, "INSERT INTO t VALUES ('committed in WAL')", 2).empty()); + + const auto image = source.SaveSnapshot(); + assert_int_equal(static_cast(image[18]), 1); + assert_int_equal(static_cast(image[19]), 1); + TSql restored(":memory:", 2); + restored.LoadSnapshot(image, 2); + assert_true(ReadText(restored, "SELECT value FROM t ORDER BY rowid") == + "value\noriginal\ncommitted in WAL\n"); +} +void test_sql_snapshot_page_size_mismatch_rejects_before_replacement(void**) { + TDatabase sourceFile, targetFile; + sourceFile.External("PRAGMA page_size=8192; VACUUM"); + TSql source(sourceFile.Path, 1); + Seed(source); + const auto image = source.SaveSnapshot(); + TSql target(targetFile.Path, 1); + Seed(target); + + Reject([&] { target.ValidateSnapshot(image, 1); }); + assert_int_equal(target.LastAppliedIndex, 1); + assert_true(ReadText(target, "SELECT value FROM t") == "value\noriginal\n"); +} +void test_sql_startup_restores_application_left_behind_raft_snapshot(void**) { + TDatabase sourceFile, targetFile; + TSql source(sourceFile.Path, 1); Seed(source); + assert_true(Apply(source, "INSERT INTO t VALUES ('committed')", 2).empty()); + TSnapshot snapshot{2, 1, {1}, source.SaveSnapshot()}; + { TSql stale(targetFile.Path, 1); Seed(stale); } + const auto raftPath = (targetFile.Directory / "raft").string(); + { TDurableState state(raftPath, 1); state.InstallSnapshot(snapshot); } + + { + auto state = std::make_shared(raftPath, 1); + auto raft = std::make_shared(state, 1, TNodeDict{}); + auto sql = std::make_shared(targetFile.Path, 1); + assert_int_equal(sql->LastAppliedIndex, 1); + TRequestProcessor processor(raft, sql, {}); + assert_int_equal(sql->LastAppliedIndex, 2); + assert_true(ReadText(*sql, "SELECT value FROM t ORDER BY rowid") == "value\noriginal\ncommitted\n"); + } + { + auto state = std::make_shared(raftPath, 1); + auto raft = std::make_shared(state, 1, TNodeDict{}); + auto sql = std::make_shared(targetFile.Path, 1); + TRequestProcessor processor(raft, sql, {}); + assert_int_equal(sql->LastAppliedIndex, 2); + assert_true(ReadText(*sql, "SELECT count(*) FROM t") == "count(*)\n2\n"); + } +} +void test_sql_write_cannot_escape_managed_transaction(void**) { + for (const std::string control : {"COMMIT", "END", "ROLLBACK", "BEGIN", "SAVEPOINT x", "RELEASE x", + "PRAGMA user_version=7", "ATTACH ':memory:' AS other"}) { + TSql sql(":memory:", 1); + Seed(sql); + auto error = Apply(sql, "INSERT INTO t VALUES ('partial'); " + control + "; INSERT INTO missing VALUES (1);", 2); + assert_false(error.empty()); + assert_true(ReadText(sql, "SELECT count(*) FROM t").find("\n1\n") != std::string::npos); + assert_int_equal(sql.LastAppliedIndex, 2); + } +} +void test_sql_write_cannot_modify_progress_metadata(void**) { + for (const std::string mutation : {"DELETE FROM raft_metadata_", "UPDATE RAFT_METADATA_ SET value='999'", + "DROP TABLE raft_metadata_", "ALTER TABLE raft_metadata_ RENAME TO stolen", + "CREATE INDEX tamper ON raft_metadata_ (value)", + "CREATE TRIGGER tamper AFTER UPDATE ON raft_metadata_ BEGIN SELECT 1; END", + "CREATE TRIGGER steal AFTER INSERT ON t BEGIN UPDATE raft_metadata_ SET value='999'; END; INSERT INTO t VALUES ('bad')"}) { + TSql sql(":memory:", 1); + Seed(sql); + auto error = Apply(sql, "INSERT INTO t VALUES ('partial'); " + mutation, 2); + assert_false(error.empty()); + assert_true(ReadText(sql, "SELECT count(*) FROM t").find("\n1\n") != std::string::npos); + assert_true(ReadText(sql, "SELECT value FROM raft_metadata_ WHERE key='LastAppliedIndex'").find("\n2\n") != std::string::npos); + assert_int_equal(sql.LastAppliedIndex, 2); + } +} +void test_sql_failed_write_progress_survives_restart(void**) { + TDatabase file; + { + TSql sql(file.Path, 1); + Seed(sql); + assert_false(Apply(sql, "INSERT INTO t VALUES ('partial'); INSERT INTO missing VALUES (1)", 2).empty()); + assert_int_equal(sql.LastAppliedIndex, 2); + assert_true(ReadText(sql, "SELECT count(*) FROM t").find("\n1\n") != std::string::npos); + } + { + TSql sql(file.Path, 1); + assert_int_equal(sql.LastAppliedIndex, 2); + Apply(sql, "INSERT INTO t VALUES ('must not replay')", 2); + assert_true(ReadText(sql, "SELECT count(*) FROM t").find("\n1\n") != std::string::npos); + assert_true(Apply(sql, "INSERT INTO t VALUES ('next')", 3).empty()); + } + TSql sql(file.Path, 1); + assert_int_equal(sql.LastAppliedIndex, 3); + assert_true(ReadText(sql, "SELECT count(*) FROM t").find("\n2\n") != std::string::npos); +} +void test_sql_write_handles_trailing_comment_without_losing_progress(void**) { + TDatabase file; + { + TSql sql(file.Path, 1); + Seed(sql); + assert_true(Apply(sql, "INSERT INTO t VALUES ('next'); -- trailing comment", 2).empty()); + assert_int_equal(sql.LastAppliedIndex, 2); + } + TSql sql(file.Path, 1); + assert_int_equal(sql.LastAppliedIndex, 2); + assert_true(ReadText(sql, "SELECT count(*) FROM t").find("\n2\n") != std::string::npos); +} +void test_sql_automatic_rollback_is_consumed_without_partial_data(void**) { + TSql sql(":memory:", 1); + assert_true(Apply(sql, "CREATE TABLE u (x INTEGER UNIQUE); INSERT INTO u VALUES (1)", 1).empty()); + assert_false(Apply(sql, "INSERT INTO u VALUES (2); INSERT OR ROLLBACK INTO u VALUES (1)", 2).empty()); + assert_int_equal(sql.LastAppliedIndex, 2); + assert_true(ReadText(sql, "SELECT count(*) FROM u").find("\n1\n") != std::string::npos); + assert_true(Apply(sql, "INSERT INTO u VALUES (3)", 3).empty()); +} +void test_sql_metadata_failure_rolls_back_data_and_does_not_advance(void**) { + TDatabase file; + { TSql sql(file.Path, 1); Seed(sql); } + file.External("CREATE TRIGGER fail_marker BEFORE UPDATE ON raft_metadata_ BEGIN SELECT RAISE(ABORT,'marker blocked'); END"); + { + TSql sql(file.Path, 1); + bool failed = false; + try { Apply(sql, "INSERT INTO t VALUES ('partial')", 2); } + catch (const std::runtime_error&) { failed = true; } + assert_true(failed); + assert_int_equal(sql.LastAppliedIndex, 1); + assert_true(ReadText(sql, "SELECT count(*) FROM t").find("\n1\n") != std::string::npos); + } + file.External("DROP TRIGGER fail_marker"); + TSql sql(file.Path, 1); + assert_int_equal(sql.LastAppliedIndex, 1); + assert_true(Apply(sql, "INSERT INTO t VALUES ('retry')", 2).empty()); +} +void test_sql_progress_restores_full_uint64_range(void**) { + TDatabase file; + { TSql sql(file.Path, 1); assert_true(Apply(sql, "CREATE TABLE t (x)", UINT64_MAX).empty()); } + TSql sql(file.Path, 1); + assert_true(sql.LastAppliedIndex == UINT64_MAX); +} +void test_sql_invalid_progress_fails_closed(void**) { + for (const std::string value : {"NULL", "'-1'", "'42junk'", "'18446744073709551616'", "''", "CAST(X'3432006a756e6b' AS TEXT)"}) { + TDatabase file; + { TSql sql(file.Path, 1); Seed(sql); } + file.External("UPDATE raft_metadata_ SET value=" + value); + bool failed = false; + try { TSql sql(file.Path, 1); } + catch (const std::exception&) { failed = true; } + assert_true(failed); + } +} + +void test_sql_busy_write_does_not_consume_entry(void**) { + for (const std::string lock : {"BEGIN IMMEDIATE", "BEGIN; SELECT * FROM t"}) { + TDatabase file; + TSql sql(file.Path, 1); + Seed(sql); + sqlite3* other = nullptr; + assert_int_equal(sqlite3_open((file.Path + ".1").c_str(), &other), SQLITE_OK); + assert_int_equal(sqlite3_exec(other, lock.c_str(), nullptr, nullptr, nullptr), SQLITE_OK); + bool failed = false; + try { Apply(sql, "INSERT INTO t VALUES ('busy')", 2); } + catch (const std::runtime_error&) { failed = true; } + sqlite3_exec(other, "ROLLBACK", nullptr, nullptr, nullptr); + sqlite3_close(other); + assert_true(failed); + assert_int_equal(sql.LastAppliedIndex, 1); + assert_true(ReadText(sql, "SELECT count(*) FROM t").find("\n1\n") != std::string::npos); + assert_true(Apply(sql, "INSERT INTO t VALUES ('retry')", 2).empty()); + assert_int_equal(sql.LastAppliedIndex, 2); + } +} +void test_sql_write_rejects_embedded_nul_before_admission(void**) { + TSql sql(":memory:", 1); + const std::string text = std::string("CREATE TABLE t (x);") + '\0' + "COMMIT"; + auto command = Query(text); + command->Flags = TCommandRequest::EWrite; + Reject([&] { sql.Validate(command); }); + Reject([&] { sql.Prepare(command); }); + auto entry = NewHoldedMessage(sizeof(TSqlLogEntry) + text.size()); + entry->QuerySize = text.size(); + memcpy(entry->Query, text.data(), text.size()); + Reject([&] { sql.Write(entry, 1); }); + assert_int_equal(sql.LastAppliedIndex, 0); +} + +const std::string ExpensiveRead = "WITH RECURSIVE n(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM n WHERE x<100000) SELECT sum(x) FROM n"; +void test_sql_read_stops_at_work_budget(void**) { + TSql sql(":memory:", 1, {.MaxVmSteps = 1000}); + Reject([&] { sql.Read(Query(ExpensiveRead), 0); }); + assert_true(ReadText(sql, "SELECT 7 AS v") == "v\n7\n"); + assert_int_equal(sql.LastAppliedIndex, 0); + TSql defaults(":memory:", 1); + Reject([&] { defaults.Read(Query(ExpensiveRead), 0); }); +} +void test_sql_read_output_budget_counts_headers_separators_and_null(void**) { + for (const auto& query : {std::string("SELECT 'abc' AS v"), std::string("SELECT NULL AS v"), + std::string("SELECT 1 AS a, 2 AS b"), std::string("SELECT 1 AS v WHERE 0")}) { + TSql reference(":memory:", 1); + const auto expected = ReadText(reference, query); + TSql exact(":memory:", 1, {.MaxResultBytes = expected.size()}); + assert_true(ReadText(exact, query) == expected); + TSql shortByOne(":memory:", 1, {.MaxResultBytes = expected.size() - 1}); + Reject([&] { shortByOne.Read(Query(query), 0); }); + } +} +void test_sql_read_bounds_accumulated_rows_and_cleans_up(void**) { + TSql sql(":memory:", 1, {.MaxResultBytes = 64}); + Reject([&] { sql.Read(Query("WITH RECURSIVE n(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM n WHERE x<10000) SELECT x FROM n"), 0); }); + assert_true(ReadText(sql, "SELECT 8 AS v") == "v\n8\n"); + assert_true(Apply(sql, "CREATE TABLE t (x); INSERT INTO t VALUES (9)", 1).empty()); + assert_true(ReadText(sql, "SELECT x FROM t") == "x\n9\n"); +} +void test_sql_read_budget_failures_recover_through_request_processor(void**) { + struct TSink : INode { + std::vector> Replies; + void Send(TMessageHolder message) override { + auto reply = message.Cast(); + Replies.emplace_back(reply->Cookie, reply->ErrorCode); + } + void Drain() override {} + }; + auto raft = std::make_shared(std::make_shared(), 1, TNodeDict{}); + raft->ProcessTimeout(ITimeSource::Time{}); + auto sql = std::make_shared(":memory:", 1, TSqlReadLimits{.MaxVmSteps = 1000, .MaxResultBytes = 64}); + TRequestProcessor processor(raft, sql, {}); + auto client = std::make_shared(); + uint32_t cookie = 1; + for (uint32_t flags : {0u, uint32_t(TCommandRequest::EConsistent)}) { + for (const auto& query : {ExpensiveRead, std::string("SELECT 'abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz' AS v"), std::string("SELECT 1 AS v")}) { + auto command = Query(query); + command->Flags = flags; + command->Cookie = cookie++; + processor.OnCommandRequest(command, client); + processor.ProcessCommitted(); + processor.ProcessWaiting(); + } + } + assert_int_equal(client->Replies.size(), 6); + for (size_t i = 0; i < client->Replies.size(); ++i) { + assert_int_equal(client->Replies[i].first, i + 1); + assert_int_equal(client->Replies[i].second, i % 3 == 2 ? 0 : 2); + } + assert_int_equal(raft->CurrentStateName(), EState::LEADER); +} +void test_sql_read_preserves_nul_bytes_and_counts_their_length(void**) { + TSql sql(":memory:", 1, {.MaxResultBytes = 6}); + assert_true(ReadText(sql, "SELECT CAST(X'610062' AS TEXT) AS v") == std::string("v\na\0b\n", 6)); + Reject([&] { sql.Read(Query("SELECT CAST(X'61006263' AS TEXT) AS v"), 0); }); +} +void test_sql_read_rejects_invalid_limits(void**) { + for (const auto limits : {TSqlReadLimits{.MaxVmSteps = 0}, + TSqlReadLimits{.MaxResultBytes = TMessageLimits{}.MaxFrameBytes}}) { + bool rejected = false; + try { TSql sql(":memory:", 1, limits); } + catch (const std::invalid_argument&) { rejected = true; } + assert_true(rejected); + } +} + +void test_sql_read_value_limit_is_restored_before_writes(void**) { + TSql sql(":memory:", 1); + const std::string value(9 * 1024 * 1024, 'x'); + assert_true(Apply(sql, "CREATE TABLE big (v TEXT); INSERT INTO big VALUES ('" + value + "')", 1).empty()); + Reject([&] { sql.Read(Query("SELECT hex(v) FROM big"), 1); }); + // Writes retain their previous SQLite length limit, even after a failed read. + assert_true(Apply(sql, "UPDATE big SET v=v||v", 2).empty()); + assert_true(Apply(sql, "UPDATE big SET v=substr(v,1,1)", 3).empty()); + assert_true(ReadText(sql, "SELECT v FROM big") == "v\nx\n"); + assert_int_equal(sql.LastAppliedIndex, 3); +} + +void test_sql_and_raft_restart_do_not_reapply_committed_insert(void**) { + TDatabase file; + const auto logPath = file.Path + "-raft"; + { + auto state = std::make_shared(logPath, 1); + auto raft = std::make_shared(state, 1, TNodeDict{}); + auto sql = std::make_shared(file.Path, 1); + TRequestProcessor processor(raft, sql, {}); + auto now = ITimeSource::Time{}; + raft->ProcessTimeout(now); + for (const auto query : {"CREATE TABLE t (value TEXT)", "INSERT INTO t VALUES ('once')"}) { + auto command = Query(query); + command->Flags = TCommandRequest::EWrite; + processor.OnCommandRequest(command, {}); + now += std::chrono::milliseconds(500); + raft->ProcessTimeout(now); + processor.ProcessCommitted(); + } + assert_int_equal(sql->LastAppliedIndex, 3); + assert_true(ReadText(*sql, "SELECT count(*) AS n FROM t") == "n\n1\n"); + } + { + auto state = std::make_shared(logPath, 1); + auto raft = std::make_shared(state, 1, TNodeDict{}); + auto sql = std::make_shared(file.Path, 1); + assert_int_equal(sql->LastAppliedIndex, 3); + TRequestProcessor processor(raft, sql, {}); + processor.ProcessCommitted(); // server timer can fire before any leader is known + const auto beforeElection = sql->LastAppliedIndex; + raft->ProcessTimeout(ITimeSource::Time{}); + processor.ProcessCommitted(); + processor.ProcessCommitted(); + assert_true(ReadText(*sql, "SELECT count(*) AS n FROM t") == "n\n1\n"); + assert_int_equal(beforeElection, 3); + assert_int_equal(sql->LastAppliedIndex, 4); // new-term stub advanced framework progress + } +} + +} +int main() { + const CMUnitTest tests[] = { + cmocka_unit_test(test_sql_prepare_rejects_short_query), + cmocka_unit_test(test_sql_read_rejects_short_query), + cmocka_unit_test(test_sql_apply_rejects_before_progress), + cmocka_unit_test(test_valid_sql_payload_roundtrip), + cmocka_unit_test(test_sql_validation_runs_before_request_admission), + cmocka_unit_test(test_sql_read_cannot_insert_update_or_delete), + cmocka_unit_test(test_sql_read_rejects_multi_statement_mutation_before_execution), + cmocka_unit_test(test_sql_read_cannot_change_schema), + cmocka_unit_test(test_sql_read_rejects_connection_control_and_unsafe_functions), + cmocka_unit_test(test_sql_read_supports_comments_ctes_and_safe_functions), + cmocka_unit_test(test_sql_read_rejects_nul_empty_and_second_select), + cmocka_unit_test(test_sql_read_policy_is_restored_after_rejection), + cmocka_unit_test(test_sql_write_cannot_escape_managed_transaction), + cmocka_unit_test(test_sql_write_cannot_modify_progress_metadata), + cmocka_unit_test(test_sql_failed_write_progress_survives_restart), + cmocka_unit_test(test_sql_write_handles_trailing_comment_without_losing_progress), + cmocka_unit_test(test_sql_automatic_rollback_is_consumed_without_partial_data), + cmocka_unit_test(test_sql_metadata_failure_rolls_back_data_and_does_not_advance), + cmocka_unit_test(test_sql_snapshot_roundtrip_and_rejection_are_atomic), + cmocka_unit_test(test_sql_snapshot_installation_busy_failure_preserves_database), + cmocka_unit_test(test_sql_snapshot_includes_committed_wal_pages), + cmocka_unit_test(test_sql_snapshot_page_size_mismatch_rejects_before_replacement), + cmocka_unit_test(test_sql_startup_restores_application_left_behind_raft_snapshot), + cmocka_unit_test(test_sql_progress_restores_full_uint64_range), + cmocka_unit_test(test_sql_invalid_progress_fails_closed), + cmocka_unit_test(test_sql_busy_write_does_not_consume_entry), + cmocka_unit_test(test_sql_write_rejects_embedded_nul_before_admission), + cmocka_unit_test(test_sql_read_stops_at_work_budget), + cmocka_unit_test(test_sql_read_output_budget_counts_headers_separators_and_null), + cmocka_unit_test(test_sql_read_bounds_accumulated_rows_and_cleans_up), + cmocka_unit_test(test_sql_read_budget_failures_recover_through_request_processor), + cmocka_unit_test(test_sql_read_preserves_nul_bytes_and_counts_their_length), + cmocka_unit_test(test_sql_read_rejects_invalid_limits), + cmocka_unit_test(test_sql_read_value_limit_is_restored_before_writes), + cmocka_unit_test(test_sql_and_raft_restart_do_not_reapply_committed_insert), + }; + return cmocka_run_group_tests(tests, nullptr, nullptr); +}