diff --git a/crates/app/src/cmd_backup.rs b/crates/app/src/cmd_backup.rs index 4af501e68..2d9165d64 100644 --- a/crates/app/src/cmd_backup.rs +++ b/crates/app/src/cmd_backup.rs @@ -169,6 +169,37 @@ //! A row `--prune-missing` leaves because its manifest is back or another //! writer moved it is reported, and the exit stays 0. //! +//! Every sync that listed the root to its end then reports the rows of the +//! stores other than the configured one: per store (kind, root, id) the +//! rows per status and each CREATING and DELETING row by ARN, which no +//! sync changes (`extenddb_engine::store_maintenance`'s module doc states +//! all three options below). `--move-store ` re-points the rows +//! of a root whose tree was copied under the configured root: it needs the +//! configured store healthy, refuses the configured root itself and a root +//! with no rows (printing the stores the catalog knows), takes the claim +//! (on SQLite too) and waits for the jobs of the configured store and of +//! every store id of the old root, checks every row against the copy and +//! names each that stops the move, moves none while any does, and then +//! syncs the configured root; it exits 1 when a row stops it, is left +//! under the old root, or lands there during that sync (the closing report +//! of other stores names it). `--forget-store ` and `--forget-backup +//! ` (repeatable, in order, up to the first refusal) remove rows +//! outright, delete no object (but for a DELETING row of the configured +//! store, whose deletion is finished first when the store's check passed), +//! and run on a store in any state and with no store configured, under the +//! claim on every backend. They open the store writing nothing to it +//! ([`Mode::Forget`]). They exit 1 for a row they leave (a restore reading +//! it, a live owner, another spelling of the ARN, an in-catalog backup, a +//! DELETING row of an unhealthy configured store, a row of a configured +//! root that has no identity file under its rows, or on SQLite of the +//! default root found missing with rows for it). After `--forget-backup` +//! the configured store's check runs once and its outcome is printed: a +//! store left unhealthy by the removed row's missing manifest reads +//! healthy, or names the next row whose manifest is gone. The three +//! options conflict with each other, with `--adopt-store` and with +//! `--prune-missing`. Each row they change is logged under +//! `extenddb::audit`. +//! //! Every `backup` command refuses an in-memory SQLite database (a //! `[storage.sqlite] path` naming no file) before it opens the catalog: it //! would open a new, empty catalog that ends with the command. A server on @@ -183,6 +214,7 @@ use extenddb_engine::maintenance::{ Interrupt, RemoveTarget, Stop, Wait, WaitOutcome, purge_rows, remove_target, wait_for_quiet, }; use extenddb_engine::{BackupRuntime, ConfiguredStore, HeldClaim, StoreHealth, StoreProblem}; +use extenddb_server::backup_store::NotConfigured; use extenddb_storage::StorageEngine; use extenddb_storage::backup_catalog::{ BackupRow, BackupStatus, JOB_HEARTBEAT_INTERVAL, JobStatus, StoreIdentity, @@ -235,9 +267,9 @@ enum BackupCommand { /// Give a catalog row to every backup of the configured backup store that has none: each /// prefix with a verified manifest (its sidecar matching, every data file present at its /// size, its ARN naming the prefix's account, table and backup id) gets an AVAILABLE row. - /// Reports the prefixes it skips and the AVAILABLE rows whose manifest is gone. Refuses on - /// an unhealthy store. Beside servers (PostgreSQL, MongoDB, Cassandra) it takes the - /// maintenance claim; on SQLite it needs the server stopped. + /// Reports the prefixes it skips, the AVAILABLE rows whose manifest is gone and the rows of + /// other stores. Refuses on an unhealthy store. Beside servers (PostgreSQL, MongoDB, + /// Cassandra) it takes the maintenance claim; on SQLite it needs the server stopped. Sync { /// Make this catalog the owner of a store root whose extenddb-store.json names another /// catalog, then sync. Stop every server of the catalog the file names first. Takes the @@ -249,6 +281,43 @@ enum BackupCommand { /// of the same path). Takes the maintenance claim first and waits for running jobs. #[arg(long)] prune_missing: bool, + /// Move the rows of OLD_ROOT (its label as the catalog stores it, as backup list and + /// the report of other stores print it) to the configured store, after the old root's + /// tree was copied under the configured root. Reconfigure every server first. Checks + /// each row against the copy and moves none while any row stops the move (all are + /// named); then syncs. Takes the maintenance claim and waits for running jobs. + #[arg( + long, + value_name = "OLD_ROOT", + conflicts_with_all = ["adopt_store", "prune_missing", "forget_store", "forget_backup"] + )] + move_store: Option, + /// Remove every catalog row of ROOT, a root other than the configured one, outright: + /// no tombstone is left and no object is deleted, so a later sync where the bytes are + /// adopts them again. Runs on a store in any state and with no store configured. Takes + /// the maintenance claim and waits for running jobs, as purge does; a row whose job + /// writes a heartbeat after the wait is left and named. + #[arg( + long, + value_name = "ROOT", + conflicts_with_all = ["adopt_store", "prune_missing", "forget_backup"] + )] + forget_store: Option, + /// Remove the catalog row of the backup ARN outright, of any status and any store: no + /// tombstone is left and no object is deleted, except that a DELETING row of the + /// configured store has its deletion finished first. Repeat the option for more + /// backups; they are removed in order, up to the first that is refused. Runs on a store + /// in any state and with no store configured, and runs the store's check afterwards + /// unless the root had no extenddb-store.json. Takes the maintenance claim and waits for + /// running jobs, as purge does; a row whose job writes a heartbeat after the wait is + /// left and named. + #[arg( + long, + value_name = "ARN", + action = clap::ArgAction::Append, + conflicts_with_all = ["adopt_store", "prune_missing"] + )] + forget_backup: Vec, }, } @@ -316,21 +385,67 @@ pub async fn run(args: BackupArgs, release: &'static str) -> anyhow::Result<()> catalog.close().await; return done; } + // The forget options run with no store configured too: the catalog is + // kept open without one. + let forgetting = matches!( + &args.command, + BackupCommand::Sync { forget_store, forget_backup, .. } + if forget_store.is_some() || !forget_backup.is_empty() + ); let (command, mode) = match args.command { BackupCommand::List { .. } => (LIST, Mode::Read), BackupCommand::Purge { confirm: false } => (PURGE, Mode::Read), BackupCommand::Purge { confirm: true } => (PURGE, Mode::Write), + BackupCommand::Sync { .. } if forgetting => (SYNC, Mode::Forget), BackupCommand::Sync { .. } => (SYNC, Mode::Sync), BackupCommand::RemoveTarget { .. } => unreachable!("handled above"), }; - let opening = open(&app_config, &args.config, release, command, mode).await?; + let opening = open( + &app_config, + &args.config, + release, + command, + mode, + forgetting, + ) + .await?; if let Some(signal) = interrupt.signalled() { - if let Opening::Store(opened) = opening { - opened.close().await; + match opening { + Opening::Store(opened) => opened.close().await, + Opening::CatalogOnly(catalog, _) => catalog.close().await, + Opening::NoStore(_) => {} } anyhow::bail!("{signal} received while opening the catalog; nothing was changed"); } match (args.command, opening) { + ( + BackupCommand::Sync { + forget_store, + forget_backup, + .. + }, + opening, + ) if forgetting => { + let (catalog, store) = match opening { + Opening::Store(opened) => { + let Opened { catalog, identity } = opened; + (catalog, Ok(identity)) + } + Opening::CatalogOnly(catalog, why) => (catalog, Err(why)), + Opening::NoStore(why) => anyhow::bail!("{why}"), + }; + let done = forget( + &catalog, + store.as_ref(), + &app_config, + forget_store.as_deref(), + &forget_backup, + interrupt, + ) + .await; + catalog.close().await; + done + } (BackupCommand::List { failed }, Opening::Store(opened)) => { let done = list(&opened, failed).await; opened.close().await; @@ -372,21 +487,39 @@ pub async fn run(args: BackupArgs, release: &'static str) -> anyhow::Result<()> BackupCommand::Sync { adopt_store, prune_missing, + move_store, + .. }, Opening::Store(opened), ) => { - let done = sync(&opened, &app_config, adopt_store, prune_missing, interrupt).await; + let done = sync( + &opened, + &app_config, + SyncOptions { + adopt_store, + prune_missing, + move_store: move_store.as_deref(), + }, + interrupt, + ) + .await; opened.close().await; done } (BackupCommand::Sync { .. }, Opening::NoStore(reason)) => anyhow::bail!("{reason}"), + (_, Opening::CatalogOnly(catalog, reason)) => { + catalog.close().await; + anyhow::bail!("{reason}") + } } } -/// What `open` found: the configured store, or why there is none. +/// What `open` found: the configured store, or why there is none (with the +/// catalog still open when the caller asked to keep it). enum Opening { Store(Opened), NoStore(String), + CatalogOnly(Catalog, NotConfigured), } /// How a command opens the catalog. @@ -411,6 +544,14 @@ enum Mode { /// (`remove-target`): the store is described as `Read` leaves it, so /// neither the default root nor an identity file is created. WriteCatalogOnly, + /// As `Write`, for `backup sync --forget-store` and `--forget-backup`, + /// which write nothing to the root: the store is described as `Read` + /// leaves it (no default root created, no `validate` and its probe + /// file), the identity file is read and never written, and when it + /// agrees with the rows the check that writes nothing + /// (`check_configured_store_read_only`) runs once and is recorded, so + /// a DELETING row of a healthy store can have its deletion finished. + Forget, } /// The open catalog: the storage engine, the backup runtime (with the @@ -443,10 +584,7 @@ async fn open_catalog( release: &'static str, command: &'static str, mode: Mode, -) -> anyhow::Result<( - Catalog, - Option, -)> { +) -> anyhow::Result<(Catalog, Option)> { let mut options = extenddb_storage::server_components::ServerComponentsOptions::default(); options.release = release; options.max_concurrent_jobs = app_config.backup.max_concurrent_jobs; @@ -492,7 +630,7 @@ async fn open_catalog( &backend, &*storage, match mode { - Mode::Read | Mode::WriteCatalogOnly => { + Mode::Read | Mode::WriteCatalogOnly | Mode::Forget => { extenddb_server::backup_store::Caller::ReadOnlyCommand } Mode::Write => extenddb_server::backup_store::Caller::Command, @@ -557,6 +695,7 @@ async fn open( release: &'static str, command: &'static str, mode: Mode, + keep_without_store: bool, ) -> anyhow::Result { let (catalog, not_configured) = open_catalog(app_config, config_path, release, command, mode).await?; @@ -565,6 +704,9 @@ async fn open( // absent or disabled, the default root not yet created, or the // default root failed its check with no rows for it. let why = not_configured.expect("a runtime without a store carries the reason"); + if keep_without_store { + return Ok(Opening::CatalogOnly(catalog, why)); + } catalog.close().await; return Ok(Opening::NoStore(why.to_string())); }; @@ -584,6 +726,32 @@ async fn open( Err(problem) => Identity::Problem(problem), } } + (Mode::Forget, _) => { + match extenddb_engine::read_identity_bounded(store, &**storage).await { + Ok(Some(identity)) => { + let check = extenddb_engine::check_configured_store_read_only( + store, + &**storage, + Some(&identity), + ) + .await; + // A file gone since the read leaves the store checking: + // not healthy, so nothing is deleted. + match check { + Ok(Some(checked)) => { + store.record_check(Ok(checked)); + } + Ok(None) => {} + Err(problem) => { + store.record_check(Err(problem)); + } + } + Identity::Known(identity) + } + Ok(None) => Identity::Empty, + Err(problem) => Identity::Problem(problem), + } + } (Mode::Write | Mode::Sync, StoreHealth::Checking) => { let outcome = extenddb_engine::check_configured_store(store, &**storage, backend).await; store.record_check(outcome.clone()); @@ -846,7 +1014,7 @@ async fn purge( // live owner is told apart by is zero. On the other backends purge runs // beside live servers, whose jobs heartbeat within `stale_after`. let owner_window = if opened.catalog.backend == "sqlite" { - resolve_dead_owners(opened).await; + resolve_dead_owners(&opened.catalog).await; Duration::ZERO } else { stale_after @@ -857,7 +1025,7 @@ async fn purge( let started = Instant::now(); let outcome = wait_for_quiet( storage, - identity, + std::slice::from_ref(identity), &claim, Wait { owner_window, @@ -1019,13 +1187,26 @@ async fn signal_name() -> &'static str { /// command's own claim would read as held only within one instant of its /// heartbeat). What was resolved is printed; a failure is printed and the /// wait decides. -async fn resolve_dead_owners(opened: &Opened) { +async fn resolve_dead_owners(catalog: &Catalog) -> Vec { + resolve_dead_owners_with( + catalog, + extenddb_engine::reconciler::ReconcilerState::for_maintenance_holder(), + ) + .await +} + +/// [`resolve_dead_owners`] with the cycle's `state`; returns the ARNs of +/// the DELETING rows whose deletion the cycle finished. +async fn resolve_dead_owners_with( + catalog: &Catalog, + mut state: extenddb_engine::reconciler::ReconcilerState, +) -> Vec { match extenddb_engine::reconciler::cycle( - &opened.catalog.storage, - &opened.catalog.runtime, + &catalog.storage, + &catalog.runtime, Duration::ZERO, - &opened.catalog.backend, - &mut extenddb_engine::reconciler::ReconcilerState::for_maintenance_holder(), + &catalog.backend, + &mut state, ) .await { @@ -1065,8 +1246,12 @@ async fn resolve_dead_owners(opened: &Opened) { command's write" ); } + report.deletions_finished + } + Err(e) => { + println!("could not resolve the jobs the stopped server left: {e}"); + Vec::new() } - Err(e) => println!("could not resolve the jobs the stopped server left: {e}"), } } @@ -1132,15 +1317,29 @@ async fn remove_restore_target( Ok(()) } -/// `backup sync [--adopt-store] [--prune-missing]` (module doc). +/// The options `sync` was given (`--forget-store` and `--forget-backup` +/// are [`forget`]'s). +#[derive(Clone, Copy)] +struct SyncOptions<'a> { + adopt_store: bool, + prune_missing: bool, + move_store: Option<&'a str>, +} + +/// `backup sync [--adopt-store] [--prune-missing] [--move-store ]` +/// (module doc). #[allow(clippy::too_many_lines)] async fn sync( opened: &Opened, app_config: &config::AppConfig, - adopt_store: bool, - prune_missing: bool, + options: SyncOptions<'_>, mut interrupt: Interrupt, ) -> anyhow::Result<()> { + let SyncOptions { + adopt_store, + prune_missing, + move_store, + } = options; let storage = &*opened.catalog.storage; let configured = opened.store(); println!("=== extenddb backup sync ==="); @@ -1185,9 +1384,10 @@ async fn sync( } Identity::Problem(problem) => anyhow::bail!( "the {} backup store {} is unhealthy: {problem}; sync changes no row while the store \ - is unhealthy", + is unhealthy{}", configured.kind, - configured.root + configured.root, + manifest_missing_hint(problem) ), Identity::Empty => anyhow::bail!( "the {} backup store {} has no {}: its check did not run", @@ -1197,53 +1397,69 @@ async fn sync( ), }; + // The old root `--move-store` names: refused before the claim when it is + // the configured root or has no rows. + let moving = match move_store { + Some(root) => { + match extenddb_engine::store_maintenance::locate_root( + storage, + root, + Some(&configured.root), + ) + .await + { + Ok(located) => Some(located), + Err(refusal) => return Err(root_refusal("--move-store", root, refusal)), + } + } + None => None, + }; + // On SQLite the command holds the serve lock, which already excludes // every server and every other command. Elsewhere every sync takes the // maintenance claim, so no purge removes a row between this sync's row // read and its insert and then deletes the bytes it adopted; plain sync // waits for no job, since its insert-if-absent orders it against a - // CreateBackup. The options wait for running jobs as purge does. + // CreateBackup. The options wait for running jobs as purge does, + // `--move-store` for those of the old root's store ids too. let sqlite = opened.catalog.backend == "sqlite"; - let waits = adopt_store || prune_missing; + let waits = adopt_store || prune_missing || moving.is_some(); let mut claim = None; if !sqlite || waits { let backup = &app_config.backup; let stale_after = Duration::from_secs(backup.stale_job_after_secs); - let holder = format!( - "extenddb backup sync on {} (pid {})", - extenddb_server::backup_store::hostname(), - std::process::id() - ); - let taken = storage - .maintenance_claim(&holder, stale_after) - .await - .map_err(|e| match e { - StorageError::AlreadyExists(m) => anyhow::anyhow!( - "{m}; one maintenance command runs at a time, and a claim whose holder died \ - expires after stale_job_after_secs ({} s)", - backup.stale_job_after_secs - ), - other => anyhow::anyhow!("cannot take the maintenance claim: {other}"), - })?; - println!("Took the maintenance claim as {holder:?}."); - let held = HeldClaim::start(taken, JOB_HEARTBEAT_INTERVAL); - if waits - && let Err(refusal) = wait_for_jobs( - opened, - configured, - known.as_ref(), + let held = take_claim(storage, app_config).await?; + if waits { + let mut identities = match &known { + Some(identity) => vec![identity.clone()], + None => match adopt_wait_identity(configured).await { + Ok(identity) => vec![identity], + Err(refusal) => { + release(held).await; + return Err(refusal); + } + }, + }; + if let Some(located) = &moving { + identities.extend(located.identities()); + } + if let Err(refusal) = wait_for_jobs( + &opened.catalog, + &identities, &held, &mut interrupt, stale_after, + false, ) .await - { - release(held).await; - return Err(refusal); + { + release(held).await; + return Err(refusal); + } } claim = Some(held); } else { - resolve_dead_owners(opened).await; + let _ = resolve_dead_owners(&opened.catalog).await; } let done = sync_held( @@ -1252,6 +1468,7 @@ async fn sync( known, adopting, prune_missing, + moving.as_ref(), claim.as_ref(), &interrupt, ) @@ -1262,59 +1479,131 @@ async fn sync( done } -/// The wait `--adopt-store` and `--prune-missing` run under the claim, as -/// purge's: on SQLite after the dead-owner resolutions with a zero window. -/// The refusal is the command's error, the claim still held. +/// Take the maintenance claim for `backup sync` and start its heartbeat. +async fn take_claim( + storage: &dyn StorageEngine, + app_config: &config::AppConfig, +) -> anyhow::Result { + let backup = &app_config.backup; + let stale_after = Duration::from_secs(backup.stale_job_after_secs); + let holder = format!( + "extenddb backup sync on {} (pid {})", + extenddb_server::backup_store::hostname(), + std::process::id() + ); + let taken = storage + .maintenance_claim(&holder, stale_after) + .await + .map_err(|e| match e { + StorageError::AlreadyExists(m) => anyhow::anyhow!( + "{m}; one maintenance command runs at a time, and a claim whose holder died \ + expires after stale_job_after_secs ({} s)", + backup.stale_job_after_secs + ), + other => anyhow::anyhow!("cannot take the maintenance claim: {other}"), + })?; + println!("Took the maintenance claim as {holder:?}."); + Ok(HeldClaim::start(taken, JOB_HEARTBEAT_INTERVAL)) +} + +/// The command's error for a root `--move-store` or `--forget-store` +/// (`option`) refused, the known stores printed first when the root has no +/// rows. +fn root_refusal( + option: &str, + root: &str, + refusal: extenddb_engine::store_maintenance::RootRefusal, +) -> anyhow::Error { + use extenddb_engine::store_maintenance::RootRefusal; + match refusal { + RootRefusal::ConfiguredRoot if option == "--move-store" => anyhow::anyhow!( + "{option} {root}: that is the configured store's root, so nothing would move. Give \ + the root the rows were written under, as backup list printed it before the \ + configuration changed or as backup sync's report of other stores prints it" + ), + RootRefusal::ConfiguredRoot => anyhow::anyhow!( + "{option} {root}: that is the configured store's root, which {option} refuses so \ + the store in use cannot lose its rows by a slip; extenddb backup purge deletes the \ + configured store's backups, and backup sync --prune-missing removes its AVAILABLE \ + rows whose manifest is gone" + ), + RootRefusal::NoRows(known) => { + println!(); + println!("--- The stores the catalog holds backup rows for:"); + for line in known_store_lines(&known) { + println!("{line}"); + } + anyhow::anyhow!( + "{option} {root}: the catalog has no backup row under that root (a root matches \ + as the catalog stores it, byte for byte: a trailing / or an S3 root's \ + ?endpoint= suffix makes another root); the roots it holds rows for are listed \ + above" + ) + } + RootRefusal::Catalog(e) => anyhow::anyhow!("the catalog could not be read: {e}"), + } +} + +/// The identity `--adopt-store`'s wait reads rows of: the root's file id, +/// before the adoption as after it. +async fn adopt_wait_identity(configured: &ConfiguredStore) -> anyhow::Result { + let file = tokio::time::timeout( + configured.check_timeout(), + extenddb_engine::store_identity::read_identity_file(&*configured.store), + ) + .await + .map_err(|_| { + anyhow::anyhow!( + "the store's {} could not be read within {} s", + extenddb_engine::STORE_IDENTITY_KEY, + configured.check_timeout().as_secs() + ) + })? + .map_err(|e| anyhow::anyhow!("{e}"))? + .ok_or_else(|| { + anyhow::anyhow!( + "the root's {} is gone; run the command again", + extenddb_engine::STORE_IDENTITY_KEY + ) + })?; + Ok(StoreIdentity { + store_kind: configured.kind, + store_root: configured.root.clone(), + store_id: file.store_id, + }) +} + +/// The wait the options run under the claim, as purge's, over the rows of +/// `identities` and every RUNNING restore: on SQLite after the dead-owner +/// resolutions with a zero window, whose cycle runs the store's check +/// unless `writes_nothing` (the forget options, which record their own +/// check that writes nothing). Returns the DELETING rows whose deletion +/// that cycle finished. The refusal is the command's error, the claim +/// still held. async fn wait_for_jobs( - opened: &Opened, - configured: &ConfiguredStore, - known: Option<&StoreIdentity>, + catalog: &Catalog, + identities: &[StoreIdentity], claim: &HeldClaim, interrupt: &mut Interrupt, stale_after: Duration, -) -> anyhow::Result<()> { - let storage = &*opened.catalog.storage; - // The identity the wait reads rows of: the file's id, before the - // adoption as after it. - let wait_identity = match known { - Some(identity) => identity.clone(), - None => { - let file = tokio::time::timeout( - configured.check_timeout(), - extenddb_engine::store_identity::read_identity_file(&*configured.store), + writes_nothing: bool, +) -> anyhow::Result> { + let storage = &*catalog.storage; + let mut finished = Vec::new(); + if catalog.backend == "sqlite" { + let state = if writes_nothing { + extenddb_engine::reconciler::ReconcilerState::for_maintenance_holder_without_store_check( ) - .await - .map_err(|_| { - anyhow::anyhow!( - "the store's {} could not be read within {} s", - extenddb_engine::STORE_IDENTITY_KEY, - configured.check_timeout().as_secs() - ) - })? - .map_err(|e| anyhow::anyhow!("{e}"))? - .ok_or_else(|| { - anyhow::anyhow!( - "the root's {} is gone; run the command again", - extenddb_engine::STORE_IDENTITY_KEY - ) - })?; - StoreIdentity { - store_kind: configured.kind, - store_root: configured.root.clone(), - store_id: file.store_id, - } - } - }; - let owner_window = if opened.catalog.backend == "sqlite" { - resolve_dead_owners(opened).await; - Duration::ZERO - } else { - stale_after - }; + } else { + extenddb_engine::reconciler::ReconcilerState::for_maintenance_holder() + }; + finished = resolve_dead_owners_with(catalog, state).await; + } + let owner_window = owner_window(&catalog.backend, stale_after); let started = Instant::now(); let outcome = wait_for_quiet( storage, - &wait_identity, + identities, claim, Wait { owner_window, @@ -1331,7 +1620,7 @@ async fn wait_for_jobs( ) .await; match outcome { - WaitOutcome::Quiet => Ok(()), + WaitOutcome::Quiet => Ok(finished), WaitOutcome::Busy(remaining) => Err(anyhow::anyhow!( "jobs were still running after waiting {} s (2 * stale_job_after_secs): {}. The \ maintenance claim is released; retry once they finish", @@ -1352,16 +1641,17 @@ async fn wait_for_jobs( } } -/// The adoption, the sync and the prune, with the claim held (on +/// The adoption, the move, the sync and the prune, with the claim held (on /// PostgreSQL, MongoDB and Cassandra) or under the serve lock (SQLite); the /// caller releases the claim. -#[allow(clippy::too_many_lines)] +#[allow(clippy::too_many_lines, clippy::too_many_arguments)] async fn sync_held( opened: &Opened, app_config: &config::AppConfig, known: Option, adopting: bool, prune_missing: bool, + moving: Option<&extenddb_engine::store_maintenance::LocatedRoot>, claim: Option<&HeldClaim>, interrupt: &Interrupt, ) -> anyhow::Result<()> { @@ -1452,6 +1742,38 @@ async fn sync_held( .map(|signal| format!("{signal} received")) }) }; + if let Some(located) = moving { + let owner_window = owner_window( + &opened.catalog.backend, + Duration::from_secs(app_config.backup.stale_job_after_secs), + ); + let report = extenddb_engine::store_maintenance::move_store( + storage, + extenddb_engine::store_maintenance::MoveTarget { + configured, + identity: &identity, + from_kind: located.kind, + from_root: &located.root, + owner_window, + stop: &stop, + }, + ) + .await + .map_err(|e| { + anyhow::anyhow!( + "--move-store: the catalog's rows could not be read: {e}; no row changed" + ) + })?; + for line in move_report_lines(&report, located) { + println!("{line}"); + } + move_outcome(&report)?; + println!(); + println!( + "Every row of {} names the configured store now; syncing it.", + located.root + ); + } let report = extenddb_engine::sync::sync( storage, extenddb_engine::sync::SyncTarget { @@ -1484,6 +1806,46 @@ async fn sync_held( report.adopted.len() ); } + // The rows of the other stores, read once the listing has ended. + let others = extenddb_engine::store_maintenance::other_stores(storage, &identity).await; + println!(); + let others_error = match &others { + Ok(others) => { + for line in other_stores_lines(others) { + println!("{line}"); + } + None + } + Err(e) => { + println!("--- Rows of other stores: not read ({e})"); + Some(e.to_string()) + } + }; + // After a move, a row of the old root in that report landed during the + // move's sync (a CreateBackup of a server still configured with the old + // root whose insert came after the move's re-list): named, and the + // command exits 1. + let landed = match (moving, &others) { + (Some(located), Ok(others)) if old_root_listed(others, located) => { + let rows = named_rows_of_root(storage, located).await; + println!(); + println!("--- Rows that landed under the old root during the move:"); + match &rows { + Ok(rows) if rows.is_empty() => println!(" (none listed now)"), + Ok(rows) => { + for row in rows { + println!( + " {} ({}, account {})", + row.backup_arn, row.status, row.account_id + ); + } + } + Err(e) => println!(" (not read: {e})"), + } + Some(rows.map_or(0, |rows| rows.len())) + } + _ => None, + }; let mut pruned = 0usize; let mut prune_left = 0usize; @@ -1560,9 +1922,105 @@ async fn sync_held( } ); } + others_outcome(others_error.as_deref())?; + if let (Some(n), Some(located)) = (landed, moving) { + anyhow::bail!( + "a row landed under the old root during the move ({n} listed above): a CreateBackup \ + of a server still configured with {}. Reconfigure every server to the new root, \ + then, once the row's job has ended, copy its prefix under the new root (or remove \ + the row with --forget-backup) and run --move-store {} again", + located.root, + located.root + ); + } complete_run(&report) } +/// Whether the report of other stores lists a row of `located`'s root (an +/// entry count with no row behind it is not one). +fn old_root_listed( + others: &[extenddb_engine::store_maintenance::OtherStore], + located: &extenddb_engine::store_maintenance::LocatedRoot, +) -> bool { + others.iter().any(|other| { + other.store.store_kind == located.kind + && other.store.store_root == located.root + && !other.counts.is_empty() + }) +} + +/// The rows of the old root, named for the operator. +async fn named_rows_of_root( + storage: &dyn StorageEngine, + located: &extenddb_engine::store_maintenance::LocatedRoot, +) -> Result, StorageError> { + storage + .list_backup_rows_by_root(located.kind, &located.root, None) + .map_ok(|row| extenddb_engine::store_maintenance::NamedRow { + status: row.status, + backup_arn: row.backup_arn, + account_id: row.account_id, + }) + .try_collect() + .await +} + +/// The exit of a sync whose report of other stores failed (`error`, its +/// read's error): 1, since the report is part of the run. +fn others_outcome(error: Option<&str>) -> anyhow::Result<()> { + match error { + Some(e) => { + anyhow::bail!("the rows of other stores could not be read: {e}; run the command again") + } + None => Ok(()), + } +} + +/// The tail of plain sync's refusal on an unhealthy store: for a missing +/// manifest, the `--forget-backup` command for that backup. +fn manifest_missing_hint(problem: &StoreProblem) -> String { + match problem { + StoreProblem::ManifestMissing { backup_arn } => format!( + "; check first that the root is the one that holds the backups (a volume mounted, \ + a root shared by every server): an unmounted volume shows this too. When that \ + backup is gone for good, remove its row with extenddb backup sync --forget-backup \ + {backup_arn}, which runs on an unhealthy store and prints the store's check \ + afterwards" + ), + _ => String::new(), + } +} + +/// The window the options tell a live owner apart by: `stale_after` beside +/// servers; zero on SQLite, where the command holds the serve lock and no +/// job of another process can be alive. +fn owner_window(backend: &str, stale_after: Duration) -> Duration { + if backend == "sqlite" { + Duration::ZERO + } else { + stale_after + } +} + +/// The stores the forget options wait on: the configured store when its +/// identity is known, otherwise every store id the catalog records for the +/// configured root (`configured_ids`), and every store id it records for +/// `--forget-store`'s root. +fn forget_wait_identities( + known: Option<&StoreIdentity>, + configured_ids: &[StoreIdentity], + located: Option<&extenddb_engine::store_maintenance::LocatedRoot>, +) -> Vec { + let mut identities: Vec = match known { + Some(known) => vec![known.clone()], + None => configured_ids.to_vec(), + }; + if let Some(located) = located { + identities.extend(located.identities()); + } + identities +} + /// The adopting sync's last read of `extenddb-store.json` (`file`, `None` /// when the read did not finish within `bound`): `Ok` when it still names /// this catalog (`catalog_id`), and otherwise the error naming what it @@ -1665,59 +2123,761 @@ fn print_sync_report( } } -#[cfg(test)] -mod tests { - use std::sync::Arc; - use std::sync::atomic::{AtomicUsize, Ordering}; - use std::time::Duration; +/// The counts of one store as a line's tail: `3 AVAILABLE, 1 DELETING`. +fn counts_text(counts: &[(BackupStatus, u64)]) -> String { + counts + .iter() + .map(|(status, n)| format!("{n} {status}")) + .collect::>() + .join(", ") +} - use async_trait::async_trait; - use extenddb_storage::hooks::{Reregistration, ServingRegistrar}; +/// The other-stores section of a sync's report: per store its counts per +/// status (on Cassandra, the status entries no row stands behind as such) +/// and its CREATING and DELETING rows, then the two options that act on +/// them. +fn other_stores_lines(others: &[extenddb_engine::store_maintenance::OtherStore]) -> Vec { + let mut lines = vec![ + "--- Rows of other stores (left as they are; sync reads and changes none of them):" + .to_owned(), + ]; + if others.is_empty() { + lines.push(" (none)".to_owned()); + return lines; + } + for other in others { + let mut tail = counts_text(&other.counts); + if !other.entries.is_empty() { + if !tail.is_empty() { + tail.push_str("; "); + } + tail.push_str(&format!( + "{} (status entries with no row of that status behind them, which readers skip; \ + with no row of the root left, --forget-store removes them)", + counts_text(&other.entries) + )); + } + lines.push(format!( + " {} {} (store id {}): {tail}", + other.store.store_kind, other.store.store_root, other.store.store_id + )); + for row in &other.unresolved { + lines.push(format!( + " {} {} (account {})", + row.status, row.backup_arn, row.account_id + )); + } + } + lines.push( + " (to keep a root's backups, copy its tree under the configured root and run \ + extenddb backup sync --move-store ; to drop its rows, extenddb backup sync \ + --forget-store )" + .to_owned(), + ); + lines +} - use super::{ - complete_run, owner_after_sync, release_registration, release_registration_within, - }; +/// The stores the catalog holds rows for, one line each, for a root that +/// has none. +fn known_store_lines(counts: &[extenddb_storage::backup_catalog::StoreRootCount]) -> Vec { + let stores = extenddb_engine::store_maintenance::group_counts(counts, None); + if stores.is_empty() { + return vec![" (none)".to_owned()]; + } + stores + .iter() + .map(|other| { + format!( + " {} {} (store id {}): {}", + other.store.store_kind, + other.store.store_root, + other.store.store_id, + counts_text(&other.counts) + ) + }) + .collect() +} - /// The identity file as a store holds it, naming `catalog_id`. - fn file(catalog_id: &str) -> extenddb_engine::StoreIdentityFile { - extenddb_engine::StoreIdentityFile { - format_version: 1, - store_id: "store-1".to_owned(), - catalog_id: catalog_id.to_owned(), - backend: "postgres".to_owned(), - created_at_ms: 0, - ephemeral_catalog: false, +/// The report of `--move-store`. +fn move_report_lines( + report: &extenddb_engine::store_maintenance::MoveReport, + located: &extenddb_engine::store_maintenance::LocatedRoot, +) -> Vec { + let mut lines = vec![ + String::new(), + format!( + "--- --move-store {} ({}): {} row(s) under the old root, checked against the copy \ + under the configured root (the copy must be complete before the move)", + located.root, located.kind, report.rows + ), + ]; + let mut section = |title: &str, items: Vec| { + lines.push(format!("--- {title}:")); + if items.is_empty() { + lines.push(" (none)".to_owned()); } + lines.extend(items.into_iter().map(|item| format!(" {item}"))); + }; + section( + "Resolved (CREATING rows whose owner is dead, against the configured store)", + report + .resolved + .iter() + .map(|(arn, status)| format!("{arn}: now {status}")) + .collect(), + ); + section( + "Finished (DELETING rows, their objects deleted under the configured root)", + report + .finished + .iter() + .map(|arn| format!("{arn}: now DELETED")) + .collect(), + ); + section( + "Stopping the move (no row is moved while any is listed here)", + report + .stopping + .iter() + .map(|(row, why)| { + format!( + "{} ({}, account {}): {why}", + row.backup_arn, row.status, row.account_id + ) + }) + .collect(), + ); + section( + "Moved (each row now names the configured store)", + report.moved.clone(), + ); + if !report.gone.is_empty() { + section( + "Gone during the move (removed by another writer; nothing moved)", + report.gone.clone(), + ); } + section( + "Left under the old root", + report + .left + .iter() + .map(|row| { + format!( + "{} ({}, account {})", + row.backup_arn, row.status, row.account_id + ) + }) + .collect(), + ); + lines.push(String::new()); + lines.push(format!( + "=== extenddb backup sync --move-store {}: {} row(s) moved, {} resolved, {} \ + deletion(s) finished, {} row(s) stopping the move, {} gone during the move, {} left \ + under the old root ===", + if report.complete() { + "complete" + } else { + "stopped" + }, + report.moved.len(), + report.resolved.len(), + report.finished.len(), + report.stopping.len(), + report.gone.len(), + report.left.len() + )); + lines +} - /// The adopting sync's last read: the file still naming this catalog - /// passes; another catalog (an adopter whose put landed during the - /// sync), no file, an unreadable file and a read past the bound each - /// fail, the first naming the new owner. - #[test] - fn the_adopting_syncs_last_read_fails_unless_the_file_names_this_catalog() { - let bound = Duration::from_secs(10); - owner_after_sync(Some(Ok(Some(file("catalog-1")))), "catalog-1", bound).unwrap(); - let err = owner_after_sync(Some(Ok(Some(file("adopter")))), "catalog-1", bound) - .unwrap_err() - .to_string(); - assert!( - err.starts_with( - "after the sync, extenddb-store.json names catalog adopter (store id \ - store-1): another catalog adopted the root during the sync" - ), - "{err}" +/// The exit of `--move-store`: `Ok` when every row of the old root was +/// moved (the sync of the configured root follows), otherwise why not. +fn move_outcome(report: &extenddb_engine::store_maintenance::MoveReport) -> anyhow::Result<()> { + if !report.stopping.is_empty() { + anyhow::bail!( + "--move-store moved no row: {} row(s) stop the move (see above). Resolve each and \ + run the command again; the CREATING rows resolved and the deletions finished are \ + kept", + report.stopping.len() ); - let err = owner_after_sync(Some(Ok(None)), "catalog-1", bound) - .unwrap_err() - .to_string(); - assert_eq!(err, "after the sync, the root has no extenddb-store.json"); - let err = owner_after_sync( - Some(Err(extenddb_engine::StoreProblem::Transport( - "down".to_owned(), - ))), - "catalog-1", - bound, + } + if let Some(reason) = &report.stopped { + anyhow::bail!("--move-store stopped before the rewrite: {reason}; no row was moved"); + } + if let Some(e) = &report.rewrite_error { + anyhow::bail!( + "--move-store's rewrite failed: {e}; {} row(s) were moved, and the rows left under \ + the old root are listed above: run the command again for them", + report.moved.len() + ); + } + if let Some(e) = &report.relist_error { + anyhow::bail!( + "{e}; the rewrite counted {} row(s). Run extenddb backup sync, which lists any row \ + still under the old root in its report of other stores and syncs the configured \ + root; run --move-store again only if that report shows rows of the old root (with \ + none, it refuses the root as having no rows)", + report.rewritten + ); + } + if !report.left.is_empty() { + anyhow::bail!( + "{} row(s) are still under the old root (see above): a CreateBackup of a server \ + still configured with that root, or a row another writer changed during the move. \ + Reconfigure every server to the new root, copy each such row's prefix under the new \ + root (or remove the row with --forget-backup), and run the command again", + report.left.len() + ); + } + Ok(()) +} + +/// The report of `--forget-store`. +fn forget_store_lines( + report: &extenddb_engine::store_maintenance::ForgetStoreReport, + located: &extenddb_engine::store_maintenance::LocatedRoot, +) -> Vec { + let mut lines = vec![ + String::new(), + format!( + "--- --forget-store {} ({}): rows removed outright; no object is deleted:", + located.root, located.kind + ), + ]; + if report.rows.is_empty() { + lines.push(" (none)".to_owned()); + } + for (row, outcome) in &report.rows { + lines.push(format!( + " {} ({}, account {}): {outcome}", + row.backup_arn, row.status, row.account_id + )); + } + let swept = report.entries_swept.unwrap_or_default(); + if swept.removed > 0 { + lines.push(format!( + " ({} status entry(ies) of the root with no row of the root behind them removed: \ + entries a deleter wrote after a move of its row, and the earlier-status entries of \ + rows removed above)", + swept.removed + )); + } + if swept.repaired > 0 { + lines.push(format!( + " ({} status entry(ies) of rows left under the root repaired: each was under an \ + earlier status than its row's, from a status change whose entry batch failed or \ + whose writer stopped before it, and is now under the row's status)", + swept.repaired + )); + } + if !report.entries_left.is_empty() { + lines.push(format!( + " (the catalog still records store id(s) {} for the root: status entries written \ + after the root was listed, by a writer still configured with it; plain sync's \ + report of other stores shows them, and running the command again removes their \ + rows)", + report.entries_left.join(", ") + )); + } + lines.push(String::new()); + lines.push(format!( + "=== extenddb backup sync --forget-store {}: {} row(s) removed, {} left ===", + if report.complete() { + "complete" + } else if report.stopped.is_some() { + "stopped" + } else { + "incomplete" + }, + report.removed(), + report + .rows_left + .map_or_else(|| "an unknown number".to_owned(), |n| n.to_string()) + )); + lines +} + +/// The exit of `--forget-store`. +fn forget_store_outcome( + report: &extenddb_engine::store_maintenance::ForgetStoreReport, +) -> anyhow::Result<()> { + if let Some(reason) = &report.stopped { + anyhow::bail!( + "--forget-store stopped before the remaining rows: {reason}; run the command again \ + for the rest" + ); + } + if let Some(e) = &report.recheck_error { + anyhow::bail!("{e}; run the command again to see what is left"); + } + if !report.complete() { + anyhow::bail!( + "{} row(s) of the root are left (see above, each with why); run the command again \ + once their jobs have ended, and for a row in use once its restore has ended or its \ + target is removed with extenddb backup remove-target --account ", + report.rows_left.unwrap_or_default() + ); + } + Ok(()) +} + +/// The report of `--forget-backup`. +fn forget_backups_lines( + report: &extenddb_engine::store_maintenance::ForgetBackupsReport, + asked: &[String], +) -> Vec { + let mut lines = vec![ + String::new(), + "--- --forget-backup (rows removed outright; no object is deleted, but a DELETING row \ + of the configured store has its deletion finished first):" + .to_owned(), + ]; + for (arn, outcome) in &report.done { + lines.push(format!(" {arn}: {outcome}")); + } + for arn in asked.iter().skip(report.done.len()) { + lines.push(format!(" {arn}: not reached")); + } + let removed = report + .done + .iter() + .filter(|(_, o)| { + matches!( + o, + extenddb_engine::store_maintenance::ForgetBackup::Row(_, forgotten) + if forgotten.is_removed() + ) + }) + .count(); + lines.push(String::new()); + lines.push(format!( + "=== extenddb backup sync --forget-backup {}: {removed} row(s) removed, {} not ===", + if report.complete(asked.len()) { + "complete" + } else { + "stopped" + }, + asked.len() - report.done.iter().filter(|(_, o)| o.is_done()).count() + )); + lines +} + +/// The exit of `--forget-backup`. +fn forget_backups_outcome( + report: &extenddb_engine::store_maintenance::ForgetBackupsReport, + asked: usize, +) -> anyhow::Result<()> { + if let Some(reason) = &report.stopped { + anyhow::bail!( + "--forget-backup stopped before the remaining ARNs: {reason}; run the command again \ + for them" + ); + } + if !report.complete(asked) { + let (arn, outcome) = report + .done + .last() + .expect("an incomplete run without a stop ends at a refused ARN"); + anyhow::bail!( + "--forget-backup did not remove the row of {arn} ({outcome}); the ARNs after it were \ + not reached" + ); + } + Ok(()) +} + +/// `backup sync --forget-store ` and `--forget-backup ` (module +/// doc): on a store in any state, or with none configured (`identity` is +/// then why), under the maintenance claim on every backend. +#[allow(clippy::too_many_lines)] +async fn forget( + catalog: &Catalog, + identity: Result<&Identity, &NotConfigured>, + app_config: &config::AppConfig, + forget_store: Option<&str>, + forget_backup: &[String], + mut interrupt: Interrupt, +) -> anyhow::Result<()> { + let storage = &*catalog.storage; + let configured = catalog.runtime.configured_store(); + // The defaulted root missing (moved away, or a link to a volume that is + // gone) while the catalog holds rows for it: its rows are refused as an + // unmounted root's, and the header says so. + let unmounted = match identity { + Err(NotConfigured::DefaultRootMissingWithRows { + label, store_ids, .. + }) => Some((label.clone(), store_ids.clone())), + _ => None, + }; + println!("=== extenddb backup sync ==="); + let check_note = if forget_store.is_none() { + "the removals read no store; the check after them does" + } else { + "the removals read no store" + }; + let known = match (configured, identity) { + (Some(store), Ok(Identity::Known(identity))) => { + match store.health() { + StoreHealth::Unhealthy(problem) => println!( + "Store: {} {} (unhealthy: {problem}; {check_note})", + store.kind, store.root + ), + _ => println!( + "Store: {} {} (store id {})", + store.kind, store.root, identity.store_id + ), + } + Some(identity.clone()) + } + (Some(store), Ok(Identity::Problem(problem))) => { + println!( + "Store: {} {} (unhealthy: {problem}; {check_note})", + store.kind, store.root + ); + None + } + (Some(store), Ok(Identity::Empty)) => { + println!( + "Store: {} {} (no {})", + store.kind, + store.root, + extenddb_engine::STORE_IDENTITY_KEY + ); + None + } + (_, Err(_)) | (None, Ok(_)) => { + match &unmounted { + Some((root, ids)) => println!( + "Store: none (the default root {root} is missing while the catalog holds \ + rows for it, store id {}: a volume that is not mounted, or a directory moved \ + away; this command refuses its rows and reads no store)", + ids.join(", ") + ), + None => println!( + "Store: none ({}; the removals read no store)", + identity + .err() + .map_or_else(|| "not configured".to_owned(), ToString::to_string) + ), + } + None + } + }; + // Every ARN's form before the claim: a typo costs no wait. + let refused = extenddb_engine::store_maintenance::check_arns(forget_backup); + if !refused.is_empty() { + println!(); + println!("--- --forget-backup: ARNs refused before any change:"); + for (arn, why) in &refused { + println!(" {arn:?}: {why}"); + } + anyhow::bail!( + "--forget-backup: {} ARN(s) given are not backup ARNs (see above); nothing was \ + changed", + refused.len() + ); + } + let located = match forget_store { + Some(root) => match extenddb_engine::store_maintenance::locate_root( + storage, + root, + configured + .map(|store| store.root.as_str()) + .or_else(|| unmounted.as_ref().map(|(root, _)| root.as_str())), + ) + .await + { + Ok(located) => Some(located), + Err(refusal) => return Err(root_refusal("--forget-store", root, refusal)), + }, + None => None, + }; + + // The configured root's store ids when its identity is not known, so + // the wait covers its rows too. + let configured_ids = match (configured, &known) { + (Some(store), None) => storage + .read_store_ids(store.kind, &store.root) + .await + .unwrap_or_default() + .into_iter() + .map(|store_id| StoreIdentity { + store_kind: store.kind, + store_root: store.root.clone(), + store_id, + }) + .collect(), + _ => Vec::new(), + }; + let stale_after = Duration::from_secs(app_config.backup.stale_job_after_secs); + let claim = take_claim(storage, app_config).await?; + let claimed_at = tokio::time::Instant::now(); + let identities = forget_wait_identities(known.as_ref(), &configured_ids, located.as_ref()); + let deletions_finished = match wait_for_jobs( + catalog, + &identities, + &claim, + &mut interrupt, + stale_after, + true, + ) + .await + { + Ok(finished) => finished, + Err(refusal) => { + release(claim).await; + return Err(refusal); + } + }; + let stop = || { + claim.lost().or_else(|| { + interrupt + .signalled() + .map(|signal| format!("{signal} received")) + }) + }; + let on_wait = |remaining: Duration| { + println!( + "Waiting {} s before the first removal of a FAILED or DELETED row (or the \ + finished deletion of a DELETING one): a deletion a server's reconciler started before the claim was taken runs at most {} s, and its \ + last store call at most {} s more.", + remaining.as_secs().max(1), + extenddb_engine::reconciler::DUTY_DELETION_BOUND.as_secs(), + extenddb_engine::S3_CHECK_TIMEOUT.as_secs() + ); + }; + // The configured store's identity when its check passed (recorded at + // the opening, writing nothing): a DELETING row of it has its deletion + // finished before its removal. + let healthy = configured.and_then(|store| store.identity().filter(|_| store.is_healthy())); + let run = extenddb_engine::store_maintenance::ForgetRun { + owner_window: owner_window(&catalog.backend, stale_after), + removals_from: removals_from(&catalog.backend, claimed_at), + on_wait: &on_wait, + stop: &stop, + configured: configured.map( + |store| extenddb_engine::store_maintenance::ForgetConfigured { + store, + healthy: healthy.as_ref(), + mismatch_row_ids: match identity { + Ok(Identity::Problem(problem)) => problem.mismatch_row_ids(), + _ => &[], + }, + file_missing_with_rows: matches!( + identity, + Ok(Identity::Problem(StoreProblem::FileMissingWithRows(_))) + ), + }, + ), + unmounted_root: unmounted.as_ref().map(|(root, _)| root.as_str()), + deletions_finished: &deletions_finished, + }; + let outcome = match &located { + Some(located) => { + let report = extenddb_engine::store_maintenance::forget_store( + storage, + located.kind, + &located.root, + run, + ) + .await; + match report { + Ok(report) => { + for line in forget_store_lines(&report, located) { + println!("{line}"); + } + forget_store_outcome(&report) + } + Err(e) => Err(anyhow::anyhow!( + "--forget-store: the root's rows could not be read: {e}; no row was removed" + )), + } + } + None => { + let report = extenddb_engine::store_maintenance::forget_backups( + storage, + forget_backup, + &app_config.server.region, + run, + ) + .await; + for line in forget_backups_lines(&report, forget_backup) { + println!("{line}"); + } + forget_backups_outcome(&report, forget_backup.len()) + } + }; + release(claim).await; + + // The store's check once more, as the next server check runs it but + // writing nothing: a removed row may have been what kept the store + // unhealthy. + if let Some(store) = configured + && forget_store.is_none() + { + println!(); + match forget_check(identity) { + ForgetCheck::Skip => { + let fresh = matches!(identity, Ok(Identity::Empty)); + for line in skipped_check_lines(&store.root, fresh) { + println!("{line}"); + } + } + ForgetCheck::Run(known) => { + let check = extenddb_engine::check_configured_store_read_only( + store, + storage, + known.as_ref(), + ) + .await; + for line in store_check_lines(&check) { + println!("{line}"); + } + } + } + } + outcome +} + +/// The instant before which a forget option removes no FAILED or DELETED +/// row: `FORGET_WAIT` after the claim was taken, beside servers; none on +/// SQLite, whose serve lock excludes every server. +fn removals_from(backend: &str, claimed_at: tokio::time::Instant) -> Option { + (backend != "sqlite").then(|| claimed_at + extenddb_engine::store_maintenance::FORGET_WAIT) +} + +/// Whether the store's check runs after `--forget-backup`, and against +/// which identity. +#[derive(Debug, Clone, PartialEq, Eq)] +enum ForgetCheck { + /// The opening found no identity file under the root (with rows for + /// it: an unmounted volume, say, whose rows the command refused; or + /// none: a fresh root): the check is not run, so nothing is written to + /// such a root. + Skip, + /// The read-only check, against the id the opening read when it read + /// one. + Run(Option), +} + +/// The check after `--forget-backup` for the store as the opening found +/// it. +fn forget_check(identity: Result<&Identity, &NotConfigured>) -> ForgetCheck { + match identity { + Ok(Identity::Known(identity)) => ForgetCheck::Run(Some(identity.clone())), + Ok(Identity::Empty | Identity::Problem(StoreProblem::FileMissingWithRows(_))) => { + ForgetCheck::Skip + } + Ok(Identity::Problem(_)) | Err(_) => ForgetCheck::Run(None), + } +} + +/// What `--forget-backup` prints when the opening found no identity file: +/// at a fresh root (`fresh`: no rows for it either), or at a root with rows +/// for it. +fn skipped_check_lines(root: &str, fresh: bool) -> Vec { + let key = extenddb_engine::STORE_IDENTITY_KEY; + let opening = format!( + "The configured store's check was not run: the root {root} had no {key} when the command \ + opened it, and this command writes nothing to a root." + ); + if fresh { + return vec![format!( + "{opening} The root is fresh (the catalog has no rows for it either): a server's \ + first check creates its {key}." + )]; + } + vec![format!( + "{opening} A server's check creates the file at a root the catalog has no rows for and \ + reports the store healthy, so mount the volume (or point [backup.store] at the root \ + that holds the backups) before the root's last row is gone." + )] +} + +/// The store's check after `--forget-backup`. +fn store_check_lines(check: &Result, StoreProblem>) -> Vec { + match check { + Ok(Some(identity)) => vec![format!( + "The configured store's check passes (store id {}); each server finds it so at its \ + next reconciler cycle.", + identity.store_id + )], + Ok(None) => vec![format!( + "The configured store's root has no {} and the catalog no rows for it; the check \ + writes nothing, and a server's next check creates the file.", + extenddb_engine::STORE_IDENTITY_KEY + )], + Err(problem @ StoreProblem::ManifestMissing { backup_arn }) => vec![ + format!("The configured store's check fails: {problem}."), + format!( + " If that backup is gone for good too, run extenddb backup sync \ + --forget-backup {backup_arn}; then backup sync --prune-missing removes the \ + remaining rows whose manifest is gone." + ), + ], + Err(problem) => vec![format!("The configured store's check fails: {problem}.")], + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::time::Duration; + + use async_trait::async_trait; + use extenddb_engine::StoreProblem; + use extenddb_engine::store_maintenance::{ + ForgetBackup, ForgetBackupsReport, ForgetStoreReport, Forgotten, LocatedRoot, MoveReport, + NamedRow, OtherStore, + }; + use extenddb_storage::backup_catalog::{BackupStatus, StoreIdentity, StoreKind}; + use extenddb_storage::hooks::{Reregistration, ServingRegistrar}; + + use super::{ + complete_run, owner_after_sync, release_registration, release_registration_within, + }; + + /// The identity file as a store holds it, naming `catalog_id`. + fn file(catalog_id: &str) -> extenddb_engine::StoreIdentityFile { + extenddb_engine::StoreIdentityFile { + format_version: 1, + store_id: "store-1".to_owned(), + catalog_id: catalog_id.to_owned(), + backend: "postgres".to_owned(), + created_at_ms: 0, + ephemeral_catalog: false, + } + } + + /// The adopting sync's last read: the file still naming this catalog + /// passes; another catalog (an adopter whose put landed during the + /// sync), no file, an unreadable file and a read past the bound each + /// fail, the first naming the new owner. + #[test] + fn the_adopting_syncs_last_read_fails_unless_the_file_names_this_catalog() { + let bound = Duration::from_secs(10); + owner_after_sync(Some(Ok(Some(file("catalog-1")))), "catalog-1", bound).unwrap(); + let err = owner_after_sync(Some(Ok(Some(file("adopter")))), "catalog-1", bound) + .unwrap_err() + .to_string(); + assert!( + err.starts_with( + "after the sync, extenddb-store.json names catalog adopter (store id \ + store-1): another catalog adopted the root during the sync" + ), + "{err}" + ); + let err = owner_after_sync(Some(Ok(None)), "catalog-1", bound) + .unwrap_err() + .to_string(); + assert_eq!(err, "after the sync, the root has no extenddb-store.json"); + let err = owner_after_sync( + Some(Err(extenddb_engine::StoreProblem::Transport( + "down".to_owned(), + ))), + "catalog-1", + bound, ) .unwrap_err() .to_string(); @@ -1824,4 +2984,542 @@ mod tests { release_registration(None).await; } + + /// `backup sync`'s options as clap parses them: the three store options + /// conflict with each other, with `--adopt-store` and with + /// `--prune-missing`; `--forget-backup` repeats, its ARNs in order. + #[test] + fn the_store_options_conflict_and_forget_backup_repeats() { + use clap::Parser; + #[derive(Parser)] + struct Cli { + #[command(subcommand)] + command: super::BackupCommand, + } + let parse = |args: &[&str]| { + Cli::try_parse_from(std::iter::once("backup").chain(args.iter().copied())) + .map(|cli| cli.command) + }; + for refused in [ + &["sync", "--move-store", "/a", "--prune-missing"][..], + &["sync", "--move-store", "/a", "--adopt-store"], + &["sync", "--move-store", "/a", "--forget-store", "/b"], + &["sync", "--move-store", "/a", "--forget-backup", "x"], + &["sync", "--forget-store", "/b", "--forget-backup", "x"], + &["sync", "--forget-store", "/b", "--prune-missing"], + &["sync", "--forget-store", "/b", "--adopt-store"], + &["sync", "--forget-backup", "x", "--prune-missing"], + &["sync", "--forget-backup", "x", "--adopt-store"], + ] { + let Err(e) = parse(refused) else { + panic!("{refused:?} parsed"); + }; + assert_eq!( + e.kind(), + clap::error::ErrorKind::ArgumentConflict, + "{refused:?}" + ); + } + match parse(&["sync", "--forget-backup", "a", "--forget-backup", "b"]).unwrap() { + super::BackupCommand::Sync { + forget_backup, + move_store: None, + forget_store: None, + adopt_store: false, + prune_missing: false, + } => assert_eq!(forget_backup, ["a", "b"]), + _ => panic!("not a sync"), + } + match parse(&["sync", "--move-store", "s3://b/p?endpoint=http://h:1"]).unwrap() { + super::BackupCommand::Sync { + move_store: Some(root), + .. + } => assert_eq!(root, "s3://b/p?endpoint=http://h:1"), + _ => panic!("not a move"), + } + } + + fn named(status: BackupStatus, arn: &str) -> NamedRow { + NamedRow { + status, + backup_arn: arn.to_owned(), + account_id: "123456789012".to_owned(), + } + } + + fn old_root() -> LocatedRoot { + LocatedRoot { + kind: StoreKind::Filesystem, + root: "/srv/old".to_owned(), + store_ids: vec!["store-0".to_owned()], + } + } + + /// The other-stores section: `(none)` without any; otherwise a line per + /// store with its counts, its CREATING and DELETING rows below it, and + /// the line naming the two options. + #[test] + fn the_other_stores_section_counts_names_and_points_to_the_options() { + assert_eq!( + super::other_stores_lines(&[]), + [ + "--- Rows of other stores (left as they are; sync reads and changes none of them):", + " (none)", + ] + ); + let lines = super::other_stores_lines(&[OtherStore { + store: StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: "/srv/old".to_owned(), + store_id: "store-0".to_owned(), + }, + counts: vec![(BackupStatus::Available, 3), (BackupStatus::Deleting, 1)], + unresolved: vec![named(BackupStatus::Deleting, "arn:d")], + entries: Vec::new(), + }]); + assert_eq!( + lines[1], + " filesystem /srv/old (store id store-0): 3 AVAILABLE, 1 DELETING" + ); + assert_eq!(lines[2], " DELETING arn:d (account 123456789012)"); + assert!( + lines[3].contains("--move-store ") && lines[3].contains("--forget-store "), + "{}", + lines[3] + ); + assert_eq!(lines.len(), 4); + } + + /// The move exits 0 only when every row moved; a stopping row, a stop, + /// a failed rewrite and a row left each exit 1 with their own sentence, + /// and the summary line says which. + #[test] + fn the_move_exits_1_unless_every_row_moved() { + let moved = MoveReport { + rows: 2, + moved: vec!["arn:1".to_owned(), "arn:2".to_owned()], + rewritten: 2, + ..MoveReport::default() + }; + super::move_outcome(&moved).unwrap(); + let summary = super::move_report_lines(&moved, &old_root()); + assert_eq!( + summary.last().unwrap(), + "=== extenddb backup sync --move-store complete: 2 row(s) moved, 0 resolved, 0 \ + deletion(s) finished, 0 row(s) stopping the move, 0 gone during the move, 0 left \ + under the old root ===" + ); + let gone = MoveReport { + rows: 3, + moved: vec!["arn:1".to_owned(), "arn:2".to_owned()], + gone: vec!["arn:3".to_owned()], + rewritten: 3, + ..MoveReport::default() + }; + assert!( + super::move_report_lines(&gone, &old_root()) + .last() + .unwrap() + .contains( + "2 row(s) moved, 0 resolved, 0 deletion(s) finished, 0 row(s) stopping \ + the move, 1 gone during the move, 0 left under the old root" + ), + "the summary line counts the rows gone during the move" + ); + let stopping = MoveReport { + stopping: vec![(named(BackupStatus::Available, "arn:1"), "why".to_owned())], + ..MoveReport::default() + }; + let err = super::move_outcome(&stopping).unwrap_err().to_string(); + assert!( + err.starts_with("--move-store moved no row: 1 row(s) stop the move"), + "{err}" + ); + assert!( + super::move_report_lines(&stopping, &old_root()) + .contains(&" arn:1 (AVAILABLE, account 123456789012): why".to_owned()) + ); + for (report, says) in [ + ( + MoveReport { + stopped: Some("SIGINT received".to_owned()), + ..MoveReport::default() + }, + "stopped before the rewrite: SIGINT received", + ), + ( + MoveReport { + rewrite_error: Some("lost".to_owned()), + ..MoveReport::default() + }, + "rewrite failed: lost", + ), + ( + MoveReport { + left: vec![named(BackupStatus::Creating, "arn:3")], + ..MoveReport::default() + }, + "1 row(s) are still under the old root", + ), + ] { + let err = super::move_outcome(&report).unwrap_err().to_string(); + assert!(err.contains(says), "{err}"); + assert!( + super::move_report_lines(&report, &old_root()) + .last() + .unwrap() + .starts_with("=== extenddb backup sync --move-store stopped:") + ); + } + } + + /// `--forget-store` exits 0 once the root lists no row, entries left + /// behind (Cassandra) reported with the exit kept; a row left, a stop + /// and a failed re-read exit 1. + #[test] + fn forget_store_exits_1_while_a_row_of_the_root_is_left() { + let done = ForgetStoreReport { + rows: vec![( + named(BackupStatus::Failed, "arn:1"), + Forgotten::Removed(BackupStatus::Failed), + )], + rows_left: Some(0), + entries_left: vec!["store-0".to_owned()], + ..ForgetStoreReport::default() + }; + super::forget_store_outcome(&done).unwrap(); + let lines = super::forget_store_lines(&done, &old_root()); + assert!( + lines.iter().any(|l| l.contains("store id(s) store-0")), + "{lines:?}" + ); + assert_eq!( + lines.last().unwrap(), + "=== extenddb backup sync --forget-store complete: 1 row(s) removed, 0 left ===" + ); + let left = ForgetStoreReport { + rows: vec![( + named(BackupStatus::Creating, "arn:2"), + Forgotten::LiveOwner(BackupStatus::Creating), + )], + rows_left: Some(1), + ..ForgetStoreReport::default() + }; + let err = super::forget_store_outcome(&left).unwrap_err().to_string(); + assert!(err.starts_with("1 row(s) of the root are left"), "{err}"); + assert!( + err.contains("extenddb backup remove-target --account"), + "{err}" + ); + assert_eq!( + super::forget_store_lines(&left, &old_root()) + .last() + .unwrap(), + "=== extenddb backup sync --forget-store incomplete: 0 row(s) removed, 1 left ===" + ); + let swept = ForgetStoreReport { + rows_left: Some(0), + entries_swept: Some(extenddb_storage::backup_catalog::StatusEntrySweep { + removed: 2, + repaired: 1, + }), + ..ForgetStoreReport::default() + }; + let lines = super::forget_store_lines(&swept, &old_root()); + assert!( + lines + .iter() + .any(|l| l.starts_with(" (2 status entry(ies) of the root")) + && lines + .iter() + .any(|l| l.starts_with(" (1 status entry(ies) of rows left")), + "{lines:?}" + ); + let stopped = ForgetStoreReport { + stopped: Some("SIGTERM received".to_owned()), + ..ForgetStoreReport::default() + }; + assert!(super::forget_store_outcome(&stopped).is_err()); + let unread = ForgetStoreReport { + recheck_error: Some("the root's rows could not be read: down".to_owned()), + ..ForgetStoreReport::default() + }; + assert!(super::forget_store_outcome(&unread).is_err()); + } + + /// `--forget-backup` exits 0 when every ARN's row is gone, and 1 naming + /// the ARN it stopped at (a legacy ARN included); the ARNs after it are + /// listed as not reached. + #[test] + fn forget_backup_exits_1_naming_the_arn_it_stopped_at() { + let asked = ["arn:1".to_owned(), "arn:2".to_owned(), "arn:3".to_owned()]; + let removed = |arn: &str| { + ( + arn.to_owned(), + ForgetBackup::Row( + BackupStatus::Available, + Forgotten::Removed(BackupStatus::Available), + ), + ) + }; + let all = ForgetBackupsReport { + done: asked.iter().map(|a| removed(a)).collect(), + stopped: None, + }; + super::forget_backups_outcome(&all, 3).unwrap(); + // An ARN no row carries counts as done: a run given again after a + // stop goes past the ARNs it removed. + let again = ForgetBackupsReport { + done: vec![ + ("arn:1".to_owned(), ForgetBackup::NoRow), + removed("arn:2"), + removed("arn:3"), + ], + stopped: None, + }; + super::forget_backups_outcome(&again, 3).unwrap(); + let lines = super::forget_backups_lines(&again, &asked); + assert!( + lines.contains(&" arn:1: no row carries this ARN; nothing to remove".to_owned()), + "{lines:?}" + ); + assert_eq!( + lines.last().unwrap(), + "=== extenddb backup sync --forget-backup complete: 2 row(s) removed, 0 not ===" + ); + let refused = ForgetBackupsReport { + done: vec![removed("arn:1"), ("arn:2".to_owned(), ForgetBackup::Legacy)], + stopped: None, + }; + let err = super::forget_backups_outcome(&refused, 3) + .unwrap_err() + .to_string(); + assert!( + err.starts_with( + "--forget-backup did not remove the row of arn:2 (an in-catalog backup" + ), + "{err}" + ); + let lines = super::forget_backups_lines(&refused, &asked); + assert!( + lines.contains(&" arn:3: not reached".to_owned()), + "{lines:?}" + ); + assert_eq!( + lines.last().unwrap(), + "=== extenddb backup sync --forget-backup stopped: 1 row(s) removed, 2 not ===" + ); + } + + /// The store's check after `--forget-backup`: passing, or the problem; + /// a manifest still missing names the next ARN to forget. + #[test] + fn the_check_after_forget_backup_names_the_next_backup_to_forget() { + let ok = super::store_check_lines(&Ok(Some(StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: "/r".to_owned(), + store_id: "store-1".to_owned(), + }))); + assert!(ok[0].starts_with("The configured store's check passes (store id store-1)")); + let empty = super::store_check_lines(&Ok(None)); + assert!( + empty[0].contains("the check writes nothing, and a server's next check creates"), + "{empty:?}" + ); + let missing = super::store_check_lines(&Err(StoreProblem::ManifestMissing { + backup_arn: "arn:next".to_owned(), + })); + assert!( + missing[1].contains("--forget-backup arn:next"), + "{missing:?}" + ); + } + + /// The check after `--forget-backup` runs against the id the opening + /// read, without one when the opening read none, and not at all when + /// the opening found no identity file (`Empty`, which the forget + /// options' opening reads for a fresh root since it writes nothing, or + /// `FileMissingWithRows`); the line printed then says to mount the + /// volume before the root's last row is gone. + #[test] + fn the_check_after_forget_backup_runs_only_where_the_opening_found_a_file() { + use super::{ForgetCheck, Identity, forget_check}; + let identity = StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: "/r".to_owned(), + store_id: "store-1".to_owned(), + }; + assert_eq!( + forget_check(Ok(&Identity::Known(identity.clone()))), + ForgetCheck::Run(Some(identity)) + ); + assert_eq!(forget_check(Ok(&Identity::Empty)), ForgetCheck::Skip); + assert_eq!( + forget_check(Ok(&Identity::Problem(StoreProblem::FileMissingWithRows( + vec!["store-1".to_owned()] + )))), + ForgetCheck::Skip + ); + assert_eq!( + forget_check(Ok(&Identity::Problem(StoreProblem::ManifestMissing { + backup_arn: "arn:gone".to_owned() + }))), + ForgetCheck::Run(None) + ); + assert_eq!( + forget_check(Err(&super::NotConfigured::Absent)), + ForgetCheck::Run(None) + ); + let skipped = &super::skipped_check_lines("/mnt/b", false)[0]; + assert!(skipped.contains("/mnt/b had no"), "{skipped}"); + assert!( + skipped.contains("before the root's last row is gone") + && !skipped.contains("each server's next check reports the store"), + "{skipped}" + ); + let fresh = &super::skipped_check_lines("/mnt/c", true)[0]; + assert!( + fresh.contains("/mnt/c had no") + && fresh.contains("The root is fresh") + && !fresh.contains("mount the volume"), + "{fresh}" + ); + } + + /// After a move, a row of the old root in the closing report of other + /// stores makes the command exit 1; counts of status entries with no + /// row behind them, or rows of another root, do not. + #[test] + fn a_row_of_the_old_root_in_the_closing_report_is_one_that_landed() { + let root = old_root(); + let other = |root: &str, counts: Vec<(BackupStatus, u64)>, entries| OtherStore { + store: StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: root.to_owned(), + store_id: "store-0".to_owned(), + }, + counts, + unresolved: Vec::new(), + entries, + }; + assert!(super::old_root_listed( + &[other( + "/srv/old", + vec![(BackupStatus::Creating, 1)], + Vec::new() + )], + &root + )); + assert!(!super::old_root_listed( + &[other( + "/srv/old", + Vec::new(), + vec![(BackupStatus::Deleting, 1)] + )], + &root + )); + assert!(!super::old_root_listed( + &[other( + "/srv/elsewhere", + vec![(BackupStatus::Available, 2)], + Vec::new() + )], + &root + )); + let lines = super::other_stores_lines(&[other( + "/srv/old", + vec![(BackupStatus::Available, 1)], + vec![(BackupStatus::Deleting, 1)], + )]); + assert_eq!( + lines[1], + " filesystem /srv/old (store id store-0): 1 AVAILABLE; 1 DELETING (status entries \ + with no row of that status behind them, which readers skip; with no row of the \ + root left, --forget-store removes them)" + ); + } + + /// The decisions the options take from the backend and the located + /// root: the owner window (zero on SQLite under the serve lock), the + /// wait before the first FAILED or DELETED removal (none on SQLite), and + /// the stores the forget options wait on (the configured store's and + /// the root's ids). + #[test] + fn the_options_take_the_window_the_wait_and_the_stores_from_the_backend() { + let stale = std::time::Duration::from_secs(300); + assert_eq!( + super::owner_window("sqlite", stale), + std::time::Duration::ZERO + ); + for backend in ["postgres", "mongodb", "cassandra"] { + assert_eq!(super::owner_window(backend, stale), stale, "{backend}"); + } + let now = tokio::time::Instant::now(); + assert_eq!(super::removals_from("sqlite", now), None); + assert_eq!( + super::removals_from("postgres", now), + Some(now + extenddb_engine::store_maintenance::FORGET_WAIT) + ); + let configured = StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: "/srv/new".to_owned(), + store_id: "store-1".to_owned(), + }; + let root = old_root(); + assert_eq!( + super::forget_wait_identities(Some(&configured), &[], Some(&root)), + [configured.clone(), root.identities()[0].clone()] + ); + assert_eq!( + super::forget_wait_identities(None, &[], Some(&root)), + root.identities() + ); + assert_eq!( + super::forget_wait_identities(Some(&configured), &[], None), + [configured] + ); + assert!(super::forget_wait_identities(None, &[], None).is_empty()); + let ids = [ + StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: "/srv/new".to_owned(), + store_id: "store-1".to_owned(), + }, + StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: "/srv/new".to_owned(), + store_id: "store-2".to_owned(), + }, + ]; + assert_eq!( + super::forget_wait_identities(None, &ids, Some(&root)), + [ids[0].clone(), ids[1].clone(), root.identities()[0].clone()], + "the configured root's ids when its identity is not known" + ); + assert_eq!( + super::forget_wait_identities(Some(&ids[0]), &ids, None), + [ids[0].clone()] + ); + } + + /// A sync whose report of other stores failed exits 1; plain sync's + /// refusal on a missing manifest names the backup's own + /// `--forget-backup` command. + #[test] + fn a_failed_report_of_other_stores_exits_1_and_the_refusal_names_the_arn() { + super::others_outcome(None).unwrap(); + let err = super::others_outcome(Some("down")).unwrap_err().to_string(); + assert!( + err.starts_with("the rows of other stores could not be read: down"), + "{err}" + ); + let hint = super::manifest_missing_hint(&StoreProblem::ManifestMissing { + backup_arn: "arn:aws:dynamodb:us-east-1:1:table/T/backup/01".to_owned(), + }); + assert!( + hint.contains("--forget-backup arn:aws:dynamodb:us-east-1:1:table/T/backup/01,"), + "{hint}" + ); + assert!(super::manifest_missing_hint(&StoreProblem::Transport("x".to_owned())).is_empty()); + } } diff --git a/crates/backup-store/src/fs.rs b/crates/backup-store/src/fs.rs index fff5547dc..ec9d24c3e 100644 --- a/crates/backup-store/src/fs.rs +++ b/crates/backup-store/src/fs.rs @@ -54,7 +54,9 @@ //! blocking pool, where dropping the future does not stop it; a guard in //! the future sets a flag when it is dropped, and the walk checks the flag //! before each removal, so a dropped `delete_prefix` stops at its next -//! entry. +//! entry. `delete_prefix_before` checks its deadline at the same point, so a +//! walk that reaches the deadline stops before its next removal, whatever +//! the caller's timer has done by then. //! //! Writes go to a temporary sibling file and rename into place, so a crash //! never leaves a truncated file under its final name. The temporary is @@ -126,7 +128,7 @@ use futures::{FutureExt, StreamExt, TryStreamExt}; use tokio::io::AsyncWriteExt; use crate::key::{normalize_prefix, validate_key, validate_prefix}; -use crate::{BackupStore, ByteStream, ObjectMeta, StoreError}; +use crate::{BackupStore, ByteStream, ObjectMeta, PrefixDeletion, StoreError}; /// Prefix and suffix marking in-flight temporary files, hidden from listings. const TEMP_PREFIX: &str = ".put-"; @@ -600,6 +602,19 @@ impl FilesystemStore { } async fn delete_prefix_impl(&self, prefix: &str) -> Result { + self.delete_walk(prefix, None) + .await + .map(|done| done.deleted) + } + + /// The walk of `delete_prefix` and `delete_prefix_before`: with a + /// `deadline`, the walk starts no removal at or after it and returns + /// what it removed with `finished` false. + async fn delete_walk( + &self, + prefix: &str, + deadline: Option, + ) -> Result { validate_prefix(prefix)?; let normalized = normalize_prefix(prefix).to_owned(); let root = self.root.clone(); @@ -614,10 +629,25 @@ impl FilesystemStore { cancel: cancel.0.clone(), removal_hook: self.removal_hook.clone(), removals: std::sync::atomic::AtomicU64::new(0), + deadline, + out_of_time: std::sync::atomic::AtomicBool::new(false), + removed: std::sync::atomic::AtomicU64::new(0), }; - tokio::task::spawn_blocking(move || delete_objects(&root, &normalized, &walk)) - .await - .map_err(|e| StoreError::Other(format!("deletion task failed: {e}")))? + tokio::task::spawn_blocking(move || { + let outcome = delete_objects(&root, &normalized, &walk); + if walk.out_of_time.load(std::sync::atomic::Ordering::Acquire) { + return Ok(PrefixDeletion { + deleted: walk.removed.load(std::sync::atomic::Ordering::Acquire), + finished: false, + }); + } + outcome.map(|deleted| PrefixDeletion { + deleted, + finished: true, + }) + }) + .await + .map_err(|e| StoreError::Other(format!("deletion task failed: {e}")))? } async fn validate_impl(&self) -> Result<(), StoreError> { @@ -742,6 +772,14 @@ impl BackupStore for FilesystemStore { self.delete_prefix_impl(prefix).boxed() } + fn delete_prefix_before<'a>( + &'a self, + prefix: &'a str, + deadline: tokio::time::Instant, + ) -> BoxFuture<'a, Result> { + self.delete_walk(prefix, Some(deadline)).boxed() + } + fn validate(&self) -> BoxFuture<'_, Result<(), StoreError>> { self.validate_impl().boxed() } @@ -1254,23 +1292,40 @@ impl Drop for CancelOnDrop { } /// What one `delete_prefix` walk carries down the tree: the flag its -/// `CancelOnDrop` sets, and the store's removal hook (set only in the -/// crate's own tests) with the count it is called with. +/// `CancelOnDrop` sets, the store's removal hook (set only in the crate's +/// own tests) with the count it is called with, and for +/// `delete_prefix_before` the deadline, whether the walk reached it, and +/// the files it removed. struct DeleteWalk { cancel: std::sync::Arc, removal_hook: Option, removals: std::sync::atomic::AtomicU64, + deadline: Option, + out_of_time: std::sync::atomic::AtomicBool, + removed: std::sync::atomic::AtomicU64, } impl DeleteWalk { - /// The error a cancelled walk ends with; nobody receives it, since the - /// caller's future is gone. + /// The error a cancelled walk, or one past its deadline, ends with; + /// nobody receives the first, since the caller's future is gone, and + /// `delete_walk` turns the second into an unfinished deletion. Called + /// before each removal. fn check_cancel(&self) -> Result<(), StoreError> { if self.cancel.load(std::sync::atomic::Ordering::Acquire) { return Err(StoreError::Other( "delete_prefix was dropped before it finished".to_owned(), )); } + if self + .deadline + .is_some_and(|deadline| tokio::time::Instant::now() >= deadline) + { + self.out_of_time + .store(true, std::sync::atomic::Ordering::Release); + return Err(StoreError::Other( + "delete_prefix_before reached its deadline".to_owned(), + )); + } Ok(()) } @@ -1368,7 +1423,11 @@ fn check_tree(dir: &Path, dir_meta: &std::fs::Metadata) -> Result<(), StoreError fn remove_file_counted(path: &Path, walk: &DeleteWalk) -> Result { walk.before_removal(); match std::fs::remove_file(path) { - Ok(()) => Ok(1), + Ok(()) => { + walk.removed + .fetch_add(1, std::sync::atomic::Ordering::AcqRel); + Ok(1) + } Err(e) if vanished(&e) => Ok(0), Err(e) => Err(StoreError::io(format!("remove {}", path.display()), e)), } diff --git a/crates/backup-store/src/lib.rs b/crates/backup-store/src/lib.rs index 23e7526ad..7724ebcc7 100644 --- a/crates/backup-store/src/lib.rs +++ b/crates/backup-store/src/lib.rs @@ -69,6 +69,17 @@ pub struct ObjectMeta { pub etag: Option, } +/// What one [`BackupStore::delete_prefix_before`] did. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PrefixDeletion { + /// The objects removed, counted as [`BackupStore::delete_prefix`] + /// counts them. + pub deleted: u64, + /// Whether the deletion ran to its end; `false` when it stopped at the + /// deadline, with the rest of the prefix left. + pub finished: bool, +} + /// A destination for backup objects. /// /// Implementations must be safe to share across tasks; the engine holds one @@ -189,6 +200,44 @@ pub trait BackupStore: Send + Sync { /// the dropped future sets). fn delete_prefix<'a>(&'a self, prefix: &'a str) -> BoxFuture<'a, Result>; + /// [`BackupStore::delete_prefix`] that starts no removal at or after + /// `deadline`: the filesystem store's walk checks the deadline beside its + /// cancel flag before each removal, the S3 store before each + /// `ListObjectsV2`, `DeleteObjects`, `ListMultipartUploads` and + /// `AbortMultipartUpload`. A call that reaches the deadline returns what + /// it removed so far with [`PrefixDeletion::finished`] false; what + /// remains is the step in flight at the deadline (one removal on a + /// filesystem root, one request on S3, up to 1,000 keys for a + /// `DeleteObjects`). The check reads the clock in the process, so it + /// bounds a process that keeps running; one paused between a check and + /// the step it admits (a stopped process, a frozen VM) runs that step + /// when it resumes. The caller still bounds the call with a timeout for + /// a step that never answers. + /// + /// The default checks the deadline once and then runs `delete_prefix` + /// whole: for a store whose deletion is one step (an in-memory test + /// store), the same rule. + fn delete_prefix_before<'a>( + &'a self, + prefix: &'a str, + deadline: tokio::time::Instant, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + if tokio::time::Instant::now() >= deadline { + return Ok(PrefixDeletion { + deleted: 0, + finished: false, + }); + } + self.delete_prefix(prefix) + .await + .map(|deleted| PrefixDeletion { + deleted, + finished: true, + }) + }) + } + /// Startup validation. The filesystem store applies the owner-only rule /// to the root and the ancestor rule to the directories above it (the /// `fs` module doc states both), checks that the root is still where diff --git a/crates/backup-store/src/s3.rs b/crates/backup-store/src/s3.rs index 465a23f32..9d724501c 100644 --- a/crates/backup-store/src/s3.rs +++ b/crates/backup-store/src/s3.rs @@ -102,7 +102,7 @@ use futures::{FutureExt, StreamExt, TryStreamExt}; use crate::config::S3StoreConfig; use crate::key::{key_matches_prefix, normalize_prefix, validate_key, validate_prefix}; -use crate::{BackupStore, ByteStream, ObjectMeta, StoreError}; +use crate::{BackupStore, ByteStream, ObjectMeta, PrefixDeletion, StoreError}; /// The smallest part size S3 accepts for any part except the last. pub const MIN_PART_SIZE_BYTES: u64 = 5 * 1024 * 1024; @@ -925,12 +925,34 @@ impl S3Store { } async fn delete_prefix_impl(&self, prefix: &str) -> Result { + self.delete_requests(prefix, None) + .await + .map(|done| done.deleted) + } + + /// The requests of `delete_prefix` and `delete_prefix_before`: with a + /// `deadline`, no request is sent at or after it, and the call returns + /// what it deleted with `finished` false. + async fn delete_requests( + &self, + prefix: &str, + deadline: Option, + ) -> Result { validate_prefix(prefix)?; let match_prefix = normalize_prefix(prefix); let request_prefix = format!("{}{match_prefix}", self.prefix); let mut deleted: u64 = 0; let mut token: Option = None; + let out_of_time = + || deadline.is_some_and(|deadline| tokio::time::Instant::now() >= deadline); + let unfinished = |deleted| PrefixDeletion { + deleted, + finished: false, + }; loop { + if out_of_time() { + return Ok(unfinished(deleted)); + } // Raw bucket-side keys, not `list_page`'s filtered view: an // object whose key fails the store rule is still under the // prefix and still has to go. @@ -964,6 +986,9 @@ impl S3Store { }) .collect::, _>>()?; let count = objects.len() as u64; + if out_of_time() { + return Ok(unfinished(deleted)); + } let delete = Delete::builder() .set_objects(Some(objects)) .quiet(true) @@ -1006,9 +1031,16 @@ impl S3Store { // does not see them and S3 bills their parts until they are aborted. // A process that died mid-put left one; the reconciler's prefix // deletion is the one place that sweeps it. - self.abort_uploads_under(&request_prefix, match_prefix) - .await?; - Ok(deleted) + if !self + .abort_uploads_under(&request_prefix, match_prefix, deadline) + .await? + { + return Ok(unfinished(deleted)); + } + Ok(PrefixDeletion { + deleted, + finished: true, + }) } /// Abort every in-progress multipart upload under the normalized store @@ -1027,14 +1059,22 @@ impl S3Store { /// backstop for such uploads, and `validate` reports either missing /// grant at startup (it probes both). A token code (`ExpiredToken` and /// the others in `is_token_code`) is not a denial and fails the call. + /// With a `deadline`, no request is sent at or after it, and the sweep + /// answers `false` (not finished); otherwise `true`. async fn abort_uploads_under( &self, request_prefix: &str, match_prefix: &str, - ) -> Result<(), StoreError> { + deadline: Option, + ) -> Result { + let out_of_time = + || deadline.is_some_and(|deadline| tokio::time::Instant::now() >= deadline); let mut key_marker: Option = None; let mut id_marker: Option = None; loop { + if out_of_time() { + return Ok(false); + } let output = match self .client .list_multipart_uploads() @@ -1056,7 +1096,7 @@ impl S3Store { there stay open until the bucket's AbortIncompleteMultipartUpload \ lifecycle rule removes them" ); - return Ok(()); + return Ok(true); } Err(e) => return Err(map_sdk_error("ListMultipartUploads", e)), }; @@ -1070,6 +1110,9 @@ impl S3Store { if !matches { continue; } + if out_of_time() { + return Ok(false); + } match self .client .abort_multipart_upload() @@ -1110,7 +1153,7 @@ impl S3Store { break; } } - Ok(()) + Ok(true) } async fn validate_impl(&self) -> Result<(), StoreError> { @@ -1325,6 +1368,14 @@ impl BackupStore for S3Store { self.delete_prefix_impl(prefix).boxed() } + fn delete_prefix_before<'a>( + &'a self, + prefix: &'a str, + deadline: tokio::time::Instant, + ) -> BoxFuture<'a, Result> { + self.delete_requests(prefix, Some(deadline)).boxed() + } + /// Lists one key under the prefix (`ListObjectsV2`, `s3:ListBucket`) /// rather than calling `HeadBucket`, since only a GET-shaped error has a /// body: a 403 keeps S3's code, a 404 names the missing bucket, a 301 the diff --git a/crates/backup-store/tests/conformance.rs b/crates/backup-store/tests/conformance.rs index 30adcdd53..f44cb6887 100644 --- a/crates/backup-store/tests/conformance.rs +++ b/crates/backup-store/tests/conformance.rs @@ -24,7 +24,7 @@ use bytes::Bytes; use extenddb_backup_store::{BackupStore, ByteStream, StoreError, byte_stream_from}; -use extenddb_backup_store::{MIN_PART_SIZE_BYTES, ObjectMeta}; +use extenddb_backup_store::{MIN_PART_SIZE_BYTES, ObjectMeta, PrefixDeletion}; use futures::TryStreamExt; /// Part size every conformance run uses: the smallest S3 accepts, so the @@ -270,6 +270,47 @@ mod suite { ); } + /// `delete_prefix_before` with its deadline passed removes nothing and + /// says it did not finish, a prefix that names one object included; + /// with a deadline ahead it deletes the prefix as `delete_prefix` does. + pub async fn delete_prefix_before_honours_its_deadline(store: &dyn BackupStore) { + for key in ["dl/a/1", "dl/a/2", "dl/b/3", "keepdl/4"] { + store.put(key, chunked_body(vec![9])).await.unwrap(); + } + let passed = tokio::time::Instant::now(); + let done = store.delete_prefix_before("dl", passed).await.unwrap(); + assert_eq!( + done, + PrefixDeletion { + deleted: 0, + finished: false + }, + "a deadline already passed starts no removal" + ); + assert_eq!(list_keys(store, "dl").await.len(), 3); + let done = store.delete_prefix_before("dl/a/1", passed).await.unwrap(); + assert_eq!( + done, + PrefixDeletion { + deleted: 0, + finished: false + }, + "a prefix that names one object, past its deadline" + ); + assert_eq!(list_keys(store, "dl").await.len(), 3); + let ahead = tokio::time::Instant::now() + std::time::Duration::from_secs(60); + let done = store.delete_prefix_before("dl", ahead).await.unwrap(); + assert_eq!( + done, + PrefixDeletion { + deleted: 3, + finished: true + } + ); + assert!(list_keys(store, "dl").await.is_empty()); + assert_eq!(list_keys(store, "keepdl").await, vec!["keepdl/4"]); + } + pub async fn missing_object_is_not_found(store: &dyn BackupStore) { let err = store.get("no/such/object").await.err().expect("get fails"); assert!(matches!(err, StoreError::NotFound), "{err}"); @@ -657,6 +698,12 @@ mod filesystem { suite::delete_prefix_removes_and_counts(&store).await; } + #[tokio::test] + async fn delete_prefix_before_honours_its_deadline() { + let (_dir, store) = fs_store().await; + suite::delete_prefix_before_honours_its_deadline(&store).await; + } + #[tokio::test] async fn missing_object_is_not_found() { let (_dir, store) = fs_store().await; @@ -931,6 +978,42 @@ mod filesystem { } } + /// A walk held past its deadline right before a removal (the hook, as a + /// process paused there would be) removes the entry in flight and no + /// other, and says it did not finish. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn delete_prefix_before_stops_at_its_deadline() { + use std::os::unix::fs::OpenOptionsExt as _; + const HOLD: std::time::Duration = std::time::Duration::from_millis(600); + let (dir, mut store) = fs_store().await; + store.set_removal_hook(std::sync::Arc::new(move |removals| { + if removals == 1 { + std::thread::sleep(HOLD); + } + })); + let prefix = dir.path().join("held"); + private_dir(&prefix); + for f in 0..6 { + std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(prefix.join(format!("f{f}"))) + .expect("create an object"); + } + let deadline = tokio::time::Instant::now() + HOLD / 2; + let done = store.delete_prefix_before("held", deadline).await.unwrap(); + assert_eq!( + done, + PrefixDeletion { + deleted: 2, + finished: false + }, + "the removal before the hold and the one in flight at the deadline" + ); + assert_eq!(std::fs::read_dir(&prefix).unwrap().count(), 4); + } + /// `validate` beside a `delete_prefix("")`, which removes temporaries /// and so the probe too: every `validate` succeeds. A round in which /// the deletion removed something (the probe is the only file under the @@ -2801,6 +2884,262 @@ mod s3 { suite::delete_prefix_removes_and_counts(&store).await; } + #[tokio::test] + async fn delete_prefix_before_honours_its_deadline() { + let Some(store) = store().await else { return }; + suite::delete_prefix_before_honours_its_deadline(&store).await; + } + + /// Holds the first request of one operation of the deletion on the + /// sending thread past the caller's deadline, as a process paused + /// between the deadline's check and the request would: `ListObjectsV2` + /// (mode 1), `DeleteObjects` (mode 2) or `AbortMultipartUpload` (mode + /// 3), once armed. Counts the `ListObjectsV2`, `DeleteObjects`, + /// `ListMultipartUploads` and `AbortMultipartUpload` requests sent. + #[derive(Debug, Default)] + struct HoldOneDeletionRequest { + mode: std::sync::Arc, + lists: std::sync::Arc, + deletes: std::sync::Arc, + upload_lists: std::sync::Arc, + aborts: std::sync::Arc, + } + + /// How long [`HoldOneDeletionRequest`] holds its request. + const DELETION_HOLD: std::time::Duration = std::time::Duration::from_millis(600); + + impl aws_sdk_s3::config::Intercept for HoldOneDeletionRequest { + fn name(&self) -> &'static str { + "HoldOneDeletionRequest" + } + + fn read_before_execution( + &self, + context: &aws_sdk_s3::config::interceptors::BeforeSerializationInterceptorContextRef< + '_, + >, + _cfg: &mut aws_sdk_s3::config::ConfigBag, + ) -> Result<(), aws_sdk_s3::error::BoxError> { + use std::sync::atomic::Ordering; + let input = context.input(); + let is_list = input + .downcast_ref::() + .is_some(); + let is_delete = input + .downcast_ref::() + .is_some(); + let is_upload_list = input + .downcast_ref::< + aws_sdk_s3::operation::list_multipart_uploads::ListMultipartUploadsInput, + >() + .is_some(); + let is_abort = input + .downcast_ref::< + aws_sdk_s3::operation::abort_multipart_upload::AbortMultipartUploadInput, + >() + .is_some(); + if is_delete { + self.deletes.fetch_add(1, Ordering::SeqCst); + } + if is_list { + self.lists.fetch_add(1, Ordering::SeqCst); + } + if is_upload_list { + self.upload_lists.fetch_add(1, Ordering::SeqCst); + } + if is_abort { + self.aborts.fetch_add(1, Ordering::SeqCst); + } + let held = match self.mode.load(Ordering::SeqCst) { + 1 => is_list, + 2 => is_delete, + 3 => is_abort, + _ => false, + }; + if held && self.mode.swap(0, Ordering::SeqCst) != 0 { + std::thread::sleep(DELETION_HOLD); + } + Ok(()) + } + } + + /// A deletion whose process is held past its deadline right after a + /// check sends the one request that check admitted and no other: held + /// at its first `ListObjectsV2`, it deletes nothing; held at its first + /// `DeleteObjects`, that batch (two keys, the page size) is deleted and + /// no further listing or batch is sent. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn delete_prefix_before_sends_no_request_after_its_deadline() { + use std::sync::atomic::Ordering; + let Some(endpoint) = endpoint_or_skip() else { + return; + }; + let config = test_config(endpoint, test_bucket(), unique_prefix()); + let hold = HoldOneDeletionRequest::default(); + let (mode, lists, deletes) = (hold.mode.clone(), hold.lists.clone(), hold.deletes.clone()); + let mut store = S3Store::open_with_interceptor(&config, hold) + .await + .expect("S3Store::open_with_interceptor"); + store.set_page_size(2); + ensure_bucket(&config).await; + for (cell, armed, deleted) in [ + ("held at its first ListObjectsV2", 1u8, 0u64), + ("held at its first DeleteObjects", 2, 2), + ] { + let prefix = format!("held{armed}"); + for n in 0..5 { + store + .put(&format!("{prefix}/{n}"), chunked_body(vec![7])) + .await + .unwrap(); + } + deletes.store(0, Ordering::SeqCst); + lists.store(0, Ordering::SeqCst); + mode.store(armed, Ordering::SeqCst); + let deadline = tokio::time::Instant::now() + DELETION_HOLD / 2; + let done = store.delete_prefix_before(&prefix, deadline).await.unwrap(); + assert_eq!( + done, + PrefixDeletion { + deleted, + finished: false + }, + "{cell}" + ); + assert_eq!( + deletes.load(Ordering::SeqCst), + usize::try_from(deleted / 2).unwrap(), + "{cell}: the DeleteObjects requests sent" + ); + assert_eq!( + lists.load(Ordering::SeqCst), + 1, + "{cell}: the ListObjectsV2 requests sent" + ); + assert_eq!( + list_keys(&store, &prefix).await.len(), + usize::try_from(5 - deleted).unwrap(), + "{cell}: the objects left" + ); + store.delete_prefix(&prefix).await.unwrap(); + } + } + + /// The multipart half of the deadline. Held at the last `DeleteObjects` + /// of a single page past its deadline, with an upload open under the + /// prefix: the page is deleted, no `ListMultipartUploads` is sent, the + /// upload stays open and the deletion answers unfinished. Held at its + /// first `AbortMultipartUpload` past the deadline, with two uploads open + /// and no object: that abort is sent, the second upload is not, and the + /// deletion answers unfinished. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn delete_prefix_before_aborts_no_upload_after_its_deadline() { + use std::sync::atomic::Ordering; + let Some(endpoint) = endpoint_or_skip() else { + return; + }; + let config = test_config(endpoint, test_bucket(), unique_prefix()); + let hold = HoldOneDeletionRequest::default(); + let (mode, upload_lists, aborts) = ( + hold.mode.clone(), + hold.upload_lists.clone(), + hold.aborts.clone(), + ); + let store = S3Store::open_with_interceptor(&config, hold) + .await + .expect("S3Store::open_with_interceptor"); + ensure_bucket(&config).await; + let client = raw_client(&config).await; + let open_uploads = |prefix: String| { + let client = client.clone(); + let bucket = config.bucket.clone(); + async move { + client + .list_multipart_uploads() + .bucket(bucket) + .prefix(prefix) + .send() + .await + .expect("list_multipart_uploads") + .uploads() + .iter() + .map(|u| { + ( + u.key().unwrap_or_default().to_owned(), + u.upload_id().unwrap_or_default().to_owned(), + ) + }) + .collect::>() + } + }; + for (cell, armed, objects, uploads, deleted, aborted) in [ + ( + "held at its last DeleteObjects", + 2u8, + 3u64, + 1usize, + 3u64, + 0usize, + ), + ("held at its first AbortMultipartUpload", 3, 0, 2, 0, 1), + ] { + let prefix = format!("mp{armed}"); + for n in 0..objects { + store + .put(&format!("{prefix}/{n}"), chunked_body(vec![7])) + .await + .unwrap(); + } + let bucket_prefix = format!("{}{prefix}/", config.prefix); + for n in 0..uploads { + client + .create_multipart_upload() + .bucket(&config.bucket) + .key(format!("{bucket_prefix}up{n}")) + .send() + .await + .expect("create an upload"); + } + assert_eq!(open_uploads(bucket_prefix.clone()).await.len(), uploads); + upload_lists.store(0, Ordering::SeqCst); + aborts.store(0, Ordering::SeqCst); + mode.store(armed, Ordering::SeqCst); + let deadline = tokio::time::Instant::now() + DELETION_HOLD / 2; + let done = store.delete_prefix_before(&prefix, deadline).await.unwrap(); + assert_eq!( + done, + PrefixDeletion { + deleted, + finished: false + }, + "{cell}" + ); + assert_eq!( + upload_lists.load(Ordering::SeqCst), + usize::from(armed == 3), + "{cell}: the ListMultipartUploads requests sent" + ); + assert_eq!( + aborts.load(Ordering::SeqCst), + aborted, + "{cell}: the AbortMultipartUpload requests sent" + ); + let left = open_uploads(bucket_prefix).await; + assert_eq!(left.len(), uploads - aborted, "{cell}: the uploads left"); + for (key, id) in left { + client + .abort_multipart_upload() + .bucket(&config.bucket) + .key(key) + .upload_id(id) + .send() + .await + .expect("abort a left upload"); + } + store.delete_prefix(&prefix).await.unwrap(); + } + } + #[tokio::test] async fn missing_object_is_not_found() { let Some(store) = store().await else { return }; diff --git a/crates/engine/src/backup.rs b/crates/engine/src/backup.rs index cc6c31760..41e820b75 100755 --- a/crates/engine/src/backup.rs +++ b/crates/engine/src/backup.rs @@ -41,7 +41,8 @@ pub use maintenance::HeldClaim; pub use restore::{OTHER_STORE, RESTORE_REGISTRATION_TIMEOUT, TARGET_NOT_EMPTY}; pub use runtime::{ BackupRuntime, ConfiguredStore, JobSettings, LOCAL_CHECK_TIMEOUT, S3_CHECK_TIMEOUT, Spool, - StoreFence, StoreHealth, StoreHealthReport, check_configured_store, read_identity_bounded, + StoreFence, StoreHealth, StoreHealthReport, check_configured_store, + check_configured_store_read_only, read_identity_bounded, }; pub use store_identity::{STORE_IDENTITY_KEY, StoreIdentityFile, StoreProblem}; @@ -555,12 +556,23 @@ where /// engine has no point between the row and the copy to check at, and the /// store-backed create checks here too rather than after its insert, since /// a row written FAILED for the refusal carried the insert's heartbeat and -/// `backup purge`'s wait counted it as a live job for the stale window. A -/// claim taken between this check and the row's insert is therefore not -/// seen by the job; the in-catalog create that slips through writes no -/// object of the store the command deletes from, and a restore or a -/// store-backed create that slips through is a RUNNING record or a CREATING -/// row the command's wait lists, so the window costs nothing. +/// `backup purge`'s wait counted it as a live job for the stale window. The +/// store-backed create reads the claim again immediately before its +/// `claim_backup`, and the store-backed restore immediately before its +/// registration (and once more after it), so a claim a command took while +/// the request opened the table or read the manifest refuses the job with +/// no row. What remains is the window from that last read to the insert, +/// however long the insert takes: a claim taken in it is not seen by the +/// job, and its CREATING row or RUNNING record lands under the claim. One +/// that lands during the command's wait is listed by it, and one that lands +/// before `--move-store`'s rewrite stays under the old root, which the +/// move's re-list reports (the rewrite skips CREATING rows). A CreateBackup +/// whose insert lands after the move's re-list, during its sync of the +/// configured root, is reported by the sync's closing report of other +/// stores if it lands before that report (and the command exits 1), and +/// only by a later plain sync otherwise. The in-catalog +/// create, which commits its row inside the backend, writes no object of +/// the store a command deletes from. async fn refuse_while_maintenance_held( storage: &S, runtime: &BackupRuntime, @@ -1986,6 +1998,7 @@ pub mod reconciler; mod restore; mod runtime; pub mod store_identity; +pub mod store_maintenance; pub mod sync; #[cfg(test)] mod tests; diff --git a/crates/engine/src/backup/job.rs b/crates/engine/src/backup/job.rs index 7c01e1081..f0ee2bda0 100644 --- a/crates/engine/src/backup/job.rs +++ b/crates/engine/src/backup/job.rs @@ -17,9 +17,11 @@ //! pipeline's item and chunk channels are the job's own), and a client that //! disconnects mid-request cannot leave a row whose owner was never //! spawned. The maintenance claim is checked by the request before the -//! permit, as the in-catalog paths check it, so a held claim refuses the -//! request and inserts no row (a FAILED row written for the refusal was -//! counted by `backup purge`'s wait as a live job for the stale window). +//! permit, as the in-catalog paths check it, and again immediately before +//! `claim_backup`, so a held claim refuses the request and inserts no row +//! (a FAILED row written for the refusal was counted by `backup purge`'s +//! wait as a live job for the stale window), and a claim taken while the +//! open waited (on a lock the table holds, say) refuses it too. //! The open and the insert are selected against the shutdown token, so a //! server that is stopping neither waits on an open the table's lock holds //! nor inserts a row it would fail at once. The `ServiceUnavailable` that @@ -96,7 +98,7 @@ use tokio::sync::oneshot; use tokio::time::Instant; use super::runtime::{BackupRuntime, ConfiguredStore, JobSettings, Spool, StoreFence}; -use super::{create_backup_storage_err_to_dynamo, storage_err_to_dynamo}; +use super::{create_backup_storage_err_to_dynamo, maintenance_held, storage_err_to_dynamo}; use failure::{FailurePath, Written}; use heartbeat::{HeartbeatHandle, HeartbeatMetric}; use writer::{DataFiles, Pipeline, Stop}; @@ -344,6 +346,21 @@ where )) })); } + // The maintenance claim once more, immediately before the insert: the + // request's check ran before the open, the TTL read and the fence, any + // of which can wait (an open behind the table's lock), and a command + // that took the claim since gets no row of this job. A claim taken + // between this read and the insert is not seen: the CREATING row lands + // under the claim, where the command's wait lists it, the move's re-list + // of the old root reports it, or (an insert after that re-list) the + // move's closing report of other stores or a later plain sync does. + if storage + .check_maintenance_claim(env.settings.stale_job_after) + .await + .map_err(storage_err_to_dynamo)? + { + return Err(maintenance_held()); + } let claimed_at = Instant::now(); // The insert too is selected against the token: a server that is // stopping inserts no row it would have to fail at once. diff --git a/crates/engine/src/backup/maintenance.rs b/crates/engine/src/backup/maintenance.rs index e76af3c1e..3889348bf 100644 --- a/crates/engine/src/backup/maintenance.rs +++ b/crates/engine/src/backup/maintenance.rs @@ -222,14 +222,21 @@ pub struct Wait { pub poll: Duration, } -/// Wait until no CREATING or FAILED backup row of `identity` has a live -/// owner and no restore job is RUNNING, re-reading every `wait.poll`, for -/// at most `wait.limit`; the claim lost, a catalog read failing, or a +/// Wait until no CREATING or FAILED backup row of any of `identities` has a +/// live owner and no restore job is RUNNING, re-reading every `wait.poll`, +/// for at most `wait.limit`; the claim lost, a catalog read failing, or a /// signal ends it early. The jobs found running are reported through -/// `announce` the first time they are seen. +/// `announce` the first time they are seen. Purge and `backup sync`'s +/// `--adopt-store` and `--prune-missing` pass the configured store; +/// `--move-store` and `--forget-store` add every store id the catalog +/// records for the old root (`read_store_ids`), so a CreateBackup of a +/// server still configured with that root is waited on too; an empty slice +/// waits on the RUNNING restores alone (`--forget-backup` with no store +/// configured, or with a store whose identity the command could not read: +/// an unhealthy store, or a root with no identity file). pub async fn wait_for_quiet( storage: &S, - identity: &StoreIdentity, + identities: &[StoreIdentity], claim: &HeldClaim, wait: Wait, interrupt: &mut Interrupt, @@ -244,7 +251,7 @@ pub async fn wait_for_quiet( if let Some(signal) = interrupt.signalled() { return WaitOutcome::Interrupted(signal); } - let remaining = match busy_jobs(storage, identity, wait.owner_window).await { + let remaining = match busy_jobs(storage, identities, wait.owner_window).await { Ok(remaining) => remaining, Err(e) => return WaitOutcome::CatalogError(e), }; @@ -275,34 +282,36 @@ pub async fn wait_for_quiet( /// (`list_running_restore_jobs`), with no age. async fn busy_jobs( storage: &S, - identity: &StoreIdentity, + identities: &[StoreIdentity], owner_window: Duration, ) -> Result, String> { let mut busy = Vec::new(); - for status in [BackupStatus::Creating, BackupStatus::Failed] { - let rows: Vec = storage - .list_backup_rows(Some(status), identity) - .try_collect() - .await - .map_err(|e| format!("cannot list the catalog's backup rows: {e}"))?; - let dead: Vec = storage - .list_backup_rows_older_than(status, identity, StaleClock::Heartbeat, owner_window) - .try_collect() - .await - .map_err(|e| format!("cannot list the catalog's backup rows by age: {e}"))?; - for row in rows { - if dead.iter().any(|d| d.backup_arn == row.backup_arn) { - continue; + for identity in identities { + for status in [BackupStatus::Creating, BackupStatus::Failed] { + let rows: Vec = storage + .list_backup_rows(Some(status), identity) + .try_collect() + .await + .map_err(|e| format!("cannot list the catalog's backup rows: {e}"))?; + let dead: Vec = storage + .list_backup_rows_older_than(status, identity, StaleClock::Heartbeat, owner_window) + .try_collect() + .await + .map_err(|e| format!("cannot list the catalog's backup rows by age: {e}"))?; + for row in rows { + if dead.iter().any(|d| d.backup_arn == row.backup_arn) { + continue; + } + busy.push(format!( + "{} {} has a live owner (heartbeat within stale_job_after_secs)", + if status == BackupStatus::Creating { + "CreateBackup" + } else { + "the FAILED backup" + }, + row.backup_arn + )); } - busy.push(format!( - "{} {} has a live owner (heartbeat within stale_job_after_secs)", - if status == BackupStatus::Creating { - "CreateBackup" - } else { - "the FAILED backup" - }, - row.backup_arn - )); } } // Every RUNNING restore job, by status alone: a record stamped in the @@ -446,7 +455,7 @@ pub async fn purge_row( .map_err(|e| format!("cannot mark the row DELETED: {e}"))?; } if storage - .remove_backup_row(&key) + .remove_backup_row(&key, row.store.store_kind, &row.store.store_root) .await .map_err(|e| format!("cannot remove the row: {e}"))? { diff --git a/crates/engine/src/backup/reconciler.rs b/crates/engine/src/backup/reconciler.rs index 963bc5a0a..6a1f33801 100644 --- a/crates/engine/src/backup/reconciler.rs +++ b/crates/engine/src/backup/reconciler.rs @@ -75,8 +75,11 @@ //! command's wait is satisfied either way: the record is no longer //! RUNNING); a record of a store other than the //! configured one (no store configured included) is failed with -//! [`OTHER_STORE`] (configure that store and restore again; no command -//! re-points a record yet), its target removed by the FAILED-target duty +//! [`OTHER_STORE`] (configure that store, or copy its tree under the +//! configured root and move the backup's rows to the configured store +//! with `extenddb backup sync --move-store`, and +//! restore again: the move re-points backup rows, never a restore +//! record), its target removed by the FAILED-target duty //! next; while the configured store is unhealthy, or not yet checked, //! the record is skipped (one line per cycle, naming which); with no free //! job permit it is skipped to the next cycle; then the backend's @@ -162,7 +165,34 @@ //! after the refusal (the dead-owner duty's reads and writes among //! them) do not run on the root; the refusing duty's later rows are //! refused with no read of the file (the FAILED-prefix and -//! DELETED-prefix duties still list each prefix first). +//! DELETED-prefix duties still list each prefix first). The FAILED, +//! DELETED and tombstone duties then read the row once more after the +//! fence passed, skip a row that is gone or whose status or store +//! changed since the duty listed it, and read the maintenance claim +//! again, starting nothing while a command holds it. The claim is read +//! once at the cycle's start, so a cycle that began just before a +//! maintenance command took it may still be running its duties when +//! the command removes a row (`backup sync --forget-backup`, +//! `--forget-store`) or moves it (`--move-store`). A deletion or +//! removal that read the claim free must end by its deadline, +//! [`DUTY_DELETION_BOUND`] after that read was sent: a read answered +//! after the deadline starts nothing, the prefix deletion checks the +//! deadline inside the store before each removal (each request on S3, +//! `delete_prefix_before`), and a call still running at it is dropped; +//! the duty then stops for this cycle, the row left for the next. A +//! command that removes FAILED or DELETED rows beside servers waits the +//! bound and one store call after it takes the claim, so what a +//! deletion duty can still do to a row such a command removes is the +//! step in flight at the deadline: one removal on a filesystem root, one +//! request on S3 (up to 1,000 keys for a `DeleteObjects`). The bound, +//! like every lease in the design, assumes the server's process keeps +//! running: one paused past the wait (SIGSTOP, a frozen cgroup, a VM +//! pause) between its last check of the deadline and a call it has +//! already sent runs that call when it resumes. The exception is a +//! DELETING row of the configured store: its deletion is finished +//! before its removal (by the command, or by `finish_deleting_rows`, +//! which runs under the claim), the deletion its DeleteBackup asked +//! for. //! - Not while the maintenance claim is held, on every backend: the //! backend's `sweep_restore_registrations` (a no-op but on Cassandra); //! the progress-row sweep (`sweep_restore_progress` with the table ids of @@ -361,6 +391,11 @@ pub struct ReconcilerState { /// takes the held branch and does not read the claim. `None` reads it /// through `check_maintenance_claim`. maintenance_held: Option, + /// Whether the cycle leaves the store's check out and reads the health + /// as the caller recorded it: a command that writes nothing to the root + /// (`backup sync --forget-backup`, `--forget-store`), whose own check + /// writes no identity file. + skip_store_check: bool, /// How many times this instance took over each store-backed restore /// record, by target table id, for the bound [`RESTORE_TAKEOVERS`]; an /// entry is removed when the record is failed by the bound. @@ -389,6 +424,18 @@ impl ReconcilerState { ..Self::default() } } + + /// [`Self::for_maintenance_holder`] for a command that writes nothing + /// to the root: the cycle runs no store check (which may write + /// `extenddb-store.json` or a probe file), and the duties read the + /// store's health as the command's own check recorded it. + #[must_use] + pub fn for_maintenance_holder_without_store_check() -> Self { + Self { + skip_store_check: true, + ..Self::for_maintenance_holder() + } + } } /// Run cycles every `settings.interval` until `shutdown` fires. One cycle @@ -510,7 +557,9 @@ where // The store's check first, so every duty of this cycle that reads the // store's health reads what this cycle's check left. - if let Some(store) = runtime.configured_store() { + if let Some(store) = runtime.configured_store() + && !state.skip_store_check + { let check = super::check_configured_store(store, &**storage, backend); let outcome = match &state.shutdown { Some(shutdown) => tokio::select! { @@ -728,6 +777,7 @@ where storage: &**storage, store: calls, fence: &fence, + stale_job_after, }; report.failed_prefixes_deleted = delete_stale_failed_prefixes(duty, &identity, stale_job_after).await?; @@ -1299,87 +1349,158 @@ async fn resolve_dead_creating_rows( let mut resolved = Vec::new(); let mut left_to_owner = Vec::new(); for row in rows { - let claim = match storage.claim_dead_backup(&row.key(), stale_job_after).await { - Ok(Some(claim)) => claim, - Ok(None) => continue, - Err(e) => { - tracing::error!( - "could not test whether the owner of CREATING backup {} (account {}) is \ - dead; the next cycle retries: {e}", - row.backup_arn, - row.account_id - ); - continue; - } - }; - let arn = match BackupArn::parse(&row.backup_arn) { - Ok(arn) => arn, - Err(e) => { - tracing::error!( - "CREATING row {} carries an ARN this build cannot parse; left as it is: {e}", - row.backup_arn - ); - continue; - } - }; - let (to, fields) = match probe_manifest(store, &arn).await { - Ok(Probe::Complete(fields)) => (JobStatus::Available, fields), - Ok(Probe::Incomplete(why)) => { - tracing::warn!( - "backup {} (account {}): its owner is dead and the store holds no complete \ - manifest ({why}); the row is FAILED and its prefix is deleted once the row's \ - heartbeat is stale", - row.backup_arn, - row.account_id - ); - ( - JobStatus::Failed, - StatusFields { - failure_reason: Some(OWNER_DEAD_NO_MANIFEST.to_owned()), - completed: true, - ..StatusFields::default() - }, - ) - } - Err(e) => { - tracing::error!( - "backup {} (account {}): its owner is dead and the store could not be \ - probed for its manifest; the next cycle retries: {e}", - row.backup_arn, - row.account_id - ); - continue; - } - }; - match write_status(&*claim, JobStatus::Creating, to, &fields).await { - Written::Matched => { - tracing::info!( - "backup {} (account {}): its owner was dead; the row is now {to}", - row.backup_arn, - row.account_id - ); - let status = match to { - JobStatus::Available => BackupStatus::Available, - _ => BackupStatus::Failed, - }; - resolved.push((row.backup_arn, status)); - } - Written::NotMatched(why) => { - tracing::info!( - "backup {} was no longer this cycle's to resolve ({why}); left to its owner", - row.backup_arn - ); - left_to_owner.push(row.backup_arn); - } - Written::Error(e) => tracing::error!( + match resolve_dead_creating_row(storage, store, &row, stale_job_after, OnIncomplete::Fail) + .await + { + CreatingOutcome::Resolved(status) => resolved.push((row.backup_arn, status)), + CreatingOutcome::LeftToOwner(_) => left_to_owner.push(row.backup_arn), + CreatingOutcome::NotHandedOver + | CreatingOutcome::Incomplete(_) + | CreatingOutcome::Failed(_) => {} + } + } + Ok((resolved, left_to_owner)) +} + +/// What [`resolve_dead_creating_row`] did with one CREATING row. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) enum CreatingOutcome { + /// The backend did not hand the row over: its owner is alive, or + /// another taker has it. + NotHandedOver, + /// The row was written AVAILABLE (the store holds its complete + /// manifest) or FAILED ([`OWNER_DEAD_NO_MANIFEST`]). + Resolved(BackupStatus), + /// The write matched no row: another owner took it over since the + /// takeover (why). + LeftToOwner(String), + /// With [`OnIncomplete::Leave`], the store holds no complete manifest + /// (why): no status was written, the row is CREATING under the + /// takeover's generation with its heartbeat as it was. + Incomplete(String), + /// A catalog or store call failed, or the row's ARN does not parse + /// (why); the row is left as it was and logged. + Failed(String), +} + +/// What [`resolve_dead_creating_row`] writes when the store holds no +/// complete manifest. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum OnIncomplete { + /// FAILED ([`OWNER_DEAD_NO_MANIFEST`]): the store probed is the row's. + Fail, + /// Nothing ([`CreatingOutcome::Incomplete`]): the store probed holds a + /// copy, and the row's own store may hold the complete backup. + Leave, +} + +/// One CREATING row whose owner may be dead: the backend's takeover +/// (`claim_dead_backup` with `stale_job_after`), the manifest probe of +/// `store` ([`probe_manifest`]), and the CREATING to AVAILABLE write under +/// the taken generation, or to FAILED when the probe finds no complete +/// manifest and `on_incomplete` is [`OnIncomplete::Fail`]. The dead-owner +/// duty runs it on the configured store's rows with `Fail`; `backup sync +/// --move-store` on the old root's rows with `Leave`, probing the +/// configured store, to which the copied prefix belongs: a copy without a +/// complete backup proves nothing about the old root's prefix (a copy's +/// order is arbitrary), so the move writes no FAILED. +pub(super) async fn resolve_dead_creating_row( + storage: &S, + store: StoreCalls<'_>, + row: &BackupRow, + stale_job_after: Duration, + on_incomplete: OnIncomplete, +) -> CreatingOutcome { + let claim = match storage.claim_dead_backup(&row.key(), stale_job_after).await { + Ok(Some(claim)) => claim, + Ok(None) => return CreatingOutcome::NotHandedOver, + Err(e) => { + tracing::error!( + "could not test whether the owner of CREATING backup {} (account {}) is dead; \ + the next cycle retries: {e}", + row.backup_arn, + row.account_id + ); + return CreatingOutcome::Failed(format!( + "could not test whether its owner is dead: {e}" + )); + } + }; + let arn = match BackupArn::parse(&row.backup_arn) { + Ok(arn) => arn, + Err(e) => { + tracing::error!( + "CREATING row {} carries an ARN this build cannot parse; left as it is: {e}", + row.backup_arn + ); + return CreatingOutcome::Failed(format!("the row's ARN does not parse: {e}")); + } + }; + let (to, fields) = match probe_manifest(store, &arn).await { + Ok(Probe::Complete(fields)) => (JobStatus::Available, fields), + Ok(Probe::Incomplete(why)) if on_incomplete == OnIncomplete::Leave => { + drop(claim); + return CreatingOutcome::Incomplete(why); + } + Ok(Probe::Incomplete(why)) => { + tracing::warn!( + "backup {} (account {}): its owner is dead and the store holds no complete \ + manifest ({why}); the row is FAILED and its prefix is deleted once the row's \ + heartbeat is stale", + row.backup_arn, + row.account_id + ); + ( + JobStatus::Failed, + StatusFields { + failure_reason: Some(OWNER_DEAD_NO_MANIFEST.to_owned()), + completed: true, + ..StatusFields::default() + }, + ) + } + Err(e) => { + tracing::error!( + "backup {} (account {}): its owner is dead and the store could not be probed \ + for its manifest; the next cycle retries: {e}", + row.backup_arn, + row.account_id + ); + return CreatingOutcome::Failed(format!( + "the store could not be probed for its manifest: {e}" + )); + } + }; + let outcome = match write_status(&*claim, JobStatus::Creating, to, &fields).await { + Written::Matched => { + tracing::info!( + "backup {} (account {}): its owner was dead; the row is now {to}", + row.backup_arn, + row.account_id + ); + CreatingOutcome::Resolved(match to { + JobStatus::Available => BackupStatus::Available, + _ => BackupStatus::Failed, + }) + } + Written::NotMatched(why) => { + tracing::info!( + "backup {} was no longer this cycle's to resolve ({why}); left to its owner", + row.backup_arn + ); + CreatingOutcome::LeftToOwner(why) + } + Written::Error(e) => { + tracing::error!( "backup {} (account {}): the {to} write failed; the next cycle retries: {e}", row.backup_arn, row.account_id - ), + ); + CreatingOutcome::Failed(format!("the {to} write failed: {e}")) } - drop(claim); - } - Ok((resolved, left_to_owner)) + }; + drop(claim); + outcome } /// The configured store as the reconciler's duties call it: each failed @@ -1431,16 +1552,22 @@ impl StoreCalls<'_> { } } - async fn delete_prefix(self, prefix: &str) -> Result { + /// The prefix deleted, no removal started at or after `deadline` + /// ([`BackupStore::delete_prefix_before`]). + async fn delete_prefix_before( + self, + prefix: &str, + deadline: tokio::time::Instant, + ) -> Result { self.store - .delete_prefix(prefix) + .delete_prefix_before(prefix, deadline) .await .map_err(|e| self.failed("delete_prefix", e)) } } /// What the manifest probe found under a dead owner's prefix. -enum Probe { +pub(super) enum Probe { /// A complete, verified manifest: the AVAILABLE write's fields. Complete(StatusFields), /// No manifest, or one the reader refuses, or a data file missing or of @@ -1450,9 +1577,14 @@ enum Probe { /// The manifest probe of the dead-owner duty (module doc): the checks of /// [`verify_prefix`], with the manifest's ARN required to be the row's (its -/// region and partition included) before the prefix is listed. A store error -/// other than an absent object is the `Err`. -async fn probe_manifest(store: StoreCalls<'_>, arn: &BackupArn) -> Result { +/// region and partition included) before the prefix is listed. `backup +/// sync --move-store` probes each AVAILABLE row of the old root with it in +/// the configured store. A store error other than an absent object is the +/// `Err`. +pub(super) async fn probe_manifest( + store: StoreCalls<'_>, + arn: &BackupArn, +) -> Result { let prefix = arn.store_prefix(); let manifest = match read_verified_manifest(store, &prefix).await? { Verified::Complete(manifest) => manifest, @@ -1660,13 +1792,37 @@ pub(super) async fn read_object( Ok(ReadObject::Bytes(bytes)) } +/// How long one FAILED-prefix or DELETED-prefix deletion, or one tombstone +/// removal, may run from the duty's read of the maintenance claim +/// ([`Duty::fenced`]), counted from before that read was sent: a read +/// answered after it starts nothing, the prefix deletion checks it inside +/// the store before each removal (the filesystem walk before each entry, +/// the S3 loop before each request: `delete_prefix_before`), and a call +/// still running at it is dropped. The duty then stops for this cycle, the +/// row left as it is for the next cycle to continue. It is the bound an S3 +/// store's calls run under ([`super::S3_CHECK_TIMEOUT`], the larger of the +/// two kinds' bounds), so a deletion of an S3 prefix stopped at it is +/// dropped before the SDK starts a retry of a request that stalled +/// ([`extenddb_backup_store::S3_ATTEMPT_TIMEOUT`]). A maintenance command +/// that removes FAILED or DELETED rows beside servers waits this bound plus +/// one store call's after it takes the claim +/// ([`super::store_maintenance::FORGET_WAIT`]), so what remains of a +/// deletion a duty began before the claim when the command removes a row is +/// the step in flight at the deadline (one removal, or one S3 request). +/// The bound assumes the duty's process keeps running: one paused past the +/// wait between its last check and a call already sent is outside it, as +/// it is for every lease in the design. +pub const DUTY_DELETION_BOUND: Duration = super::S3_CHECK_TIMEOUT; + /// What the duties that delete a prefix or remove a tombstone act through: -/// the catalog, the store's calls, and the deletion fence each runs right -/// before its deletion or removal. +/// the catalog, the store's calls, the deletion fence each runs right +/// before its deletion or removal, and the dead-owner window the +/// maintenance claim is read with. struct Duty<'a, S: ?Sized> { storage: &'a S, store: StoreCalls<'a>, fence: &'a StoreFence, + stale_job_after: Duration, } impl Clone for Duty<'_, S> { @@ -1677,19 +1833,142 @@ impl Clone for Duty<'_, S> { impl Copy for Duty<'_, S> {} +/// What [`Duty::fenced`] decided for one row. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Fenced { + /// Act, ending by this instant. + Until(tokio::time::Instant), + /// Act on nothing for this row; the duty goes on to the next. + Skip, + /// The claim's read answered after the deadline: the duty stops for + /// this cycle. + OutOfTime, +} + +/// What one deletion or removal of a duty came to. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Acted { + /// The prefix was deleted, or the row removed. + Done, + /// Nothing was done for this row; the duty goes on to the next. + Skipped, + /// The call did not end within [`DUTY_DELETION_BOUND`]: the duty stops + /// for this cycle. + OutOfTime, +} + impl Duty<'_, S> { - /// The fence before `row`'s deletion or removal: `true` when it passed; - /// a refusal is logged and acts on nothing. - async fn fenced(self, row: &BackupRow, status: &str, what: &str) -> bool { - match self.fence.check_catalog(self.storage).await { - Ok(()) => true, - Err(problem) => { + /// The fence before `row`'s deletion or removal, the row read once + /// more, then the maintenance claim read again: the instant the call + /// must end by ([`DUTY_DELETION_BOUND`] after the claim's read) when the + /// fence passed, the row is still of the status and the store the duty + /// listed it with, and no maintenance command holds the claim + /// ([`Fenced::Until`]); [`Fenced::OutOfTime`] when the claim's read + /// answered at or after that instant; [`Fenced::Skip`] otherwise, logged, + /// acting on nothing. The cycle reads the claim once, + /// at its start, so a duty that listed the row before a maintenance + /// command took the claim may reach its row after the command has + /// removed it (`backup sync --forget-backup`, `--forget-store`) or moved + /// it to another store (`--move-store`): the re-read skips a row that is + /// gone, of another status or of another store, and the claim's read + /// after it starts no deletion once a command holds the claim. A + /// deletion that read the claim free just before a command took it ends + /// by the bound ([`DUTY_DELETION_BOUND`], counted from before the read + /// was sent, so a read answered late starts nothing), and a command that + /// removes FAILED or DELETED rows waits the bound and one store call + /// before its first removal (`store_maintenance::FORGET_WAIT`). A failed + /// read is left for the next cycle. + async fn fenced(self, row: &BackupRow, status: &str, what: &str) -> Fenced { + if let Err(problem) = self.fence.check_catalog(self.storage).await { + tracing::error!( + "did not {what} of {status} backup {} (account {}): {problem}", + row.backup_arn, + row.account_id + ); + return Fenced::Skip; + } + let arn = match BackupArn::parse(&row.backup_arn) { + Ok(arn) => arn, + Err(e) => { + tracing::error!( + "did not {what} of {status} backup {}: its ARN does not parse: {e}", + row.backup_arn + ); + return Fenced::Skip; + } + }; + match self.storage.read_backup_row(&arn).await { + Ok(Some(now)) if now.status == row.status && now.store == row.store => {} + Ok(Some(now)) => { + tracing::info!( + "did not {what} of {status} backup {} (account {}): the row is now {} in \ + the {} store {} (store id {}); left to whoever changed it", + row.backup_arn, + row.account_id, + now.status, + now.store.store_kind, + now.store.store_root, + now.store.store_id + ); + return Fenced::Skip; + } + Ok(None) => { + tracing::info!( + "did not {what} of {status} backup {} (account {}): the row is gone (removed \ + since the duty listed it); its objects are left", + row.backup_arn, + row.account_id + ); + return Fenced::Skip; + } + Err(e) => { tracing::error!( - "did not {what} of {status} backup {} (account {}): {problem}", + "did not {what} of {status} backup {} (account {}): the row could not be read \ + again; the next cycle retries: {e}", row.backup_arn, row.account_id ); - false + return Fenced::Skip; + } + } + // The bound counts from before the claim's read: a command that + // takes the claim after the read was answered waits out the bound, + // and a read answered after it (a slow catalog) acts on nothing. + let deadline = tokio::time::Instant::now() + DUTY_DELETION_BOUND; + match self + .storage + .check_maintenance_claim(self.stale_job_after) + .await + { + Ok(false) if tokio::time::Instant::now() >= deadline => { + tracing::warn!( + "did not {what} of {status} backup {} (account {}): the maintenance claim's \ + read answered after the {} s bound; the duty stops for this cycle and the \ + next cycle continues", + row.backup_arn, + row.account_id, + DUTY_DELETION_BOUND.as_secs() + ); + Fenced::OutOfTime + } + Ok(false) => Fenced::Until(deadline), + Ok(true) => { + tracing::info!( + "did not {what} of {status} backup {} (account {}): a maintenance claim is \ + held now; the next cycle without one acts on it", + row.backup_arn, + row.account_id + ); + Fenced::Skip + } + Err(e) => { + tracing::error!( + "did not {what} of {status} backup {} (account {}): the maintenance claim \ + could not be read again; the next cycle retries: {e}", + row.backup_arn, + row.account_id + ); + Fenced::Skip } } } @@ -1714,8 +1993,10 @@ async fn delete_stale_failed_prefixes( .await?; let mut deleted = Vec::new(); for row in rows { - if delete_prefix_if_listed(duty, &row, "FAILED").await { - deleted.push(row.backup_arn); + match delete_prefix_if_listed(duty, &row, "FAILED").await { + Acted::Done => deleted.push(row.backup_arn), + Acted::Skipped => {} + Acted::OutOfTime => break, } } Ok(deleted) @@ -1734,21 +2015,24 @@ async fn sweep_deleted_prefixes( .await?; let mut swept = Vec::new(); for row in rows { - if delete_prefix_if_listed(duty, &row, "DELETED").await { - swept.push(row.backup_arn); + match delete_prefix_if_listed(duty, &row, "DELETED").await { + Acted::Done => swept.push(row.backup_arn), + Acted::Skipped => {} + Acted::OutOfTime => break, } } Ok(swept) } -/// Delete `row`'s prefix when it lists objects, once the fence passed; -/// returns whether a deletion ran. A store failure or the fence's refusal -/// is logged and left for the next cycle. +/// Delete `row`'s prefix when it lists objects, once [`Duty::fenced`] +/// passed, within [`DUTY_DELETION_BOUND`]. A store failure or a refusal is +/// logged and left for the next cycle; a deletion that outlasts the bound +/// is dropped ([`Acted::OutOfTime`]). async fn delete_prefix_if_listed( duty: Duty<'_, S>, row: &BackupRow, status: &str, -) -> bool { +) -> Acted { let store = duty.store; let prefix = match BackupArn::parse(&row.backup_arn) { Ok(arn) => arn.store_prefix(), @@ -1757,47 +2041,77 @@ async fn delete_prefix_if_listed( "{status} row {} carries an ARN this build cannot parse; left as it is: {e}", row.backup_arn ); - return false; + return Acted::Skipped; } }; match store.prefix_has_objects(&prefix).await { Ok(true) => {} - Ok(false) => return false, + Ok(false) => return Acted::Skipped, Err(e) => { tracing::error!( "could not list the prefix of {status} backup {}; the next cycle retries: {e}", row.backup_arn ); - return false; + return Acted::Skipped; } } - if !duty.fenced(row, status, "delete the prefix").await { - return false; - } - match store.delete_prefix(&prefix).await { - Ok(n) => { + let deadline = match duty.fenced(row, status, "delete the prefix").await { + Fenced::Until(deadline) => deadline, + Fenced::Skip => return Acted::Skipped, + Fenced::OutOfTime => return Acted::OutOfTime, + }; + // The store checks the deadline before each removal, so a call that + // resumes past it (a paused process) starts no other; the timeout ends + // a step that does not answer. + match tokio::time::timeout_at(deadline, store.delete_prefix_before(&prefix, deadline)).await { + Ok(Ok(done)) if done.finished => { tracing::info!( - "deleted {n} object(s) left under the prefix of {status} backup {} (account {})", + "deleted {} object(s) left under the prefix of {status} backup {} (account {})", + done.deleted, row.backup_arn, row.account_id ); - true + Acted::Done } - Err(e) => { + Ok(Ok(done)) => { + tracing::warn!( + "the deletion of the prefix of {status} backup {} (account {}) reached its {} s \ + bound after {} object(s) and stopped; the duty stops for this cycle and the \ + next cycle continues", + row.backup_arn, + row.account_id, + DUTY_DELETION_BOUND.as_secs(), + done.deleted + ); + Acted::OutOfTime + } + Ok(Err(e)) => { tracing::error!( "could not delete the prefix of {status} backup {} (account {}); the next cycle \ retries: {e}", row.backup_arn, row.account_id ); - false + Acted::Skipped + } + Err(_) => { + tracing::warn!( + "the deletion of the prefix of {status} backup {} (account {}) did not end \ + within {} s and was stopped; the duty stops for this cycle and the next cycle \ + continues", + row.backup_arn, + row.account_id, + DUTY_DELETION_BOUND.as_secs() + ); + Acted::OutOfTime } } } /// The tombstone duty (module doc): FAILED and DELETED rows whose /// `completed_at` is older than `retention` and whose prefix lists nothing -/// are removed. +/// are removed, each removal within [`DUTY_DELETION_BOUND`] of the claim's +/// read; one that outlasts it stops the duty for this cycle. async fn remove_tombstones( duty: Duty<'_, S>, identity: &StoreIdentity, @@ -1836,14 +2150,22 @@ async fn remove_tombstones( } // The empty listing is a fact about this catalog's root only // while the root is this catalog's. - if !duty + let deadline = match duty .fenced(&row, &status.to_string(), "remove the row") .await { - continue; - } - match storage.remove_backup_row(&row.key()).await { - Ok(true) => { + Fenced::Until(deadline) => deadline, + Fenced::Skip => continue, + Fenced::OutOfTime => return Ok(removed), + }; + // Conditioned on the store the duty listed the row with: a row + // `--move-store` moved since is the configured store's to + // finish. + let key = row.key(); + let removal = + storage.remove_backup_row(&key, row.store.store_kind, &row.store.store_root); + match tokio::time::timeout_at(deadline, removal).await { + Ok(Ok(true)) => { tracing::info!( "removed the {status} row of backup {} (account {}): its prefix is empty \ and its retention has passed", @@ -1852,13 +2174,24 @@ async fn remove_tombstones( ); removed.push(row.backup_arn); } - Ok(false) => {} - Err(e) => tracing::error!( + Ok(Ok(false)) => {} + Ok(Err(e)) => tracing::error!( "could not remove the {status} row of backup {} (account {}); the next cycle \ retries: {e}", row.backup_arn, row.account_id ), + Err(_) => { + tracing::warn!( + "the removal of the {status} row of backup {} (account {}) did not end \ + within {} s and was stopped; the duty stops for this cycle and the next \ + cycle continues", + row.backup_arn, + row.account_id, + DUTY_DELETION_BOUND.as_secs() + ); + return Ok(removed); + } } } } diff --git a/crates/engine/src/backup/restore.rs b/crates/engine/src/backup/restore.rs index 2233858d0..b026f7c67 100644 --- a/crates/engine/src/backup/restore.rs +++ b/crates/engine/src/backup/restore.rs @@ -23,8 +23,10 @@ //! the detached task ([`start`]) whose answer it awaits for at most //! [`RESTORE_REGISTRATION_TIMEOUT`]; past that it answers //! `InternalServerError` while the task goes on (the target then appears -//! through DescribeTable). The task registers the target and its RUNNING -//! record in one write (`claim_restore`, under the backup row's lock), sends +//! through DescribeTable). The task reads the maintenance claim (held, the +//! request is refused with `LimitExceededException` and nothing is +//! registered, so no record lands while a command holds the claim), then +//! registers the target and its RUNNING record in one write (`claim_restore`, under the backup row's lock), sends //! the job's first beat through the claim (the registration's own //! `heartbeat_at` stamp can be old: the design's 4(e) states each backend's //! clock, PostgreSQL's `now()` being the transaction's start, so a @@ -145,8 +147,9 @@ pub const RESTORE_REGISTRATION_TIMEOUT: Duration = Duration::from_secs(60); pub const TARGET_NOT_EMPTY: &str = "target_not_empty"; /// The reason the reconciler writes on a RUNNING record whose store is not -/// the configured one. -pub const OTHER_STORE: &str = "the restore reads a store other than the configured one; configure that store in [backup.store] and restore again"; +/// the configured one. `--move-store` re-points the backup's rows, not the +/// record, so the restore is run again after the move. +pub const OTHER_STORE: &str = "the restore reads a store other than the configured one; configure that store in [backup.store], or copy its tree under the configured root and move the backup's rows to the configured store with `extenddb backup sync --move-store `, and restore again"; /// What the request resolved before it spawned the task. pub(crate) struct RestoreRequestParts { @@ -451,6 +454,27 @@ pub(crate) async fn start( spool_host: spool_host.clone(), owner_generation: UNOWNED_GENERATION, }; + // The maintenance claim immediately before the registration: the + // request's check ran before the manifest reads and the permit, and a + // command that took the claim since gets no record of this restore. A + // claim taken in the gap between this read and the registration is met + // by the check after it, below, and by the command's wait. + let refused = match storage + .check_maintenance_claim(env.settings.stale_job_after) + .await + { + Ok(false) => None, + Ok(true) => Some(maintenance_held()), + Err(e) => Some(storage_err_to_dynamo(e)), + }; + if let Some(error) = refused { + drop(permit); + if let Some(registration) = registration { + registration.finish(Err(error.clone())); + } + let _ = reply.send(Err(error)); + return; + } let registered = storage .claim_restore(&account_id, input, ®istration_row) .await; diff --git a/crates/engine/src/backup/runtime.rs b/crates/engine/src/backup/runtime.rs index b765ba2a1..6210a6265 100644 --- a/crates/engine/src/backup/runtime.rs +++ b/crates/engine/src/backup/runtime.rs @@ -1027,6 +1027,51 @@ pub async fn read_identity_bounded( read } +/// The store's check as a command that writes nothing runs it after its +/// removals (`backup sync --forget-backup`): [`store_identity::read_identity`] +/// (no identity file is created, and `validate`, which writes a probe file, +/// does not run), the file's id compared with `known` when the command read +/// one when it opened the store (a root replaced since is +/// [`StoreProblem::IdMismatch`], as [`store_identity::recheck`] has it), then +/// the manifest probe, every store call under +/// [`ConfiguredStore::check_timeout`]. `Ok(None)` for a root with no file +/// and no rows for it, left as it is: a server's next check creates the +/// file. +/// +/// # Errors +/// +/// The problem that would make the store unhealthy +/// ([`store_identity`]'s module doc), or the bound. +pub async fn check_configured_store_read_only( + store: &ConfiguredStore, + catalog: &C, + known: Option<&StoreIdentity>, +) -> Result, StoreProblem> { + let via = store.calls("check"); + let outcome = async { + let Some(identity) = + store_identity::read_identity(&via, store.kind, &store.root, catalog).await? + else { + return Ok(None); + }; + if let Some(known) = known + && known.store_id != identity.store_id + { + return Err(StoreProblem::IdMismatch { + file_id: identity.store_id, + row_ids: vec![known.store_id.clone()], + }); + } + store_identity::manifest_probe(&via, &identity, catalog).await?; + Ok(Some(identity)) + } + .await; + if via.timed_out() { + return Err(via.did_not_finish_problem()); + } + outcome +} + /// The configured store as the checks, a command's identity read, the /// deletion fence and the deletions call it (module doc): each call runs under `bound`, and a call /// that does not finish is a transport failure naming `what` and the bound. diff --git a/crates/engine/src/backup/store_identity.rs b/crates/engine/src/backup/store_identity.rs index 889b443ac..5532ff64e 100644 --- a/crates/engine/src/backup/store_identity.rs +++ b/crates/engine/src/backup/store_identity.rs @@ -203,6 +203,17 @@ pub enum StoreProblem { } impl StoreProblem { + /// The row ids an identity mismatch is about (`IdMismatch`'s row ids, + /// `MultipleRowIds`' ids); empty for every other problem. + #[must_use] + pub fn mismatch_row_ids(&self) -> &[String] { + match self { + Self::IdMismatch { row_ids, .. } => row_ids, + Self::MultipleRowIds(ids) => ids, + _ => &[], + } + } + /// The problem's category, one or two words, for an API refusal. #[must_use] pub fn category(&self) -> &'static str { diff --git a/crates/engine/src/backup/store_maintenance.rs b/crates/engine/src/backup/store_maintenance.rs new file mode 100644 index 000000000..0aadcc555 --- /dev/null +++ b/crates/engine/src/backup/store_maintenance.rs @@ -0,0 +1,1554 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! The options of `backup sync` that act on the rows of a root other than +//! the configured one: `--move-store ` ([`move_store`]), +//! `--forget-store ` ([`forget_store`]) and `--forget-backup ` +//! ([`forget_backups`]), and plain sync's report of the rows of the other +//! stores ([`other_stores`]). Each is written against the catalog's trait +//! and the configured store, so the engine's tests run them against a +//! double; the command takes the maintenance claim, waits for running jobs +//! and prints what they return. +//! +//! A root is given as its label, as the catalog stores it and as +//! `backup sync` prints it (the canonical directory of a filesystem store, +//! `s3:///` for S3 with its `?endpoint=` suffix when the +//! store has one), and matches byte for byte. [`locate_root`] finds the +//! kind of the rows under it (an S3 label starts with `s3://` and a +//! filesystem label is an absolute path, so the two never meet) and the +//! store ids the catalog records for it, which the command's wait reads +//! rows of besides the configured store's; a root with no rows is refused +//! with the stores the catalog knows ([`RootRefusal::NoRows`]), the help an +//! operator needs after a label that differs by a trailing `/` or by its +//! `?endpoint=` suffix. Both options refuse the configured root itself. +//! +//! [`move_store`] is the way a tree copied to a new root keeps its rows: +//! the operator copies the old root's tree under the configured root +//! first. It reads the old root's rows (each row's +//! `source_table_details_json` is dropped as it is read) and the configured +//! root's (account, backup id) pairs, so a failed read stops the move +//! before anything changed. The rows are grouped by (account, backup id), +//! and every old-root row of a group of more than one (with a row of the +//! configured root, or with another spelling of its ARN among the moved +//! rows, which only Cassandra holds) stops the move and is set aside before +//! any row is checked, so none of them is checked further: on Cassandra two +//! spellings of an ARN share one prefix, which a DELETING row's finished +//! deletion would otherwise empty. Every other row is checked against the +//! configured store, the store the copy is in, and every row that stops +//! the move is named in one pass: +//! - A CREATING row whose owner is dead is resolved AVAILABLE when the copy +//! holds its complete backup (`reconciler::resolve_dead_creating_row` +//! with `OnIncomplete::Leave`: the takeover on the catalog's clock, the +//! manifest probe of the configured store, the AVAILABLE write). Any +//! other probe result stops the move and writes no status: a copy without +//! the complete backup proves nothing about the old root's prefix (a +//! copy's order is arbitrary, so a copy without the manifest may come +//! from a complete prefix), and a FAILED row would have the reconciler +//! delete the copy, and any server still on the old root delete the +//! original. The row is left CREATING for a reconciler on the old root +//! to resolve there, or for `--forget-backup`. A row the backend does not +//! hand over has a live owner, or another taker, and stops the move. +//! - A DELETING row whose copied prefix holds a verified manifest +//! (`reconciler::verify_prefix`) stops the move. Otherwise its deletion +//! is finished in the configured store behind the deletion fence +//! (`finish_backup_delete`), which deletes what the copy brought of it +//! and sets the row DELETED: that is the deletion the row's DeleteBackup +//! asked for. +//! - A DELETED or FAILED row whose copied prefix holds a verified manifest +//! stops the move, naming `--forget-backup`: moving the tombstone would +//! have the reconciler delete a complete backup. Otherwise it moves, and +//! the reconciler of the configured store cleans its leftovers there. +//! - An AVAILABLE row is probed in the configured store +//! (`reconciler::probe_manifest`: the manifest and its sidecar, the +//! digest, the reader's acceptance, the row's full ARN, every data file +//! at its recorded size), and the manifest's size and item total must be +//! the row's; anything else stops the move with the reason. +//! +//! The AVAILABLE resolutions and the finished deletions are kept when the +//! move stops; each is idempotent, and the next run sees the statuses they +//! wrote. With no row stopping, the fence re-reads the configured root's +//! identity file once (no row is moved into a root another catalog adopted +//! since the check), then `rewrite_store` moves the rows, under the +//! condition that each is still of the old kind and root and not CREATING, +//! and the old root is listed once more: a row still there (a CreateBackup +//! of a server still configured with the old root whose row landed after +//! the wait, a row `rewrite_store` did not count, a rewrite that stopped +//! part way) is reported and the command exits 1. Every row moved, +//! resolved or finished is logged under the `extenddb::audit` target. No +//! object is copied by the move, and none of the old root is touched. +//! +//! [`forget_store`] and [`forget_backups`] remove rows outright, leaving no +//! DELETED tombstone; the removals delete no object and read no store, so +//! they run on a store in any state and with none configured. A deletion +//! duty starts no deletion once it reads the maintenance claim held +//! (`reconciler`'s `Duty::fenced`), each deletion it started before ends by +//! [`super::reconciler::DUTY_DELETION_BOUND`] from its read of the claim +//! (the store checks the deadline before each removal), and beside servers +//! the command waits [`FORGET_WAIT`] from the claim before its first +//! removal of a FAILED or DELETED row (on SQLite the serve lock excludes +//! every server, and nothing waits). What a duty can still do to a removed +//! row's bytes is the step in flight at its deadline (one removal, or one +//! S3 request), and only a server process paused past the wait between its +//! last check and that step runs it after the removal: the assumption every +//! lease in the design shares. The exception is a DELETING row of the +//! configured store: the command finishes its deletion behind the fence +//! (`finish_backup_delete`, the deletion its DeleteBackup asked for) before +//! it removes the row, which needs the store's check, run at the command's +//! opening and writing nothing, to have passed; with the store unhealthy or +//! its id unknown the row is refused ([`Forgotten::DeletionPending`]), and +//! when the check refused on an identity mismatch that the row's own store +//! id takes part in, the refusal says so and names the way out, the same +//! command with no store configured ([`Forgotten::DeletionBehindMismatch`]). +//! On SQLite the command's own dead-owner pass may finish it first, under the +//! claim, and the removal of its DELETED row says so. A DELETING row of +//! another store keeps its bytes, and a sync of that root after its return +//! adopts the backup again. While the configured root has no +//! `extenddb-store.json` and the catalog holds rows for it (an unmounted +//! volume), every row of that root is refused +//! ([`Forgotten::RootWithoutFile`]): a server's next check would write a +//! new identity file at the empty root once its last row is gone. Each +//! status has its removal ([`forget_row`]): AVAILABLE through +//! `forget_available_backup_row` (the row's lock, the status check and the +//! in-use rule in one step: a row a CREATING restore target reads is left); +//! DELETED and FAILED through `remove_backup_row`; DELETING through +//! `set_backup_deleted` then `remove_backup_row`, the DELETED state between +//! the two met by no duty that starts after the claim, and by none that +//! listed the row before it (the row was DELETING then); CREATING through +//! `remove_dead_creating_backup_row` on the catalog's clock, a row left +//! CREATING having a live owner. A FAILED row +//! whose heartbeat is within the owner window (a job that has not yet seen +//! its claim lost may still write under the prefix) is left as having a +//! live owner. A removal that finds another status re-reads the row once +//! and applies that status's removal. `--forget-backup` takes the ARN as +//! the row carries it: a row under another spelling of the ARN (the +//! server's region) is named and left, an earlier release's in-catalog +//! backup is reported as one, with nothing changed, and an ARN no row and +//! no in-catalog backup carries counts as done (a run given again after a +//! stop goes past the ARNs it removed). The command checks every ARN's +//! form ([`check_arns`]) before it takes the claim. After `--forget-store`'s +//! removals, the root's status entries are swept under the claim +//! (`remove_orphan_status_entries`, each verified against its row; only +//! Cassandra has them): an entry whose row is gone or names another store +//! (a deleter whose entry batch landed after a move of its row) is +//! deleted, and one whose row is of the root under another status (a +//! status change whose entry batch failed, or whose writer died between +//! its row and its batch) is repaired, each counted; the store ids the +//! catalog still records for the root are then read once more and +//! reported. +//! +//! [`other_stores`] is the report plain `backup sync` prints after its +//! listing: per store other than the configured one, the rows per status +//! (`summarize_store_roots`; on Cassandra a count of status entries, +//! checked against the root's listing, and what no row of its status +//! stands behind is reported as status entries), each CREATING and +//! DELETING row by ARN (the +//! rows 4(c) leaves alone), so the operator can move them with +//! `--move-store` or drop them with `--forget-store`. + +use std::collections::{HashMap, HashSet}; +use std::time::Duration; + +use extenddb_core::backup_format::BackupArn; +use extenddb_storage::backup_catalog::{ + BackupKey, BackupRow, BackupStatus, ForgetRow, StaleClock, StatusEntrySweep, StoreIdentity, + StoreKind, StoreRootCount, is_backup_not_found, +}; +use extenddb_storage::error::StorageError; +use extenddb_storage::{BackupCatalog, LegacyBackupReader}; +use futures::TryStreamExt; + +use super::ConfiguredStore; +use super::reconciler::{ + CreatingOutcome, OnIncomplete, Probe, StoreCalls, Verified, probe_manifest, + resolve_dead_creating_row, verify_prefix, +}; +use super::sync::Stop; + +/// A row named in a report: its status, ARN and account. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct NamedRow { + pub status: BackupStatus, + pub backup_arn: String, + pub account_id: String, +} + +impl NamedRow { + fn of(row: &BackupRow) -> Self { + Self { + status: row.status, + backup_arn: row.backup_arn.clone(), + account_id: row.account_id.clone(), + } + } +} + +// --- the report of other stores ------------------------------------------ + +/// The rows the catalog holds for one store other than the configured one. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OtherStore { + /// The store: kind, root and id. + pub store: StoreIdentity, + /// The rows per status, each status with at least one row, in + /// [`BackupStatus::ALL`]'s order. + pub counts: Vec<(BackupStatus, u64)>, + /// The store's CREATING and DELETING rows, in that order and each in + /// listing order. + pub unresolved: Vec, + /// Per status, the status entries no listed row of that status stands + /// behind, each status with at least one, in [`BackupStatus::ALL`]'s + /// order: on a backend whose summary counts status entries + /// (Cassandra), the summary's count less the store's rows of that + /// status in the root's listing (`list_backup_rows_by_root`, which + /// lists a row whose entry is under an earlier status under its own). + /// Empty elsewhere. + pub entries: Vec<(BackupStatus, u64)>, +} + +/// The counts of `summarize_store_roots` (in its order) grouped by store, +/// `except` left out; no row is named. +#[must_use] +pub fn group_counts(counts: &[StoreRootCount], except: Option<&StoreIdentity>) -> Vec { + let mut stores: Vec = Vec::new(); + for count in counts { + if except == Some(&count.store) { + continue; + } + match stores.last_mut() { + Some(last) if last.store == count.store => { + last.counts.push((count.status, count.rows)); + } + _ => stores.push(OtherStore { + store: count.store.clone(), + counts: vec![(count.status, count.rows)], + unresolved: Vec::new(), + entries: Vec::new(), + }), + } + } + stores +} + +/// Every store other than `configured` that the catalog holds rows for +/// (module doc), in [`StoreRootCount::sort`]'s order. A store of the +/// configured root under another id is another store too. On a backend +/// whose summary counts status entries (Cassandra), the store's rows are +/// read from its root's listing (`list_backup_rows_by_root`, kept to the +/// store's id), which lists each row under its own status whatever +/// partition its entry is in: `counts` holds those rows, and +/// [`OtherStore::entries`] the entries no row of their status stands +/// behind. +/// +/// # Errors +/// +/// The backend's error for the summary or a listing. +pub async fn other_stores( + storage: &S, + configured: &StoreIdentity, +) -> Result, StorageError> { + let counts = storage.summarize_store_roots().await?; + let mut stores = group_counts(&counts, Some(configured)); + if storage.summary_counts_status_entries() { + for other in &mut stores { + // The root's listing, which reads every status partition, so a + // row whose entry is under an earlier status counts under its + // own status; kept to the store's id. + let (rows, unresolved) = storage + .list_backup_rows_by_root(other.store.store_kind, &other.store.store_root, None) + .try_fold( + (Vec::<(BackupStatus, u64)>::new(), Vec::new()), + |(mut rows, mut unresolved), row| { + if row.store == other.store { + match rows.iter_mut().find(|(s, _)| *s == row.status) { + Some((_, n)) => *n += 1, + None => rows.push((row.status, 1)), + } + if matches!(row.status, BackupStatus::Creating | BackupStatus::Deleting) + { + unresolved.push(NamedRow::of(&row)); + } + } + std::future::ready(Ok((rows, unresolved))) + }, + ) + .await?; + for (status, entries) in &other.counts { + let listed = rows + .iter() + .find(|(s, _)| s == status) + .map_or(0, |(_, n)| *n); + if *entries > listed { + other.entries.push((*status, entries - listed)); + } + } + other.counts = BackupStatus::ALL + .iter() + .filter_map(|status| rows.iter().find(|(s, _)| s == status).copied()) + .collect(); + other.unresolved = [BackupStatus::Creating, BackupStatus::Deleting] + .iter() + .flat_map(|status| unresolved.iter().filter(|r| r.status == *status).cloned()) + .collect(); + } + return Ok(stores); + } + for other in &mut stores { + for status in [BackupStatus::Creating, BackupStatus::Deleting] { + if !other.counts.iter().any(|(s, n)| *s == status && *n > 0) { + continue; + } + let mut rows = storage.list_backup_rows(Some(status), &other.store); + while let Some(row) = rows.try_next().await? { + other.unresolved.push(NamedRow::of(&row)); + } + } + } + Ok(stores) +} + +// --- the root an option names -------------------------------------------- + +/// The root an option names, as the catalog records it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LocatedRoot { + pub kind: StoreKind, + pub root: String, + /// The store ids the catalog records for the root (`read_store_ids`). + pub store_ids: Vec, +} + +impl LocatedRoot { + /// The root's stores, one per id: what the command's wait reads rows + /// of besides the configured store's. + #[must_use] + pub fn identities(&self) -> Vec { + self.store_ids + .iter() + .map(|id| StoreIdentity { + store_kind: self.kind, + store_root: self.root.clone(), + store_id: id.clone(), + }) + .collect() + } +} + +/// Why [`locate_root`] refused a root. +#[derive(Debug)] +pub enum RootRefusal { + /// The root is the configured store's own. + ConfiguredRoot, + /// The catalog records no row under the root, of either kind; the + /// stores it does record rows for. + NoRows(Vec), + /// The catalog could not be read. + Catalog(StorageError), +} + +/// The kind and store ids of the rows the catalog records under `root` +/// (module doc). `configured_root` is the configured store's root label, +/// when a store is configured: that root is refused. +/// +/// # Errors +/// +/// [`RootRefusal`]. +pub async fn locate_root( + storage: &S, + root: &str, + configured_root: Option<&str>, +) -> Result { + if configured_root == Some(root) { + return Err(RootRefusal::ConfiguredRoot); + } + for kind in [StoreKind::Filesystem, StoreKind::S3] { + let store_ids = storage + .read_store_ids(kind, root) + .await + .map_err(RootRefusal::Catalog)?; + if !store_ids.is_empty() { + return Ok(LocatedRoot { + kind, + root: root.to_owned(), + store_ids, + }); + } + } + let known = storage + .summarize_store_roots() + .await + .map_err(RootRefusal::Catalog)?; + Err(RootRefusal::NoRows(known)) +} + +// --- the removals ---------------------------------------------------------- + +/// What [`forget_row`] did with one row. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Forgotten { + /// The row is gone; it had this status. No object was deleted. + Removed(BackupStatus), + /// A DELETING row of the configured store: this command finished its + /// deletion (its objects are deleted, the deletion its DeleteBackup + /// asked for), then removed the row. Also a DELETED row whose deletion + /// the command's own dead-owner pass finished (SQLite). + RemovedAfterDeletion, + /// Left: a DELETING row of the configured store whose deletion this + /// command cannot finish, since the store's check did not pass (or the + /// store's id is unknown). + DeletionPending, + /// Left: a DELETING row of the configured store that carries this + /// store id, one of the row ids the store's check refused on as an + /// identity mismatch (`IdMismatch`, `MultipleRowIds`), so the deletion + /// waits on a check the row itself takes part in. + DeletionBehindMismatch(String), + /// Left: a row of the configured root, whose `extenddb-store.json` the + /// command found missing while the catalog holds rows for the root (a + /// volume that is not mounted, say), or of the defaulted root found + /// missing with rows for it ([`ForgetRun::unmounted_root`]). + RootWithoutFile, + /// The row was gone before the removal. + Gone, + /// Left: a restore target registered from the backup is CREATING (its + /// name, and the row's account). + InUse(String, String), + /// Left: a CREATING or FAILED row whose owner is alive (its heartbeat + /// is within the owner window). + LiveOwner(BackupStatus), + /// Left: the row changed under the removal again after the re-read (its + /// status and store now). + Changed(BackupStatus, StoreIdentity), + /// Left: a catalog call failed (why). + Failed(String), + /// Not reached: the run stopped (why) during the wait before the + /// first FAILED or DELETED removal, or before a DELETING row's + /// finished deletion ([`FORGET_WAIT`]); the row is as it was. + Stopped(String), +} + +impl Forgotten { + /// Whether the row is still in the catalog: the command exits 1. + #[must_use] + pub fn is_left(&self) -> bool { + !matches!( + self, + Self::Removed(_) | Self::RemovedAfterDeletion | Self::Gone + ) + } + + /// Whether this run removed the row. + #[must_use] + pub fn is_removed(&self) -> bool { + matches!(self, Self::Removed(_) | Self::RemovedAfterDeletion) + } +} + +impl std::fmt::Display for Forgotten { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Removed(status) => write!( + f, + "removed: the row ({status}) is gone, and no object was deleted" + ), + Self::RemovedAfterDeletion => f.write_str( + "removed: the row is gone, and this command deleted its objects first (the row \ + was DELETING in the configured store: the deletion its DeleteBackup asked for)", + ), + Self::DeletionPending => f.write_str( + "left: a DELETING row of the configured store, whose deletion this command \ + finishes before it removes the row, and the store's check did not pass; let a \ + server finish the DeleteBackup, or run again once the store's check passes", + ), + Self::DeletionBehindMismatch(id) => write!( + f, + "left: a DELETING row of the configured store carrying store id {id}, one of the \ + ids the store's identity check refuses, so neither this command nor a server \ + finishes its deletion while that check fails; remove the row by running this \ + command with [backup.store] kind = \"disabled\" (its bytes stay), then, with the \ + store configured again, let backup sync adopt the backup once the check passes \ + and DeleteBackup it again" + ), + Self::RootWithoutFile => write!( + f, + "left: the configured root has no {} while the catalog holds rows for it (a \ + volume that is not mounted?); mount it, or point [backup.store] at the root \ + that holds these backups, then run again", + super::STORE_IDENTITY_KEY + ), + Self::Gone => f.write_str("the row was already gone"), + Self::InUse(table, account) => write!( + f, + "left: the restore target {table} is CREATING from this backup: wait for its \ + restore to end, or, when that restore failed or its owner is dead, remove the \ + target with extenddb backup remove-target --account {account} {table}; then run \ + again" + ), + Self::LiveOwner(status) => write!( + f, + "left: the {status} row has a live owner (its heartbeat is within \ + stale_job_after_secs); run the command again once its job has ended" + ), + Self::Changed(status, store) => write!( + f, + "left: the row changed under the removal; it is now {status} in the {} store {} \ + (store id {})", + store.store_kind, store.store_root, store.store_id + ), + Self::Failed(why) => write!(f, "left: {why}"), + Self::Stopped(why) => write!(f, "not reached: {why}"), + } + } +} + +/// How long `--forget-store` and `--forget-backup` beside servers +/// (PostgreSQL, MongoDB, Cassandra) wait after taking the maintenance claim +/// before their first removal of a FAILED or DELETED row (and before +/// finishing the deletion of a DELETING row of the configured store, whose +/// DELETED row is removed next): a deletion duty +/// that read the claim free just before the command took it starts no +/// removal after [`super::reconciler::DUTY_DELETION_BOUND`] from that read +/// (the store checks the deadline before each one), and its step in flight +/// at the deadline, one removal or one S3 request, is answered or abandoned +/// within one call's bound ([`super::S3_CHECK_TIMEOUT`], the larger of the +/// two kinds', since the servers' stores need not be the command's). The +/// wait assumes the servers' processes keep running: one paused past it +/// between its last check of the deadline and a step it has already sent +/// runs that step when it resumes, the assumption every lease in the design +/// shares. On SQLite the serve lock excludes every server, and the command +/// does not wait. +pub const FORGET_WAIT: Duration = + super::reconciler::DUTY_DELETION_BOUND.saturating_add(super::S3_CHECK_TIMEOUT); + +/// How often the wait before the first FAILED or DELETED removal asks the +/// run's stop. +const FORGET_WAIT_STEP: Duration = Duration::from_secs(1); + +/// What a run of `--forget-store` or `--forget-backup` removes rows under. +#[derive(Clone, Copy)] +pub struct ForgetRun<'a> { + /// The window a live owner is told apart by: `stale_job_after` beside + /// servers, zero on SQLite under the serve lock. + pub owner_window: Duration, + /// The instant before which no FAILED or DELETED row is removed: the + /// claim's taking plus [`FORGET_WAIT`] beside servers, `None` on + /// SQLite. + pub removals_from: Option, + /// Told how long the run waits, once, when it does. + pub on_wait: &'a (dyn Fn(Duration) + Sync), + /// Asked before each row or ARN, and during the wait. + pub stop: Stop<'a>, + /// The configured store as the command opened it, when one is + /// configured. + pub configured: Option>, + /// A filesystem root the command found missing as it opened the + /// store, with rows for it in the catalog: the defaulted root moved + /// away, or a symbolic link to a volume that is gone. Every row of it + /// is refused ([`Forgotten::RootWithoutFile`]), as the rows of a + /// configured root without its identity file are. + pub unmounted_root: Option<&'a str>, + /// The DELETING rows whose deletion the command's own dead-owner pass + /// finished before the removals (SQLite), by ARN: their DELETED rows' + /// removal says the command deleted the objects. + pub deletions_finished: &'a [String], +} + +/// The configured store as a forget option meets its rows. +#[derive(Clone, Copy)] +pub struct ForgetConfigured<'a> { + pub store: &'a ConfiguredStore, + /// Its identity when the command's check of the store, which writes + /// nothing, passed: a DELETING row of this store has its deletion + /// finished behind the fence before its removal. `None` otherwise, and + /// such a row is refused ([`Forgotten::DeletionPending`]). + pub healthy: Option<&'a StoreIdentity>, + /// The root's row ids when the opening's check failed on an identity + /// mismatch (`IdMismatch`, `MultipleRowIds`), empty otherwise: a + /// DELETING row carrying one of them is refused as + /// [`Forgotten::DeletionBehindMismatch`], which names the way out. + pub mismatch_row_ids: &'a [String], + /// Whether the opening found no identity file under the root while the + /// catalog holds rows for it: every row of the root is refused + /// ([`Forgotten::RootWithoutFile`]), since a server's next check + /// would write a new identity file at the root once its last row is + /// gone. + pub file_missing_with_rows: bool, +} + +impl ForgetConfigured<'_> { + /// Whether `row` is of the configured root (any id). + fn holds(&self, row: &BackupRow) -> bool { + row.store.store_kind == self.store.kind && row.store.store_root == self.store.root + } +} + +/// One run's removals: its [`ForgetRun`], the FAILED rows whose owner is +/// dead (read once per store), and whether the wait is over. +struct Removals<'a> { + run: ForgetRun<'a>, + failed_owners: FailedOwners, + waited: bool, +} + +impl<'a> Removals<'a> { + fn new(run: ForgetRun<'a>) -> Self { + Self { + run, + failed_owners: FailedOwners::new(run.owner_window), + waited: run.removals_from.is_none(), + } + } + + /// The wait before the first FAILED or DELETED removal ([`FORGET_WAIT`]); + /// `Err` is the stop that ended it. + async fn wait_for_duties(&mut self) -> Result<(), String> { + if self.waited { + return Ok(()); + } + let until = self + .run + .removals_from + .expect("an unfinished wait has an end"); + let now = tokio::time::Instant::now(); + if until > now { + (self.run.on_wait)(until - now); + } + loop { + if let Some(reason) = (self.run.stop)() { + return Err(reason); + } + let now = tokio::time::Instant::now(); + if now >= until { + break; + } + tokio::time::sleep((until - now).min(FORGET_WAIT_STEP)).await; + } + self.waited = true; + Ok(()) + } +} + +/// The form of each ARN `--forget-backup` is given, before the command +/// takes the claim: the ARNs refused, each with why. +#[must_use] +pub fn check_arns(arns: &[String]) -> Vec<(String, String)> { + arns.iter() + .filter_map(|given| { + let why = match extenddb_storage::util::check_backup_arn(given) { + Err(e) => e.to_string(), + Ok(_) => match BackupArn::parse(given) { + Err(e) => e.to_string(), + Ok(_) => return None, + }, + }; + Some((given.clone(), why)) + }) + .collect() +} + +/// The FAILED rows whose owner is dead, per store, read once per store: +/// a FAILED row the backend lists as older than the owner window on the +/// heartbeat clock. +struct FailedOwners { + window: Duration, + dead: HashMap>, +} + +impl FailedOwners { + fn new(window: Duration) -> Self { + Self { + window, + dead: HashMap::new(), + } + } + + async fn is_dead( + &mut self, + storage: &S, + row: &BackupRow, + ) -> Result { + if !self.dead.contains_key(&row.store) { + let dead: HashSet = storage + .list_backup_rows_older_than( + BackupStatus::Failed, + &row.store, + StaleClock::Heartbeat, + self.window, + ) + .map_ok(|r| r.backup_arn) + .try_collect() + .await?; + self.dead.insert(row.store.clone(), dead); + } + Ok(self.dead[&row.store].contains(&row.backup_arn)) + } +} + +/// Remove `row` outright by its status's removal (module doc); a removal +/// that finds another status re-reads the row once and applies that +/// status's. +async fn forget_row( + storage: &S, + row: &BackupRow, + removals: &mut Removals<'_>, +) -> Forgotten { + let mut current = row.clone(); + for attempt in 0..2 { + if let Some(done) = remove_by_status(storage, ¤t, removals).await { + return done; + } + // The removal matched no row of that status: another writer moved + // it, or (CREATING) its owner is alive. Read it once more. + let arn = match BackupArn::parse(&row.backup_arn) { + Ok(arn) => arn, + Err(e) => return Forgotten::Failed(format!("the row's ARN does not parse: {e}")), + }; + let now = match storage.read_backup_row(&arn).await { + Ok(Some(now)) => lean(now), + Ok(None) => return Forgotten::Gone, + Err(e) => return Forgotten::Failed(format!("the row could not be read again: {e}")), + }; + if now.store != row.store { + return Forgotten::Changed(now.status, now.store); + } + if now.status == BackupStatus::Creating && current.status == BackupStatus::Creating { + return Forgotten::LiveOwner(BackupStatus::Creating); + } + current = now; + if attempt == 1 { + break; + } + } + Forgotten::Changed(current.status, current.store) +} + +/// The removal of `row`'s status (module doc): `None` when it matched no +/// row of that status. +async fn remove_by_status( + storage: &S, + row: &BackupRow, + removals: &mut Removals<'_>, +) -> Option { + let owner_window = removals.run.owner_window; + let key = row.key(); + let failed = |what: &str, e: StorageError| Some(Forgotten::Failed(format!("{what}: {e}"))); + let status = row.status; + let configured = removals + .run + .configured + .filter(|configured| configured.holds(row)); + let unmounted = removals.run.unmounted_root.is_some_and(|root| { + row.store.store_kind == StoreKind::Filesystem && row.store.store_root == root + }); + if unmounted || configured.is_some_and(|configured| configured.file_missing_with_rows) { + return Some(Forgotten::RootWithoutFile); + } + match status { + BackupStatus::Available => match storage.forget_available_backup_row(&key).await { + Ok(ForgetRow::Removed) => Some(Forgotten::Removed(status)), + Ok(ForgetRow::InUse(table)) => Some(Forgotten::InUse(table, row.account_id.clone())), + Ok(ForgetRow::Status(_)) => None, + Err(e) if is_backup_not_found(&e) => Some(Forgotten::Gone), + Err(e) => failed("the row could not be removed", e), + }, + BackupStatus::Creating => match storage + .remove_dead_creating_backup_row(&key, owner_window) + .await + { + Ok(true) => Some(Forgotten::Removed(status)), + Ok(false) => None, + Err(e) => failed("the row could not be removed", e), + }, + BackupStatus::Failed | BackupStatus::Deleted => { + if status == BackupStatus::Failed { + match removals.failed_owners.is_dead(storage, row).await { + Ok(true) => {} + Ok(false) => return Some(Forgotten::LiveOwner(status)), + Err(e) => return failed("the row's owner could not be tested", e), + } + } + if let Err(reason) = removals.wait_for_duties().await { + return Some(Forgotten::Stopped(reason)); + } + match storage + .remove_backup_row(&key, row.store.store_kind, &row.store.store_root) + .await + { + Ok(true) + if status == BackupStatus::Deleted + && removals.run.deletions_finished.contains(&row.backup_arn) => + { + Some(Forgotten::RemovedAfterDeletion) + } + Ok(true) => Some(Forgotten::Removed(status)), + Ok(false) => None, + Err(e) => failed("the row could not be removed", e), + } + } + BackupStatus::Deleting if configured.is_some() => { + let configured = configured.expect("tested above"); + if configured.healthy != Some(&row.store) { + if configured.mismatch_row_ids.contains(&row.store.store_id) { + return Some(Forgotten::DeletionBehindMismatch( + row.store.store_id.clone(), + )); + } + return Some(Forgotten::DeletionPending); + } + finish_then_remove(storage, configured.store, row, removals).await + } + BackupStatus::Deleting => match storage.set_backup_deleted(&key).await { + Ok(true) => match storage + .remove_backup_row(&key, row.store.store_kind, &row.store.store_root) + .await + { + Ok(true) => Some(Forgotten::Removed(status)), + Ok(false) => None, + Err(e) => failed( + "the row was set DELETED and could not be removed (a second run removes it)", + e, + ), + }, + Ok(false) => None, + Err(e) => failed("the row could not be set DELETED", e), + }, + } +} + +/// A DELETING row of the configured store (healthy, its id the row's): +/// after the wait for the duties, its deletion finished behind the fence +/// (`finish_backup_delete`, which deletes the prefix and sets the row +/// DELETED), then the row removed. The wait comes first, so a stop during +/// it leaves the row DELETING and its objects untouched. `None` when the +/// row was no longer DELETING. +async fn finish_then_remove( + storage: &S, + configured: &ConfiguredStore, + row: &BackupRow, + removals: &mut Removals<'_>, +) -> Option { + let arn = match BackupArn::parse(&row.backup_arn) { + Ok(arn) => arn, + Err(e) => { + return Some(Forgotten::Failed(format!( + "the row's ARN does not parse: {e}" + ))); + } + }; + if let Err(reason) = removals.wait_for_duties().await { + return Some(Forgotten::Stopped(reason)); + } + let fence = configured.fence(); + match super::finish_backup_delete(storage, &configured.calls("deletion"), &fence, &arn).await { + Ok(true) => { + tracing::info!( + target: "extenddb::audit", + action = "backup sync --forget-backup", + account_id = %row.account_id, + backup_arn = %row.backup_arn, + status = "DELETED", + store_root = %row.store.store_root, + "finished a DELETING row's deletion in the configured store before its removal" + ); + } + Ok(false) => return None, + Err(e) => { + return Some(Forgotten::Failed(format!( + "its deletion in the configured store did not finish ({e}); the row is left \ + DELETING" + ))); + } + } + match storage + .remove_backup_row(&row.key(), row.store.store_kind, &row.store.store_root) + .await + { + Ok(true) => Some(Forgotten::RemovedAfterDeletion), + Ok(false) => None, + Err(e) => Some(Forgotten::Failed(format!( + "the row's deletion is finished and the DELETED row could not be removed (a second \ + run removes it): {e}" + ))), + } +} + +/// The audit line of a removal. +fn audit_removal(action: &str, row: &BackupRow, outcome: &Forgotten) { + let (status, message) = match outcome { + Forgotten::Removed(status) => ( + *status, + "removed a backup row outright; no object was deleted", + ), + Forgotten::RemovedAfterDeletion => ( + row.status, + "removed a backup row after this command finished its deletion", + ), + _ => return, + }; + tracing::info!( + target: "extenddb::audit", + action = action, + account_id = %row.account_id, + backup_arn = %row.backup_arn, + status = %status, + store_kind = %row.store.store_kind, + store_root = %row.store.store_root, + store_id = %row.store.store_id, + "{message}" + ); +} + +/// What [`forget_store`] did. +#[derive(Debug, Default)] +pub struct ForgetStoreReport { + /// Each row of the root and what was done with it, in listing order. + pub rows: Vec<(NamedRow, Forgotten)>, + /// Why the run stopped before the remaining rows, when it did. + pub stopped: Option, + /// The rows the root still lists after the removals; `None` when the + /// listing failed (its error in `recheck_error`). + pub rows_left: Option, + /// What the sweep of the root's status entries after the removals did + /// (Cassandra; zero elsewhere): the entries with no row of the root + /// behind them deleted, and those whose row is of the root under + /// another status repaired. `None` when it did not run (a stop, or a + /// listing that failed). + pub entries_swept: Option, + /// The store ids the catalog still records for the root after the + /// sweep, read when the root's listing after the removals returned no + /// row: status entries written after that listing (a writer still + /// configured with the root). + pub entries_left: Vec, + /// Why the read after the removals failed, when it did. + pub recheck_error: Option, +} + +impl ForgetStoreReport { + /// The rows removed. + #[must_use] + pub fn removed(&self) -> usize { + self.rows.iter().filter(|(_, f)| f.is_removed()).count() + } + + /// Whether every row is gone: no row left, no stop, the read after the + /// removals answered and listed nothing. Entries left behind do not + /// count: no row stands behind them. + #[must_use] + pub fn complete(&self) -> bool { + self.stopped.is_none() + && self.recheck_error.is_none() + && self.rows_left == Some(0) + && !self.rows.iter().any(|(_, f)| f.is_left()) + } +} + +/// `--forget-store ` (module doc): every row of `root` (of kind +/// `kind`, from [`locate_root`]) removed outright under `run`. Its stop is +/// asked before each row (the claim lost, a signal). Each removal is logged +/// under the `extenddb::audit` target. +/// +/// # Errors +/// +/// The backend's error for the root's listing; nothing was removed. +pub async fn forget_store( + storage: &S, + kind: StoreKind, + root: &str, + run: ForgetRun<'_>, +) -> Result { + let rows: Vec = storage + .list_backup_rows_by_root(kind, root, None) + .map_ok(lean) + .try_collect() + .await?; + let mut report = ForgetStoreReport::default(); + let mut removals = Removals::new(run); + for row in &rows { + if let Some(reason) = (run.stop)() { + report.stopped = Some(reason); + return Ok(report); + } + let outcome = forget_row(storage, row, &mut removals).await; + if let Forgotten::Stopped(reason) = outcome { + report.stopped = Some(reason); + return Ok(report); + } + audit_removal("backup sync --forget-store", row, &outcome); + report.rows.push((NamedRow::of(row), outcome)); + } + match storage + .list_backup_rows_by_root(kind, root, None) + .try_fold(0usize, |n, _| std::future::ready(Ok(n + 1))) + .await + { + Ok(left) => report.rows_left = Some(left), + Err(e) => report.recheck_error = Some(format!("the root's rows could not be read: {e}")), + } + // The root's status entries (Cassandra), under the claim: each verified + // against its row as it is read, deleted or repaired. + if report.recheck_error.is_none() { + match storage.remove_orphan_status_entries(kind, root).await { + Ok(sweep) => report.entries_swept = Some(sweep), + Err(e) => { + report.recheck_error = + Some(format!("the root's status entries could not be swept: {e}")); + } + } + } + if report.rows_left == Some(0) && report.recheck_error.is_none() { + match storage.read_store_ids(kind, root).await { + Ok(ids) => report.entries_left = ids, + Err(e) => { + report.recheck_error = Some(format!("the root's store ids could not be read: {e}")); + } + } + } + Ok(report) +} + +/// What `--forget-backup` did with one ARN. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ForgetBackup { + /// The ARN's row, and what [`forget_row`] did with it. + Row(BackupStatus, Forgotten), + /// No row carries the ARN as given; a row under this other spelling of + /// it does (its status), and was left. + OtherSpelling(String, BackupStatus), + /// The ARN is an earlier release's in-catalog backup: nothing changed. + Legacy, + /// No row and no in-catalog backup carries the ARN, under either + /// spelling: there is nothing to remove, and the ARN counts as done. + NoRow, + /// The ARN is not one (why). + Invalid(String), + /// A catalog call failed (why); nothing changed. + Failed(String), +} + +impl ForgetBackup { + /// Whether the ARN's row is gone (removed now, or already, or no row + /// carries it): the run goes on to the next ARN; anything else stops it + /// and exits 1. + #[must_use] + pub fn is_done(&self) -> bool { + matches!( + self, + Self::Row( + _, + Forgotten::Removed(_) | Forgotten::RemovedAfterDeletion | Forgotten::Gone + ) | Self::NoRow + ) + } +} + +impl std::fmt::Display for ForgetBackup { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Row(_, outcome) => write!(f, "{outcome}"), + Self::OtherSpelling(arn, status) => write!( + f, + "left: no row carries this ARN as written; the row of this backup is {arn} \ + ({status}): run --forget-backup with that ARN" + ), + Self::Legacy => f.write_str( + "an in-catalog backup; --forget-backup changes nothing; DeleteBackup removes it", + ), + Self::NoRow => f.write_str("no row carries this ARN; nothing to remove"), + Self::Invalid(why) => write!(f, "refused: {why}"), + Self::Failed(why) => write!(f, "left: {why}"), + } + } +} + +/// What [`forget_backups`] did. +#[derive(Debug, Default)] +pub struct ForgetBackupsReport { + /// Each ARN reached, as given, and its outcome, in the order given. + pub done: Vec<(String, ForgetBackup)>, + /// Why the run stopped before the remaining ARNs: a signal, the claim + /// lost. A refusal stops the run too and is the last of `done`. + pub stopped: Option, +} + +impl ForgetBackupsReport { + /// Whether every ARN's row is gone. + #[must_use] + pub fn complete(&self, asked: usize) -> bool { + self.stopped.is_none() + && self.done.len() == asked + && self.done.iter().all(|(_, outcome)| outcome.is_done()) + } +} + +/// `--forget-backup `, once per ARN in the order given under `run`, +/// stopping at the first ARN whose row is not removed (module doc). +/// `region` is the server's, for the other spelling of the ARN. The run's +/// stop is asked before each ARN. Each removal is logged under the +/// `extenddb::audit` target. +pub async fn forget_backups( + storage: &S, + arns: &[String], + region: &str, + run: ForgetRun<'_>, +) -> ForgetBackupsReport { + let mut report = ForgetBackupsReport::default(); + let mut removals = Removals::new(run); + for given in arns { + if let Some(reason) = (run.stop)() { + report.stopped = Some(reason); + return report; + } + let outcome = forget_backup(storage, given, region, &mut removals).await; + if let ForgetBackup::Row(_, Forgotten::Stopped(reason)) = &outcome { + report.stopped = Some(reason.clone()); + return report; + } + let done = outcome.is_done(); + report.done.push((given.clone(), outcome)); + if !done { + break; + } + } + report +} + +async fn forget_backup( + storage: &S, + given: &str, + region: &str, + removals: &mut Removals<'_>, +) -> ForgetBackup { + let parts = match extenddb_storage::util::check_backup_arn(given) { + Ok(parts) => parts, + Err(e) => return ForgetBackup::Invalid(e.to_string()), + }; + let arn = match BackupArn::parse(given) { + Ok(arn) => arn, + Err(e) => return ForgetBackup::Invalid(e.to_string()), + }; + match storage.read_backup_row(&arn).await { + Ok(Some(row)) => { + let row = lean(row); + let outcome = forget_row(storage, &row, removals).await; + audit_removal("backup sync --forget-backup", &row, &outcome); + return ForgetBackup::Row(row.status, outcome); + } + Ok(None) => {} + Err(e) => return ForgetBackup::Failed(format!("the row could not be read: {e}")), + } + let spellings = super::BackupArnSpellings::new(&parts, region); + for spelling in spellings.arns() { + if *spelling == arn { + continue; + } + match storage.read_backup_row(spelling).await { + Ok(Some(row)) => return ForgetBackup::OtherSpelling(row.backup_arn, row.status), + Ok(None) => {} + Err(e) => return ForgetBackup::Failed(format!("the row could not be read: {e}")), + } + } + let mut legacy = vec![spellings.canonical().to_owned()]; + if let Some(as_written) = &spellings.as_written { + legacy.push(as_written.clone()); + } + for spelling in legacy { + match storage.describe(parts.account_id, &spelling).await { + Ok(_) => return ForgetBackup::Legacy, + Err(e) if is_backup_not_found(&e) => {} + Err(StorageError::Unsupported(_)) => {} + Err(e) => { + return ForgetBackup::Failed(format!( + "the in-catalog backups could not be read: {e}" + )); + } + } + } + ForgetBackup::NoRow +} + +/// A row with its parsed `source_table_details_json` dropped: the options +/// keep the rows of a whole root, and read no column of that one. +fn lean(mut row: BackupRow) -> BackupRow { + row.source_table_details_json = serde_json::Value::Null; + row +} + +// --- the move -------------------------------------------------------------- + +/// What [`move_store`] moves rows from and to. +#[derive(Clone, Copy)] +pub struct MoveTarget<'a> { + /// The configured store, healthy: the copy is under its root. + pub configured: &'a ConfiguredStore, + /// Its identity, the one the rows get. + pub identity: &'a StoreIdentity, + /// The old root, from [`locate_root`]. + pub from_kind: StoreKind, + pub from_root: &'a str, + /// The window a live owner is told apart by (`stale_job_after` beside + /// servers, zero on SQLite under the serve lock). + pub owner_window: Duration, + /// Asked before each row and before the rewrite. + pub stop: Stop<'a>, +} + +/// What [`move_store`] did. +#[derive(Debug, Default)] +pub struct MoveReport { + /// How many rows the old root listed. + pub rows: usize, + /// The CREATING rows resolved, with the status written (AVAILABLE: the + /// move writes no other). + pub resolved: Vec<(String, BackupStatus)>, + /// The DELETING rows whose deletion was finished in the configured + /// store (now DELETED). + pub finished: Vec, + /// Every row that stops the move, with the reason; non-empty means no + /// row was moved. + pub stopping: Vec<(NamedRow, String)>, + /// Why the move stopped before the rewrite otherwise: a signal, the + /// claim lost, the fence's refusal. + pub stopped: Option, + /// The rows moved, by ARN: rows the old root no longer lists and the + /// configured store's listing now holds (every key counted, when the + /// old root's listing after the rewrite failed). + pub moved: Vec, + /// The rows neither root lists after the rewrite: removed by another + /// writer during the move (a tombstone duty of a server on the old + /// root); nothing was moved for them. + pub gone: Vec, + /// What `rewrite_store` counted. + pub rewritten: u64, + /// The rewrite's error, when it failed (it may have moved some rows). + pub rewrite_error: Option, + /// The rows the old root still lists after the rewrite. + pub left: Vec, + /// Why the listing after the rewrite failed, when it did. + pub relist_error: Option, +} + +impl MoveReport { + /// Whether every row of the old root was moved. + #[must_use] + pub fn complete(&self) -> bool { + self.stopping.is_empty() + && self.stopped.is_none() + && self.rewrite_error.is_none() + && self.relist_error.is_none() + && self.left.is_empty() + } +} + +/// `--move-store ` (module doc). The caller holds the maintenance +/// claim and has waited for the jobs of the configured store and of the old +/// root's store ids. +/// +/// # Errors +/// +/// The backend's error for the listing of the old root or of the +/// configured root, both read before any row is checked; nothing was +/// changed. +pub async fn move_store( + storage: &S, + target: MoveTarget<'_>, +) -> Result { + let mut rows: Vec = storage + .list_backup_rows_by_root(target.from_kind, target.from_root, None) + .map_ok(lean) + .try_collect() + .await?; + let mut report = MoveReport { + rows: rows.len(), + ..MoveReport::default() + }; + // One backup id per account after the move, checked first, so no row of + // a group that stops for it gets a resolution or a deletion (on + // Cassandra two spellings of an ARN share one prefix). The rows are + // grouped by (account, backup id) over the configured root's rows and + // the old root's, and every old-root row of a group of more than one is + // set aside before any row is checked. + let mut groups: HashMap<(String, String), Vec> = HashMap::new(); + let mut configured_rows = storage.list_backup_rows_by_root( + target.identity.store_kind, + &target.identity.store_root, + None, + ); + while let Some(row) = configured_rows.try_next().await? { + groups + .entry((row.account_id, row.backup_id)) + .or_default() + .push(row.backup_arn); + } + drop(configured_rows); + for row in &rows { + groups + .entry((row.account_id.clone(), row.backup_id.clone())) + .or_default() + .push(row.backup_arn.clone()); + } + let mut two_rows: HashSet = HashSet::new(); + for row in &rows { + let group = &groups[&(row.account_id.clone(), row.backup_id.clone())]; + if group.len() < 2 { + continue; + } + two_rows.insert(row.backup_arn.clone()); + let others: Vec<&str> = group + .iter() + .filter(|arn| **arn != row.backup_arn) + .map(String::as_str) + .collect(); + report.stopping.push(( + NamedRow::of(row), + format!( + "the move would give account {} two rows of backup id {}: {} holds it as well; \ + forget one of them with --forget-backup", + row.account_id, + row.backup_id, + others.join(", ") + ), + )); + } + let calls = StoreCalls { + store: &*target.configured.store, + metrics: None, + }; + for row in &mut rows { + if two_rows.contains(&row.backup_arn) { + continue; + } + if let Some(reason) = (target.stop)() { + report.stopped = Some(reason); + return Ok(report); + } + if let Err(why) = check_row(storage, target, calls, row, &mut report).await { + report.stopping.push((NamedRow::of(row), why)); + } + } + if !report.stopping.is_empty() { + return Ok(report); + } + if let Some(reason) = (target.stop)() { + report.stopped = Some(reason); + return Ok(report); + } + if let Err(problem) = target.configured.fence().check_catalog(storage).await { + report.stopped = Some(format!( + "the configured store's fence refused the move: {problem}" + )); + return Ok(report); + } + let keys: Vec = rows.iter().map(BackupRow::key).collect(); + match storage + .rewrite_store(target.from_kind, target.from_root, &keys, target.identity) + .await + { + Ok(n) => report.rewritten = n, + Err(e) => report.rewrite_error = Some(e.to_string()), + } + let left: Result, StorageError> = storage + .list_backup_rows_by_root(target.from_kind, target.from_root, None) + .map_ok(|row| NamedRow::of(&row)) + .try_collect() + .await; + match left { + Ok(left) => { + let still: HashSet<&str> = left.iter().map(|r| r.backup_arn.as_str()).collect(); + // A key is moved when its row now names the configured store. + let mut under_to: HashSet = HashSet::new(); + let mut listed = storage.list_backup_rows(None, target.identity); + loop { + match listed.try_next().await { + Ok(Some(row)) => { + under_to.insert(row.backup_arn); + } + Ok(None) => break, + Err(e) => { + report.relist_error = Some(format!( + "the configured store's rows could not be read after the rewrite: \ + {e}" + )); + break; + } + } + } + drop(listed); + for key in keys + .iter() + .filter(|k| !still.contains(k.backup_arn.as_str())) + { + if report.relist_error.is_some() || under_to.contains(&key.backup_arn) { + report.moved.push(key.backup_arn.clone()); + } else { + report.gone.push(key.backup_arn.clone()); + } + } + report.left = left; + } + Err(e) => { + report.relist_error = Some(format!( + "the old root's rows could not be read after the rewrite: {e}" + )); + // The count is all there is; with every key counted, every + // row moved. + if report.rewrite_error.is_none() && report.rewritten == keys.len() as u64 { + report.moved = keys.iter().map(|k| k.backup_arn.clone()).collect(); + } + } + } + let moved: HashSet<&str> = report.moved.iter().map(String::as_str).collect(); + for row in rows + .iter() + .filter(|r| moved.contains(r.backup_arn.as_str())) + { + tracing::info!( + target: "extenddb::audit", + action = "backup sync --move-store", + account_id = %row.account_id, + backup_arn = %row.backup_arn, + status = %row.status, + old_store_root = %target.from_root, + old_store_id = %row.store.store_id, + store_kind = %target.identity.store_kind, + store_root = %target.identity.store_root, + store_id = %target.identity.store_id, + "moved a backup row to the configured store; no object was copied or deleted" + ); + } + Ok(report) +} + +/// Why a CREATING row the backend does not hand over stops the move. +const LIVE_CREATING: &str = "CREATING with a live owner (or another taker): wait for its \ + CreateBackup to end and run the move again"; + +/// One row of the old root against the configured store (module doc): +/// `Err` is why it stops the move. A CREATING row resolved, and a DELETING +/// row finished, carry their new status in `row` from here on. +async fn check_row( + storage: &S, + target: MoveTarget<'_>, + calls: StoreCalls<'_>, + row: &mut BackupRow, + report: &mut MoveReport, +) -> Result<(), String> { + let arn = BackupArn::parse(&row.backup_arn) + .map_err(|e| format!("the row's ARN does not parse: {e}"))?; + let prefix = arn.store_prefix(); + if row.status == BackupStatus::Creating { + return match resolve_dead_creating_row( + storage, + calls, + row, + target.owner_window, + OnIncomplete::Leave, + ) + .await + { + CreatingOutcome::Resolved(status) => { + tracing::info!( + target: "extenddb::audit", + action = "backup sync --move-store", + account_id = %row.account_id, + backup_arn = %row.backup_arn, + status = %status, + store_root = %row.store.store_root, + "resolved a CREATING row whose owner is dead against the configured store" + ); + report.resolved.push((row.backup_arn.clone(), status)); + row.status = status; + // The probe that resolved it is the AVAILABLE check. + Ok(()) + } + CreatingOutcome::Incomplete(why) => Err(format!( + "CREATING, its owner dead, and the copy under the configured root holds no \ + complete backup ({why}); the move writes no status for it, since the old root \ + may hold the complete backup. Run a server configured with the old root (or, \ + on SQLite, plain backup sync there), whose reconciler resolves the row against \ + the old root, then copy the prefix again; or remove the row with \ + --forget-backup" + )), + CreatingOutcome::NotHandedOver => Err(LIVE_CREATING.to_owned()), + CreatingOutcome::LeftToOwner(why) => Err(format!( + "CREATING, taken over by another owner during the move ({why})" + )), + CreatingOutcome::Failed(why) => Err(format!("CREATING: {why}")), + }; + } + let verified = |what: &str| { + format!( + "{what} whose prefix under the configured root holds a verified manifest: moving it \ + would have the reconciler delete a complete backup. Remove that prefix from the \ + configured root, or remove the row with --forget-backup and let backup sync adopt \ + the prefix" + ) + }; + match row.status { + BackupStatus::Creating => unreachable!("returned above"), + BackupStatus::Deleting => { + match verify_prefix(calls, &prefix, None).await { + Ok(Verified::Complete(_)) => return Err(verified("DELETING")), + Ok(Verified::Incomplete(_)) => {} + Err(e) => return Err(format!("the configured store could not be read: {e}")), + } + let fence = target.configured.fence(); + match super::finish_backup_delete( + storage, + &target.configured.calls("deletion"), + &fence, + &arn, + ) + .await + { + Ok(true) => { + tracing::info!( + target: "extenddb::audit", + action = "backup sync --move-store", + account_id = %row.account_id, + backup_arn = %row.backup_arn, + status = "DELETED", + store_root = %target.identity.store_root, + "finished a DELETING row's deletion in the configured store" + ); + report.finished.push(row.backup_arn.clone()); + row.status = BackupStatus::Deleted; + Ok(()) + } + Ok(false) => Err( + "DELETING: the row was no longer DELETING when its objects were gone" + .to_owned(), + ), + Err(e) => Err(format!( + "DELETING: its deletion in the configured store did not finish: {e}" + )), + } + } + BackupStatus::Deleted | BackupStatus::Failed => { + match verify_prefix(calls, &prefix, None).await { + Ok(Verified::Complete(_)) => Err(verified(row.status.as_str())), + Ok(Verified::Incomplete(_)) => Ok(()), + Err(e) => Err(format!("the configured store could not be read: {e}")), + } + } + BackupStatus::Available => match probe_manifest(calls, &arn).await { + Ok(Probe::Complete(fields)) => { + if fields.size_bytes == Some(row.size_bytes) + && fields.item_count == Some(row.item_count) + { + Ok(()) + } else { + Err(format!( + "the manifest under the configured root records {} bytes and {} items \ + and the row {} bytes and {} items: another backup under the same ARN", + fields.size_bytes.unwrap_or_default(), + fields.item_count.unwrap_or_default(), + row.size_bytes, + row.item_count + )) + } + } + Ok(Probe::Incomplete(why)) => Err(format!( + "not a complete copy under the configured root: {why}; copy the prefix again \ + (or remove the row with --forget-backup)" + )), + Err(e) => Err(format!("the configured store could not be read: {e}")), + }, + } +} diff --git a/crates/engine/src/backup/sync.rs b/crates/engine/src/backup/sync.rs index e96b70dfd..855809249 100644 --- a/crates/engine/src/backup/sync.rs +++ b/crates/engine/src/backup/sync.rs @@ -84,6 +84,16 @@ //! them. A later sync where the bytes are adopts them again. Without //! `--prune-missing` such rows stay AVAILABLE. //! +//! [`sync`] neither lists nor changes the rows of the other stores (another +//! kind or root, or the configured root under another store id): a prefix +//! whose backup id such a row holds meets it at the read under the ARN's +//! spellings or at the insert, and is skipped naming that row's status, +//! kind, root and store id. The command reports +//! those rows as a whole once a listing has reached its end +//! ([`super::store_maintenance::other_stores`]: the rows per status of +//! each other store, and each CREATING and DELETING row by ARN), with the +//! two options that act on them, `--move-store` and `--forget-store`. +//! //! The command takes the maintenance claim for every sync on PostgreSQL, //! MongoDB and Cassandra, where it runs beside servers (design 4(f)), and //! runs under the serve lock on SQLite; so no purge runs between its row diff --git a/crates/engine/src/backup/tests.rs b/crates/engine/src/backup/tests.rs index 077dc5d96..70503597a 100644 --- a/crates/engine/src/backup/tests.rs +++ b/crates/engine/src/backup/tests.rs @@ -23,8 +23,8 @@ use extenddb_core::types::{ use extenddb_storage::backup_catalog::{ AbandonedLegacyBackup, BackupKey, BackupRow, BackupStatus, DeleteStart, ForgetRow, Heartbeat, JobClaim, JobRow, JobStatus, RestoreJobRow, RestoreKind, RestoreRegistrationRow, - RestoreTargetKey, StaleClock, StatusFields, StoreIdentity, StoreKind, TimeBounds, - UNOWNED_GENERATION, WriteOutcome, new_generation, + RestoreTargetKey, StaleClock, StatusEntrySweep, StatusFields, StoreIdentity, StoreKind, + StoreRootCount, TimeBounds, UNOWNED_GENERATION, WriteOutcome, new_generation, }; use extenddb_storage::backup_definition::{BACKUP_DEFINITION_VERSION, BackupTableDefinition}; use extenddb_storage::error::StorageError; @@ -327,7 +327,74 @@ struct Fake { /// ARN, and `read_backup_row` finds a row by its ARN; otherwise by /// backup id, as on PostgreSQL, SQLite and MongoDB. unique_by_arn: std::sync::atomic::AtomicBool, -} + /// Rows `rewrite_store` lands just before it decides, by ARN: a + /// CreateBackup of a server still on the old root whose row lands after + /// the move's checks, or another writer's change to a row the move + /// checked. + lands_before_rewrite: Mutex>, + /// When set, `rewrite_store` rewrites at most this many rows and then + /// fails, leaving the rest of the keys as they were: a move that stops + /// part way (MongoDB between chunks, Cassandra between keys). + fail_rewrite_after: Mutex>, + /// Every `rewrite_store` call: the old kind and root, the keys and the + /// new store. + rewrites_seen: Mutex>, + /// When set, run (once) inside `list_backup_rows_older_than` of this + /// status and clock after it builds its answer: a maintenance command's + /// removal or move of a row landing between a duty's listing and its + /// deletion. + after_older_than: Mutex>, + /// When set, run (once) right after `read_backup_row` of this ARN + /// builds its answer: a maintenance command acting between a duty's + /// re-read of the row and what follows it. + after_row_read: Mutex>, + /// When set, run (once) right after the `n`th `check_maintenance_claim` + /// (counted over `maintenance_checks`) builds its answer: a command + /// taking the claim just after a reader found it free. + after_maintenance_check: Mutex>, + /// When set, `claim_dead_backup` hands over a CREATING row whose + /// `heartbeat_at` is older than its `stale_after` on the catalog's + /// clock, as the backends judge it, instead of the rows in `dead`. + dead_by_heartbeat: std::sync::atomic::AtomicBool, + /// When set, `list_backup_rows_by_root` fails once `rewrite_store` has + /// been called: the move's re-list of the old root failing. + by_root_error_after_rewrite: std::sync::atomic::AtomicBool, + /// Rows `rewrite_store` removes just before it decides, by ARN: a + /// tombstone duty of a server on the old root removing a row the move + /// checked. + removed_before_rewrite: Mutex>, + /// When set, `remove_backup_row` waits this long before it removes + /// anything (a catalog write held on a lock); a call dropped during the + /// wait removes nothing. + remove_delay: Mutex>, + /// Status entries with no row behind them, as Cassandra can hold + /// (store and status): `read_store_ids` and `summarize_store_roots` + /// count them, and `remove_orphan_status_entries` deletes a root's. + orphan_entries: Mutex>, + /// What `summary_counts_status_entries` answers (Cassandra's `true`). + counts_entries: std::sync::atomic::AtomicBool, + /// Status entries a writer still configured with a root writes after + /// `remove_orphan_status_entries` read that root: counted by + /// `read_store_ids`, not seen by the sweep. + late_entries: Mutex>, + /// Rows whose one status entry is under an earlier status, by ARN, as + /// Cassandra holds after a status change whose entry batch failed (or + /// whose writer died before it): the entry's status, and the row's + /// status and store when the knob was set. While the row keeps that + /// status and store, `list_backup_rows` does not list it (its readers + /// read one status partition and skip the entry) and + /// `summarize_store_roots` counts it under the entry's status; + /// `list_backup_rows_by_root` lists it under its own, and + /// `remove_orphan_status_entries` of its root repairs the entry (or + /// removes it, once the row is gone or names another store). + stale_entries: Mutex>, + /// When set, `list_backup_rows_by_root` of this root fails. + by_root_error_for: Mutex>, +} + +/// One `rewrite_store` call the `Fake` received: the old kind and root, the +/// keys and the new store. +type RewriteCall = (StoreKind, String, Vec, StoreIdentity); /// A closure the `Fake` runs inside `sweep_restore_progress`. type SweepHook = Box; @@ -482,6 +549,27 @@ impl JobClaim for FakeRestoreClaim { } impl Fake { + /// The status of `row`'s stale status entry (`stale_entries`), while + /// the row keeps the status and store it had when the entry was set. + fn stale_entry(&self, row: &BackupRow) -> Option { + self.stale_entries + .lock() + .unwrap() + .get(&row.backup_arn) + .filter(|(_, status, store)| *status == row.status && *store == row.store) + .map(|(entry, _, _)| *entry) + } + + /// Leave the seeded row of `arn` with its one status entry under + /// `entry` (`stale_entries`). + fn set_stale_entry(&self, arn: &str, entry: BackupStatus) { + let row = self.rows.lock().unwrap()[arn].clone(); + self.stale_entries + .lock() + .unwrap() + .insert(arn.to_owned(), (entry, row.status, row.store)); + } + fn restore_claim_on(&self, table_id: &str, generation: u64) -> Box { Box::new(FakeRestoreClaim { records: Arc::clone(&self.restore_jobs), @@ -888,6 +976,9 @@ struct MemStore { panic_body_of_get: std::sync::atomic::AtomicUsize, /// Every `get`'s key, in order, failed ones included. gets: Mutex>, + /// When set, `delete_prefix_before` deletes nothing and answers that it + /// reached its deadline (a walk stopped at its next entry). + deadline_reached: std::sync::atomic::AtomicBool, /// Called with the key of every `put` that stored its object, so a test /// can act at a point of the job (after the sidecar's put, before the /// claim re-check). @@ -1386,6 +1477,37 @@ impl extenddb_backup_store::BackupStore for MemStore { }) } + /// The trait's rule (the deadline checked once, then the whole + /// deletion); `deadline_reached` answers unfinished with nothing + /// deleted. + fn delete_prefix_before<'a>( + &'a self, + prefix: &'a str, + deadline: tokio::time::Instant, + ) -> BoxedFuture< + 'a, + Result, + > { + Box::pin(async move { + if self + .deadline_reached + .load(std::sync::atomic::Ordering::Acquire) + || tokio::time::Instant::now() >= deadline + { + return Ok(extenddb_backup_store::PrefixDeletion { + deleted: 0, + finished: false, + }); + } + self.delete_prefix(prefix) + .await + .map(|deleted| extenddb_backup_store::PrefixDeletion { + deleted, + finished: true, + }) + }) + } + fn validate(&self) -> BoxedFuture<'_, Result<(), extenddb_backup_store::StoreError>> { Box::pin(async move { self.stall().await; @@ -1824,10 +1946,20 @@ impl BackupCatalog for Fake { fn claim_dead_backup<'a>( &'a self, key: &'a BackupKey, - _stale_after: Duration, + stale_after: Duration, ) -> BoxedFuture<'a, Result>, StorageError>> { - let dead = self.dead.lock().unwrap().contains(&key.backup_arn); let mut rows = self.rows.lock().unwrap(); + let dead = if self + .dead_by_heartbeat + .load(std::sync::atomic::Ordering::Acquire) + { + let cutoff = seconds_since_epoch() + *self.catalog_clock_offset_secs.lock().unwrap() + - stale_after.as_secs_f64(); + rows.get(&key.backup_arn) + .is_some_and(|row| row.heartbeat_at < cutoff) + } else { + self.dead.lock().unwrap().contains(&key.backup_arn) + }; let claim = match rows.get_mut(&key.backup_arn) { Some(row) if dead && row.status == BackupStatus::Creating => { row.owner_generation = new_generation(); @@ -1872,6 +2004,57 @@ impl BackupCatalog for Fake { .filter(|r| r.store == *store) .filter(|r| status.is_none_or(|s| r.status == s)) .filter(|r| !self.unlisted.lock().unwrap().contains(&r.backup_arn)) + .filter(|r| self.stale_entry(r).is_none()) + .cloned() + .collect(); + rows.sort_by(|a, b| { + a.created_at + .total_cmp(&b.created_at) + .then_with(|| a.backup_arn.cmp(&b.backup_arn)) + }); + Box::pin(futures::stream::iter(rows.into_iter().map(Ok))) + } + + /// The backend's listing of a root: rows of the kind and root, whatever + /// their id, of the status when given, in the order `list_backup_rows` + /// uses; `unlisted` rows and `list_error` as there. + fn list_backup_rows_by_root<'a>( + &'a self, + store_kind: StoreKind, + store_root: &'a str, + status: Option, + ) -> BoxStream<'a, Result> { + if self + .by_root_error_after_rewrite + .load(std::sync::atomic::Ordering::Acquire) + && !self.rewrites_seen.lock().unwrap().is_empty() + { + return Box::pin(futures::stream::once(async { + Err(StorageError::Internal( + "the root's listing failed".to_owned(), + )) + })); + } + if let Some(e) = self.list_error.lock().unwrap().clone() { + return Box::pin(futures::stream::once(async move { + Err(StorageError::Internal(e)) + })); + } + if self.by_root_error_for.lock().unwrap().as_deref() == Some(store_root) { + return Box::pin(futures::stream::once(async { + Err(StorageError::Internal( + "the root's listing failed".to_owned(), + )) + })); + } + let mut rows: Vec = self + .rows + .lock() + .unwrap() + .values() + .filter(|r| r.store.store_kind == store_kind && r.store.store_root == store_root) + .filter(|r| status.is_none_or(|s| r.status == s)) + .filter(|r| !self.unlisted.lock().unwrap().contains(&r.backup_arn)) .cloned() .collect(); rows.sort_by(|a, b| { @@ -1882,6 +2065,114 @@ impl BackupCatalog for Fake { Box::pin(futures::stream::iter(rows.into_iter().map(Ok))) } + /// The backends' rewrite: `lands_before_rewrite` lands first; then each + /// key's row that is still of the old kind and root and not CREATING + /// gets the new store, counted, until `fail_rewrite_after` rows are + /// rewritten, when the call fails with the rest left. + fn rewrite_store<'a>( + &'a self, + from_kind: StoreKind, + from_root: &'a str, + keys: &'a [BackupKey], + to: &'a StoreIdentity, + ) -> BoxedFuture<'a, Result> { + self.rewrites_seen.lock().unwrap().push(( + from_kind, + from_root.to_owned(), + keys.to_vec(), + to.clone(), + )); + let mut rows = self.rows.lock().unwrap(); + for landed in self.lands_before_rewrite.lock().unwrap().drain(..) { + rows.insert(landed.backup_arn.clone(), landed); + } + for removed in self.removed_before_rewrite.lock().unwrap().drain(..) { + rows.remove(&removed); + } + let limit = *self.fail_rewrite_after.lock().unwrap(); + let mut rewritten = 0_usize; + for key in keys { + let Some(row) = rows.get_mut(&key.backup_arn) else { + continue; + }; + if row.account_id != key.account_id + || row.store.store_kind != from_kind + || row.store.store_root != from_root + || row.status == BackupStatus::Creating + { + continue; + } + if limit.is_some_and(|limit| rewritten >= limit) { + return Box::pin(async move { + Err(StorageError::Internal(format!( + "the rewrite stopped after {rewritten} rows" + ))) + }); + } + row.store = to.clone(); + rewritten += 1; + } + Box::pin(async move { Ok(rewritten as u64) }) + } + + /// One count per store and status of the rows, sorted as the backends + /// sort them. + fn summarize_store_roots(&self) -> BoxedFuture<'_, Result, StorageError>> { + let mut seen: Vec<(StoreIdentity, BackupStatus)> = self + .rows + .lock() + .unwrap() + .values() + .map(|r| (r.store.clone(), self.stale_entry(r).unwrap_or(r.status))) + .collect(); + seen.extend(self.orphan_entries.lock().unwrap().iter().cloned()); + let counts = StoreRootCount::tally(seen); + Box::pin(async move { Ok(counts) }) + } + + fn summary_counts_status_entries(&self) -> bool { + self.counts_entries + .load(std::sync::atomic::Ordering::Acquire) + } + + /// Delete the root's entries of `orphan_entries`, and settle its + /// `stale_entries`: repaired while the row is of the root under + /// another status, removed once it is gone or names another store. + fn remove_orphan_status_entries<'a>( + &'a self, + store_kind: StoreKind, + store_root: &'a str, + ) -> BoxedFuture<'a, Result> { + let at_root = |store: &StoreIdentity| { + store.store_kind == store_kind && store.store_root == store_root + }; + let mut sweep = StatusEntrySweep::default(); + let mut entries = self.orphan_entries.lock().unwrap(); + let before = entries.len(); + entries.retain(|(store, _)| !at_root(store)); + sweep.removed = (before - entries.len()) as u64; + drop(entries); + let rows = self.rows.lock().unwrap(); + self.stale_entries + .lock() + .unwrap() + .retain(|arn, (entry, _, store)| { + if !at_root(store) { + return true; + } + match rows.get(arn) { + Some(row) if at_root(&row.store) => { + if row.status != *entry { + sweep.repaired += 1; + } + } + _ => sweep.removed += 1, + } + false + }); + Box::pin(async move { Ok(sweep) }) + } + /// The last AVAILABLE row of the store in the listing's order, read as /// one row; `list_error` fails it as it fails the listing. fn newest_available_backup_row<'a>( @@ -1956,6 +2247,15 @@ impl BackupCatalog for Fake { }) .filter(|r| r.table_name == arn.table_name() && r.backup_arn == arn.to_arn_string()) .cloned(); + let mut hook = self.after_row_read.lock().unwrap(); + if hook + .as_ref() + .is_some_and(|(a, _)| *a == arn.to_arn_string()) + { + let (_, run) = hook.take().expect("checked"); + drop(hook); + run(self); + } Box::pin(async move { Ok(found) }) } @@ -2012,18 +2312,32 @@ impl BackupCatalog for Fake { Box::pin(async move { Ok(moved) }) } - /// Remove a DELETED or FAILED tombstone, as the backends do. + /// Remove a DELETED or FAILED tombstone still of the store given, as + /// the backends do. fn remove_backup_row<'a>( &'a self, key: &'a BackupKey, + store_kind: StoreKind, + store_root: &'a str, ) -> BoxedFuture<'a, Result> { self.remove_attempts .lock() .unwrap() .push(key.backup_arn.clone()); + if let Some(delay) = *self.remove_delay.lock().unwrap() { + return Box::pin(async move { + tokio::time::sleep(delay).await; + *self.remove_delay.lock().unwrap() = None; + self.remove_backup_row(key, store_kind, store_root).await + }); + } let mut rows = self.rows.lock().unwrap(); let removed = match rows.get(&key.backup_arn) { - Some(row) if matches!(row.status, BackupStatus::Deleted | BackupStatus::Failed) => { + Some(row) + if matches!(row.status, BackupStatus::Deleted | BackupStatus::Failed) + && row.store.store_kind == store_kind + && row.store.store_root == store_root => + { rows.remove(&key.backup_arn); true } @@ -2124,6 +2438,15 @@ impl BackupCatalog for Fake { .total_cmp(&b.created_at) .then_with(|| a.backup_arn.cmp(&b.backup_arn)) }); + let mut hook = self.after_older_than.lock().unwrap(); + if hook + .as_ref() + .is_some_and(|(s, c, _)| *s == status && *c == clock) + { + let (_, _, run) = hook.take().expect("checked"); + drop(hook); + run(self); + } Box::pin(futures::stream::iter(rows.into_iter().map(Ok))) } @@ -2311,6 +2634,31 @@ impl BackupCatalog for Fake { .filter(|row| row.store.store_kind == store_kind && row.store.store_root == store_root) .map(|row| row.store.store_id.clone()) .collect(); + ids.extend( + self.orphan_entries + .lock() + .unwrap() + .iter() + .filter(|(store, _)| { + store.store_kind == store_kind && store.store_root == store_root + }) + .map(|(store, _)| store.store_id.clone()), + ); + ids.extend( + self.late_entries + .lock() + .unwrap() + .iter() + .chain( + self.stale_entries + .lock() + .unwrap() + .values() + .map(|(_, _, s)| s), + ) + .filter(|store| store.store_kind == store_kind && store.store_root == store_root) + .map(|store| store.store_id.clone()), + ); ids.sort(); ids.dedup(); Box::pin(async move { Ok(ids) }) @@ -2349,12 +2697,22 @@ impl BackupCatalog for Fake { &self, stale_after: Duration, ) -> BoxedFuture<'_, Result> { - self.maintenance_checks.lock().unwrap().push(stale_after); + let checks = { + let mut checks = self.maintenance_checks.lock().unwrap(); + checks.push(stale_after); + checks.len() + }; let held = match *self.maintenance_beat_age.lock().unwrap() { Some(age) => age < stale_after, None => *self.maintenance_held.lock().unwrap(), }; let error = self.maintenance_error.lock().unwrap().clone(); + let mut hook = self.after_maintenance_check.lock().unwrap(); + if hook.as_ref().is_some_and(|(n, _)| *n == checks) { + let (_, run) = hook.take().expect("checked"); + drop(hook); + run(self); + } let delay = *self.maintenance_delay.lock().unwrap(); Box::pin(async move { if let Some(delay) = delay { @@ -8113,7 +8471,7 @@ async fn the_wait_reports_each_outcome_as_itself() { let (_tx, mut quiet) = Interrupt::new(); assert_eq!( - wait_for_quiet(&fake, &store(), &claim, wait, &mut quiet, announce).await, + wait_for_quiet(&fake, &[store()], &claim, wait, &mut quiet, announce).await, WaitOutcome::Quiet ); // A FAILED record whose target is still CREATING is listed by the @@ -8129,7 +8487,7 @@ async fn the_wait_reports_each_outcome_as_itself() { .insert(arn_n(3), row(&arn_n(3), BackupStatus::Failed)); let started = tokio::time::Instant::now(); assert_eq!( - wait_for_quiet(&fake, &store(), &claim, wait, &mut quiet, announce).await, + wait_for_quiet(&fake, &[store()], &claim, wait, &mut quiet, announce).await, WaitOutcome::Quiet, "a FAILED restore record and a FAILED row with a dead owner are not waited on" ); @@ -8150,7 +8508,7 @@ async fn the_wait_reports_each_outcome_as_itself() { .unwrap() .push(dead_job(RestoreKind::Legacy, "Target")); let started = tokio::time::Instant::now(); - let busy = wait_for_quiet(&fake, &store(), &claim, wait, &mut quiet, announce).await; + let busy = wait_for_quiet(&fake, &[store()], &claim, wait, &mut quiet, announce).await; let expected = vec![ format!( "CreateBackup {} has a live owner (heartbeat within stale_job_after_secs)", @@ -8174,7 +8532,7 @@ async fn the_wait_reports_each_outcome_as_itself() { }); let started = tokio::time::Instant::now(); assert_eq!( - wait_for_quiet(&fake, &store(), &claim, wait, &mut interrupted, announce).await, + wait_for_quiet(&fake, &[store()], &claim, wait, &mut interrupted, announce).await, WaitOutcome::Interrupted("SIGTERM") ); assert!( @@ -8185,14 +8543,14 @@ async fn the_wait_reports_each_outcome_as_itself() { // One already received ends it before the first read. assert_eq!(interrupted.signalled(), Some("SIGTERM")); assert_eq!( - wait_for_quiet(&fake, &store(), &claim, wait, &mut interrupted, announce).await, + wait_for_quiet(&fake, &[store()], &claim, wait, &mut interrupted, announce).await, WaitOutcome::Interrupted("SIGTERM") ); // The catalog unreadable. *fake.list_error.lock().unwrap() = Some("pool exhausted".to_owned()); assert_eq!( - wait_for_quiet(&fake, &store(), &claim, wait, &mut quiet, announce).await, + wait_for_quiet(&fake, &[store()], &claim, wait, &mut quiet, announce).await, WaitOutcome::CatalogError( "cannot list the catalog's backup rows: Internal error: pool exhausted".to_owned() ) @@ -8203,7 +8561,7 @@ async fn the_wait_reports_each_outcome_as_itself() { *holder.lost.lock().unwrap() = true; tokio::time::sleep(Duration::from_millis(1_500)).await; assert_eq!( - wait_for_quiet(&fake, &store(), &claim, wait, &mut quiet, announce).await, + wait_for_quiet(&fake, &[store()], &claim, wait, &mut quiet, announce).await, WaitOutcome::Lost( "the maintenance claim was lost (another holder took it over)".to_owned() ) @@ -8446,7 +8804,7 @@ async fn the_wait_reads_a_live_owner_on_the_catalogs_clock() { r }); assert_eq!( - wait_for_quiet(&fake, &store(), &claim, wait, &mut quiet, |_| {}).await, + wait_for_quiet(&fake, &[store()], &claim, wait, &mut quiet, |_| {}).await, WaitOutcome::Busy(vec![format!( "CreateBackup {} has a live owner (heartbeat within stale_job_after_secs)", arn_n(1) @@ -8462,7 +8820,7 @@ async fn the_wait_reads_a_live_owner_on_the_catalogs_clock() { .unwrap() .heartbeat_at = seconds_since_epoch(); assert_eq!( - wait_for_quiet(&fake, &store(), &claim, wait, &mut quiet, |_| {}).await, + wait_for_quiet(&fake, &[store()], &claim, wait, &mut quiet, |_| {}).await, WaitOutcome::Quiet, "an owner whose heartbeat is old on the catalog's clock is dead" ); @@ -8544,8 +8902,9 @@ async fn a_cycle_run_by_the_claim_holder_takes_the_held_branch_whatever_its_wind assert!(!report.maintenance_held); assert_eq!( *fake.maintenance_checks.lock().unwrap(), - [Duration::ZERO], - "a default state reads the claim through the window it is given" + [Duration::ZERO, Duration::ZERO], + "a default state reads the claim through the window it is given, at the cycle's start \ + and again before the FAILED row's deletion" ); assert_eq!( report.failed_prefixes_deleted, @@ -10585,7 +10944,7 @@ mod review_round_2 { poll: Duration::from_secs(2), }; let (_tx, mut quiet) = Interrupt::new(); - let outcome = wait_for_quiet(&fake, &store(), &claim, wait, &mut quiet, |_| {}).await; + let outcome = wait_for_quiet(&fake, &[store()], &claim, wait, &mut quiet, |_| {}).await; assert_eq!( outcome, WaitOutcome::Busy(vec![format!( @@ -10594,7 +10953,7 @@ mod review_round_2 { ); fake.running_jobs.lock().unwrap().clear(); assert_eq!( - wait_for_quiet(&fake, &store(), &claim, wait, &mut quiet, |_| {}).await, + wait_for_quiet(&fake, &[store()], &claim, wait, &mut quiet, |_| {}).await, WaitOutcome::Quiet ); claim.release().await.unwrap(); @@ -11221,8 +11580,9 @@ async fn a_store_backed_create_answers_creating_and_its_job_writes_the_backup() ); assert_eq!( fake.maintenance_checks.lock().unwrap().as_slice(), - [Duration::from_secs(300), Duration::from_secs(300)], - "the maintenance claim is read before the permit, by the refused request too" + [Duration::from_secs(300); 3], + "the maintenance claim is read before the permit, by the refused request too, and by \ + the job again before its insert" ); // Three heartbeat periods inside the stream, so the timer beats before @@ -12630,8 +12990,9 @@ async fn a_stopped_heartbeat_timer_starts_no_put() { ); } -/// The maintenance check runs before the permit and the row, so a slow -/// check delays the request and not the job's first put: with a period of +/// The maintenance checks (the request's before the permit, the job's right +/// before its insert) run before the row, so a slow check delays the +/// request and not the job's first put: with a period of /// 100 ms, `stale_job_after` of 300 ms (the floor's ratio) and a check of /// 400 ms, the request takes at least the check, the row is inserted after /// it (its heartbeat the insert's, within two periods of the first put), and @@ -12655,7 +13016,11 @@ async fn a_slow_maintenance_check_delays_the_request_and_not_the_first_put() { asked.elapsed() >= 4 * period, "the request waited for the check" ); - assert_eq!(fake.maintenance_checks.lock().unwrap().len(), 1); + assert_eq!( + fake.maintenance_checks.lock().unwrap().len(), + 2, + "the request's check and the job's before its insert" + ); wait_for_jobs(&runtime).await; let row = fake.rows.lock().unwrap()[&details.backup_arn].clone(); assert_eq!( @@ -13671,8 +14036,8 @@ async fn a_store_backed_restore_answers_creating_and_its_job_loads_the_backup() assert_eq!(record.spool_host, None, "a filesystem store has no spool"); assert_eq!( fake.maintenance_checks.lock().unwrap().as_slice(), - [job_settings().stale_job_after], - "the maintenance claim was read once, after the registration" + [job_settings().stale_job_after; 2], + "the maintenance claim was read before the registration and after it" ); wait_for_jobs(&runtime).await; @@ -13896,12 +14261,15 @@ async fn the_manifest_request_refuses_in_order_before_the_registration() { assert_eq!(runtime.running_jobs(), 0); } -/// The maintenance claim is checked after the registration, not before: a -/// held claim leaves a record that was FAILED `maintenance` from RUNNING -/// under the job's claim, its target removed, and the request answered -/// `LimitExceededException`; the registration ran (the row is what the -/// command's wait sees). A claim that cannot be read fails the record with -/// the read's error and answers it. +/// The maintenance claim is read immediately before the registration and +/// again after it. Held before, the request is answered +/// `LimitExceededException` and nothing is registered, so no record lands +/// while a command holds the claim. Taken between the read before and the +/// registration, the record is FAILED `maintenance` from RUNNING under the +/// job's claim, its target removed, and the request answered +/// `LimitExceededException` (the record is what the command's wait sees). +/// A claim that cannot be read after the registration fails the record +/// with the read's error and answers it. #[tokio::test] async fn a_manifest_restore_under_a_maintenance_claim_fails_its_record_after_registering() { let mem = Arc::new(MemStore::default()); @@ -13915,6 +14283,14 @@ async fn a_manifest_restore_under_a_maintenance_claim_fails_its_record_after_reg true, ); let runtime = job_runtime(&mem); + // Held before the registration. + let checks = |fake: &Fake| fake.maintenance_checks.lock().unwrap().len(); + let take_claim_after = |fake: &Fake, n: usize| { + *fake.after_maintenance_check.lock().unwrap() = Some(( + n, + Box::new(|fake: &Fake| *fake.maintenance_held.lock().unwrap() = true), + )); + }; *fake.maintenance_held.lock().unwrap() = true; let err = restore_on(&fake, &runtime, &restore_body(json!({}))) .await @@ -13924,6 +14300,24 @@ async fn a_manifest_restore_under_a_maintenance_claim_fails_its_record_after_reg "{err:?}" ); wait_for_jobs(&runtime).await; + assert!( + fake.restore_claims.lock().unwrap().is_empty(), + "nothing is registered while the claim is held" + ); + assert!(restore_writes(&fake).is_empty()); + assert_eq!(runtime.free_slots(), 4); + + // Taken after the read before the registration. + *fake.maintenance_held.lock().unwrap() = false; + take_claim_after(&fake, checks(&fake) + 1); + let err = restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap_err(); + assert!( + matches!(err, DynamoDbError::LimitExceededException(ref m) if m.contains("maintenance claim")), + "{err:?}" + ); + wait_for_jobs(&runtime).await; assert_eq!( fake.restore_claims.lock().unwrap().len(), 1, @@ -13970,7 +14364,13 @@ async fn a_manifest_restore_under_a_maintenance_claim_fails_its_record_after_reg ); *fake.maintenance_held.lock().unwrap() = false; - *fake.maintenance_error.lock().unwrap() = Some("pool exhausted".to_owned()); + let fail_after = checks(&fake) + 1; + *fake.after_maintenance_check.lock().unwrap() = Some(( + fail_after, + Box::new(|fake: &Fake| { + *fake.maintenance_error.lock().unwrap() = Some("pool exhausted".to_owned()); + }), + )); let err = restore_on(&fake, &runtime, &restore_body(json!({}))) .await .unwrap_err(); @@ -22300,3 +22700,2864 @@ async fn a_fence_refuses_a_file_of_this_catalog_under_another_store_id() { BackupStatus::Available ); } + +// --- backup sync --move-store, --forget-store, --forget-backup --------------- + +/// The root the rows were written under before the store moved. +fn old_store() -> StoreIdentity { + StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: "/srv/old".to_owned(), + store_id: "store-0".to_owned(), + } +} + +/// A row of `arn` under the old root. +fn old_row(arn: &str, status: BackupStatus) -> BackupRow { + let mut row = row(arn, status); + row.store = old_store(); + row +} + +fn seed_rows(fake: &Fake, rows: &[BackupRow]) { + let mut held = fake.rows.lock().unwrap(); + for row in rows { + held.insert(row.backup_arn.clone(), row.clone()); + } +} + +/// The stop of a run nothing stops. +fn never() -> Option { + None +} + +/// What a run told of its wait ignores. +fn no_wait(_: Duration) {} + +/// A forget run with `owner_window` and `stop` that waits for nothing +/// before its FAILED and DELETED removals (as on SQLite). +fn forget_run<'a>( + owner_window: Duration, + stop: super::sync::Stop<'a>, +) -> super::store_maintenance::ForgetRun<'a> { + super::store_maintenance::ForgetRun { + owner_window, + removals_from: None, + on_wait: &no_wait, + stop, + configured: None, + unmounted_root: None, + deletions_finished: &[], + } +} + +/// `--move-store /srv/old` into the store `runtime` is configured with, +/// with the servers' 300 s owner window. +async fn move_old(fake: &Fake, runtime: &BackupRuntime) -> super::store_maintenance::MoveReport { + let configured = runtime.configured_store().unwrap(); + let identity = configured.identity().unwrap(); + super::store_maintenance::move_store( + fake, + super::store_maintenance::MoveTarget { + configured, + identity: &identity, + from_kind: StoreKind::Filesystem, + from_root: "/srv/old", + owner_window: Duration::from_secs(300), + stop: &never, + }, + ) + .await + .unwrap() +} + +fn sorted(mut arns: Vec) -> Vec { + arns.sort(); + arns +} + +/// `--move-store` over a copied tree, one row of each status the move +/// resolves or carries: a dead CREATING row whose copied prefix is complete +/// is resolved AVAILABLE against the configured store; a DELETING row whose copy holds no verified manifest has its +/// deletion finished in the configured store (the copied leftover deleted) +/// and moves DELETED; DELETED and FAILED rows without a verified manifest +/// and a complete AVAILABLE row move. Every row then names the configured +/// store, which serves it (`is_store_of`), the rewrite is one call from the +/// old root, and the old root lists nothing. A reconciler cycle afterwards +/// (no claim) cleans the moved tombstones' leftovers in the configured +/// store and removes the tombstones past their retention. +#[tokio::test] +async fn move_store_resolves_checks_and_moves_each_status_of_the_old_root() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let runtime = runtime_on(&mem, store()); + + let resolved_available = seed_dead_creating_with_manifest(&fake, &mem, true); + fake.rows + .lock() + .unwrap() + .get_mut(&resolved_available) + .unwrap() + .store = old_store(); + let deleting = sync_arn(2); + let deleting_leftover = format!("{}data/000001.json.gz", prefix_of(&deleting)); + mem.insert(&deleting_leftover, b"x"); + let deleted = sync_arn(3); + let deleted_leftover = format!("{}data/000001.json.gz", prefix_of(&deleted)); + mem.insert(&deleted_leftover, b"x"); + let failed = sync_arn(4); + let failed_leftover = format!("{}data/000001.json.gz", prefix_of(&failed)); + mem.insert(&failed_leftover, b"x"); + let available = sync_arn(5); + put_backup(&mem, &available); + let mut rows = vec![ + old_row(&deleting, BackupStatus::Deleting), + old_row(&deleted, BackupStatus::Deleted), + old_row(&failed, BackupStatus::Failed), + old_row(&available, BackupStatus::Available), + ]; + for row in &mut rows[1..3] { + row.completed_at = Some(1.0); + } + seed_rows(&fake, &rows); + + let report = move_old(&fake, &runtime).await; + assert!(report.complete(), "{report:?}"); + assert_eq!(report.rows, 5); + assert_eq!( + report.resolved, + [(resolved_available.clone(), BackupStatus::Available)] + ); + assert_eq!(report.finished, std::slice::from_ref(&deleting)); + assert_eq!(report.rewritten, 5); + assert!(report.left.is_empty()); + assert!(report.gone.is_empty()); + let all = sorted(vec![ + resolved_available.clone(), + deleting.clone(), + deleted.clone(), + failed.clone(), + available.clone(), + ]); + assert_eq!(sorted(report.moved.clone()), all); + let configured = runtime.configured_store().unwrap(); + { + let held = fake.rows.lock().unwrap(); + for arn in &all { + assert_eq!(held[arn].store, store(), "{arn}"); + assert!(configured.is_store_of(&held[arn].store), "{arn}"); + } + assert_eq!(held[&resolved_available].status, BackupStatus::Available); + assert_eq!(held[&deleting].status, BackupStatus::Deleted); + assert_eq!(held[&available].status, BackupStatus::Available); + } + let calls = fake.rewrites_seen.lock().unwrap().clone(); + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].0, StoreKind::Filesystem); + assert_eq!(calls[0].1, "/srv/old"); + assert_eq!(calls[0].3, store()); + assert!( + !mem.keys().contains(&deleting_leftover), + "the deletion was finished" + ); + assert!(mem.keys().contains(&deleted_leftover)); + assert!(mem.keys().contains(&failed_leftover)); + + // The configured store's reconciler cleans the moved tombstones. + let cycle = cycle_on(&fake, &runtime).await; + assert_eq!(cycle.store_healthy, Some(true)); + assert!(!mem.keys().contains(&deleted_leftover), "{cycle:?}"); + assert!(!mem.keys().contains(&failed_leftover), "{cycle:?}"); + let held = fake.rows.lock().unwrap(); + assert!( + !held.contains_key(&deleted), + "the DELETED tombstone is removed" + ); + assert!( + !held.contains_key(&failed), + "the FAILED tombstone is removed" + ); + assert_eq!(held[&available].status, BackupStatus::Available); +} + +/// Every row that stops the move is named in one pass and no row is +/// moved: a CREATING row with a live owner; a DELETING, a DELETED and a +/// FAILED row whose copied prefix holds a verified manifest (the +/// DELETING row's copy is not deleted); AVAILABLE rows with no copy, a data +/// file missing, a data file of another size, no sidecar, a manifest naming +/// another backup, and a manifest whose size is not the row's. The one +/// complete AVAILABLE row is not moved either, and `rewrite_store` is never +/// called. +#[tokio::test] +async fn move_store_names_every_row_that_stops_it_and_moves_none() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let runtime = runtime_on(&mem, store()); + + let live = sync_arn(1); + let deleting = sync_arn(2); + let deleted = sync_arn(3); + let failed = sync_arn(4); + for arn in [&deleting, &deleted, &failed] { + put_backup(&mem, arn); + } + let no_copy = sync_arn(5); + let missing_file = sync_arn(6); + let mut objects = backup_objects(&missing_file); + let dropped = objects + .iter() + .position(|(k, _)| k.contains("/data/")) + .unwrap(); + objects.remove(dropped); + for (key, bytes) in objects { + mem.insert(&key, &bytes); + } + let short_file = sync_arn(7); + for (key, mut bytes) in backup_objects(&short_file) { + if key.contains("/data/") && bytes.len() > 1 { + bytes.pop(); + } + mem.insert(&key, &bytes); + } + let no_sidecar = sync_arn(8); + for (key, bytes) in backup_objects(&no_sidecar) { + if !key.ends_with("extenddb-manifest.sha256") { + mem.insert(&key, &bytes); + } + } + let other_backup = sync_arn(9); + for (key, bytes) in backup_objects(&sync_arn(19)) { + mem.insert( + &key.replacen(&prefix_of(&sync_arn(19)), &prefix_of(&other_backup), 1), + &bytes, + ); + } + let other_size = sync_arn(10); + put_backup(&mem, &other_size); + let complete = sync_arn(11); + put_backup(&mem, &complete); + + let mut sized = old_row(&other_size, BackupStatus::Available); + sized.size_bytes += 1; + let rows = vec![ + old_row(&live, BackupStatus::Creating), + old_row(&deleting, BackupStatus::Deleting), + old_row(&deleted, BackupStatus::Deleted), + old_row(&failed, BackupStatus::Failed), + old_row(&no_copy, BackupStatus::Available), + old_row(&missing_file, BackupStatus::Available), + old_row(&short_file, BackupStatus::Available), + old_row(&no_sidecar, BackupStatus::Available), + old_row(&other_backup, BackupStatus::Available), + sized, + old_row(&complete, BackupStatus::Available), + ]; + seed_rows(&fake, &rows); + let keys_before = mem.keys(); + + let report = move_old(&fake, &runtime).await; + assert!(!report.complete()); + let why: HashMap = report + .stopping + .iter() + .map(|(row, why)| (row.backup_arn.clone(), why.clone())) + .collect(); + assert_eq!(report.stopping.len(), 10, "{why:#?}"); + assert!(why[&live].contains("live owner"), "{}", why[&live]); + for (arn, status) in [ + (&deleting, "DELETING"), + (&deleted, "DELETED"), + (&failed, "FAILED"), + ] { + assert!( + why[arn].starts_with(&format!( + "{status} whose prefix under the configured root holds a verified manifest" + )), + "{}", + why[arn] + ); + assert!(why[arn].contains("--forget-backup"), "{}", why[arn]); + } + assert!(why[&no_copy].contains("no manifest"), "{}", why[&no_copy]); + assert!( + why[&missing_file].contains("is not in the store"), + "{}", + why[&missing_file] + ); + assert!( + why[&short_file].contains("bytes in the store"), + "{}", + why[&short_file] + ); + assert!( + why[&no_sidecar].contains("without its sidecar"), + "{}", + why[&no_sidecar] + ); + assert!( + why[&other_backup].contains("not the row's backup"), + "{}", + why[&other_backup] + ); + assert!( + why[&other_size].contains("another backup under the same ARN"), + "{}", + why[&other_size] + ); + assert!(!why.contains_key(&complete)); + assert!(report.moved.is_empty()); + assert!( + fake.rewrites_seen.lock().unwrap().is_empty(), + "nothing is rewritten after a stop" + ); + let held = fake.rows.lock().unwrap(); + for row in &rows { + assert_eq!(held[&row.backup_arn], *row, "{}", row.backup_arn); + } + assert_eq!(mem.keys(), keys_before, "no object was deleted"); +} + +/// The rewrite moves only what is still the old root's: a CreateBackup of a +/// server still on the old root whose CREATING row lands after the checks +/// is left there and reported (the move is not complete), and a rewrite +/// that stops part way leaves the rest under the old root, listed, with +/// the rows moved before the stop counted as moved. +#[tokio::test] +async fn rows_the_rewrite_does_not_move_are_listed_as_left() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let runtime = runtime_on(&mem, store()); + let rows: Vec = (1..=3) + .map(|n| { + let arn = sync_arn(n); + put_backup(&mem, &arn); + old_row(&arn, BackupStatus::Available) + }) + .collect(); + seed_rows(&fake, &rows); + let landed = old_row(&sync_arn(4), BackupStatus::Creating); + fake.lands_before_rewrite + .lock() + .unwrap() + .push(landed.clone()); + + let report = move_old(&fake, &runtime).await; + assert!(report.stopping.is_empty(), "{:?}", report.stopping); + assert_eq!(report.rewritten, 3); + assert_eq!(report.moved.len(), 3); + assert_eq!( + report.left, + [super::store_maintenance::NamedRow { + status: BackupStatus::Creating, + backup_arn: landed.backup_arn.clone(), + account_id: ACCOUNT.to_owned(), + }] + ); + assert!(!report.complete()); + assert_eq!( + fake.rows.lock().unwrap()[&landed.backup_arn].store, + old_store() + ); + + // A rewrite that stops after one row. + let fake = Arc::new(Fake::default()); + seed_rows(&fake, &rows); + *fake.fail_rewrite_after.lock().unwrap() = Some(1); + let report = move_old(&fake, &runtime).await; + assert!(report.rewrite_error.is_some()); + assert_eq!(report.moved.len(), 1); + assert_eq!(report.left.len(), 2); + assert!(!report.complete()); +} + +/// The fence runs once before the rewrite: a root another catalog adopted +/// after the store's check gets no row, though every row passed its check. +#[tokio::test] +async fn move_store_on_a_root_another_catalog_adopted_moves_nothing() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let runtime = runtime_on(&mem, store()); + let arn = sync_arn(1); + put_backup(&mem, &arn); + seed_rows(&fake, &[old_row(&arn, BackupStatus::Available)]); + mem.insert(STORE_IDENTITY_KEY, &adopted_file()); + + let report = move_old(&fake, &runtime).await; + assert!(report.stopping.is_empty(), "{:?}", report.stopping); + let stopped = report.stopped.clone().unwrap(); + assert!(stopped.contains("fence refused"), "{stopped}"); + assert!(fake.rewrites_seen.lock().unwrap().is_empty()); + assert_eq!(fake.rows.lock().unwrap()[&arn].store, old_store()); + assert_fenced(&runtime); +} + +/// Where rows are unique by ARN alone (Cassandra), a moved row whose +/// account and backup id a row of the configured root holds under another +/// spelling stops the move, naming that row: no account holds two rows of +/// one backup id. +#[tokio::test] +async fn move_store_refuses_a_second_row_of_one_backup_id() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + fake.unique_by_arn + .store(true, std::sync::atomic::Ordering::Release); + let runtime = runtime_on(&mem, store()); + let ours = sync_arn(1); + let other_spelling = ours.replace(REGION, "eu-west-1"); + put_backup(&mem, &other_spelling); + seed_rows( + &fake, + &[ + row(&ours, BackupStatus::Available), + old_row(&other_spelling, BackupStatus::Available), + ], + ); + + let report = move_old(&fake, &runtime).await; + assert_eq!(report.stopping.len(), 1, "{:?}", report.stopping); + assert_eq!(report.stopping[0].0.backup_arn, other_spelling); + assert!( + report.stopping[0].1.contains("two rows of backup id") + && report.stopping[0].1.contains(&ours), + "{}", + report.stopping[0].1 + ); + assert!(fake.rewrites_seen.lock().unwrap().is_empty()); +} + +/// The wait of `--move-store` and `--forget-store` covers the old root's +/// store ids as well as the configured store's: a live CreateBackup of a +/// server still configured with the old root is waited on. +#[tokio::test(start_paused = true)] +async fn the_wait_covers_every_store_it_is_given() { + use maintenance::{Interrupt, Wait, WaitOutcome, wait_for_quiet}; + let fake = Fake::default(); + let mut live = old_row(&sync_arn(1), BackupStatus::Creating); + live.heartbeat_at = seconds_since_epoch(); + seed_rows(&fake, &[live.clone()]); + let holder = Arc::new(FakeHolder::default()); + let claim = held(&holder); + let wait = Wait { + owner_window: Duration::from_secs(300), + limit: Duration::ZERO, + poll: Duration::from_secs(1), + }; + let (_tx, mut quiet) = Interrupt::new(); + assert_eq!( + wait_for_quiet(&fake, &[store()], &claim, wait, &mut quiet, |_| {}).await, + WaitOutcome::Quiet + ); + assert_eq!( + wait_for_quiet( + &fake, + &[store(), old_store()], + &claim, + wait, + &mut quiet, + |_| {} + ) + .await, + WaitOutcome::Busy(vec![format!( + "CreateBackup {} has a live owner (heartbeat within stale_job_after_secs)", + live.backup_arn + )]) + ); + assert_eq!( + wait_for_quiet(&fake, &[], &claim, wait, &mut quiet, |_| {}).await, + WaitOutcome::Quiet + ); +} + +/// The root an option names: the configured root is refused; a root with +/// no row, of either kind, is refused with the stores the catalog knows, +/// a root that differs from one by a trailing `/` or by the `?endpoint=` +/// suffix among them; an S3 label's rows are found as S3 rows, with their +/// store ids. +#[tokio::test] +async fn locate_root_finds_the_kind_and_ids_and_refuses_the_rest() { + use super::store_maintenance::{RootRefusal, locate_root}; + let fake = Fake::default(); + let s3_root = "s3://bucket/prefix?endpoint=http://127.0.0.1:9000".to_owned(); + let mut s3_row = row(&sync_arn(1), BackupStatus::Available); + s3_row.store = StoreIdentity { + store_kind: StoreKind::S3, + store_root: s3_root.clone(), + store_id: "s3-id".to_owned(), + }; + seed_rows( + &fake, + &[ + s3_row, + old_row(&sync_arn(2), BackupStatus::Failed), + row(&sync_arn(3), BackupStatus::Available), + ], + ); + let configured = store().store_root; + assert!(matches!( + locate_root(&fake, &configured, Some(&configured)).await, + Err(RootRefusal::ConfiguredRoot) + )); + let found = locate_root(&fake, &s3_root, Some(&configured)) + .await + .unwrap(); + assert_eq!(found.kind, StoreKind::S3); + assert_eq!(found.store_ids, ["s3-id"]); + let found = locate_root(&fake, "/srv/old", None).await.unwrap(); + assert_eq!(found.kind, StoreKind::Filesystem); + assert_eq!(found.identities(), [old_store()]); + for absent in ["/srv/old/", "s3://bucket/prefix", "/srv/none"] { + match locate_root(&fake, absent, Some(&configured)).await { + Err(RootRefusal::NoRows(known)) => { + let roots: Vec<&str> = known.iter().map(|c| c.store.store_root.as_str()).collect(); + assert_eq!( + roots, + ["/srv/old", "/var/lib/extenddb/backups", s3_root.as_str()], + "{absent}" + ); + } + other => panic!("{absent}: {other:?}"), + } + } +} + +/// `--forget-store`: every row of the root is removed outright, each status +/// by its removal (no DELETED tombstone is left, and no object is deleted), +/// except a row a restore is reading, a FAILED row and a CREATING row whose +/// owner is alive, which are left and reported; the configured store's row +/// is untouched. Once the root is configured again, a reconciler cycle +/// deletes none of its bytes: no row is left for a duty to act on. +#[tokio::test] +async fn forget_store_removes_the_roots_rows_outright_and_reports_what_it_left() { + use super::store_maintenance::{Forgotten, forget_store}; + let fake = Arc::new(Fake::default()); + let now = seconds_since_epoch(); + let available = sync_arn(1); + let in_use = sync_arn(2); + let deleting = sync_arn(3); + let deleted = sync_arn(4); + let failed = sync_arn(5); + let failed_live = sync_arn(6); + let creating = sync_arn(7); + let creating_live = sync_arn(8); + let ours = sync_arn(9); + let mut rows = vec![ + old_row(&available, BackupStatus::Available), + old_row(&in_use, BackupStatus::Available), + old_row(&deleting, BackupStatus::Deleting), + old_row(&deleted, BackupStatus::Deleted), + old_row(&failed, BackupStatus::Failed), + old_row(&failed_live, BackupStatus::Failed), + old_row(&creating, BackupStatus::Creating), + old_row(&creating_live, BackupStatus::Creating), + ]; + rows[5].heartbeat_at = now; + rows[7].heartbeat_at = now; + for row in &mut rows { + row.completed_at = Some(1.0); + } + seed_rows(&fake, &rows); + seed_rows(&fake, &[row(&ours, BackupStatus::Available)]); + fake.forget_answers + .lock() + .unwrap() + .insert(in_use.clone(), ForgetRow::InUse("Target".to_owned())); + let old_mem = Arc::new(MemStore::default()); + for arn in [&available, &in_use, &deleting, &deleted, &failed, &creating] { + put_backup(&old_mem, arn); + } + + let report = forget_store( + &*fake, + StoreKind::Filesystem, + "/srv/old", + forget_run(Duration::from_secs(300), &never), + ) + .await + .unwrap(); + let outcome: HashMap = report + .rows + .iter() + .map(|(row, f)| (row.backup_arn.clone(), f.clone())) + .collect(); + assert_eq!( + outcome[&available], + Forgotten::Removed(BackupStatus::Available) + ); + assert_eq!( + outcome[&deleting], + Forgotten::Removed(BackupStatus::Deleting) + ); + assert_eq!(outcome[&deleted], Forgotten::Removed(BackupStatus::Deleted)); + assert_eq!(outcome[&failed], Forgotten::Removed(BackupStatus::Failed)); + assert_eq!( + outcome[&creating], + Forgotten::Removed(BackupStatus::Creating) + ); + assert_eq!( + outcome[&in_use], + Forgotten::InUse("Target".to_owned(), ACCOUNT.to_owned()) + ); + assert_eq!( + outcome[&failed_live], + Forgotten::LiveOwner(BackupStatus::Failed) + ); + assert_eq!( + outcome[&creating_live], + Forgotten::LiveOwner(BackupStatus::Creating) + ); + assert_eq!(report.removed(), 5); + assert_eq!(report.rows_left, Some(3)); + assert!(!report.complete()); + { + let held = fake.rows.lock().unwrap(); + for arn in [&available, &deleting, &deleted, &failed, &creating] { + assert!(!held.contains_key(arn), "{arn}: no tombstone is left"); + } + assert_eq!(held[&ours].store, store()); + assert_eq!(held[&ours].status, BackupStatus::Available); + } + + // The store's return: no duty acts on its bytes. + let runtime = runtime_on(&old_mem, old_store()); + let keys_before = old_mem.keys(); + let cycle = cycle_on(&fake, &runtime).await; + assert_eq!(cycle.store_healthy, Some(true), "{cycle:?}"); + assert_eq!(old_mem.keys(), keys_before); +} + +/// `--forget-backup`, in the order given, up to the first ARN whose row is +/// not removed: an AVAILABLE, a DELETING, a dead FAILED and a dead CREATING +/// row are removed outright; a CREATING row with a live owner stops the +/// run, and the ARN after it is not reached. Alone, each refusal leaves its +/// row: a backup a restore target is reading, a FAILED row whose owner is +/// alive, a row under another spelling of the ARN (named), an earlier +/// release's in-catalog backup (said so, nothing changed), an ARN with no +/// row and a string that is no ARN. +#[tokio::test] +async fn forget_backup_removes_each_row_in_order_and_stops_at_the_first_refusal() { + use super::store_maintenance::{ForgetBackup, Forgotten, forget_backups}; + let fake = Arc::new(Fake::default()); + let now = seconds_since_epoch(); + let [available, deleting, failed, creating, live, after] = [1, 2, 3, 4, 5, 6].map(sync_arn); + let mut live_row = old_row(&live, BackupStatus::Creating); + live_row.heartbeat_at = now; + seed_rows( + &fake, + &[ + row(&available, BackupStatus::Available), + old_row(&deleting, BackupStatus::Deleting), + row(&failed, BackupStatus::Failed), + old_row(&creating, BackupStatus::Creating), + live_row, + row(&after, BackupStatus::Available), + ], + ); + let asked = [&available, &deleting, &failed, &creating, &live, &after].map(|a| a.clone()); + let report = forget_backups( + &*fake, + &asked, + REGION, + forget_run(Duration::from_secs(300), &never), + ) + .await; + assert_eq!( + report.done, + [ + ( + available.clone(), + ForgetBackup::Row( + BackupStatus::Available, + Forgotten::Removed(BackupStatus::Available) + ) + ), + ( + deleting.clone(), + ForgetBackup::Row( + BackupStatus::Deleting, + Forgotten::Removed(BackupStatus::Deleting) + ) + ), + ( + failed.clone(), + ForgetBackup::Row( + BackupStatus::Failed, + Forgotten::Removed(BackupStatus::Failed) + ) + ), + ( + creating.clone(), + ForgetBackup::Row( + BackupStatus::Creating, + Forgotten::Removed(BackupStatus::Creating) + ) + ), + ( + live.clone(), + ForgetBackup::Row( + BackupStatus::Creating, + Forgotten::LiveOwner(BackupStatus::Creating) + ) + ), + ] + ); + assert!(!report.complete(asked.len())); + { + let held = fake.rows.lock().unwrap(); + for arn in [&available, &deleting, &failed, &creating] { + assert!(!held.contains_key(arn), "{arn}"); + } + assert!(held.contains_key(&live)); + assert!(held.contains_key(&after), "not reached"); + } + + let one = |arn: &str| vec![arn.to_owned()]; + let in_use = sync_arn(7); + let failed_live = sync_arn(8); + let stored = sync_arn(9); + let legacy = sync_arn(10); + let mut live_failed = row(&failed_live, BackupStatus::Failed); + live_failed.heartbeat_at = now; + seed_rows( + &fake, + &[ + row(&in_use, BackupStatus::Available), + live_failed, + row(&stored, BackupStatus::Available), + ], + ); + fake.forget_answers + .lock() + .unwrap() + .insert(in_use.clone(), ForgetRow::InUse("Target".to_owned())); + fake.descriptions.lock().unwrap().insert( + legacy.clone(), + backup_description("AVAILABLE", "PAY_PER_REQUEST"), + ); + let written = stored.replace(REGION, "eu-west-1"); + for (asked, expected) in [ + ( + in_use.clone(), + ForgetBackup::Row( + BackupStatus::Available, + Forgotten::InUse("Target".to_owned(), ACCOUNT.to_owned()), + ), + ), + ( + failed_live.clone(), + ForgetBackup::Row( + BackupStatus::Failed, + Forgotten::LiveOwner(BackupStatus::Failed), + ), + ), + ( + written.clone(), + ForgetBackup::OtherSpelling(stored.clone(), BackupStatus::Available), + ), + (legacy.clone(), ForgetBackup::Legacy), + ] { + let report = forget_backups( + &*fake, + &one(&asked), + REGION, + forget_run(Duration::from_secs(300), &never), + ) + .await; + assert_eq!(report.done, [(asked.clone(), expected)], "{asked}"); + assert!(!report.complete(1)); + } + let report = forget_backups( + &*fake, + &one("not-an-arn"), + REGION, + forget_run(Duration::from_secs(300), &never), + ) + .await; + assert!( + matches!(&report.done[0].1, ForgetBackup::Invalid(_)), + "{:?}", + report.done + ); + let none = forget_backups( + &*fake, + &one(&sync_arn(11)), + REGION, + forget_run(Duration::from_secs(300), &never), + ) + .await; + assert_eq!(none.done, [(sync_arn(11), ForgetBackup::NoRow)]); + assert!(none.complete(1), "an ARN no row carries counts as done"); + let held = fake.rows.lock().unwrap(); + for arn in [&in_use, &failed_live, &stored] { + assert!(held.contains_key(arn), "{arn}"); + } + assert!(fake.descriptions.lock().unwrap().contains_key(&legacy)); +} + +/// A stop asked before the first ARN or row changes nothing. +#[tokio::test] +async fn a_stop_before_the_first_row_forgets_nothing() { + use super::store_maintenance::{forget_backups, forget_store}; + let fake = Arc::new(Fake::default()); + let arn = sync_arn(1); + seed_rows(&fake, &[old_row(&arn, BackupStatus::Available)]); + let stop = || Some("SIGINT received".to_owned()); + let report = forget_backups( + &*fake, + std::slice::from_ref(&arn), + REGION, + forget_run(Duration::ZERO, &stop), + ) + .await; + assert_eq!(report.stopped.as_deref(), Some("SIGINT received")); + assert!(report.done.is_empty()); + let report = forget_store( + &*fake, + StoreKind::Filesystem, + "/srv/old", + forget_run(Duration::ZERO, &stop), + ) + .await + .unwrap(); + assert_eq!(report.stopped.as_deref(), Some("SIGINT received")); + assert!(report.rows.is_empty()); + assert!(fake.rows.lock().unwrap().contains_key(&arn)); + assert!(fake.forgets_seen.lock().unwrap().is_empty()); +} + +/// A restored catalog whose newest AVAILABLE row's backup was deleted after +/// the dump: the store's check fails naming it ("manifest probe failed"); +/// `--forget-backup` of that ARN removes the row, and the next check passes +/// on the next-newest row, whose manifest is there. +#[tokio::test] +async fn forget_backup_of_the_newest_rows_deleted_backup_heals_the_store() { + use super::store_maintenance::forget_backups; + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let kept = sync_arn(1); + let gone = sync_arn(2); + put_backup(&mem, &kept); + seed_rows( + &fake, + &[ + row(&kept, BackupStatus::Available), + row(&gone, BackupStatus::Available), + ], + ); + let runtime = runtime_on(&mem, store()); + let configured = runtime.configured_store().unwrap(); + let check = super::check_configured_store(configured, &*fake, "test").await; + assert!( + matches!(&check, Err(StoreProblem::ManifestMissing { backup_arn }) if *backup_arn == gone), + "{check:?}" + ); + configured.record_check(check); + assert!(!configured.is_healthy()); + + let report = forget_backups( + &*fake, + std::slice::from_ref(&gone), + REGION, + forget_run(Duration::from_secs(300), &never), + ) + .await; + assert!(report.complete(1), "{:?}", report.done); + let check = super::check_configured_store(configured, &*fake, "test").await; + assert_eq!(check.unwrap(), store()); + assert!(fake.rows.lock().unwrap().contains_key(&kept)); +} + +/// A deletion duty that listed a row before a maintenance command removed +/// it, or moved it to another store, acts on nothing (the row is read once +/// more after the fence): the FAILED-prefix duty keeps the prefix of a row +/// `--forget-backup` removed after the listing, and the tombstone duty +/// keeps a row `--move-store` moved after the listing. +#[tokio::test] +async fn a_duty_whose_row_is_forgotten_or_moved_after_its_listing_acts_on_nothing() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let runtime = runtime_on(&mem, store()); + let forgotten = sync_arn(1); + let moved = sync_arn(2); + let leftover = format!("{}data/000001.json.gz", prefix_of(&forgotten)); + mem.insert(&leftover, b"x"); + seed_rows(&fake, &[row(&forgotten, BackupStatus::Failed)]); + let gone = forgotten.clone(); + *fake.after_older_than.lock().unwrap() = Some(( + BackupStatus::Failed, + StaleClock::Heartbeat, + Box::new(move |fake: &Fake| { + fake.rows.lock().unwrap().remove(&gone); + }), + )); + let cycle = cycle_on(&fake, &runtime).await; + assert!(cycle.failed_prefixes_deleted.is_empty(), "{cycle:?}"); + assert!( + mem.keys().contains(&leftover), + "the forgotten row's bytes are kept" + ); + + let mut tombstone = row(&moved, BackupStatus::Deleted); + tombstone.completed_at = Some(1.0); + seed_rows(&fake, &[tombstone]); + let elsewhere = moved.clone(); + *fake.after_older_than.lock().unwrap() = Some(( + BackupStatus::Deleted, + StaleClock::Completed, + Box::new(move |fake: &Fake| { + fake.rows.lock().unwrap().get_mut(&elsewhere).unwrap().store = old_store(); + }), + )); + let cycle = cycle_on(&fake, &runtime).await; + assert!(cycle.tombstones_removed.is_empty(), "{cycle:?}"); + assert_eq!(fake.rows.lock().unwrap()[&moved].store, old_store()); + assert!( + fake.after_older_than.lock().unwrap().is_none(), + "the hook ran" + ); +} + +/// Plain sync's report of other stores: per store other than the +/// configured one (another root, another kind, the configured root under +/// another id), the rows per status and each CREATING and DELETING row by +/// ARN; the configured store's rows are not in it. A reconciler cycle of +/// the configured store deletes none of the other stores' prefixes. +#[tokio::test] +async fn the_report_of_other_stores_counts_and_names_their_rows_and_nothing_acts_on_them() { + use super::store_maintenance::{NamedRow, other_stores}; + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let runtime = runtime_on(&mem, store()); + let s3 = StoreIdentity { + store_kind: StoreKind::S3, + store_root: "s3://bucket/p".to_owned(), + store_id: "s3-id".to_owned(), + }; + let other_id = StoreIdentity { + store_id: "store-9".to_owned(), + ..store() + }; + let mut s3_failed = row(&sync_arn(5), BackupStatus::Failed); + s3_failed.store = s3.clone(); + let mut other_id_row = row(&sync_arn(6), BackupStatus::Available); + other_id_row.store = other_id.clone(); + let mut old_failed = old_row(&sync_arn(3), BackupStatus::Failed); + old_failed.completed_at = Some(1.0); + let mut old_deleted = old_row(&sync_arn(4), BackupStatus::Deleted); + old_deleted.completed_at = Some(1.0); + seed_rows( + &fake, + &[ + row(&sync_arn(10), BackupStatus::Available), + row(&sync_arn(11), BackupStatus::Creating), + old_row(&sync_arn(1), BackupStatus::Creating), + old_row(&sync_arn(2), BackupStatus::Deleting), + old_failed, + old_deleted, + old_row(&sync_arn(7), BackupStatus::Available), + old_row(&sync_arn(8), BackupStatus::Available), + s3_failed, + other_id_row, + ], + ); + for n in [3, 4, 5] { + mem.insert( + &format!("{}data/000001.json.gz", prefix_of(&sync_arn(n))), + b"x", + ); + } + put_backup(&mem, &sync_arn(10)); + + let others = other_stores(&*fake, &store()).await.unwrap(); + let stores: Vec<&StoreIdentity> = others.iter().map(|o| &o.store).collect(); + assert_eq!(stores, [&old_store(), &other_id, &s3]); + assert_eq!( + others[0].counts, + [ + (BackupStatus::Creating, 1), + (BackupStatus::Available, 2), + (BackupStatus::Deleting, 1), + (BackupStatus::Deleted, 1), + (BackupStatus::Failed, 1), + ] + ); + let named = |status, n| NamedRow { + status, + backup_arn: sync_arn(n), + account_id: ACCOUNT.to_owned(), + }; + assert_eq!( + others[0].unresolved, + [ + named(BackupStatus::Creating, 1), + named(BackupStatus::Deleting, 2) + ] + ); + assert_eq!(others[1].counts, [(BackupStatus::Available, 1)]); + assert!(others[1].unresolved.is_empty()); + assert_eq!(others[2].counts, [(BackupStatus::Failed, 1)]); + + // The configured root under a second id would make the store unhealthy; + // the cycle is about the other roots' prefixes. + fake.rows.lock().unwrap().remove(&sync_arn(6)); + let keys_before = mem.keys(); + let cycle = cycle_on(&fake, &runtime).await; + assert_eq!(cycle.store_healthy, Some(true)); + assert_eq!( + mem.keys(), + keys_before, + "no prefix of another store is deleted" + ); +} + +/// A dead CREATING row of the old root whose copy under the configured root +/// holds no complete backup stops the move, and the move writes no status +/// for it: the copy proves nothing about the old root's prefix, and a FAILED +/// row would have the reconciler delete the copy (and a server still on the +/// old root the original). Each shape of copy: none at all, a data file +/// missing, one short, one long, no sidecar, a manifest of another backup. +/// Each row is named with the reason and the two ways out, keeps its +/// status, heartbeat and store, and nothing is rewritten or deleted. +#[tokio::test] +async fn a_dead_creating_row_whose_copy_is_incomplete_stops_the_move_unwritten() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let runtime = runtime_on(&mem, store()); + let no_copy = sync_arn(1); + let missing = sync_arn(2); + let short = sync_arn(3); + let long = sync_arn(4); + let no_sidecar = sync_arn(5); + let other = sync_arn(6); + let mut objects = backup_objects(&missing); + let dropped = objects + .iter() + .position(|(k, _)| k.contains("/data/")) + .unwrap(); + objects.remove(dropped); + for (key, bytes) in objects { + mem.insert(&key, &bytes); + } + for (key, mut bytes) in backup_objects(&short) { + if key.contains("/data/") && bytes.len() > 1 { + bytes.pop(); + } + mem.insert(&key, &bytes); + } + for (key, mut bytes) in backup_objects(&long) { + if key.contains("/data/") { + bytes.push(0); + } + mem.insert(&key, &bytes); + } + for (key, bytes) in backup_objects(&no_sidecar) { + if !key.ends_with("extenddb-manifest.sha256") { + mem.insert(&key, &bytes); + } + } + for (key, bytes) in backup_objects(&sync_arn(19)) { + mem.insert( + &key.replacen(&prefix_of(&sync_arn(19)), &prefix_of(&other), 1), + &bytes, + ); + } + let arns = [&no_copy, &missing, &short, &long, &no_sidecar, &other]; + let rows: Vec = arns + .iter() + .map(|arn| { + let mut row = old_row(arn, BackupStatus::Creating); + row.heartbeat_at = 1.0; + fake.dead.lock().unwrap().insert((*arn).clone()); + row + }) + .collect(); + seed_rows(&fake, &rows); + let keys_before = mem.keys(); + + let report = move_old(&fake, &runtime).await; + assert!(!report.complete()); + assert!(report.resolved.is_empty(), "{:?}", report.resolved); + let why: HashMap = report + .stopping + .iter() + .map(|(row, why)| (row.backup_arn.clone(), why.clone())) + .collect(); + assert_eq!(report.stopping.len(), 6, "{why:#?}"); + for (arn, detail) in [ + (&no_copy, "no manifest"), + (&missing, "is not in the store"), + (&short, "bytes in the store"), + (&long, "bytes in the store"), + (&no_sidecar, "without its sidecar"), + (&other, "not the row's backup"), + ] { + let why = &why[arn]; + assert!( + why.starts_with("CREATING, its owner dead, and the copy under the configured root") + && why.contains(detail) + && why.contains("the move writes no status for it") + && why.contains("configured with the old root") + && why.contains("--forget-backup"), + "{arn}: {why}" + ); + } + assert!(fake.rewrites_seen.lock().unwrap().is_empty()); + let held = fake.rows.lock().unwrap(); + for row in &rows { + let now = &held[&row.backup_arn]; + assert_eq!( + ( + now.status, + now.heartbeat_at, + &now.store, + &now.failure_reason + ), + (BackupStatus::Creating, 1.0, &old_store(), &None), + "{}", + row.backup_arn + ); + } + drop(held); + assert_eq!(mem.keys(), keys_before, "no object was deleted"); +} + +/// The owner window the move is given is the one the takeover reads: a +/// CREATING row whose last beat is 100 s old has a live owner under the +/// servers' 300 s window and stops the move as such, and is resolved under +/// SQLite's zero window (the serve lock held, no job of another process can +/// be alive), its complete copy making it AVAILABLE. +#[tokio::test] +async fn the_moves_owner_window_is_the_one_the_takeover_reads() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + fake.dead_by_heartbeat + .store(true, std::sync::atomic::Ordering::Release); + let runtime = runtime_on(&mem, store()); + let arn = seed_dead_creating_with_manifest(&fake, &mem, true); + { + let mut rows = fake.rows.lock().unwrap(); + let row = rows.get_mut(&arn).unwrap(); + row.store = old_store(); + row.heartbeat_at = seconds_since_epoch() - 100.0; + } + let report = move_old(&fake, &runtime).await; + assert_eq!(report.stopping.len(), 1, "{report:?}"); + assert!(report.stopping[0].1.contains("live owner"), "{report:?}"); + assert_eq!( + fake.rows.lock().unwrap()[&arn].status, + BackupStatus::Creating + ); + + let configured = runtime.configured_store().unwrap(); + let identity = configured.identity().unwrap(); + let report = super::store_maintenance::move_store( + &*fake, + super::store_maintenance::MoveTarget { + configured, + identity: &identity, + from_kind: StoreKind::Filesystem, + from_root: "/srv/old", + owner_window: Duration::ZERO, + stop: &never, + }, + ) + .await + .unwrap(); + assert!(report.complete(), "{report:?}"); + assert_eq!(report.resolved, [(arn.clone(), BackupStatus::Available)]); + let row = fake.rows.lock().unwrap()[&arn].clone(); + assert_eq!((row.status, row.store), (BackupStatus::Available, store())); +} + +/// The manifest's item total must be the row's too: a copy whose manifest +/// records the row's size and another item total is another backup under +/// the same ARN, and stops the move. +#[tokio::test] +async fn an_available_row_whose_manifest_counts_other_items_stops_the_move() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let runtime = runtime_on(&mem, store()); + let arn = sync_arn(1); + put_backup(&mem, &arn); + let mut row = old_row(&arn, BackupStatus::Available); + row.item_count += 1; + seed_rows(&fake, &[row]); + let report = move_old(&fake, &runtime).await; + assert_eq!(report.stopping.len(), 1, "{report:?}"); + assert!( + report.stopping[0] + .1 + .contains("another backup under the same ARN"), + "{}", + report.stopping[0].1 + ); + assert!(fake.rewrites_seen.lock().unwrap().is_empty()); +} + +/// Where rows are unique by ARN alone (Cassandra), two rows of one backup +/// id among the old root's rows (two spellings of the ARN, two FAILED rows +/// with no copy, each of which passes its own check) stop the move: each +/// is named with the other. +#[tokio::test] +async fn two_rows_of_one_backup_id_among_the_moved_stop_the_move() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + fake.unique_by_arn + .store(true, std::sync::atomic::Ordering::Release); + let runtime = runtime_on(&mem, store()); + let ours = sync_arn(1); + let other_spelling = ours.replace(REGION, "eu-west-1"); + seed_rows( + &fake, + &[ + old_row(&ours, BackupStatus::Failed), + old_row(&other_spelling, BackupStatus::Failed), + ], + ); + let report = move_old(&fake, &runtime).await; + assert_eq!(report.stopping.len(), 2, "{:?}", report.stopping); + let both = sorted(vec![ours.clone(), other_spelling.clone()]); + for ((row, why), (arn, other)) in report + .stopping + .iter() + .zip([(&both[0], &both[1]), (&both[1], &both[0])]) + { + assert_eq!(&row.backup_arn, arn); + assert!( + why.contains("two rows of backup id") && why.contains(other.as_str()), + "{why}" + ); + } + assert!(fake.rewrites_seen.lock().unwrap().is_empty()); +} + +/// The stop is asked before each row: a stop that fires after the first +/// row's check leaves the second unchecked (its dead CREATING row not taken +/// over, its DELETING row's deletion not finished) and nothing rewritten. +#[tokio::test] +async fn a_stop_during_the_checks_leaves_the_later_rows_unchecked() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let runtime = runtime_on(&mem, store()); + let first = sync_arn(1); + let deleting = sync_arn(2); + put_backup(&mem, &first); + let leftover = format!("{}data/000001.json.gz", prefix_of(&deleting)); + mem.insert(&leftover, b"x"); + let mut first_row = old_row(&first, BackupStatus::Available); + first_row.created_at = 1.0; + let mut deleting_row = old_row(&deleting, BackupStatus::Deleting); + deleting_row.created_at = 2.0; + seed_rows(&fake, &[first_row, deleting_row]); + let asked = std::sync::atomic::AtomicUsize::new(0); + let stop = || { + (asked.fetch_add(1, std::sync::atomic::Ordering::AcqRel) >= 1) + .then(|| "SIGTERM received".to_owned()) + }; + let configured = runtime.configured_store().unwrap(); + let identity = configured.identity().unwrap(); + let report = super::store_maintenance::move_store( + &*fake, + super::store_maintenance::MoveTarget { + configured, + identity: &identity, + from_kind: StoreKind::Filesystem, + from_root: "/srv/old", + owner_window: Duration::from_secs(300), + stop: &stop, + }, + ) + .await + .unwrap(); + assert_eq!(report.stopped.as_deref(), Some("SIGTERM received")); + assert!(report.finished.is_empty(), "{report:?}"); + assert!( + mem.keys().contains(&leftover), + "the second row was not checked" + ); + assert_eq!( + fake.rows.lock().unwrap()[&deleting].status, + BackupStatus::Deleting + ); + assert!(fake.rewrites_seen.lock().unwrap().is_empty()); +} + +/// A key the old root no longer lists counts as moved only when the +/// configured store's listing holds its row: a row another writer removed +/// during the move (a tombstone duty of a server on the old root) is gone, +/// not moved, and gets no audit line. A re-list of the old root that fails +/// after a rewrite that counted every key takes every key as moved, and +/// the move is not complete. +#[tokio::test] +async fn a_key_counts_as_moved_only_when_its_row_names_the_configured_store() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let runtime = runtime_on(&mem, store()); + let kept = sync_arn(1); + let removed = sync_arn(2); + put_backup(&mem, &kept); + seed_rows( + &fake, + &[ + old_row(&kept, BackupStatus::Available), + old_row(&removed, BackupStatus::Failed), + ], + ); + fake.removed_before_rewrite + .lock() + .unwrap() + .push(removed.clone()); + let report = move_old(&fake, &runtime).await; + assert_eq!(report.rewritten, 1); + assert_eq!(report.moved, std::slice::from_ref(&kept)); + assert_eq!(report.gone, std::slice::from_ref(&removed)); + assert!(report.complete(), "{report:?}"); + + let fake = Arc::new(Fake::default()); + seed_rows( + &fake, + &[ + old_row(&kept, BackupStatus::Available), + old_row(&removed, BackupStatus::Failed), + ], + ); + fake.by_root_error_after_rewrite + .store(true, std::sync::atomic::Ordering::Release); + let report = move_old(&fake, &runtime).await; + assert_eq!(report.rewritten, 2); + assert!(report.relist_error.is_some(), "{report:?}"); + assert_eq!( + sorted(report.moved.clone()), + sorted(vec![kept.clone(), removed.clone()]) + ); + assert!(!report.complete()); + + // With a key the rewrite did not count, a failed re-list takes none as + // moved. + let fake = Arc::new(Fake::default()); + seed_rows( + &fake, + &[ + old_row(&kept, BackupStatus::Available), + old_row(&removed, BackupStatus::Failed), + ], + ); + fake.removed_before_rewrite + .lock() + .unwrap() + .push(removed.clone()); + fake.by_root_error_after_rewrite + .store(true, std::sync::atomic::Ordering::Release); + let report = move_old(&fake, &runtime).await; + assert_eq!(report.rewritten, 1); + assert!(report.moved.is_empty(), "{report:?}"); + assert!(!report.complete()); +} + +/// A removal that meets another status reads the row once more and applies +/// that status's removal: an AVAILABLE row a DeleteBackup moves to DELETING +/// under `--forget-backup` is removed as DELETING. A row moved to another +/// store under the removal is left and named as changed. +#[tokio::test] +async fn a_removal_that_meets_another_status_or_store_reads_the_row_again() { + use super::store_maintenance::{ForgetBackup, Forgotten, forget_backups}; + let fake = Arc::new(Fake::default()); + let arn = sync_arn(1); + seed_rows(&fake, &[row(&arn, BackupStatus::Available)]); + let deleting = arn.clone(); + *fake.after_row_read.lock().unwrap() = Some(( + arn.clone(), + Box::new(move |fake: &Fake| { + fake.rows.lock().unwrap().get_mut(&deleting).unwrap().status = BackupStatus::Deleting; + }), + )); + let report = forget_backups( + &*fake, + std::slice::from_ref(&arn), + REGION, + forget_run(Duration::from_secs(300), &never), + ) + .await; + assert_eq!( + report.done, + [( + arn.clone(), + ForgetBackup::Row( + BackupStatus::Available, + Forgotten::Removed(BackupStatus::Deleting) + ) + )] + ); + assert!(!fake.rows.lock().unwrap().contains_key(&arn)); + + let moved = sync_arn(2); + seed_rows(&fake, &[row(&moved, BackupStatus::Available)]); + let elsewhere = moved.clone(); + *fake.after_row_read.lock().unwrap() = Some(( + moved.clone(), + Box::new(move |fake: &Fake| { + let mut rows = fake.rows.lock().unwrap(); + let row = rows.get_mut(&elsewhere).unwrap(); + row.status = BackupStatus::Deleting; + row.store = old_store(); + }), + )); + let report = forget_backups( + &*fake, + std::slice::from_ref(&moved), + REGION, + forget_run(Duration::from_secs(300), &never), + ) + .await; + assert_eq!( + report.done, + [( + moved.clone(), + ForgetBackup::Row( + BackupStatus::Available, + Forgotten::Changed(BackupStatus::Deleting, old_store()) + ) + )] + ); + assert!(fake.rows.lock().unwrap().contains_key(&moved)); +} + +/// `--forget-backup` given again after a stop: an ARN no row and no +/// in-catalog backup carries counts as done ("nothing to remove") and the +/// run goes on to the next ARN, whose row is removed. +#[tokio::test] +async fn forget_backup_goes_past_an_arn_no_row_carries() { + use super::store_maintenance::{ForgetBackup, Forgotten, forget_backups}; + let fake = Arc::new(Fake::default()); + let gone = sync_arn(1); + let kept = sync_arn(2); + seed_rows(&fake, &[row(&kept, BackupStatus::Available)]); + let asked = [gone.clone(), kept.clone()]; + let report = forget_backups( + &*fake, + &asked, + REGION, + forget_run(Duration::from_secs(300), &never), + ) + .await; + assert_eq!( + report.done, + [ + (gone.clone(), ForgetBackup::NoRow), + ( + kept.clone(), + ForgetBackup::Row( + BackupStatus::Available, + Forgotten::Removed(BackupStatus::Available) + ) + ) + ] + ); + assert!(report.complete(2)); + assert_eq!( + ForgetBackup::NoRow.to_string(), + "no row carries this ARN; nothing to remove" + ); + assert!(!fake.rows.lock().unwrap().contains_key(&kept)); +} + +/// The form of every ARN is checked before the command takes the claim: +/// a string that is no ARN and an ARN whose backup id is no id are refused, +/// each with why, said once; a well-formed ARN passes. +#[test] +fn the_arns_are_checked_before_the_claim() { + use super::store_maintenance::check_arns; + let bad_id = sync_arn(1).replace("/backup/", "/backup/x"); + let refused = check_arns(&[ + "not-an-arn".to_owned(), + String::new(), + sync_arn(2), + bad_id.clone(), + ]); + assert_eq!( + refused, + [ + ("not-an-arn".to_owned(), "not a backup ARN".to_owned()), + (String::new(), "not a backup ARN".to_owned()), + (bad_id, "not a backup id".to_owned()), + ] + ); + assert_eq!( + super::store_maintenance::ForgetBackup::Invalid("not a backup ARN".to_owned()).to_string(), + "refused: not a backup ARN" + ); +} + +/// A deletion duty that listed a FAILED row and re-read it before a +/// maintenance command took the claim reads the claim once more after the +/// re-read, finds it held, and deletes nothing; so does a tombstone duty +/// whose listing comes after the command took it. The claim's read is the +/// last step before the call. +#[tokio::test] +async fn a_duty_that_finds_the_claim_held_after_its_reread_acts_on_nothing() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let runtime = runtime_on(&mem, store()); + let failed = sync_arn(1); + put_backup(&mem, &failed); + let mut failed_row = row(&failed, BackupStatus::Failed); + failed_row.heartbeat_at = 1.0; + seed_rows(&fake, &[failed_row]); + *fake.after_row_read.lock().unwrap() = Some(( + failed.clone(), + Box::new(|fake: &Fake| *fake.maintenance_held.lock().unwrap() = true), + )); + let cycle = cycle_on(&fake, &runtime).await; + assert!(!cycle.maintenance_held, "the cycle began without the claim"); + assert!(cycle.failed_prefixes_deleted.is_empty(), "{cycle:?}"); + assert!( + objects_under(&mem, &failed).len() > 1, + "the row's bytes are kept" + ); + assert!( + fake.after_row_read.lock().unwrap().is_none(), + "the hook ran" + ); + + *fake.maintenance_held.lock().unwrap() = false; + fake.rows.lock().unwrap().clear(); + let tombstone = sync_arn(2); + let mut tombstone_row = row(&tombstone, BackupStatus::Deleted); + tombstone_row.completed_at = Some(1.0); + seed_rows(&fake, &[tombstone_row]); + *fake.after_older_than.lock().unwrap() = Some(( + BackupStatus::Deleted, + StaleClock::Completed, + Box::new(|fake: &Fake| *fake.maintenance_held.lock().unwrap() = true), + )); + let cycle = cycle_on(&fake, &runtime).await; + assert!(cycle.tombstones_removed.is_empty(), "{cycle:?}"); + assert!(fake.rows.lock().unwrap().contains_key(&tombstone)); +} + +/// The status half of the duty's re-read: a FAILED row forgotten and its +/// complete prefix adopted again (an AVAILABLE row under the same ARN) +/// between the FAILED-prefix duty's listing and its deletion keeps its +/// prefix. +#[tokio::test] +async fn a_duty_keeps_the_prefix_of_a_row_that_turned_available_after_its_listing() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let runtime = runtime_on(&mem, store()); + let arn = sync_arn(1); + put_backup(&mem, &arn); + let mut failed = row(&arn, BackupStatus::Failed); + failed.heartbeat_at = 1.0; + seed_rows(&fake, &[failed]); + let adopted = arn.clone(); + *fake.after_older_than.lock().unwrap() = Some(( + BackupStatus::Failed, + StaleClock::Heartbeat, + Box::new(move |fake: &Fake| { + let mut rows = fake.rows.lock().unwrap(); + let row = rows.get_mut(&adopted).unwrap(); + row.status = BackupStatus::Available; + row.owner_generation = UNOWNED_GENERATION; + }), + )); + let whole = objects_under(&mem, &arn).len(); + let cycle = cycle_on(&fake, &runtime).await; + assert!(cycle.failed_prefixes_deleted.is_empty(), "{cycle:?}"); + assert_eq!( + objects_under(&mem, &arn).len(), + whole, + "the backup is whole" + ); +} + +/// A deletion that read the claim free just before a command took it runs +/// at most `DUTY_DELETION_BOUND`: a prefix deletion that would take longer +/// is dropped at the bound, nothing of the prefix is deleted, and the duty +/// stops for this cycle (the next FAILED row's prefix is not even listed); +/// the next cycle continues. +#[tokio::test(start_paused = true)] +async fn a_duty_deletion_that_outlasts_the_bound_stops_the_duty_for_the_cycle() { + use super::reconciler::DUTY_DELETION_BOUND; + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let runtime = runtime_on(&mem, store()); + let first = sync_arn(1); + let second = sync_arn(2); + for (n, arn) in [&first, &second].into_iter().enumerate() { + put_backup(&mem, arn); + let mut failed = row(arn, BackupStatus::Failed); + failed.heartbeat_at = 1.0; + failed.created_at = n as f64; + seed_rows(&fake, &[failed]); + } + *mem.delete_delay.lock().unwrap() = Some(DUTY_DELETION_BOUND + Duration::from_secs(10)); + // The cycle's own read is the first; the first row's after its re-read + // the second: a command takes the claim right after it. + *fake.after_maintenance_check.lock().unwrap() = Some(( + 2, + Box::new(|fake: &Fake| *fake.maintenance_held.lock().unwrap() = true), + )); + let whole = objects_under(&mem, &first).len(); + let started = tokio::time::Instant::now(); + let cycle = cycle_on(&fake, &runtime).await; + let took = started.elapsed(); + assert!( + took >= DUTY_DELETION_BOUND && took < DUTY_DELETION_BOUND + Duration::from_secs(10), + "{took:?}" + ); + assert!(cycle.failed_prefixes_deleted.is_empty(), "{cycle:?}"); + assert_eq!( + objects_under(&mem, &first).len(), + whole, + "nothing was deleted" + ); + assert_eq!(objects_under(&mem, &second).len(), whole); + assert!( + !mem.lists.lock().unwrap().contains(&prefix_of(&second)), + "the duty stopped before the next row" + ); + + // The next cycle, without the claim and with a store that answers, + // continues. + *fake.maintenance_held.lock().unwrap() = false; + *mem.delete_delay.lock().unwrap() = None; + let cycle = cycle_on(&fake, &runtime).await; + assert_eq!( + sorted(cycle.failed_prefixes_deleted.clone()), + sorted(vec![first.clone(), second.clone()]) + ); +} + +/// Beside servers the forget options wait `FORGET_WAIT` from the claim +/// before their first FAILED or DELETED removal, told once; an AVAILABLE +/// row's removal does not wait; a stop during the wait leaves the FAILED +/// row and ends the run; with no wait (SQLite) nothing waits. The wait is +/// the duty's bound and one store call's. +#[tokio::test(start_paused = true)] +async fn the_forget_options_wait_out_the_duties_before_a_failed_removal() { + use super::reconciler::DUTY_DELETION_BOUND; + use super::store_maintenance::{FORGET_WAIT, ForgetRun, Forgotten, forget_store}; + assert_eq!(FORGET_WAIT, DUTY_DELETION_BOUND + super::S3_CHECK_TIMEOUT); + assert!(DUTY_DELETION_BOUND < extenddb_backup_store::S3_ATTEMPT_TIMEOUT); + let told = Mutex::new(Vec::new()); + let tell = |remaining: Duration| told.lock().unwrap().push(remaining); + let seed = |fake: &Fake| { + let mut available = old_row(&sync_arn(1), BackupStatus::Available); + available.created_at = 1.0; + let mut failed = old_row(&sync_arn(2), BackupStatus::Failed); + failed.created_at = 2.0; + seed_rows(fake, &[available, failed]); + }; + + let fake = Arc::new(Fake::default()); + seed(&fake); + let claimed = tokio::time::Instant::now(); + let run = ForgetRun { + owner_window: Duration::ZERO, + removals_from: Some(claimed + FORGET_WAIT), + on_wait: &tell, + stop: &never, + configured: None, + unmounted_root: None, + deletions_finished: &[], + }; + let report = forget_store(&*fake, StoreKind::Filesystem, "/srv/old", run) + .await + .unwrap(); + assert!(report.complete(), "{report:?}"); + assert!(claimed.elapsed() >= FORGET_WAIT); + assert_eq!(*told.lock().unwrap(), [FORGET_WAIT]); + + // Only an AVAILABLE row: no wait. + told.lock().unwrap().clear(); + let fake = Arc::new(Fake::default()); + seed_rows(&fake, &[old_row(&sync_arn(1), BackupStatus::Available)]); + let claimed = tokio::time::Instant::now(); + let run = ForgetRun { + removals_from: Some(claimed + FORGET_WAIT), + ..run + }; + let report = forget_store(&*fake, StoreKind::Filesystem, "/srv/old", run) + .await + .unwrap(); + assert!(report.complete()); + assert!(claimed.elapsed() < FORGET_WAIT); + assert!(told.lock().unwrap().is_empty()); + + // A stop during the wait. + let fake = Arc::new(Fake::default()); + seed(&fake); + let claimed = tokio::time::Instant::now(); + let stop = + || (claimed.elapsed() >= Duration::from_secs(5)).then(|| "SIGINT received".to_owned()); + let run = ForgetRun { + removals_from: Some(claimed + FORGET_WAIT), + stop: &stop, + ..run + }; + let report = forget_store(&*fake, StoreKind::Filesystem, "/srv/old", run) + .await + .unwrap(); + assert_eq!(report.stopped.as_deref(), Some("SIGINT received")); + assert_eq!( + report + .rows + .iter() + .map(|(_, f)| f.clone()) + .collect::>(), + [Forgotten::Removed(BackupStatus::Available)] + ); + assert!(fake.rows.lock().unwrap().contains_key(&sync_arn(2))); + assert!(claimed.elapsed() < FORGET_WAIT); + + // No wait: SQLite, under the serve lock. + let fake = Arc::new(Fake::default()); + seed(&fake); + let claimed = tokio::time::Instant::now(); + let report = forget_store( + &*fake, + StoreKind::Filesystem, + "/srv/old", + forget_run(Duration::ZERO, &never), + ) + .await + .unwrap(); + assert!(report.complete()); + assert_eq!(claimed.elapsed(), Duration::ZERO); +} + +/// The store-backed create reads the maintenance claim again immediately +/// before its insert: a claim a command took after the request's check +/// (while the job opened the table, say) refuses the request as a held +/// claim does, with no row inserted and every slot free. +#[tokio::test] +async fn a_claim_taken_after_the_requests_check_refuses_the_create_before_its_row() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + snapshot_fixture(&fake, &music_table(), songs(3), true, None); + *fake.after_maintenance_check.lock().unwrap() = Some(( + 1, + Box::new(|fake: &Fake| *fake.maintenance_held.lock().unwrap() = true), + )); + let runtime = job_runtime(&mem); + let err = create_music_backup(&fake, &runtime).await.unwrap_err(); + assert!( + matches!(&err, DynamoDbError::LimitExceededException(m) + if m.contains("A backup maintenance command holds")), + "{err:?}" + ); + wait_for_jobs(&runtime).await; + assert_eq!(fake.maintenance_checks.lock().unwrap().len(), 2); + assert!(fake.rows.lock().unwrap().is_empty(), "no row is inserted"); + assert!(fake.claims.lock().unwrap().is_empty()); + assert!(mem.puts.lock().unwrap().is_empty()); + assert_eq!(runtime.free_slots(), 4); +} + +/// The read-only check after `--forget-backup` writes nothing: on a root +/// with no identity file and no rows it creates no file (`Ok(None)`); on +/// a root whose file holds another id than the one the command opened it +/// with it is an identity mismatch; otherwise it passes, the manifest +/// probe included. +#[tokio::test] +async fn the_read_only_check_writes_nothing_to_the_root() { + let fake = Arc::new(Fake::default()); + let mem = Arc::new(MemStore::default()); + let configured = ConfiguredStore::new( + store().store_kind, + store().store_root, + Arc::clone(&mem) as Arc, + None, + Ok(store()), + ); + let check = super::check_configured_store_read_only(&configured, &*fake, None).await; + assert_eq!(check, Ok(None)); + assert!(mem.keys().is_empty(), "no identity file is created"); + assert!(mem.puts.lock().unwrap().is_empty()); + + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let check = super::check_configured_store_read_only(&configured, &*fake, Some(&store())).await; + assert_eq!(check, Ok(Some(store()))); + let other = StoreIdentity { + store_id: "store-9".to_owned(), + ..store() + }; + let check = super::check_configured_store_read_only(&configured, &*fake, Some(&other)).await; + assert!( + matches!(&check, Err(StoreProblem::IdMismatch { file_id, .. }) if file_id == "store-1"), + "{check:?}" + ); + let gone = sync_arn(1); + seed_rows(&fake, &[row(&gone, BackupStatus::Available)]); + let check = super::check_configured_store_read_only(&configured, &*fake, Some(&store())).await; + assert!( + matches!(&check, Err(StoreProblem::ManifestMissing { backup_arn }) if *backup_arn == gone), + "{check:?}" + ); + assert!(mem.puts.lock().unwrap().is_empty()); +} + +/// The DELETED-prefix deletion and the tombstone removal run under the same +/// bound: a deletion of a DELETED row's prefix that would outlast it is +/// dropped with nothing deleted and the DELETED-prefix duty stops for the +/// cycle (the next DELETED row's prefix is not listed); a tombstone +/// removal held past it is dropped with the row kept and the tombstone duty +/// stops (the next tombstone is not tried). +#[tokio::test(start_paused = true)] +async fn the_deleted_prefix_and_tombstone_calls_stop_their_duty_at_the_bound() { + use super::reconciler::DUTY_DELETION_BOUND; + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let runtime = runtime_on(&mem, store()); + let swept = [sync_arn(1), sync_arn(2)]; + for (n, arn) in swept.iter().enumerate() { + put_backup(&mem, arn); + let mut deleted = row(arn, BackupStatus::Deleted); + deleted.created_at = n as f64; + seed_rows(&fake, &[deleted]); + } + *mem.delete_delay.lock().unwrap() = Some(DUTY_DELETION_BOUND + Duration::from_secs(10)); + let whole = objects_under(&mem, &swept[0]).len(); + let cycle = cycle_on(&fake, &runtime).await; + assert!(cycle.deleted_prefixes_swept.is_empty(), "{cycle:?}"); + assert_eq!(objects_under(&mem, &swept[0]).len(), whole); + assert!( + !mem.lists.lock().unwrap().contains(&prefix_of(&swept[1])), + "the duty stopped before the next row" + ); + + *mem.delete_delay.lock().unwrap() = None; + fake.rows.lock().unwrap().clear(); + let tombstones = [sync_arn(3), sync_arn(4)]; + for (n, arn) in tombstones.iter().enumerate() { + let mut failed = row(arn, BackupStatus::Failed); + failed.completed_at = Some(1.0); + failed.heartbeat_at = seconds_since_epoch(); + failed.created_at = n as f64; + seed_rows(&fake, &[failed]); + } + *fake.remove_delay.lock().unwrap() = Some(DUTY_DELETION_BOUND + Duration::from_secs(10)); + let cycle = cycle_on(&fake, &runtime).await; + assert!(cycle.tombstones_removed.is_empty(), "{cycle:?}"); + let held = fake.rows.lock().unwrap(); + assert!(held.contains_key(&tombstones[0]) && held.contains_key(&tombstones[1])); + drop(held); + assert_eq!( + *fake.remove_attempts.lock().unwrap(), + [tombstones[0].clone()], + "the duty stopped before the next tombstone" + ); +} + +// --- the duties' deadline, the move's order, the forget's configured store --- + +/// A duty's claim read that answers "free" after its deadline (the query +/// ran before a command took the claim; its answer came two minutes later, +/// after the command had waited and removed the row) starts no prefix +/// deletion and no tombstone removal: the duty stops for this cycle. +#[tokio::test(start_paused = true)] +async fn a_claim_read_answered_after_the_bound_starts_no_deletion() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let runtime = runtime_on(&mem, store()); + let arn = sync_arn(1); + put_backup(&mem, &arn); + let mut failed = row(&arn, BackupStatus::Failed); + failed.heartbeat_at = 1.0; + seed_rows(&fake, &[failed]); + let gone = arn.clone(); + // The cycle's own read is the first; the duty's, after its re-read, the + // second. + *fake.after_maintenance_check.lock().unwrap() = Some(( + 2, + Box::new(move |fake: &Fake| { + fake.rows.lock().unwrap().remove(&gone); + *fake.maintenance_delay.lock().unwrap() = Some(Duration::from_secs(120)); + }), + )); + let whole = objects_under(&mem, &arn).len(); + let cycle = cycle_on(&fake, &runtime).await; + assert!(cycle.failed_prefixes_deleted.is_empty(), "{cycle:?}"); + assert_eq!( + objects_under(&mem, &arn).len(), + whole, + "a deletion whose claim read answered after the bound must not start" + ); + + *fake.maintenance_delay.lock().unwrap() = None; + let tombstone = sync_arn(2); + let mut tombstone_row = row(&tombstone, BackupStatus::Deleted); + tombstone_row.completed_at = Some(1.0); + seed_rows(&fake, &[tombstone_row]); + let checks = fake.maintenance_checks.lock().unwrap().len(); + *fake.after_maintenance_check.lock().unwrap() = Some(( + checks + 2, + Box::new(|fake: &Fake| { + *fake.maintenance_delay.lock().unwrap() = Some(Duration::from_secs(120)); + }), + )); + let cycle = cycle_on(&fake, &runtime).await; + assert!(cycle.tombstones_removed.is_empty(), "{cycle:?}"); + assert!(fake.rows.lock().unwrap().contains_key(&tombstone)); +} + +/// On a catalog keyed by ARN (Cassandra), an old-root DELETING row under +/// another spelling of a configured-root row's ARN shares that row's +/// prefix: the two-rows check stops it before any check, so the configured +/// row's prefix, incomplete, keeps every object. A configured root whose +/// rows cannot be read stops the move before anything changed: the dead +/// CREATING row of the old root is not resolved, and the DELETING row not +/// finished. +#[tokio::test] +async fn the_two_rows_check_comes_before_any_row_of_the_old_root_is_checked() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + fake.unique_by_arn + .store(true, std::sync::atomic::Ordering::Release); + let runtime = runtime_on(&mem, store()); + let ours = sync_arn(1); + let other_spelling = ours.replace(REGION, "eu-west-1"); + let mut objects = backup_objects(&ours); + let dropped = objects + .iter() + .position(|(k, _)| k.contains("/data/")) + .unwrap(); + objects.remove(dropped); + for (key, bytes) in objects { + mem.insert(&key, &bytes); + } + let before = objects_under(&mem, &ours).len(); + seed_rows( + &fake, + &[ + row(&ours, BackupStatus::Available), + old_row(&other_spelling, BackupStatus::Deleting), + ], + ); + let report = move_old(&fake, &runtime).await; + assert!(report.finished.is_empty(), "{report:?}"); + assert_eq!(report.stopping.len(), 1, "{report:?}"); + assert!( + report.stopping[0].1.contains("two rows of backup id"), + "{report:?}" + ); + assert_eq!(objects_under(&mem, &ours).len(), before); + assert_eq!( + fake.rows.lock().unwrap()[&other_spelling].status, + BackupStatus::Deleting + ); + + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let runtime = runtime_on(&mem, store()); + let creating = sync_arn(2); + let deleting = sync_arn(3); + put_backup(&mem, &creating); + let leftover = format!("{}data/000001.json.gz", prefix_of(&deleting)); + mem.insert(&leftover, b"x"); + let mut creating_row = old_row(&creating, BackupStatus::Creating); + creating_row.heartbeat_at = 1.0; + fake.dead_by_heartbeat + .store(true, std::sync::atomic::Ordering::Release); + seed_rows( + &fake, + &[creating_row, old_row(&deleting, BackupStatus::Deleting)], + ); + *fake.by_root_error_for.lock().unwrap() = Some(store().store_root); + let configured = runtime.configured_store().unwrap(); + let identity = configured.identity().unwrap(); + let outcome = super::store_maintenance::move_store( + &*fake, + super::store_maintenance::MoveTarget { + configured, + identity: &identity, + from_kind: StoreKind::Filesystem, + from_root: "/srv/old", + owner_window: Duration::from_secs(300), + stop: &never, + }, + ) + .await; + assert!(outcome.is_err(), "{outcome:?}"); + let held = fake.rows.lock().unwrap(); + assert_eq!(held[&creating].status, BackupStatus::Creating); + assert_eq!(held[&deleting].status, BackupStatus::Deleting); + assert!(mem.keys().contains(&leftover)); +} + +/// The stop asked after the last row's check and before the fence ends the +/// move with nothing rewritten (the claim lost, or a signal, during the +/// configured root's listing or the last check). +#[tokio::test] +async fn a_stop_after_the_last_check_rewrites_nothing() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let runtime = runtime_on(&mem, store()); + let arn = sync_arn(1); + put_backup(&mem, &arn); + seed_rows(&fake, &[old_row(&arn, BackupStatus::Available)]); + let asked = std::sync::atomic::AtomicUsize::new(0); + // Asked before the one row, then before the fence. + let stop = || { + (asked.fetch_add(1, std::sync::atomic::Ordering::AcqRel) >= 1) + .then(|| "the maintenance claim was lost".to_owned()) + }; + let configured = runtime.configured_store().unwrap(); + let identity = configured.identity().unwrap(); + let report = super::store_maintenance::move_store( + &*fake, + super::store_maintenance::MoveTarget { + configured, + identity: &identity, + from_kind: StoreKind::Filesystem, + from_root: "/srv/old", + owner_window: Duration::from_secs(300), + stop: &stop, + }, + ) + .await + .unwrap(); + assert_eq!(asked.load(std::sync::atomic::Ordering::Acquire), 2); + assert_eq!( + report.stopped.as_deref(), + Some("the maintenance claim was lost") + ); + assert!(report.stopping.is_empty(), "{report:?}"); + assert!(fake.rewrites_seen.lock().unwrap().is_empty()); + assert_eq!(fake.rows.lock().unwrap()[&arn].store.store_root, "/srv/old"); +} + +/// The wait before the first FAILED or DELETED removal covers a DELETED +/// row as it covers a FAILED one. +#[tokio::test(start_paused = true)] +async fn the_forget_options_wait_out_the_duties_before_a_deleted_removal() { + use super::store_maintenance::{FORGET_WAIT, ForgetRun, forget_store}; + let told = Mutex::new(Vec::new()); + let tell = |remaining: Duration| told.lock().unwrap().push(remaining); + let fake = Arc::new(Fake::default()); + let mut available = old_row(&sync_arn(1), BackupStatus::Available); + available.created_at = 1.0; + let mut deleted = old_row(&sync_arn(2), BackupStatus::Deleted); + deleted.created_at = 2.0; + seed_rows(&fake, &[available, deleted]); + let claimed = tokio::time::Instant::now(); + let run = ForgetRun { + owner_window: Duration::ZERO, + removals_from: Some(claimed + FORGET_WAIT), + on_wait: &tell, + stop: &never, + configured: None, + unmounted_root: None, + deletions_finished: &[], + }; + let report = forget_store(&*fake, StoreKind::Filesystem, "/srv/old", run) + .await + .unwrap(); + assert!(report.complete(), "{report:?}"); + assert!(claimed.elapsed() >= FORGET_WAIT); + assert_eq!(*told.lock().unwrap(), [FORGET_WAIT]); +} + +/// A failed read in the duty's fence is left for the next cycle: the claim +/// read failing, or the row's re-read failing, deletes nothing. +#[tokio::test] +async fn a_duty_whose_claim_read_or_reread_fails_deletes_nothing() { + for cell in ["the claim read fails", "the re-read fails"] { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let runtime = runtime_on(&mem, store()); + let arn = sync_arn(1); + put_backup(&mem, &arn); + let mut failed = row(&arn, BackupStatus::Failed); + failed.heartbeat_at = 1.0; + seed_rows(&fake, &[failed]); + if cell == "the claim read fails" { + *fake.after_row_read.lock().unwrap() = Some(( + arn.clone(), + Box::new(|fake: &Fake| { + *fake.maintenance_error.lock().unwrap() = Some("timed out".to_owned()); + }), + )); + } else { + *fake.after_older_than.lock().unwrap() = Some(( + BackupStatus::Failed, + StaleClock::Heartbeat, + Box::new(|fake: &Fake| { + *fake.read_row_error.lock().unwrap() = Some("timed out".to_owned()); + }), + )); + } + let whole = objects_under(&mem, &arn).len(); + let cycle = cycle_on(&fake, &runtime).await; + assert!( + cycle.failed_prefixes_deleted.is_empty(), + "{cell}: {cycle:?}" + ); + assert_eq!(objects_under(&mem, &arn).len(), whole, "{cell}"); + } +} + +/// `--forget-backup` of an earlier release's in-catalog backup given under +/// a spelling of its ARN that is not the server's region reads the legacy +/// backup under the ARN as written, and changes nothing. +#[tokio::test] +async fn forget_backup_reads_a_legacy_backup_under_the_arn_as_written() { + use super::store_maintenance::{ForgetBackup, forget_backups}; + let fake = Arc::new(Fake::default()); + let written = sync_arn(12).replace(REGION, "eu-west-1"); + fake.descriptions.lock().unwrap().insert( + written.clone(), + backup_description("AVAILABLE", "PAY_PER_REQUEST"), + ); + let report = forget_backups( + &*fake, + std::slice::from_ref(&written), + REGION, + forget_run(Duration::from_secs(300), &never), + ) + .await; + assert_eq!(report.done, [(written.clone(), ForgetBackup::Legacy)]); + assert!(!report.complete(1)); + assert!(fake.descriptions.lock().unwrap().contains_key(&written)); +} + +/// The FAILED live-owner rule reads each store's dead rows: dead FAILED +/// rows of two stores in one run are both removed. +#[tokio::test] +async fn forget_backup_tells_dead_failed_rows_of_two_stores_apart() { + use super::store_maintenance::{ForgetBackup, Forgotten, forget_backups}; + let fake = Arc::new(Fake::default()); + let ours = sync_arn(1); + let theirs = sync_arn(2); + seed_rows( + &fake, + &[ + row(&ours, BackupStatus::Failed), + old_row(&theirs, BackupStatus::Failed), + ], + ); + let asked = [ours.clone(), theirs.clone()]; + let report = forget_backups( + &*fake, + &asked, + REGION, + forget_run(Duration::from_secs(300), &never), + ) + .await; + let removed = ForgetBackup::Row( + BackupStatus::Failed, + Forgotten::Removed(BackupStatus::Failed), + ); + assert_eq!( + report.done, + [(ours, removed.clone()), (theirs, removed)], + "{report:?}" + ); + assert!(fake.rows.lock().unwrap().is_empty()); +} + +/// `--forget-store` lists a row whose one status entry is under an +/// earlier status (a DELETING row with an AVAILABLE entry) and removes it. +/// After its removals it sweeps the root's status entries: the entry with +/// no row behind it and the removed row's stale entry are deleted and +/// counted; the entry a row left in use keeps under an earlier status is +/// repaired and counted apart. An entry written after the root's listing +/// (`late_entries`) has its id reported. +#[tokio::test] +async fn forget_store_removes_the_roots_entries_with_no_row_and_reports_the_rest() { + use super::store_maintenance::{Forgotten, forget_store}; + let fake = Arc::new(Fake::default()); + let available = sync_arn(1); + let deleting = sync_arn(2); + seed_rows( + &fake, + &[ + old_row(&available, BackupStatus::Available), + old_row(&deleting, BackupStatus::Deleting), + ], + ); + fake.set_stale_entry(&deleting, BackupStatus::Available); + fake.orphan_entries + .lock() + .unwrap() + .push((old_store(), BackupStatus::Deleting)); + let mut late = old_store(); + late.store_id = "old-2".to_owned(); + fake.late_entries.lock().unwrap().push(late); + let report = forget_store( + &*fake, + StoreKind::Filesystem, + "/srv/old", + forget_run(Duration::ZERO, &never), + ) + .await + .unwrap(); + assert!(report.complete(), "{report:?}"); + assert_eq!(report.removed(), 2); + assert_eq!( + report.entries_swept, + Some(StatusEntrySweep { + removed: 2, + repaired: 0 + }) + ); + assert_eq!(report.entries_left, ["old-2"]); + assert!(fake.orphan_entries.lock().unwrap().is_empty()); + assert!(fake.stale_entries.lock().unwrap().is_empty()); + + // A row left in use, its entry under an earlier status: repaired. + let in_use = sync_arn(3); + seed_rows(&fake, &[old_row(&in_use, BackupStatus::Available)]); + fake.set_stale_entry(&in_use, BackupStatus::Creating); + fake.forget_answers + .lock() + .unwrap() + .insert(in_use.clone(), ForgetRow::InUse("Target".to_owned())); + let report = forget_store( + &*fake, + StoreKind::Filesystem, + "/srv/old", + forget_run(Duration::ZERO, &never), + ) + .await + .unwrap(); + assert!(!report.complete(), "{report:?}"); + assert_eq!( + report.rows[0].1, + Forgotten::InUse("Target".to_owned(), ACCOUNT.to_owned()) + ); + assert_eq!( + report.entries_swept, + Some(StatusEntrySweep { + removed: 0, + repaired: 1 + }) + ); + assert!( + fake.list_backup_rows(Some(BackupStatus::Available), &old_store()) + .try_collect::>() + .await + .unwrap() + .iter() + .any(|r| r.backup_arn == in_use), + "the repaired row is listed under its own status" + ); +} + +/// A DELETING row of the configured store has its deletion finished behind +/// the fence before its removal when the store's check passed, and is left +/// otherwise; a DELETING row of another store is removed and keeps its +/// bytes; a DELETED row whose deletion the command's own pass finished says +/// so. +#[tokio::test] +async fn forget_backup_finishes_a_deleting_row_of_the_configured_store_first() { + use super::store_maintenance::{ + ForgetBackup, ForgetConfigured, ForgetRun, Forgotten, forget_backups, + }; + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let runtime = runtime_on(&mem, store()); + let configured = runtime.configured_store().unwrap(); + let identity = configured.identity().unwrap(); + let identity = &identity; + fn run<'a>( + store: &'a ConfiguredStore, + healthy: Option<&'a StoreIdentity>, + finished: &'a [String], + ) -> ForgetRun<'a> { + ForgetRun { + configured: Some(ForgetConfigured { + store, + healthy, + mismatch_row_ids: &[], + file_missing_with_rows: false, + }), + deletions_finished: finished, + ..forget_run(Duration::from_secs(300), &never) + } + } + + // The store's check passed: the deletion is finished, then the row goes. + let deleting = sync_arn(1); + put_backup(&mem, &deleting); + seed_rows(&fake, &[row(&deleting, BackupStatus::Deleting)]); + let report = forget_backups( + &*fake, + std::slice::from_ref(&deleting), + REGION, + run(configured, Some(identity), &[]), + ) + .await; + assert_eq!( + report.done, + [( + deleting.clone(), + ForgetBackup::Row(BackupStatus::Deleting, Forgotten::RemovedAfterDeletion) + )] + ); + assert!( + objects_under(&mem, &deleting).is_empty(), + "the prefix is deleted" + ); + assert!(!fake.rows.lock().unwrap().contains_key(&deleting)); + + // Unhealthy (or the id unknown): left, bytes and row. + let pending = sync_arn(2); + put_backup(&mem, &pending); + seed_rows(&fake, &[row(&pending, BackupStatus::Deleting)]); + let whole = objects_under(&mem, &pending).len(); + let report = forget_backups( + &*fake, + std::slice::from_ref(&pending), + REGION, + run(configured, None, &[]), + ) + .await; + assert_eq!( + report.done, + [( + pending.clone(), + ForgetBackup::Row(BackupStatus::Deleting, Forgotten::DeletionPending) + )] + ); + assert_eq!(objects_under(&mem, &pending).len(), whole); + assert_eq!( + fake.rows.lock().unwrap()[&pending].status, + BackupStatus::Deleting + ); + + // Another store's DELETING row: removed, nothing deleted. + let theirs = sync_arn(3); + put_backup(&mem, &theirs); + seed_rows(&fake, &[old_row(&theirs, BackupStatus::Deleting)]); + let whole = objects_under(&mem, &theirs).len(); + let report = forget_backups( + &*fake, + std::slice::from_ref(&theirs), + REGION, + run(configured, Some(identity), &[]), + ) + .await; + assert_eq!( + report.done, + [( + theirs.clone(), + ForgetBackup::Row( + BackupStatus::Deleting, + Forgotten::Removed(BackupStatus::Deleting) + ) + )] + ); + assert_eq!(objects_under(&mem, &theirs).len(), whole); + + // A DELETED row the command's own pass finished. + let finished = sync_arn(4); + seed_rows(&fake, &[row(&finished, BackupStatus::Deleted)]); + let list = [finished.clone()]; + let report = forget_backups( + &*fake, + std::slice::from_ref(&finished), + REGION, + run(configured, Some(identity), &list), + ) + .await; + assert_eq!( + report.done, + [( + finished.clone(), + ForgetBackup::Row(BackupStatus::Deleted, Forgotten::RemovedAfterDeletion) + )] + ); +} + +/// While the configured root has no identity file and the catalog holds +/// rows for it (an unmounted volume), its rows are refused and kept; a row +/// of another root in the same run is removed. +#[tokio::test] +async fn forget_backup_refuses_the_rows_of_a_configured_root_without_its_file() { + use super::store_maintenance::{ + ForgetBackup, ForgetConfigured, ForgetRun, Forgotten, forget_backups, + }; + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let runtime = runtime_unhealthy( + &mem, + StoreProblem::FileMissingWithRows(vec!["store-1".to_owned()]), + ); + let configured = runtime.configured_store().unwrap(); + let theirs = sync_arn(1); + let ours = sync_arn(2); + seed_rows( + &fake, + &[ + old_row(&theirs, BackupStatus::Available), + row(&ours, BackupStatus::Available), + ], + ); + let run = ForgetRun { + configured: Some(ForgetConfigured { + store: configured, + healthy: None, + mismatch_row_ids: &[], + file_missing_with_rows: true, + }), + ..forget_run(Duration::from_secs(300), &never) + }; + let asked = [theirs.clone(), ours.clone()]; + let report = forget_backups(&*fake, &asked, REGION, run).await; + assert_eq!( + report.done, + [ + ( + theirs.clone(), + ForgetBackup::Row( + BackupStatus::Available, + Forgotten::Removed(BackupStatus::Available) + ) + ), + ( + ours.clone(), + ForgetBackup::Row(BackupStatus::Available, Forgotten::RootWithoutFile) + ), + ] + ); + assert!(fake.rows.lock().unwrap().contains_key(&ours)); + assert!( + Forgotten::RootWithoutFile + .to_string() + .contains("point [backup.store] at the root that holds these backups") + ); +} + +/// On a backend whose summary counts status entries (Cassandra), the report +/// of other stores checks each count against the store's listing: what no +/// row stands behind is reported as status entries, not rows. +#[tokio::test] +async fn the_report_of_other_stores_names_status_entries_with_no_row() { + let fake = Arc::new(Fake::default()); + fake.counts_entries + .store(true, std::sync::atomic::Ordering::Release); + seed_rows(&fake, &[old_row(&sync_arn(1), BackupStatus::Available)]); + fake.orphan_entries + .lock() + .unwrap() + .push((old_store(), BackupStatus::Deleting)); + let others = super::store_maintenance::other_stores(&*fake, &store()) + .await + .unwrap(); + assert_eq!(others.len(), 1, "{others:?}"); + assert_eq!(others[0].counts, [(BackupStatus::Available, 1)]); + assert_eq!(others[0].entries, [(BackupStatus::Deleting, 1)]); + assert!(others[0].unresolved.is_empty()); + + // Elsewhere the counts are rows, taken as they are. + fake.counts_entries + .store(false, std::sync::atomic::Ordering::Release); + fake.orphan_entries.lock().unwrap().clear(); + let others = super::store_maintenance::other_stores(&*fake, &store()) + .await + .unwrap(); + assert!(others[0].entries.is_empty()); +} + +/// The tombstone duty's removal names the store it listed the row with: a +/// row a move rewrote to another store after the duty's fence is left for +/// that store's duties. +#[tokio::test] +async fn the_tombstone_duty_leaves_a_row_moved_after_its_fence() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let runtime = runtime_on(&mem, store()); + let tombstone = sync_arn(1); + let mut tombstone_row = row(&tombstone, BackupStatus::Deleted); + tombstone_row.completed_at = Some(1.0); + seed_rows(&fake, &[tombstone_row]); + let moved = tombstone.clone(); + *fake.after_maintenance_check.lock().unwrap() = Some(( + 2, + Box::new(move |fake: &Fake| { + fake.rows.lock().unwrap().get_mut(&moved).unwrap().store = old_store(); + }), + )); + let cycle = cycle_on(&fake, &runtime).await; + assert!(cycle.tombstones_removed.is_empty(), "{cycle:?}"); + assert_eq!(fake.rows.lock().unwrap()[&tombstone].store, old_store()); + assert_eq!(*fake.remove_attempts.lock().unwrap(), [tombstone]); +} + +/// A prefix deletion the store stopped at its deadline (it answers what it +/// removed, unfinished) stops the duty for this cycle: the row is not +/// reported deleted, and the next row's prefix is not listed. +#[tokio::test] +async fn a_deletion_the_store_stopped_at_its_deadline_stops_the_duty() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let runtime = runtime_on(&mem, store()); + let first = sync_arn(1); + let second = sync_arn(2); + for (n, arn) in [&first, &second].into_iter().enumerate() { + put_backup(&mem, arn); + let mut failed = row(arn, BackupStatus::Failed); + failed.heartbeat_at = 1.0; + failed.created_at = n as f64; + seed_rows(&fake, &[failed]); + } + mem.deadline_reached + .store(true, std::sync::atomic::Ordering::Release); + let cycle = cycle_on(&fake, &runtime).await; + assert!(cycle.failed_prefixes_deleted.is_empty(), "{cycle:?}"); + assert!( + !mem.lists.lock().unwrap().contains(&prefix_of(&second)), + "the duty stopped before the next row" + ); +} + +/// The cycle a forget option runs on SQLite (the claim held, no store +/// check) writes nothing to the root: a root that lost its identity file +/// is not given a new one, and the store's health stays as the command +/// recorded it. +#[tokio::test] +async fn a_cycle_without_the_store_check_writes_nothing_to_the_root() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let runtime = runtime_on(&mem, store()); + mem.objects.lock().unwrap().remove(STORE_IDENTITY_KEY); + super::reconciler::cycle( + &fake, + &runtime, + Duration::ZERO, + "sqlite", + &mut super::reconciler::ReconcilerState::for_maintenance_holder_without_store_check(), + ) + .await + .unwrap(); + assert!( + !mem.objects.lock().unwrap().contains_key(STORE_IDENTITY_KEY), + "no identity file is written" + ); + assert!(mem.puts.lock().unwrap().is_empty(), "nothing is put"); + assert!(runtime.configured_store().unwrap().is_healthy()); +} + +/// Every row of one backup id among the moved (rows unique by ARN alone, as +/// on Cassandra) is set aside before any row is checked: the first in +/// listing order is DELETING and the second AVAILABLE, and the copy of +/// their shared prefix lacks a data file. Neither is checked, so the +/// DELETING row's deletion is not finished and the shared prefix keeps +/// every object; both are named. +#[tokio::test] +async fn every_row_of_one_backup_id_among_the_moved_is_checked_no_further() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + fake.unique_by_arn + .store(true, std::sync::atomic::Ordering::Release); + let runtime = runtime_on(&mem, store()); + let ours = sync_arn(1); + let other = ours.replace(REGION, "eu-west-1"); + let mut objects = backup_objects(&other); + let dropped = objects + .iter() + .position(|(k, _)| k.contains("/data/")) + .unwrap(); + objects.remove(dropped); + for (key, bytes) in objects { + mem.insert(&key, &bytes); + } + let before = objects_under(&mem, &other).len(); + let mut deleting = old_row(&ours, BackupStatus::Deleting); + deleting.created_at = 1.0; + let mut available = old_row(&other, BackupStatus::Available); + available.created_at = 2.0; + seed_rows(&fake, &[deleting, available]); + let report = move_old(&fake, &runtime).await; + assert!(report.finished.is_empty(), "{report:?}"); + let named: Vec<&str> = report + .stopping + .iter() + .map(|(row, _)| row.backup_arn.as_str()) + .collect(); + assert_eq!(named, [ours.as_str(), other.as_str()], "{report:?}"); + assert!( + report + .stopping + .iter() + .all(|(_, why)| why.contains("two rows of backup id")), + "{report:?}" + ); + assert_eq!(objects_under(&mem, &other).len(), before); + assert_eq!( + fake.rows.lock().unwrap()[&ours].status, + BackupStatus::Deleting + ); + assert!(fake.rewrites_seen.lock().unwrap().is_empty()); +} + +/// The FAILED-prefix duty keeps the prefix of a row that a move rewrote to +/// another store after the duty listed it, its status unchanged: only the +/// store half of `Duty::fenced`'s re-read stands between them (the prefix +/// deletion has no store condition of its own). +#[tokio::test] +async fn the_failed_prefix_duty_keeps_a_row_moved_after_its_listing() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let runtime = runtime_on(&mem, store()); + let moved = sync_arn(1); + let leftover = format!("{}data/000001.json.gz", prefix_of(&moved)); + mem.insert(&leftover, b"x"); + seed_rows(&fake, &[row(&moved, BackupStatus::Failed)]); + let elsewhere = moved.clone(); + *fake.after_older_than.lock().unwrap() = Some(( + BackupStatus::Failed, + StaleClock::Heartbeat, + Box::new(move |fake: &Fake| { + fake.rows.lock().unwrap().get_mut(&elsewhere).unwrap().store = old_store(); + }), + )); + let cycle = cycle_on(&fake, &runtime).await; + assert!( + fake.after_older_than.lock().unwrap().is_none(), + "the hook ran" + ); + assert!(cycle.failed_prefixes_deleted.is_empty(), "{cycle:?}"); + assert!( + mem.keys().contains(&leftover), + "the moved row's prefix is kept" + ); +} + +/// A row of the old root whose one status entry is under an earlier status +/// (on Cassandra, a DeleteBackup whose entry batch failed: the row +/// DELETING, its entry AVAILABLE) is listed by the root's listing and +/// checked like any other: with a complete copy under the configured root +/// it stops the move; with none, its deletion is finished and it moves. +#[tokio::test] +async fn move_store_checks_a_row_whose_status_entry_is_under_an_earlier_status() { + let deleting = sync_arn(1); + + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let runtime = runtime_on(&mem, store()); + put_backup(&mem, &deleting); + seed_rows(&fake, &[old_row(&deleting, BackupStatus::Deleting)]); + fake.set_stale_entry(&deleting, BackupStatus::Available); + let report = move_old(&fake, &runtime).await; + assert_eq!(report.rows, 1, "{report:?}"); + assert_eq!(report.stopping.len(), 1, "{report:?}"); + assert!( + report.stopping[0].1.contains("verified manifest"), + "{report:?}" + ); + assert!(fake.rewrites_seen.lock().unwrap().is_empty()); + + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let runtime = runtime_on(&mem, store()); + seed_rows(&fake, &[old_row(&deleting, BackupStatus::Deleting)]); + fake.set_stale_entry(&deleting, BackupStatus::Available); + let report = move_old(&fake, &runtime).await; + assert!(report.complete(), "{report:?}"); + assert_eq!(report.finished, std::slice::from_ref(&deleting)); + assert_eq!(report.moved, std::slice::from_ref(&deleting)); + let row = fake.rows.lock().unwrap()[&deleting].clone(); + assert_eq!((row.status, row.store), (BackupStatus::Deleted, store())); +} + +/// On a backend whose summary counts status entries (Cassandra), a row +/// whose one entry is under an earlier status counts as a row under its own +/// status in the report of other stores (and a DELETING one is named), +/// while its stale entry is reported as a status entry. +#[tokio::test] +async fn the_report_of_other_stores_counts_a_row_whose_entry_is_under_an_earlier_status() { + let fake = Arc::new(Fake::default()); + fake.counts_entries + .store(true, std::sync::atomic::Ordering::Release); + let available = sync_arn(1); + let deleting = sync_arn(2); + seed_rows( + &fake, + &[ + old_row(&available, BackupStatus::Available), + old_row(&deleting, BackupStatus::Deleting), + ], + ); + fake.set_stale_entry(&deleting, BackupStatus::Available); + let others = super::store_maintenance::other_stores(&*fake, &store()) + .await + .unwrap(); + assert_eq!(others.len(), 1, "{others:?}"); + assert_eq!( + others[0].counts, + [(BackupStatus::Available, 1), (BackupStatus::Deleting, 1)] + ); + assert_eq!(others[0].entries, [(BackupStatus::Available, 1)]); + let named: Vec<&str> = others[0] + .unresolved + .iter() + .map(|r| r.backup_arn.as_str()) + .collect(); + assert_eq!(named, [deleting.as_str()]); +} + +/// A stop during the wait before a DELETING row's finished deletion leaves +/// the row DELETING and its objects untouched: the ARN is not reached. +#[tokio::test] +async fn a_stop_during_the_wait_before_a_deleting_rows_finish_deletes_nothing() { + use super::store_maintenance::{ForgetConfigured, ForgetRun, forget_backups}; + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let runtime = runtime_on(&mem, store()); + let configured = runtime.configured_store().unwrap(); + let identity = configured.identity().unwrap(); + let deleting = sync_arn(1); + put_backup(&mem, &deleting); + seed_rows(&fake, &[row(&deleting, BackupStatus::Deleting)]); + let whole = objects_under(&mem, &deleting).len(); + let waiting = std::sync::atomic::AtomicBool::new(false); + let on_wait = |_: Duration| waiting.store(true, std::sync::atomic::Ordering::Release); + let stop = || { + waiting + .load(std::sync::atomic::Ordering::Acquire) + .then(|| "SIGTERM received".to_owned()) + }; + let run = ForgetRun { + removals_from: Some(tokio::time::Instant::now() + Duration::from_secs(60)), + on_wait: &on_wait, + configured: Some(ForgetConfigured { + store: configured, + healthy: Some(&identity), + mismatch_row_ids: &[], + file_missing_with_rows: false, + }), + ..forget_run(Duration::from_secs(300), &stop) + }; + let report = forget_backups(&*fake, std::slice::from_ref(&deleting), REGION, run).await; + assert_eq!(report.stopped.as_deref(), Some("SIGTERM received")); + assert!(report.done.is_empty(), "{report:?}"); + assert_eq!(objects_under(&mem, &deleting).len(), whole); + assert_eq!( + fake.rows.lock().unwrap()[&deleting].status, + BackupStatus::Deleting + ); +} + +/// The backup manual's copied-identity-file case with a DELETING row among +/// the new root's rows: the root's file names the old root's id +/// (`store-old`) and its rows carry `store-1`, so the store's check fails +/// (`IdMismatch`). `--forget-backup` removes the AVAILABLE row and refuses +/// the DELETING one, saying that its own id takes part in the mismatch and +/// naming the way out (the same command with no store configured); the +/// check still fails afterwards. +#[tokio::test] +async fn a_deleting_row_behind_an_identity_mismatch_is_refused_naming_the_way_out() { + use super::store_maintenance::{ + ForgetBackup, ForgetConfigured, ForgetRun, Forgotten, forget_backups, + }; + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-old", "catalog-1")); + let fake = Arc::new(Fake::default()); + let runtime = runtime_on(&mem, store()); + let configured = runtime.configured_store().unwrap(); + let available = sync_arn(1); + let deleting = sync_arn(2); + put_backup(&mem, &available); + put_backup(&mem, &deleting); + seed_rows( + &fake, + &[ + row(&available, BackupStatus::Available), + row(&deleting, BackupStatus::Deleting), + ], + ); + let problem = super::check_configured_store_read_only(configured, &*fake, None) + .await + .unwrap_err(); + assert!( + matches!(problem, StoreProblem::IdMismatch { .. }), + "{problem:?}" + ); + let run = ForgetRun { + configured: Some(ForgetConfigured { + store: configured, + healthy: None, + mismatch_row_ids: problem.mismatch_row_ids(), + file_missing_with_rows: false, + }), + ..forget_run(Duration::from_secs(300), &never) + }; + let asked = [available.clone(), deleting.clone()]; + let report = forget_backups(&*fake, &asked, REGION, run).await; + let refused = Forgotten::DeletionBehindMismatch("store-1".to_owned()); + assert_eq!( + report.done, + [ + ( + available.clone(), + ForgetBackup::Row( + BackupStatus::Available, + Forgotten::Removed(BackupStatus::Available) + ) + ), + ( + deleting.clone(), + ForgetBackup::Row(BackupStatus::Deleting, refused.clone()) + ), + ] + ); + let text = refused.to_string(); + assert!( + text.contains("store id store-1") + && text.contains("kind = \"disabled\"") + && text.contains("DeleteBackup it again"), + "{text}" + ); + assert_eq!( + objects_under(&mem, &deleting).len(), + backup_objects(&deleting).len() + ); + assert!(matches!( + super::check_configured_store_read_only(configured, &*fake, None).await, + Err(StoreProblem::IdMismatch { .. }) + )); +} + +/// A filesystem root the command found missing with rows for it (the +/// defaulted root moved away, or a link to a volume that is gone): its rows +/// are refused as an unmounted root's, with no store configured; a row of +/// another root in the same run is removed. +#[tokio::test] +async fn forget_backup_refuses_the_rows_of_a_missing_root_with_rows() { + use super::store_maintenance::{ForgetBackup, ForgetRun, Forgotten, forget_backups}; + let fake = Arc::new(Fake::default()); + let theirs = sync_arn(1); + let ours = sync_arn(2); + seed_rows( + &fake, + &[ + old_row(&theirs, BackupStatus::Available), + row(&ours, BackupStatus::Available), + ], + ); + let configured = store(); + let run = ForgetRun { + unmounted_root: Some(&configured.store_root), + ..forget_run(Duration::from_secs(300), &never) + }; + let asked = [theirs.clone(), ours.clone()]; + let report = forget_backups(&*fake, &asked, REGION, run).await; + assert_eq!( + report.done, + [ + ( + theirs.clone(), + ForgetBackup::Row( + BackupStatus::Available, + Forgotten::Removed(BackupStatus::Available) + ) + ), + ( + ours.clone(), + ForgetBackup::Row(BackupStatus::Available, Forgotten::RootWithoutFile) + ), + ] + ); + assert!(fake.rows.lock().unwrap().contains_key(&ours)); +} diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 501720e57..fd7a2faa4 100755 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -12,7 +12,8 @@ pub use backup::{ BackupRuntime, ConfiguredStore, HeldClaim, JobSettings, KillPoint, LOCAL_CHECK_TIMEOUT, OTHER_STORE, RESTORE_REGISTRATION_TIMEOUT, S3_CHECK_TIMEOUT, STORE_IDENTITY_KEY, Spool, StoreFence, StoreHealth, StoreHealthReport, StoreIdentityFile, StoreProblem, TARGET_NOT_EMPTY, - check_configured_store, maintenance, read_identity_bounded, reconciler, store_identity, sync, + check_configured_store, check_configured_store_read_only, maintenance, read_identity_bounded, + reconciler, store_identity, store_maintenance, sync, }; mod batch_get_item; mod batch_write_item; diff --git a/crates/server/src/backup_store.rs b/crates/server/src/backup_store.rs index 2743ea835..02b9b34af 100644 --- a/crates/server/src/backup_store.rs +++ b/crates/server/src/backup_store.rs @@ -138,8 +138,10 @@ pub enum Caller { /// run): as `Command`, and no check of the store runs, so a root is left /// as it is found (neither the default directory nor an identity file is /// created; a default root that does not exist yet leaves the store - /// unconfigured, logged); the store is returned `checking` and the - /// command reads what it needs itself. + /// unconfigured, logged, as [`NotConfigured::DefaultRootMissing`] or, + /// when the catalog holds rows for it, as + /// [`NotConfigured::DefaultRootMissingWithRows`]); the store is + /// returned `checking` and the command reads what it needs itself. ReadOnlyCommand, } @@ -151,9 +153,19 @@ pub enum NotConfigured { Absent, /// `kind = "disabled"`. Disabled, - /// A read-only command found that the default root does not exist yet: - /// no server has run with this database file. + /// A read-only command found that the default root does not exist yet, + /// and the catalog holds no rows for it: no server has run with this + /// database file. DefaultRootMissing(PathBuf), + /// A read-only command found the default root missing while the catalog + /// holds rows for it: moved away, or a symbolic link to a volume that + /// is gone. `label` is the root as the rows record it (the link's + /// target, for a link), `store_ids` the ids they carry. + DefaultRootMissingWithRows { + root: PathBuf, + label: String, + store_ids: Vec, + }, /// The defaulted root could not be created, opened, validated or checked /// while the catalog has no rows for it (module doc): the warning path. DefaultRootUnusable { @@ -181,6 +193,18 @@ impl std::fmt::Display for NotConfigured { file, so there are no store-backed backups to list, purge or sync", root.display() ), + Self::DefaultRootMissingWithRows { + root, + label, + store_ids, + } => write!( + f, + "the default root {} is missing while the catalog holds rows for {label} (store \ + id {}): a volume that is not mounted, a directory moved away, or a link to a \ + volume that is gone; put the root back before listing, purging or syncing it", + root.display(), + store_ids.join(", ") + ), Self::DefaultRootUnusable { root, problem } => write!( f, "the default root {root} failed its check ({problem}) and the catalog has no \ @@ -335,6 +359,15 @@ pub async fn configure_described( && let Some(fs) = &config.filesystem && !fs.path.exists() { + if let Some((label, store_ids)) = missing_root_with_rows(catalog, &fs.path).await? { + let why = NotConfigured::DefaultRootMissingWithRows { + root: fs.path.clone(), + label, + store_ids, + }; + tracing::warn!("backup store: {why}"); + return Ok(StoreSetup::without_store(runtime, why)); + } tracing::info!( "backup store: the default root {} does not exist yet (no server has run with this \ database file); there are no store-backed backups to list", @@ -768,6 +801,63 @@ fn banner_target(config: &extenddb_backup_store::BackupStoreConfig) -> String { } } +/// The label of a defaulted filesystem root that does not exist, and the +/// store ids the catalog records for it, when it records any. The labels +/// tried: the path as configured, its canonical parent joined with its name +/// (the spelling a server's rows record), and for a dangling symbolic link +/// its target the same two ways (rows recorded through the link carry the +/// target's canonical directory). One level of link is followed. +async fn missing_root_with_rows( + catalog: &C, + path: &Path, +) -> anyhow::Result)>> { + let mut paths = vec![path.to_path_buf()]; + if let Ok(target) = std::fs::read_link(path) { + paths.push(match path.parent() { + Some(parent) if target.is_relative() => parent.join(target), + _ => target, + }); + } + let mut labels: Vec = Vec::new(); + for path in &paths { + for label in [Some(path.clone()), canonical_ancestor(path)] + .into_iter() + .flatten() + { + let label = label.display().to_string(); + if !labels.contains(&label) { + labels.push(label); + } + } + } + for label in labels { + let ids = catalog + .read_store_ids(StoreKind::Filesystem, &label) + .await + .map_err(|e| { + anyhow::anyhow!("the catalog's rows for the default root could not be read: {e}") + })?; + if !ids.is_empty() { + return Ok(Some((label, ids))); + } + } + Ok(None) +} + +/// `path` with its longest existing ancestor canonicalized and the rest of +/// its components joined back on; `None` when no ancestor resolves. +fn canonical_ancestor(path: &Path) -> Option { + let mut rest = Vec::new(); + let mut current = path; + loop { + if let Ok(canonical) = std::fs::canonicalize(current) { + return Some(rest.iter().rev().fold(canonical, |p, c| p.join(c))); + } + rest.push(current.file_name()?.to_owned()); + current = current.parent()?; + } +} + /// A path for an operator's line: quoted when it is empty or begins or ends /// with whitespace, so the line shows where it ends. fn shown_path(path: &Path) -> String { @@ -1016,7 +1106,7 @@ mod tests { use extenddb_storage::BoxedFuture; use extenddb_storage::backup_catalog::{ BackupKey, BackupRow, BackupStatus, DeleteStart, JobClaim, MaintenanceClaim, RestoreJobRow, - RestoreTargetKey, StoreIdentity, TimeBounds, + RestoreTargetKey, StoreIdentity, StoreRootCount, TimeBounds, }; use extenddb_storage::error::StorageError; use futures::stream::BoxStream; @@ -1142,6 +1232,8 @@ mod tests { fn remove_backup_row<'a>( &'a self, _key: &'a BackupKey, + _store_kind: StoreKind, + _store_root: &'a str, ) -> BoxedFuture<'a, Result> { unused() } @@ -1210,6 +1302,28 @@ mod tests { let ids = self.ids.clone(); Box::pin(async move { Ok(ids) }) } + fn list_backup_rows_by_root<'a>( + &'a self, + _store_kind: StoreKind, + _store_root: &'a str, + _status: Option, + ) -> BoxStream<'a, Result> { + Box::pin(futures::stream::once(unused())) + } + fn rewrite_store<'a>( + &'a self, + _from_kind: StoreKind, + _from_root: &'a str, + _keys: &'a [BackupKey], + _to: &'a StoreIdentity, + ) -> BoxedFuture<'a, Result> { + unused() + } + fn summarize_store_roots( + &self, + ) -> BoxedFuture<'_, Result, StorageError>> { + unused() + } fn maintenance_claim<'a>( &'a self, _holder: &'a str, @@ -1641,6 +1755,17 @@ mod tests { .to_string() .contains("/srv/db/backups does not exist yet") ); + let with_rows = NotConfigured::DefaultRootMissingWithRows { + root: PathBuf::from("/srv/db/backups"), + label: "/mnt/vol/backups".to_owned(), + store_ids: vec!["store-1".to_owned()], + } + .to_string(); + assert!( + with_rows.contains("/srv/db/backups is missing while the catalog holds rows for /mnt/vol/backups (store id store-1)") + && !with_rows.contains("no server has run"), + "{with_rows}" + ); let unusable = NotConfigured::DefaultRootUnusable { root: "/srv/db/backups".to_owned(), problem: "extenddb-store.json names catalog other".to_owned(), diff --git a/crates/storage-cassandra/src/backup_catalog.rs b/crates/storage-cassandra/src/backup_catalog.rs index 61e7ee05e..381557b20 100644 --- a/crates/storage-cassandra/src/backup_catalog.rs +++ b/crates/storage-cassandra/src/backup_catalog.rs @@ -32,21 +32,63 @@ //! DeleteBackup's lock is the lightweight transaction on the `backups_by_arn` //! row, `IF backup_status = 'AVAILABLE' AND active_restores = null`, on the //! same partition a restore registers on, so the two order wholly one before -//! the other, as the in-catalog paths do. +//! the other, as the in-catalog paths do. The condition also carries the +//! store kind and root the delete read, as does `set_backup_deleted`'s +//! `IF backup_status = 'DELETING'`: each moves the row's status entry under +//! the root it read, so a row `rewrite_store` moved after the read fails the +//! condition, is read again, and gets its entry under the root it names. //! //! A status change moves the row's `backup_index_by_status` entry with a //! plain batch after the lightweight transaction on the row. A process that -//! dies between the two leaves an entry whose status the row no longer -//! matches; every reader of the status table re-reads the row and skips such -//! an entry, one wasted read each. `remove_backup_row` deletes the entries -//! under the status it read, so an entry under an earlier status stays until -//! a sweep of orphan entries exists, and no sweep removes them yet. One path -//! writes such an entry by design: `insert_backup_row`'s row is AVAILABLE -//! from its lightweight transaction on, so a DeleteBackup between that -//! transaction and the insert's entry batch moves a status entry the batch -//! has not written yet, and the batch then writes an AVAILABLE entry that no -//! later write removes (an insert completed by a later sync has the same -//! window). +//! dies between the two, or a batch that fails (it is logged, not sent +//! again), leaves the row's only entry under the earlier status; every +//! reader of one status partition re-reads the row and skips such an entry, +//! one wasted read each, so `list_backup_rows` does not list that row until +//! its entry is repaired. `list_backup_rows_by_root` reads all five +//! partitions of the root and lists the row under its own status. +//! `remove_backup_row` deletes the entries under the status it read, so an +//! entry under an earlier status stays. +//! Three paths leave such entries without a crash or a failed batch: +//! - `insert_backup_row`'s row is AVAILABLE from its lightweight transaction +//! on, so a DeleteBackup between that transaction and the insert's entry +//! batch moves a status entry the batch has not written yet, and the batch +//! then writes an AVAILABLE entry that no later write removes (an insert +//! completed by a later sync has the same window). +//! - A deleter (`start_backup_delete`'s DELETING, `set_backup_deleted`'s +//! DELETED) whose lightweight transaction lands before a `rewrite_store` +//! of the row and whose entry batch lands after it writes the new status's +//! entry under the old root, where no row stands behind it. +//! - A `rewrite_store` whose answer was lost and whose row changed status +//! before the retry leaves the new root's entry under the earlier status. +//! +//! Every reader skips them. Plain sync's report of other stores verifies +//! the summary's counts against the rows on this backend +//! (`summary_counts_status_entries`, through `list_backup_rows_by_root`) +//! and names the counts no row of that status stands behind as status +//! entries, and `backup sync --forget-store `, under the maintenance +//! claim, deletes the root's entries whose row is gone or names another +//! store, and repairs those whose row is of the root under another status +//! (`remove_orphan_status_entries`). The second case +//! cannot arise for the tombstone removal: `remove_backup_row`'s +//! transaction is conditioned on the store it read. +//! +//! `rewrite_store` (`backup sync --move-store`) is the one writer that +//! writes an entry before its row: per row it writes the new root's status +//! entry, then the lightweight transaction that rewrites the row's kind, +//! root and id (conditioned on the old kind and root and the status it +//! read), then deletes the old root's entry. With the row first, a stop +//! before the new entry would leave a moved row that no listing of the +//! configured store returns, so the reconciler, sync's row read and the +//! manifest probe would not see it. With the entry first, a stop before the +//! transaction leaves an entry under the new root whose row still names the +//! old root, which every reader skips, and the row still listed under the +//! old root, so the move runs again; a stop after it leaves the old root's +//! entry, which readers skip and `read_store_ids` of the old root counts, +//! until `--forget-store` of the old root deletes it. A transaction that does not +//! apply (another writer changed the row after the read) is followed by a +//! serial read of the row: one that already names the new store keeps the +//! new entry and counts as moved, and only otherwise is the new entry +//! removed. //! //! A conditional write whose error is the driver's `Io` or `Timeout`, or the //! server's `WriteTimeout` or `WriteFailure`, has an unknown outcome @@ -83,8 +125,9 @@ use extenddb_core::types::CreateTableInput; use extenddb_storage::backup_catalog::{ BackupCatalog, BackupKey, BackupRow, BackupStatus, DeleteStart, ForgetRow, Heartbeat, JobClaim, JobRow, JobStatus, MaintenanceClaim, RestoreJobRow, RestoreKind, RestoreRegistered, - RestoreRegistrationRow, RestoreTargetKey, StaleClock, StatusFields, StoreIdentity, StoreKind, - TimeBounds, UNOWNED_GENERATION, WriteOutcome, billing_mode_name, new_generation, + RestoreRegistrationRow, RestoreTargetKey, StaleClock, StatusEntrySweep, StatusFields, + StoreIdentity, StoreKind, StoreRootCount, TimeBounds, UNOWNED_GENERATION, WriteOutcome, + billing_mode_name, new_generation, }; use extenddb_storage::error::StorageError; use futures::future::BoxFuture; @@ -99,8 +142,13 @@ use crate::engine::CassandraSession; const DELETE_CLAIM_ATTEMPTS: usize = 3; /// The condition of the DELETING write and of the outright removal: the row -/// is AVAILABLE and no restore is registered on it. -const DELETE_CLAIM_CONDITION: &str = "IF backup_status = 'AVAILABLE' AND active_restores = null"; +/// is AVAILABLE, no restore is registered on it, and it still names the +/// store (kind and root) the writer read, bound after the key. With the +/// store in the condition, a row `rewrite_store` moved after the read is not +/// matched, and the re-read moves its status entry under the root it now +/// names, not the one read. +const DELETE_CLAIM_CONDITION: &str = "IF backup_status = 'AVAILABLE' AND active_restores = null \ + AND store_kind = ? AND store_root = ?"; /// The `owner_generation` column value of a generation: the column is a /// signed 64-bit integer and `new_generation` never exceeds `i64::MAX`. @@ -622,8 +670,10 @@ impl CassandraEngine { /// Move the `backup_index_by_status` entry of a row from one status /// partition to another, after the authoritative row changed. A failure - /// is logged: a reader of the status index re-reads the authoritative - /// row and skips an entry whose status it no longer matches. + /// is logged and the batch is not sent again: the row's entry stays + /// under `from`, readers of one status partition re-read the row and + /// skip it, `list_backup_rows_by_root` lists the row under its own + /// status, and `remove_orphan_status_entries` of the root repairs it. async fn move_status_index_row( &self, key: &BackupKey, @@ -675,7 +725,10 @@ impl CassandraEngine { /// [`DELETE_CLAIM_CONDITION`] that writes DELETING or, with `remove`, /// deletes the row (`Deleting` then means removed); then the row's index /// entries. Every writer of `active_restores` is conditional, so the - /// condition orders the write wholly before or after a registration. + /// condition orders the write wholly before or after a registration; the + /// condition also carries the store read, so a row `rewrite_store` + /// moved between the read and the write is read again and its entry is + /// moved under the root it names. /// /// A lost answer of the removal is settled by a serial read: no row /// means removed, a row means the loop runs again (a lost answer on @@ -729,8 +782,16 @@ impl CassandraEngine { None }; crate::test_hooks::before_delete_claim(&key.backup_arn).await; - let values = - || cdrs_tokio::query_values!(key.account_id.as_str(), key.backup_arn.as_str()); + let read_kind = stored.store_kind.as_deref().unwrap_or(""); + let read_root = stored.store_root.as_deref().unwrap_or(""); + let values = || { + cdrs_tokio::query_values!( + key.account_id.as_str(), + key.backup_arn.as_str(), + read_kind, + read_root + ) + }; let fault = if remove { crate::test_hooks::forget_write_fault(&key.backup_arn) } else { @@ -771,6 +832,7 @@ impl CassandraEngine { ); } } else { + crate::test_hooks::after_delete_claim(&key.backup_arn).await; let store = StoreIdentity { store_kind: StoreKind::parse(stored.store_kind.as_deref().unwrap_or("")) .unwrap_or(StoreKind::Filesystem), @@ -790,8 +852,9 @@ impl CassandraEngine { key.backup_arn ))); } - // A registration or another writer landed between the check - // and the write on every attempt. + // A registration or another writer (a move of the row to + // another store included) landed between the check and the + // write on every attempt. let stored = self .load_stored_backup(&key.account_id, &key.backup_arn) .await? @@ -816,6 +879,24 @@ impl CassandraEngine { &self, store: &StoreIdentity, status: BackupStatus, + ) -> Result, StorageError> { + self.root_index_arns( + store.store_kind, + &store.store_root, + Some(&store.store_id), + status, + ) + .await + } + + /// The ARNs the status index lists under `store_root` and `status` whose + /// entry carries `store_kind`, and `store_id` when given. + async fn root_index_arns( + &self, + store_kind: StoreKind, + store_root: &str, + store_id: Option<&str>, + status: BackupStatus, ) -> Result, StorageError> { let query = format!( "SELECT backup_arn, account_id, store_kind, store_id FROM {}.backup_index_by_status \ @@ -825,16 +906,16 @@ impl CassandraEngine { let rows = crate::cassandra_util::query_rows_quorum::( &self.session, &query, - cdrs_tokio::query_values!(store.store_root.as_str(), status.as_str()), + cdrs_tokio::query_values!(store_root, status.as_str()), "status index", ) .await?; let mut arns = Vec::with_capacity(rows.len()); for row in rows { - let store_kind: Option = row.get_by_name("store_kind").ok().flatten(); - let store_id: Option = row.get_by_name("store_id").ok().flatten(); - if store_kind.as_deref() != Some(store.store_kind.as_str()) - || store_id.as_deref() != Some(store.store_id.as_str()) + let entry_kind: Option = row.get_by_name("store_kind").ok().flatten(); + let entry_id: Option = row.get_by_name("store_id").ok().flatten(); + if entry_kind.as_deref() != Some(store_kind.as_str()) + || store_id.is_some_and(|id| entry_id.as_deref() != Some(id)) { continue; } @@ -847,6 +928,205 @@ impl CassandraEngine { Ok(arns) } + /// Write one `backup_index_by_status` entry, or with `remove` delete it, + /// at `LOCAL_QUORUM`: `rewrite_store`'s two entry writes. + async fn write_status_entry( + &self, + key: &BackupKey, + store: &StoreIdentity, + status: BackupStatus, + remove: bool, + ) -> Result<(), StorageError> { + let catalog = self.catalog_keyspace(); + if remove { + crate::cassandra_util::query_rows_quorum::( + &self.session, + &format!( + "DELETE FROM {catalog}.backup_index_by_status \ + WHERE store_root = ? AND backup_status = ? AND backup_arn = ?" + ), + cdrs_tokio::query_values!( + store.store_root.as_str(), + status.as_str(), + key.backup_arn.as_str() + ), + "remove a status entry", + ) + .await?; + } else { + crate::cassandra_util::query_rows_quorum::( + &self.session, + &format!( + "INSERT INTO {catalog}.backup_index_by_status \ + (store_root, backup_status, backup_arn, account_id, store_kind, store_id) \ + VALUES (?, ?, ?, ?, ?, ?)" + ), + cdrs_tokio::query_values!( + store.store_root.as_str(), + status.as_str(), + key.backup_arn.as_str(), + key.account_id.as_str(), + store.store_kind.as_str(), + store.store_id.as_str() + ), + "write a status entry", + ) + .await?; + } + Ok(()) + } + + /// `rewrite_store` for one key, in the order the module doc gives: the + /// row read at `LOCAL_SERIAL` and skipped unless it is of the old kind + /// and root and not CREATING; the new root's status entry; the + /// lightweight transaction on the row, conditioned on the old kind and + /// root and the status read; then the old root's entry deleted. + /// `NotMatched` (another writer changed the row in between) reads the + /// row again at `LOCAL_SERIAL`: a row that already carries `to` counts + /// as this call's write and keeps the new entry; otherwise the new entry + /// is removed and the loop reads again, up to [`DELETE_CLAIM_ATTEMPTS`]; + /// a lost answer reads again, and a row that then carries `to`'s store + /// is this call's write. `true` when this call rewrote the row. + async fn rewrite_store_row( + &self, + from_kind: StoreKind, + from_root: &str, + key: &BackupKey, + to: &StoreIdentity, + ) -> Result { + let update = format!( + "UPDATE {}.backups_by_arn SET store_kind = ?, store_root = ?, store_id = ? \ + WHERE account_id = ? AND backup_arn = ? \ + IF store_kind = ? AND store_root = ? AND backup_status = ?", + self.catalog_keyspace() + ); + let mut lost: Option = None; + for _ in 0..DELETE_CLAIM_ATTEMPTS { + let Some(row) = self + .read_index_row_serial(&key.account_id, &key.backup_arn) + .await? + else { + return Ok(false); + }; + if let Some(status) = lost + && row.store == *to + { + // The lost answer was this call's write. + self.remove_old_status_entry(key, from_kind, from_root, status) + .await; + return Ok(true); + } + if row.store.store_kind != from_kind + || row.store.store_root != from_root + || row.status == BackupStatus::Creating + { + return Ok(false); + } + // The new root's entry before the row: a stop after it leaves an + // entry readers skip, and the row still listed under the old root. + self.write_status_entry(key, to, row.status, false).await?; + crate::test_hooks::rewrite_stop( + &key.backup_arn, + crate::test_hooks::RewriteStop::AfterNewEntry, + )?; + crate::test_hooks::before_rewrite_write(&key.backup_arn).await; + let values = || { + cdrs_tokio::query_values!( + to.store_kind.as_str(), + to.store_root.as_str(), + to.store_id.as_str(), + key.account_id.as_str(), + key.backup_arn.as_str(), + from_kind.as_str(), + from_root, + row.status.as_str() + ) + }; + let fault = crate::test_hooks::rewrite_write_fault(&key.backup_arn); + let outcome = if fault == Some(crate::test_hooks::ProgressWriteFault::Lost) { + WriteOutcome::Unknown + } else { + let outcome = conditional_write(&self.session, &update, values()).await?; + if fault.is_some() { + WriteOutcome::Unknown + } else { + outcome + } + }; + match outcome { + WriteOutcome::Matched => { + crate::test_hooks::rewrite_stop( + &key.backup_arn, + crate::test_hooks::RewriteStop::AfterRow, + )?; + self.remove_old_status_entry(key, from_kind, from_root, row.status) + .await; + return Ok(true); + } + WriteOutcome::NotMatched => { + lost = None; + // Read again before the new entry goes: a row that + // already carries `to` is this call's write (an earlier + // answer lost in transit whose proposal the retry's own + // Paxos prepare completed) or another move to the same store, + // and the new entry is its only correct one. + if self + .read_index_row_serial(&key.account_id, &key.backup_arn) + .await? + .is_some_and(|now| now.store == *to) + { + self.remove_old_status_entry(key, from_kind, from_root, row.status) + .await; + return Ok(true); + } + if let Err(error) = self.write_status_entry(key, to, row.status, true).await { + tracing::warn!( + "backup {}: the status entry written under {} for a move that \ + did not apply is left ({error}); readers verify against the row", + key.backup_arn, + to.store_root + ); + } + } + WriteOutcome::Unknown => lost = Some(row.status), + } + } + if lost.is_some() { + return Err(StorageError::Internal(format!( + "the store rewrite of backup {} has an unknown outcome on every attempt", + key.backup_arn + ))); + } + // Another writer changed the row between the read and the write on + // every attempt: left, and the caller's re-list names it. + Ok(false) + } + + /// Delete the old root's status entry of a row `rewrite_store` moved, + /// under the status the move read. A failure is logged: readers skip + /// the entry, and `read_store_ids` of the old root counts it until + /// `--forget-store` of the old root deletes it. + async fn remove_old_status_entry( + &self, + key: &BackupKey, + from_kind: StoreKind, + from_root: &str, + status: BackupStatus, + ) { + let old = StoreIdentity { + store_kind: from_kind, + store_root: from_root.to_owned(), + store_id: String::new(), + }; + if let Err(error) = self.write_status_entry(key, &old, status, true).await { + tracing::warn!( + "backup {}: moved, and its status entry under {from_root} is left ({error}); \ + readers verify against the row", + key.backup_arn + ); + } + } + /// Re-apply the target's `active_restores[table_id] = table_name` entry /// on the backup's row when it is absent, under the takeover of a dead /// manifest record (the trait doc of `claim_restore_dead`): one serial @@ -1410,6 +1690,43 @@ impl BackupCatalog for CassandraEngine { }) } + fn list_backup_rows_by_root<'a>( + &'a self, + store_kind: StoreKind, + store_root: &'a str, + status: Option, + ) -> BoxStream<'a, Result> { + rows_stream(async move { + // Every status partition of the root, whatever `status` is: the + // entry is a hint and the authoritative row decides, so a row + // whose entry is under an earlier status (a status change whose + // entry batch failed, or whose writer died between its + // transaction and its batch) is found and listed under its own + // status. Each row is read once, however many entries it has. + let mut seen = std::collections::HashSet::new(); + let mut rows = Vec::new(); + for entry_status in BackupStatus::ALL { + for (account_id, backup_arn) in self + .root_index_arns(store_kind, store_root, None, entry_status) + .await? + { + if !seen.insert((account_id.clone(), backup_arn.clone())) { + continue; + } + if let Some(row) = self.read_index_row(&account_id, &backup_arn).await? + && status.is_none_or(|status| row.status == status) + && row.store.store_kind == store_kind + && row.store.store_root == store_root + { + rows.push(row); + } + } + } + sort_listing(&mut rows); + Ok(rows) + }) + } + fn newest_available_backup_row<'a>( &'a self, store: &'a StoreIdentity, @@ -1548,6 +1865,27 @@ impl BackupCatalog for CassandraEngine { }) } + fn rewrite_store<'a>( + &'a self, + from_kind: StoreKind, + from_root: &'a str, + keys: &'a [BackupKey], + to: &'a StoreIdentity, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let mut rewritten = 0; + for key in keys { + if self + .rewrite_store_row(from_kind, from_root, key, to) + .await? + { + rewritten += 1; + } + } + Ok(rewritten) + }) + } + fn set_backup_deleted<'a>( &'a self, key: &'a BackupKey, @@ -1559,30 +1897,64 @@ impl BackupCatalog for CassandraEngine { else { return Ok(false); }; + // The store read is in the condition, so a row `rewrite_store` + // moved after the read is not matched: it is read once more at + // `LOCAL_SERIAL` and, still DELETING under the root it now + // names, written again with that root, and its status entry + // moves there. let query = format!( "UPDATE {}.backups_by_arn SET backup_status = 'DELETED', \ completed_at = toTimestamp(now()) \ - WHERE account_id = ? AND backup_arn = ? IF backup_status = 'DELETING'", + WHERE account_id = ? AND backup_arn = ? \ + IF backup_status = 'DELETING' AND store_kind = ? AND store_root = ?", self.catalog_keyspace() ); - let result = crate::cassandra_util::query_lwt( - &self.session, - &query, - cdrs_tokio::query_values!(key.account_id.as_str(), key.backup_arn.as_str()), - ) - .await?; - let applied = crate::cassandra_util::lwt_applied(&result)?; - if applied { - self.move_status_index_row(key, &row.store, "DELETING", "DELETED") - .await; + crate::test_hooks::before_deleted_write(&key.backup_arn).await; + let mut row = row; + for attempt in 0..2 { + let result = crate::cassandra_util::query_lwt( + &self.session, + &query, + cdrs_tokio::query_values!( + key.account_id.as_str(), + key.backup_arn.as_str(), + row.store.store_kind.as_str(), + row.store.store_root.as_str() + ), + ) + .await?; + if crate::cassandra_util::lwt_applied(&result)? { + crate::test_hooks::after_deleted_write(&key.backup_arn).await; + self.move_status_index_row(key, &row.store, "DELETING", "DELETED") + .await; + return Ok(true); + } + if attempt > 0 { + break; + } + match self + .read_index_row_serial(&key.account_id, &key.backup_arn) + .await? + { + Some(now) + if now.status == BackupStatus::Deleting + && (now.store.store_kind != row.store.store_kind + || now.store.store_root != row.store.store_root) => + { + row = now; + } + _ => break, + } } - Ok(applied) + Ok(false) }) } fn remove_backup_row<'a>( &'a self, key: &'a BackupKey, + store_kind: StoreKind, + store_root: &'a str, ) -> BoxFuture<'a, Result> { Box::pin(async move { let Some(row) = self @@ -1591,14 +1963,20 @@ impl BackupCatalog for CassandraEngine { else { return Ok(false); }; - if !matches!(row.status, BackupStatus::Deleted | BackupStatus::Failed) { + if !matches!(row.status, BackupStatus::Deleted | BackupStatus::Failed) + || row.store.store_kind != store_kind + || row.store.store_root != store_root + { return Ok(false); } + crate::test_hooks::before_tombstone_removal(&key.backup_arn).await; // The authoritative row first, conditioned on the tombstone - // status read; then its index entries. + // status and the store read, so a row `rewrite_store` moved + // after the read is left (`NotMatched` answers `false`); then + // its index entries. let delete = format!( "DELETE FROM {}.backups_by_arn WHERE account_id = ? AND backup_arn = ? \ - IF backup_status = ?", + IF backup_status = ? AND store_kind = ? AND store_root = ?", self.catalog_keyspace() ); let result = crate::cassandra_util::query_lwt( @@ -1607,7 +1985,9 @@ impl BackupCatalog for CassandraEngine { cdrs_tokio::query_values!( key.account_id.as_str(), key.backup_arn.as_str(), - row.status.as_str() + row.status.as_str(), + store_kind.as_str(), + store_root ), ) .await?; @@ -2360,7 +2740,8 @@ impl BackupCatalog for CassandraEngine { // `backup_index_by_status` is partitioned by (root, status), so a // root's rows are its five status partitions; the entry carries // the kind and id, which is all this read needs (an orphan entry - // left by a crash names the same store its row did). + // left by a crash names the store its row had or was being + // moved to). let query = format!( "SELECT store_kind, store_id FROM {}.backup_index_by_status \ WHERE store_root = ? AND backup_status = ?", @@ -2397,6 +2778,120 @@ impl BackupCatalog for CassandraEngine { }) } + fn summary_counts_status_entries(&self) -> bool { + true + } + + fn remove_orphan_status_entries<'a>( + &'a self, + store_kind: StoreKind, + store_root: &'a str, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + // The root's five status partitions; each entry's row is read at + // `LOCAL_SERIAL` (a lightweight transaction in progress is + // seen). An entry whose row is gone or names another store is + // deleted. An entry whose row is of this root under another + // status (a status change whose entry batch failed, or whose + // writer died between its transaction and its batch) is + // repaired: the entry under the row's status is written (an + // idempotent upsert), then the stale one is deleted. + let delete = format!( + "DELETE FROM {}.backup_index_by_status \ + WHERE store_root = ? AND backup_status = ? AND backup_arn = ?", + self.catalog_keyspace() + ); + let mut sweep = StatusEntrySweep::default(); + for status in BackupStatus::ALL { + for (account_id, backup_arn) in self + .root_index_arns(store_kind, store_root, None, status) + .await? + { + match self.read_index_row_serial(&account_id, &backup_arn).await? { + Some(row) + if row.store.store_kind == store_kind + && row.store.store_root == store_root => + { + if row.status == status { + continue; + } + self.write_status_entry(&row.key(), &row.store, row.status, false) + .await?; + sweep.repaired += 1; + } + _ => sweep.removed += 1, + } + crate::cassandra_util::query_rows_quorum::( + &self.session, + &delete, + cdrs_tokio::query_values!(store_root, status.as_str(), backup_arn.as_str()), + "remove an orphan status entry", + ) + .await?; + } + } + Ok(sweep) + }) + } + + fn summarize_store_roots(&self) -> BoxFuture<'_, Result, StorageError>> { + Box::pin(async move { + // The status table's partitions are (root, status): a scan of + // its partition keys, then each partition's entries counted by + // the kind and id they carry. Hints, not verified against the + // rows (the trait doc). + let catalog = self.catalog_keyspace(); + let partitions = crate::cassandra_util::query_rows_quorum::( + &self.session, + &format!("SELECT DISTINCT store_root, backup_status FROM {catalog}.backup_index_by_status"), + cdrs_tokio::query_values!(), + "status index partitions", + ) + .await?; + let entries_query = format!( + "SELECT store_kind, store_id FROM {catalog}.backup_index_by_status \ + WHERE store_root = ? AND backup_status = ?" + ); + let mut entries = Vec::new(); + for partition in partitions { + let root: String = + crate::cassandra_util::get_column(&partition, "store_root", "status index")?; + let status: String = + crate::cassandra_util::get_column(&partition, "backup_status", "status index")?; + let status = BackupStatus::parse(&status)?; + let rows = crate::cassandra_util::query_rows_quorum::( + &self.session, + &entries_query, + cdrs_tokio::query_values!(root.as_str(), status.as_str()), + "status index", + ) + .await?; + for row in rows { + let kind: Option = row.get_by_name("store_kind").ok().flatten(); + let id: Option = row.get_by_name("store_id").ok().flatten(); + let (Some(kind), Some(id)) = (kind, id) else { + continue; + }; + let store_kind = StoreKind::parse(&kind).ok_or_else(|| { + StorageError::Internal(format!( + "a status entry of root {root:?} carries an unknown store kind \ + {kind:?}" + )) + })?; + entries.push(( + StoreIdentity { + store_kind, + store_root: root.clone(), + store_id: id, + }, + status, + )); + } + } + Ok(StoreRootCount::tally(entries)) + }) + } + fn maintenance_claim<'a>( &'a self, holder: &'a str, diff --git a/crates/storage-cassandra/src/test_hooks.rs b/crates/storage-cassandra/src/test_hooks.rs index 62f318c95..67e698b85 100644 --- a/crates/storage-cassandra/src/test_hooks.rs +++ b/crates/storage-cassandra/src/test_hooks.rs @@ -51,7 +51,18 @@ //! * a fault on the lightweight transaction of `forget_available_backup_row`, //! and an error on the logged batch that removes a row's index entries, so //! a test can see the serial re-read settle a lost removal and a removal -//! whose entries stay answer that the row is removed. +//! whose entries stay answer that the row is removed; +//! * a fault on the lightweight transaction of `rewrite_store`, and a stop +//! of one row's rewrite after its first write or after its transaction, so +//! a test can see the serial re-read settle a lost rewrite and see where a +//! stopped rewrite leaves the row listed; +//! * a pause in one row's `rewrite_store` between the new root's status +//! entry and the lightweight transaction, so a test can delete or move +//! the row in that gap and see the transaction's conditions and the +//! serial re-read after it; +//! * a pause in `set_backup_deleted` between its read of the row and its +//! lightweight transaction, so a test can move the row in that gap and see +//! the transaction's store condition send it under the root it names. use std::time::Duration; @@ -69,6 +80,16 @@ pub enum ProgressWriteFault { Lost, } +/// Where a test stops one row's `rewrite_store`, with an error, as a process +/// that dies there would: after the write that precedes the lightweight +/// transaction (the new root's status entry), or after the transaction, +/// before the old root's entry is deleted. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RewriteStop { + AfterNewEntry, + AfterRow, +} + #[cfg(feature = "test-hooks")] mod armed { use std::collections::{HashMap, VecDeque}; @@ -77,7 +98,7 @@ mod armed { use tokio::sync::Notify; - use super::ProgressWriteFault; + use super::{ProgressWriteFault, RewriteStop}; /// A pause a test arms at one point of one operation: the operation /// signals `reached` when it gets there and waits for `released`. @@ -128,6 +149,13 @@ mod armed { pub(super) index_entry_batch_errors: HashMap, pub(super) forget_write_faults: HashMap>, pub(super) index_entry_removal_errors: HashMap, + pub(super) rewrite_write_faults: HashMap>, + pub(super) rewrite_stops: HashMap, + pub(super) before_rewrite_write: HashMap>, + pub(super) before_deleted_write: HashMap>, + pub(super) after_delete_claim: HashMap>, + pub(super) after_deleted_write: HashMap>, + pub(super) before_tombstone_removal: HashMap>, } pub(super) static HOOKS: LazyLock> = LazyLock::new(Mutex::default); @@ -184,6 +212,76 @@ pub fn pause_before_delete_claim(backup_arn: &str) -> std::sync::Arc { pause } +/// Arm a pause in the next `rewrite_store` of the row of `backup_arn`, +/// between the new root's status entry and the lightweight transaction on +/// the row. The pause is taken once and then disarmed. +#[cfg(feature = "test-hooks")] +pub fn pause_before_rewrite_write(backup_arn: &str) -> std::sync::Arc { + let pause = std::sync::Arc::new(Pause::new()); + armed::with_hooks(|hooks| { + hooks + .before_rewrite_write + .insert(backup_arn.to_owned(), std::sync::Arc::clone(&pause)); + }); + pause +} + +/// Arm a pause in the next `set_backup_deleted` of `backup_arn`, between +/// its read of the row and its lightweight transaction. The pause is taken +/// once and then disarmed. +#[cfg(feature = "test-hooks")] +pub fn pause_before_deleted_write(backup_arn: &str) -> std::sync::Arc { + let pause = std::sync::Arc::new(Pause::new()); + armed::with_hooks(|hooks| { + hooks + .before_deleted_write + .insert(backup_arn.to_owned(), std::sync::Arc::clone(&pause)); + }); + pause +} + +/// Arm a pause in the next delete of `backup_arn`, after its applied +/// DELETING claim and before the batch that moves its status entry. The +/// pause is taken once and then disarmed. +#[cfg(feature = "test-hooks")] +pub fn pause_after_delete_claim(backup_arn: &str) -> std::sync::Arc { + let pause = std::sync::Arc::new(Pause::new()); + armed::with_hooks(|hooks| { + hooks + .after_delete_claim + .insert(backup_arn.to_owned(), std::sync::Arc::clone(&pause)); + }); + pause +} + +/// Arm a pause in the next `set_backup_deleted` of `backup_arn`, after its +/// applied lightweight transaction and before the batch that moves its +/// status entry. The pause is taken once and then disarmed. +#[cfg(feature = "test-hooks")] +pub fn pause_after_deleted_write(backup_arn: &str) -> std::sync::Arc { + let pause = std::sync::Arc::new(Pause::new()); + armed::with_hooks(|hooks| { + hooks + .after_deleted_write + .insert(backup_arn.to_owned(), std::sync::Arc::clone(&pause)); + }); + pause +} + +/// Arm a pause in the next `remove_backup_row` of `backup_arn`, between its +/// read of the row and its lightweight transaction. The pause is taken once +/// and then disarmed. +#[cfg(feature = "test-hooks")] +pub fn pause_before_tombstone_removal(backup_arn: &str) -> std::sync::Arc { + let pause = std::sync::Arc::new(Pause::new()); + armed::with_hooks(|hooks| { + hooks + .before_tombstone_removal + .insert(backup_arn.to_owned(), std::sync::Arc::clone(&pause)); + }); + pause +} + /// Arm a pause in the next abandoned-create scan that reaches the CREATING /// row `backup_arn`, between the read of the row and the fence that moves it /// to DELETING. The pause is taken once and then disarmed. @@ -345,6 +443,69 @@ pub fn fail_index_entry_removals(backup_arn: &str, count: usize) { }); } +/// Make the next lightweight transactions of `rewrite_store` on the row of +/// `backup_arn` answer `WriteOutcome::Unknown`, one fault per write in the +/// order given, as [`fail_insert_writes`] does for the insert. +#[cfg(feature = "test-hooks")] +pub fn fail_rewrite_writes(backup_arn: &str, faults: &[ProgressWriteFault]) { + armed::with_hooks(|hooks| { + hooks + .rewrite_write_faults + .insert(backup_arn.to_owned(), faults.iter().copied().collect()); + }); +} + +/// Stop the next `rewrite_store` of the row of `backup_arn` at `at` with an +/// error, which ends the whole call. Taken once, then disarmed. +#[cfg(feature = "test-hooks")] +pub fn stop_rewrite(backup_arn: &str, at: RewriteStop) { + armed::with_hooks(|hooks| { + hooks.rewrite_stops.insert(backup_arn.to_owned(), at); + }); +} + +/// The fault the next rewrite lightweight transaction of `backup_arn` takes, +/// if a test armed one. +pub(crate) fn rewrite_write_fault(backup_arn: &str) -> Option { + #[cfg(feature = "test-hooks")] + { + armed::with_hooks(|hooks| { + hooks + .rewrite_write_faults + .get_mut(backup_arn) + .and_then(std::collections::VecDeque::pop_front) + }) + } + #[cfg(not(feature = "test-hooks"))] + { + let _ = backup_arn; + None + } +} + +/// The error a rewrite of `backup_arn` that reached `at` stops with, if a +/// test armed [`stop_rewrite`] there. +pub(crate) fn rewrite_stop(backup_arn: &str, at: RewriteStop) -> Result<(), StorageError> { + #[cfg(feature = "test-hooks")] + { + let armed_here = armed::with_hooks(|hooks| { + if hooks.rewrite_stops.get(backup_arn) == Some(&at) { + hooks.rewrite_stops.remove(backup_arn); + true + } else { + false + } + }); + if armed_here { + return Err(StorageError::Internal(format!( + "test hook: the rewrite of {backup_arn} stopped at {at:?}" + ))); + } + } + let _ = (backup_arn, at); + Ok(()) +} + /// The fault the next removal lightweight transaction of `backup_arn` takes, /// if a test armed one. pub(crate) fn forget_write_fault(backup_arn: &str) -> Option { @@ -477,6 +638,56 @@ pub(crate) async fn before_delete_claim(backup_arn: &str) { let _ = backup_arn; } +/// Hold here if a test armed a pause for the rewrite of `backup_arn`. +pub(crate) async fn before_rewrite_write(backup_arn: &str) { + #[cfg(feature = "test-hooks")] + if let Some(pause) = armed::with_hooks(|hooks| hooks.before_rewrite_write.remove(backup_arn)) { + pause.hold().await; + } + let _ = backup_arn; +} + +/// Hold here if a test armed a pause for the DELETED write of `backup_arn`. +pub(crate) async fn before_deleted_write(backup_arn: &str) { + #[cfg(feature = "test-hooks")] + if let Some(pause) = armed::with_hooks(|hooks| hooks.before_deleted_write.remove(backup_arn)) { + pause.hold().await; + } + let _ = backup_arn; +} + +/// Hold here if a test armed a pause after the DELETING claim of +/// `backup_arn`. +pub(crate) async fn after_delete_claim(backup_arn: &str) { + #[cfg(feature = "test-hooks")] + if let Some(pause) = armed::with_hooks(|hooks| hooks.after_delete_claim.remove(backup_arn)) { + pause.hold().await; + } + let _ = backup_arn; +} + +/// Hold here if a test armed a pause after the DELETED write of +/// `backup_arn`. +pub(crate) async fn after_deleted_write(backup_arn: &str) { + #[cfg(feature = "test-hooks")] + if let Some(pause) = armed::with_hooks(|hooks| hooks.after_deleted_write.remove(backup_arn)) { + pause.hold().await; + } + let _ = backup_arn; +} + +/// Hold here if a test armed a pause for the tombstone removal of +/// `backup_arn`. +pub(crate) async fn before_tombstone_removal(backup_arn: &str) { + #[cfg(feature = "test-hooks")] + if let Some(pause) = + armed::with_hooks(|hooks| hooks.before_tombstone_removal.remove(backup_arn)) + { + pause.hold().await; + } + let _ = backup_arn; +} + /// Hold here if a test armed a pause for the abandoned-create fence of /// `backup_arn`. pub(crate) async fn before_abandoned_fence(backup_arn: &str) { diff --git a/crates/storage-cassandra/tests/direct/backup_engine.rs b/crates/storage-cassandra/tests/direct/backup_engine.rs index ff08c8972..d874b5892 100644 --- a/crates/storage-cassandra/tests/direct/backup_engine.rs +++ b/crates/storage-cassandra/tests/direct/backup_engine.rs @@ -2552,7 +2552,10 @@ async fn test_start_backup_delete_checks_the_status_and_the_in_use_rule_under_th Err(StorageError::Validation(m)) if m.contains("Backup not found") )); assert!( - !engine.remove_backup_row(&key).await.unwrap(), + !engine + .remove_backup_row(&key, row.store.store_kind, &row.store.store_root) + .await + .unwrap(), "DELETING is not a tombstone" ); assert!(engine.set_backup_deleted(&key).await.unwrap()); @@ -2570,8 +2573,82 @@ async fn test_start_backup_delete_checks_the_status_and_the_in_use_rule_under_th .await .unwrap(); assert_eq!(deleted.len(), 1, "the status index followed: {deleted:?}"); - assert!(engine.remove_backup_row(&key).await.unwrap()); - assert!(!engine.remove_backup_row(&key).await.unwrap()); + // A row another writer moved to another store after the caller read it + // is left by a removal that names the store read. + let moved_to = StoreIdentity { + store_kind: row.store.store_kind, + store_root: format!("{}-moved", row.store.store_root), + store_id: "moved-1".to_owned(), + }; + assert_eq!( + engine + .rewrite_store( + row.store.store_kind, + &row.store.store_root, + std::slice::from_ref(&key), + &moved_to + ) + .await + .unwrap(), + 1 + ); + assert!( + !engine + .remove_backup_row(&key, row.store.store_kind, &row.store.store_root) + .await + .unwrap(), + "a row moved to another store is left" + ); + assert_eq!( + engine + .read_backup_row(&arn_of(&row)) + .await + .unwrap() + .map(|r| (r.status, r.store)), + Some((BackupStatus::Deleted, moved_to.clone())) + ); + // The same root under the other kind is another store: a removal that + // names the kind it read leaves the row. + let other_kind = StoreIdentity { + store_kind: if moved_to.store_kind == StoreKind::S3 { + StoreKind::Filesystem + } else { + StoreKind::S3 + }, + ..moved_to.clone() + }; + assert_eq!( + engine + .rewrite_store( + moved_to.store_kind, + &moved_to.store_root, + std::slice::from_ref(&key), + &other_kind + ) + .await + .unwrap(), + 1 + ); + assert!( + !engine + .remove_backup_row(&key, moved_to.store_kind, &moved_to.store_root) + .await + .unwrap(), + "a row moved to the other kind is left" + ); + let moved_to = other_kind; + assert!( + engine + .remove_backup_row(&key, moved_to.store_kind, &moved_to.store_root) + .await + .unwrap() + ); + assert!( + !engine + .remove_backup_row(&key, moved_to.store_kind, &moved_to.store_root) + .await + .unwrap() + ); assert!( engine .read_backup_row(&arn_of(&row)) @@ -4273,7 +4350,12 @@ async fn test_a_restore_registering_between_the_in_use_read_and_the_claim_keeps_ DeleteStart::Deleting ); assert!(engine.set_backup_deleted(&key).await.unwrap()); - assert!(engine.remove_backup_row(&key).await.unwrap()); + assert!( + engine + .remove_backup_row(&key, row.store.store_kind, &row.store.store_root) + .await + .unwrap() + ); drop(claim); engine .delete_table( @@ -4889,7 +4971,12 @@ async fn test_a_stale_creating_index_row_is_skipped_by_the_abandoned_create_scan ), "the legacy reader does not see an index row" ); - assert!(engine.remove_backup_row(&row.key()).await.is_ok()); + assert!( + engine + .remove_backup_row(&row.key(), row.store.store_kind, &row.store.store_root) + .await + .is_ok() + ); } /// `fail_restore_job` is conditioned on the `owner_generation` and @@ -5007,67 +5094,1136 @@ async fn test_the_failed_write_matches_only_the_listed_generation_and_heartbeat( fenced, generation, "the FAILED write carries a fresh generation" ); - assert!(completed.is_some()); - assert!(engine.remove_restore_target(&key).await.expect("remove")); - engine - .delete(&account_id, &backup_arn) - .await - .expect("the backup is free"); + assert!(completed.is_some()); + assert!(engine.remove_restore_target(&key).await.expect("remove")); + engine + .delete(&account_id, &backup_arn) + .await + .expect("the backup is free"); +} + +/// `read_store_ids`: the distinct ids of the rows for a root, of every +/// status, scoped to the kind and the root, read from the status partitions. +#[tokio::test] +async fn test_read_store_ids_lists_the_distinct_ids_of_a_root() { + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + let store = unique_store(); + let root = store.store_root.clone(); + let ids = |kind: StoreKind| { + let engine = &engine; + let root = root.clone(); + async move { engine.read_store_ids(kind, &root).await.expect("ids") } + }; + assert!(ids(StoreKind::Filesystem).await.is_empty()); + let first = index_row(&account_id, "Music", "a", &store); + let _c1 = engine.claim_backup(&first).await.expect("claim"); + let second = index_row(&account_id, "Music", "b", &store); + let c2 = engine.claim_backup(&second).await.expect("claim"); + assert_eq!( + c2.set_status( + JobStatus::Creating, + JobStatus::Failed, + &StatusFields::default() + ) + .await + .expect("fail"), + WriteOutcome::Matched + ); + assert_eq!( + ids(StoreKind::Filesystem).await, + ["store-1"], + "two rows of one store, of two statuses, are one id" + ); + let mut other_store = store.clone(); + other_store.store_id = "store-2".to_owned(); + let other = index_row(&account_id, "Songs", "c", &other_store); + let _c3 = engine.claim_backup(&other).await.expect("claim"); + assert_eq!(ids(StoreKind::Filesystem).await, ["store-1", "store-2"]); + assert!( + ids(StoreKind::S3).await.is_empty(), + "another kind at the same root is another store" + ); + assert!( + engine + .read_store_ids(StoreKind::Filesystem, "/elsewhere") + .await + .expect("ids") + .is_empty() + ); +} + +/// A row of `index_row(account, table, name, ..)` with `status`, under the +/// store of `kind`, `root` and `id`. +fn row_at( + account: &str, + name: &str, + status: BackupStatus, + (kind, root, id): (StoreKind, &str, &str), +) -> BackupRow { + let store = StoreIdentity { + store_kind: kind, + store_root: root.to_owned(), + store_id: id.to_owned(), + }; + let mut row = index_row(account, "Music", name, &store); + row.status = status; + row.size_bytes = 10; + row.item_count = 2; + row +} + +/// The ARNs of `rows` in the listings' (`created_at`, `backup_arn`) order. +fn in_listing_order(rows: &[&BackupRow]) -> Vec { + let mut rows = rows.to_vec(); + rows.sort_by(|a, b| { + a.created_at + .total_cmp(&b.created_at) + .then_with(|| a.backup_arn.cmp(&b.backup_arn)) + }); + rows.into_iter().map(|r| r.backup_arn.clone()).collect() +} + +/// The kind and id the status entry of `arn` under `root` and `status` +/// carries, read at quorum; `None` when there is no such entry. +async fn status_entry( + engine: &CassandraEngine, + root: &str, + status: BackupStatus, + arn: &str, +) -> Option<(String, String)> { + let rows = extenddb_storage_cassandra::cassandra_util::query_rows_quorum::( + &engine.session_arc(), + &format!( + "SELECT store_kind, store_id FROM {}.backup_index_by_status \ + WHERE store_root = ? AND backup_status = ? AND backup_arn = ?", + engine.catalog_keyspace() + ), + cdrs_tokio::query_values!(root, status.as_str(), arn), + "status entry", + ) + .await + .expect("read a status entry"); + rows.first().map(|row| { + use cdrs_tokio::types::IntoRustByName; + let kind: Option = row.get_by_name("store_kind").ok().flatten(); + let id: Option = row.get_by_name("store_id").ok().flatten(); + (kind.unwrap_or_default(), id.unwrap_or_default()) + }) +} + +/// The rows a by-root listing returns, by ARN. +async fn by_root(engine: &CassandraEngine, kind: StoreKind, root: &str) -> Vec { + let rows: Vec = engine + .list_backup_rows_by_root(kind, root, None) + .try_collect() + .await + .expect("listing"); + rows.into_iter().map(|r| r.backup_arn).collect() +} + +/// `list_backup_rows_by_root`: every row of the kind and root, whatever its +/// id, of every status or of one, whole and in the listings' order, read +/// from the root's status partitions and re-read; nothing of a root that +/// differs by one byte (a trailing `/`, one character fewer), of an S3 root +/// that differs by its `?endpoint=` suffix, or of the other kind at the same +/// label. +#[tokio::test] +async fn test_list_backup_rows_by_root_returns_every_id_and_status_of_the_root_and_nothing_else() { + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + let tag = crate::helpers::unique_test_id(); + let root = format!("/srv/backups-{tag}"); + let at = |id| (StoreKind::Filesystem, root.as_str(), id); + let mut of_root = Vec::new(); + for (i, status) in [ + BackupStatus::Available, + BackupStatus::Deleting, + BackupStatus::Deleted, + BackupStatus::Failed, + ] + .into_iter() + .enumerate() + { + let id = if i % 2 == 0 { "store-1" } else { "store-2" }; + of_root.push(row_at(&account_id, &format!("r{i}"), status, at(id))); + } + // Inserted newest first: the order asserted is the listing's. + for row in of_root.iter().rev() { + assert!(engine.insert_backup_row(row).await.expect("insert")); + } + let creating = row_at(&account_id, "job", BackupStatus::Creating, at("store-1")); + let claim = engine.claim_backup(&creating).await.expect("claim"); + let slash = format!("{root}/"); + let shorter = root[..root.len() - 1].to_owned(); + let s3 = format!("s3://bucket-{tag}/prefix"); + let s3_endpoint = format!("{s3}?endpoint=http://127.0.0.1:9200"); + let others = [ + row_at( + &account_id, + "slash", + BackupStatus::Available, + (StoreKind::Filesystem, &slash, "store-1"), + ), + row_at( + &account_id, + "shorter", + BackupStatus::Available, + (StoreKind::Filesystem, &shorter, "store-1"), + ), + row_at( + &account_id, + "kind", + BackupStatus::Available, + (StoreKind::S3, &root, "store-1"), + ), + row_at( + &account_id, + "s3", + BackupStatus::Available, + (StoreKind::S3, &s3, "store-3"), + ), + row_at( + &account_id, + "endpoint", + BackupStatus::Available, + (StoreKind::S3, &s3_endpoint, "store-3"), + ), + ]; + for row in &others { + assert!(engine.insert_backup_row(row).await.expect("insert")); + } + let listed = |kind: StoreKind, root: String, status: Option| { + let engine = &engine; + async move { + let rows: Vec = engine + .list_backup_rows_by_root(kind, &root, status) + .try_collect() + .await + .expect("listing"); + rows + } + }; + let arns = + |rows: Vec| -> Vec { rows.into_iter().map(|r| r.backup_arn).collect() }; + let mut all: Vec<&BackupRow> = of_root.iter().collect(); + all.push(&creating); + let whole = listed(StoreKind::Filesystem, root.clone(), None).await; + assert_eq!(arns(whole.clone()), in_listing_order(&all)); + for row in &whole { + assert_eq!( + Some(row), + engine + .read_backup_row(&arn_of(row)) + .await + .expect("read") + .as_ref(), + "the listing returns rows whole" + ); + } + for status in BackupStatus::ALL { + let of_status: Vec<&BackupRow> = + all.iter().copied().filter(|r| r.status == status).collect(); + assert_eq!( + arns(listed(StoreKind::Filesystem, root.clone(), Some(status)).await), + in_listing_order(&of_status), + "{status}" + ); + } + for (kind, label, row) in [ + (StoreKind::Filesystem, slash.as_str(), &others[0]), + (StoreKind::Filesystem, shorter.as_str(), &others[1]), + (StoreKind::S3, root.as_str(), &others[2]), + (StoreKind::S3, s3.as_str(), &others[3]), + (StoreKind::S3, s3_endpoint.as_str(), &others[4]), + ] { + assert_eq!( + arns(listed(kind, label.to_owned(), None).await), + std::slice::from_ref(&row.backup_arn), + "{kind} {label}" + ); + } + drop(claim); + let mut every: Vec<&BackupRow> = all.clone(); + every.extend(others.iter()); + remove_index_rows(&engine, &every).await; +} + +/// `rewrite_store`: of the keys given, the rows still of the old kind and +/// root and not CREATING get the new store, with every other column left, +/// and are counted; their status entries move from the old root's +/// partition to the new root's, under the same status, carrying the new +/// kind and id. A CREATING row, a row of another root, a row of the other +/// kind at the same label and a key with no row are left and not counted; a +/// second call counts 0; a row another writer moved to a third root before +/// the call is left there. +#[tokio::test] +async fn test_rewrite_store_moves_only_the_old_roots_rows_that_are_not_creating() { + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + let tag = crate::helpers::unique_test_id(); + let old_root = format!("/srv/backups-old-{tag}"); + let old = (StoreKind::Filesystem, old_root.as_str(), "old-1"); + let to = StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: format!("/srv/backups-new-{tag}"), + store_id: "new-1".to_owned(), + }; + let mut movable = Vec::new(); + for (i, status) in [ + BackupStatus::Available, + BackupStatus::Deleting, + BackupStatus::Deleted, + BackupStatus::Failed, + ] + .into_iter() + .enumerate() + { + movable.push(row_at(&account_id, &format!("m{i}"), status, old)); + } + let moved_away = row_at(&account_id, "away", BackupStatus::Available, old); + let other_root_label = format!("/srv/backups-other-{tag}"); + let other_root = row_at( + &account_id, + "other-root", + BackupStatus::Available, + (StoreKind::Filesystem, &other_root_label, "old-1"), + ); + let other_kind = row_at( + &account_id, + "other-kind", + BackupStatus::Available, + (StoreKind::S3, &old_root, "old-1"), + ); + for row in movable + .iter() + .chain([&moved_away, &other_root, &other_kind]) + { + assert!(engine.insert_backup_row(row).await.expect("insert")); + } + let creating = row_at(&account_id, "job", BackupStatus::Creating, old); + let claim = engine.claim_backup(&creating).await.expect("claim"); + let stored = |row: &BackupRow| { + let engine = &engine; + let arn = arn_of(row); + async move { + engine + .read_backup_row(&arn) + .await + .expect("read") + .expect("the row exists") + } + }; + let mut before = Vec::new(); + for row in &movable { + before.push(stored(row).await); + } + // Another writer moves one row to a third root first. + let third = StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: format!("/srv/backups-third-{tag}"), + store_id: "third-1".to_owned(), + }; + assert_eq!( + engine + .rewrite_store(old.0, old.1, &[moved_away.key()], &third) + .await + .expect("rewrite"), + 1 + ); + let mut ghost = row_at(&account_id, "ghost", BackupStatus::Available, old).key(); + ghost.backup_arn = format!("{}-ghost", ghost.backup_arn); + let keys: Vec = movable + .iter() + .chain([&creating, &moved_away, &other_root, &other_kind]) + .map(BackupRow::key) + .chain([ghost]) + .collect(); + assert_eq!( + engine + .rewrite_store(old.0, old.1, &keys, &to) + .await + .expect("rewrite"), + 4, + "the four rows of the old root that are not CREATING" + ); + for (row, before) in movable.iter().zip(&before) { + assert_eq!( + stored(row).await, + BackupRow { + store: to.clone(), + ..before.clone() + }, + "only the store changed" + ); + assert_eq!( + status_entry(&engine, &to.store_root, row.status, &row.backup_arn).await, + Some(("filesystem".to_owned(), "new-1".to_owned())), + "{}: the new root's entry", + row.status + ); + assert_eq!( + status_entry(&engine, old.1, row.status, &row.backup_arn).await, + None, + "{}: the old root's entry is gone", + row.status + ); + } + let creating_now = stored(&creating).await; + assert_eq!( + (creating_now.status, creating_now.store.store_root.as_str()), + (BackupStatus::Creating, old.1) + ); + assert_eq!(creating_now.owner_generation, claim.generation()); + assert_eq!(stored(&moved_away).await.store, third); + assert_eq!(stored(&other_root).await.store, other_root.store); + assert_eq!(stored(&other_kind).await.store, other_kind.store); + assert_eq!( + engine + .rewrite_store(old.0, old.1, &keys, &to) + .await + .expect("rewrite"), + 0, + "a second call finds nothing left to move" + ); + let listed: Vec = engine + .list_backup_rows(None, &to) + .try_collect() + .await + .expect("listing"); + let movable_refs: Vec<&BackupRow> = movable.iter().collect(); + assert_eq!( + listed.into_iter().map(|r| r.backup_arn).collect::>(), + in_listing_order(&movable_refs) + ); + assert_eq!( + by_root(&engine, old.0, old.1).await, + std::slice::from_ref(&creating.backup_arn) + ); + assert_eq!( + engine.read_store_ids(old.0, old.1).await.expect("ids"), + ["old-1"], + "the CREATING row keeps the old root, and no other entry is left there" + ); + drop(claim); + let mut cleanup: Vec = Vec::new(); + for row in movable.iter().chain([&moved_away]) { + for store in [&to, &third] { + cleanup.push(BackupRow { + store: store.clone(), + ..row.clone() + }); + } + } + let mut every: Vec<&BackupRow> = cleanup.iter().collect(); + every.extend( + movable + .iter() + .chain([&creating, &moved_away, &other_root, &other_kind]), + ); + remove_index_rows(&engine, &every).await; +} + +/// `summarize_store_roots`: one count per kind, root, id and status entry, +/// in the one order; two ids of one root, two statuses of one id, two roots +/// and the two kinds are each counted apart. The catalog keyspace is shared, +/// so only this test's roots are compared. +#[tokio::test] +async fn test_summarize_store_roots_counts_the_rows_of_each_store_and_status() { + use extenddb_storage::backup_catalog::StoreRootCount; + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + let tag = crate::helpers::unique_test_id(); + let (root_a, root_b, root_s3) = ( + format!("/srv/a-{tag}"), + format!("/srv/b-{tag}"), + format!("s3://bucket-{tag}/a"), + ); + let a1 = (StoreKind::Filesystem, root_a.as_str(), "id-1"); + let a2 = (StoreKind::Filesystem, root_a.as_str(), "id-2"); + let b1 = (StoreKind::Filesystem, root_b.as_str(), "id-1"); + let s3 = (StoreKind::S3, root_s3.as_str(), "id-1"); + let rows = [ + row_at(&account_id, "1", BackupStatus::Available, a1), + row_at(&account_id, "2", BackupStatus::Available, a1), + row_at(&account_id, "3", BackupStatus::Failed, a1), + row_at(&account_id, "4", BackupStatus::Available, a2), + row_at(&account_id, "5", BackupStatus::Deleting, b1), + row_at(&account_id, "7", BackupStatus::Available, b1), + row_at(&account_id, "6", BackupStatus::Available, s3), + ]; + for row in &rows { + assert!(engine.insert_backup_row(row).await.expect("insert")); + } + let creating = row_at(&account_id, "job", BackupStatus::Creating, b1); + let claim = engine.claim_backup(&creating).await.expect("claim"); + let line = |(kind, root, id): (StoreKind, &str, &str), status, rows| StoreRootCount { + store: StoreIdentity { + store_kind: kind, + store_root: root.to_owned(), + store_id: id.to_owned(), + }, + status, + rows, + }; + let ours: Vec = engine + .summarize_store_roots() + .await + .expect("summary") + .into_iter() + .filter(|c| [&root_a, &root_b, &root_s3].contains(&&c.store.store_root)) + .collect(); + assert_eq!( + ours, + [ + line(a1, BackupStatus::Available, 2), + line(a1, BackupStatus::Failed, 1), + line(a2, BackupStatus::Available, 1), + line(b1, BackupStatus::Creating, 1), + line(b1, BackupStatus::Available, 1), + line(b1, BackupStatus::Deleting, 1), + line(s3, BackupStatus::Available, 1), + ] + ); + drop(claim); + let mut every: Vec<&BackupRow> = rows.iter().collect(); + every.push(&creating); + remove_index_rows(&engine, &every).await; +} + +/// A rewrite whose answer is lost is settled by a serial read: one whose +/// transaction ran finds the row under the new root, counts it and removes +/// the old root's entry; one whose transaction never ran finds the row under +/// the old root, sends it once more and counts it; a lost answer on every +/// attempt is an error that leaves the row under the old root and listed +/// there. +#[cfg(feature = "test-hooks")] +#[tokio::test] +async fn test_an_unknown_rewrite_outcome_is_settled_by_a_serial_read() { + use extenddb_storage_cassandra::test_hooks::{ProgressWriteFault, fail_rewrite_writes}; + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + let tag = crate::helpers::unique_test_id(); + let old_root = format!("/srv/backups-old-{tag}"); + let old = (StoreKind::Filesystem, old_root.as_str(), "old-1"); + let to = StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: format!("/srv/backups-new-{tag}"), + store_id: "new-1".to_owned(), + }; + let mut cleanup = Vec::new(); + for (name, fault) in [ + ("landed", ProgressWriteFault::Landed), + ("lost", ProgressWriteFault::Lost), + ] { + let row = row_at(&account_id, name, BackupStatus::Available, old); + assert!(engine.insert_backup_row(&row).await.expect("insert")); + fail_rewrite_writes(&row.backup_arn, &[fault]); + assert_eq!( + engine + .rewrite_store(old.0, old.1, &[row.key()], &to) + .await + .expect("rewrite"), + 1, + "{name}" + ); + let stored = engine.read_backup_row(&arn_of(&row)).await.unwrap(); + assert_eq!(stored.map(|r| r.store), Some(to.clone()), "{name}"); + assert_eq!( + status_entry(&engine, old.1, row.status, &row.backup_arn).await, + None, + "{name}: the old root's entry is gone" + ); + assert!( + status_entry(&engine, &to.store_root, row.status, &row.backup_arn) + .await + .is_some(), + "{name}: the new root's entry" + ); + cleanup.push(row); + } + + let unknown = row_at(&account_id, "unknown", BackupStatus::Available, old); + assert!(engine.insert_backup_row(&unknown).await.expect("insert")); + fail_rewrite_writes(&unknown.backup_arn, &[ProgressWriteFault::Lost; 3]); + let failed = engine + .rewrite_store(old.0, old.1, &[unknown.key()], &to) + .await + .expect_err("a lost answer on every attempt"); + assert!( + matches!(&failed, StorageError::Internal(m) if m.contains("unknown outcome")), + "{failed}" + ); + assert_eq!( + engine + .read_backup_row(&arn_of(&unknown)) + .await + .unwrap() + .map(|r| r.store.store_root), + Some(old_root.clone()) + ); + assert_eq!( + by_root(&engine, old.0, old.1).await, + std::slice::from_ref(&unknown.backup_arn) + ); + cleanup.push(unknown); + let moved: Vec = cleanup + .iter() + .map(|row| BackupRow { + store: to.clone(), + ..row.clone() + }) + .collect(); + let every: Vec<&BackupRow> = cleanup.iter().chain(moved.iter()).collect(); + remove_index_rows(&engine, &every).await; +} + +/// A rewrite that stops between its writes leaves the row listed under one +/// root: stopped after its first write (the new root's entry), the row +/// still names the old root and is listed there and not under the new one, +/// and the next call moves it; stopped after its transaction, the row names +/// the new root and is listed there, and the old root's entry is left, which +/// the old root's listing skips and `read_store_ids` of the old root still +/// counts. +#[cfg(feature = "test-hooks")] +#[tokio::test] +async fn test_a_stopped_rewrite_leaves_the_row_listed_under_one_root() { + use extenddb_storage_cassandra::test_hooks::{RewriteStop, stop_rewrite}; + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + let tag = crate::helpers::unique_test_id(); + let old_root = format!("/srv/backups-old-{tag}"); + let old = (StoreKind::Filesystem, old_root.as_str(), "old-1"); + let to = StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: format!("/srv/backups-new-{tag}"), + store_id: "new-1".to_owned(), + }; + let listed_under_new = |row: &BackupRow| { + let engine = &engine; + let to = &to; + let arn = row.backup_arn.clone(); + async move { + let rows: Vec = engine + .list_backup_rows(None, to) + .try_collect() + .await + .expect("listing"); + rows.iter().any(|r| r.backup_arn == arn) + } + }; + + let first = row_at(&account_id, "first", BackupStatus::Failed, old); + assert!(engine.insert_backup_row(&first).await.expect("insert")); + stop_rewrite(&first.backup_arn, RewriteStop::AfterNewEntry); + engine + .rewrite_store(old.0, old.1, &[first.key()], &to) + .await + .expect_err("stopped after the first write"); + assert_eq!( + engine + .read_backup_row(&arn_of(&first)) + .await + .unwrap() + .map(|r| r.store.store_root), + Some(old_root.clone()) + ); + assert_eq!( + by_root(&engine, old.0, old.1).await, + std::slice::from_ref(&first.backup_arn), + "still listed under the old root" + ); + assert!( + !listed_under_new(&first).await, + "the new root's entry is skipped" + ); + assert_eq!( + engine + .rewrite_store(old.0, old.1, &[first.key()], &to) + .await + .expect("rewrite"), + 1, + "the next call moves it" + ); + assert!(listed_under_new(&first).await); + assert!(by_root(&engine, old.0, old.1).await.is_empty()); + + let second = row_at(&account_id, "second", BackupStatus::Available, old); + assert!(engine.insert_backup_row(&second).await.expect("insert")); + stop_rewrite(&second.backup_arn, RewriteStop::AfterRow); + engine + .rewrite_store(old.0, old.1, &[second.key()], &to) + .await + .expect_err("stopped after the transaction"); + assert_eq!( + engine + .read_backup_row(&arn_of(&second)) + .await + .unwrap() + .map(|r| r.store), + Some(to.clone()) + ); + assert!(listed_under_new(&second).await, "listed under the new root"); + assert!( + by_root(&engine, old.0, old.1).await.is_empty(), + "the old root's listing skips the entry left" + ); + assert!( + status_entry(&engine, old.1, second.status, &second.backup_arn) + .await + .is_some(), + "the old root's entry is left" + ); + assert_eq!( + engine.read_store_ids(old.0, old.1).await.expect("ids"), + ["old-1"], + "an entry left counts for the old root's ids" + ); + let moved: Vec = [&first, &second] + .into_iter() + .map(|row| BackupRow { + store: to.clone(), + ..row.clone() + }) + .collect(); + let every: Vec<&BackupRow> = [&first, &second].into_iter().chain(moved.iter()).collect(); + remove_index_rows(&engine, &every).await; +} + +/// The stores of a test's rows moved from `old` to `to`, for the cleanup: +/// each row under both roots, so every status entry of either is removed. +#[cfg(feature = "test-hooks")] +fn under_both(rows: &[&BackupRow], to: &StoreIdentity) -> Vec { + rows.iter() + .flat_map(|row| { + [ + (*row).clone(), + BackupRow { + store: to.clone(), + ..(*row).clone() + }, + ] + }) + .collect() +} + +/// Whether `list_backup_rows(Some(status), store)` returns `arn`. +#[cfg(feature = "test-hooks")] +async fn listed_with( + engine: &CassandraEngine, + status: BackupStatus, + store: &StoreIdentity, + arn: &str, +) -> bool { + let rows: Vec = engine + .list_backup_rows(Some(status), store) + .try_collect() + .await + .expect("listing"); + rows.iter().any(|r| r.backup_arn == arn) +} + +/// A DeleteBackup or the deletion's DELETED write that read the row before +/// `rewrite_store` moved it, and writes after: each one's lightweight +/// transaction carries the store it read, so it does not apply to the moved +/// row; the row is read again and written under the root it now names, and +/// its status entry moves there. The row ends DELETING (then DELETED) under +/// the new root and is listed by the new store's listing of that status, +/// with no entry left under either root for the status it had. +#[cfg(feature = "test-hooks")] +#[tokio::test] +async fn test_a_delete_that_read_the_row_before_a_rewrite_moves_its_entry_under_the_new_root() { + use extenddb_storage_cassandra::test_hooks::{ + pause_before_delete_claim, pause_before_deleted_write, + }; + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = std::sync::Arc::new(setup_engine().await); + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + let tag = crate::helpers::unique_test_id(); + let old_root = format!("/srv/backups-old-{tag}"); + let old = (StoreKind::Filesystem, old_root.as_str(), "old-1"); + let to = StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: format!("/srv/backups-new-{tag}"), + store_id: "new-1".to_owned(), + }; + let new_entry = Some(("filesystem".to_owned(), "new-1".to_owned())); + + // DeleteBackup: read AVAILABLE under the old root, then the move. + let available = row_at(&account_id, "deleted", BackupStatus::Available, old); + assert!(engine.insert_backup_row(&available).await.expect("insert")); + let pause = pause_before_delete_claim(&available.backup_arn); + let start = { + let engine = std::sync::Arc::clone(&engine); + let key = available.key(); + tokio::spawn(async move { engine.start_backup_delete(&key).await }) + }; + pause.reached().await; + assert_eq!( + engine + .rewrite_store(old.0, old.1, &[available.key()], &to) + .await + .expect("rewrite"), + 1 + ); + pause.release(); + assert_eq!( + start.await.expect("delete task").expect("delete"), + DeleteStart::Deleting + ); + let stored = engine + .read_backup_row(&arn_of(&available)) + .await + .unwrap() + .expect("the row exists"); + assert_eq!( + (stored.status, &stored.store), + (BackupStatus::Deleting, &to) + ); + assert!( + listed_with(&engine, BackupStatus::Deleting, &to, &available.backup_arn).await, + "the new store's DELETING listing returns the row" + ); + let arn = available.backup_arn.as_str(); + assert_eq!( + status_entry(&engine, &to.store_root, BackupStatus::Deleting, arn).await, + new_entry + ); + for (root, status) in [ + (to.store_root.as_str(), BackupStatus::Available), + (old.1, BackupStatus::Available), + (old.1, BackupStatus::Deleting), + ] { + assert_eq!( + status_entry(&engine, root, status, arn).await, + None, + "{root} {status}" + ); + } + + // The DELETED write: read DELETING under the old root, then the move. + let deleting = row_at(&account_id, "finished", BackupStatus::Deleting, old); + assert!(engine.insert_backup_row(&deleting).await.expect("insert")); + let pause = pause_before_deleted_write(&deleting.backup_arn); + let finish = { + let engine = std::sync::Arc::clone(&engine); + let key = deleting.key(); + tokio::spawn(async move { engine.set_backup_deleted(&key).await }) + }; + pause.reached().await; + assert_eq!( + engine + .rewrite_store(old.0, old.1, &[deleting.key()], &to) + .await + .expect("rewrite"), + 1 + ); + pause.release(); + assert!(finish.await.expect("task").expect("set DELETED")); + let stored = engine + .read_backup_row(&arn_of(&deleting)) + .await + .unwrap() + .expect("the row exists"); + assert_eq!((stored.status, &stored.store), (BackupStatus::Deleted, &to)); + assert!( + listed_with(&engine, BackupStatus::Deleted, &to, &deleting.backup_arn).await, + "the new store's DELETED listing returns the row" + ); + let arn = deleting.backup_arn.as_str(); + assert_eq!( + status_entry(&engine, &to.store_root, BackupStatus::Deleted, arn).await, + new_entry + ); + for (root, status) in [ + (to.store_root.as_str(), BackupStatus::Deleting), + (old.1, BackupStatus::Deleting), + (old.1, BackupStatus::Deleted), + ] { + assert_eq!( + status_entry(&engine, root, status, arn).await, + None, + "{root} {status}" + ); + } + let every = under_both(&[&available, &deleting], &to); + remove_index_rows(&engine, &every.iter().collect::>()).await; +} + +/// A rewrite whose row another writer changes between the new root's entry +/// and the lightweight transaction. A DeleteBackup in the gap: the +/// transaction, conditioned on the status read, does not apply; the serial +/// read finds the row still under the old root, the new entry is removed, +/// and the next attempt moves the DELETING row, which is then listed under +/// the new root with its one entry there. Another move of the row to the +/// same store in the gap: the serial read after the transaction that did not +/// apply finds the row already under the new store, so the call counts it +/// and keeps the new entry, the row's only correct one. +#[cfg(feature = "test-hooks")] +#[tokio::test] +async fn test_a_rewrite_whose_row_changes_before_its_transaction_reads_the_row_again() { + use extenddb_storage_cassandra::test_hooks::pause_before_rewrite_write; + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = std::sync::Arc::new(setup_engine().await); + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + let tag = crate::helpers::unique_test_id(); + let old_root = format!("/srv/backups-old-{tag}"); + let old = (StoreKind::Filesystem, old_root.as_str(), "old-1"); + let to = StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: format!("/srv/backups-new-{tag}"), + store_id: "new-1".to_owned(), + }; + let rewrite = |row: &BackupRow| { + let engine = std::sync::Arc::clone(&engine); + let key = row.key(); + let (to, root) = (to.clone(), old_root.clone()); + tokio::spawn(async move { + engine + .rewrite_store(StoreKind::Filesystem, &root, &[key], &to) + .await + }) + }; + + let deleted = row_at(&account_id, "deleted", BackupStatus::Available, old); + assert!(engine.insert_backup_row(&deleted).await.expect("insert")); + let pause = pause_before_rewrite_write(&deleted.backup_arn); + let moving = rewrite(&deleted); + pause.reached().await; + assert_eq!( + engine + .start_backup_delete(&deleted.key()) + .await + .expect("delete"), + DeleteStart::Deleting + ); + pause.release(); + assert_eq!(moving.await.expect("task").expect("rewrite"), 1); + let stored = engine + .read_backup_row(&arn_of(&deleted)) + .await + .unwrap() + .expect("the row exists"); + assert_eq!( + (stored.status, &stored.store), + (BackupStatus::Deleting, &to) + ); + assert!(listed_with(&engine, BackupStatus::Deleting, &to, &deleted.backup_arn).await); + let arn = deleted.backup_arn.as_str(); + assert!( + status_entry(&engine, &to.store_root, BackupStatus::Deleting, arn) + .await + .is_some() + ); + for (root, status) in [ + (to.store_root.as_str(), BackupStatus::Available), + (old.1, BackupStatus::Available), + (old.1, BackupStatus::Deleting), + ] { + assert_eq!( + status_entry(&engine, root, status, arn).await, + None, + "{root} {status}: only the new root's DELETING entry is left" + ); + } + + let raced = row_at(&account_id, "raced", BackupStatus::Available, old); + assert!(engine.insert_backup_row(&raced).await.expect("insert")); + let pause = pause_before_rewrite_write(&raced.backup_arn); + let first = rewrite(&raced); + pause.reached().await; + assert_eq!( + engine + .rewrite_store(old.0, old.1, &[raced.key()], &to) + .await + .expect("the other move"), + 1 + ); + pause.release(); + assert_eq!( + first.await.expect("task").expect("rewrite"), + 1, + "the row already under the new store counts as this call's write" + ); + assert!( + listed_with(&engine, BackupStatus::Available, &to, &raced.backup_arn).await, + "the new root's entry is kept" + ); + assert_eq!( + status_entry(&engine, old.1, BackupStatus::Available, &raced.backup_arn).await, + None + ); + let every = under_both(&[&deleted, &raced], &to); + remove_index_rows(&engine, &every.iter().collect::>()).await; +} + +/// `list_backup_rows_by_root` lists a row once, under its own status and +/// kind: a second entry of the row under another status of the same root +/// is passed over (each row is read once), and an entry whose kind is not +/// its row's is skipped on the re-read. +#[tokio::test] +async fn test_list_backup_rows_by_root_skips_an_entry_whose_row_has_another_status_or_kind() { + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + let tag = crate::helpers::unique_test_id(); + let root = format!("/srv/backups-{tag}"); + let row = row_at( + &account_id, + "row", + BackupStatus::Available, + (StoreKind::Filesystem, &root, "id-1"), + ); + let s3_row = row_at( + &account_id, + "s3", + BackupStatus::Available, + (StoreKind::S3, &root, "id-1"), + ); + for r in [&row, &s3_row] { + assert!(engine.insert_backup_row(r).await.expect("insert")); + } + let catalog = engine.catalog_keyspace(); + let entry = |status: BackupStatus, arn: &str| { + ( + format!( + "INSERT INTO {catalog}.backup_index_by_status \ + (store_root, backup_status, backup_arn, account_id, store_kind, store_id) \ + VALUES (?, ?, ?, ?, 'filesystem', 'id-1')" + ), + cdrs_tokio::query_values!( + root.as_str(), + status.as_str(), + arn.to_owned(), + account_id.as_str() + ), + ) + }; + // A stale entry of the filesystem row under FAILED, and the S3 row's + // AVAILABLE entry overwritten with the filesystem kind. + for (cql, values) in [ + entry(BackupStatus::Failed, &row.backup_arn), + entry(BackupStatus::Available, &s3_row.backup_arn), + ] { + engine + .session() + .query_with_values(&cql, values) + .await + .expect("write an entry"); + } + assert_eq!( + by_root(&engine, StoreKind::Filesystem, &root).await, + std::slice::from_ref(&row.backup_arn), + "listed once, and the S3 row not at all" + ); + for status in [BackupStatus::Failed, BackupStatus::Available] { + let rows: Vec = engine + .list_backup_rows_by_root(StoreKind::Filesystem, &root, Some(status)) + .try_collect() + .await + .expect("listing"); + let expected: &[String] = if status == BackupStatus::Available { + std::slice::from_ref(&row.backup_arn) + } else { + &[] + }; + assert_eq!( + rows.into_iter().map(|r| r.backup_arn).collect::>(), + expected, + "{status}" + ); + } + remove_index_rows(&engine, &[&row, &s3_row]).await; } -/// `read_store_ids`: the distinct ids of the rows for a root, of every -/// status, scoped to the kind and the root, read from the status partitions. +/// After a rewrite stopped between its transaction and the old entry's +/// removal, `summarize_store_roots` counts the row under both roots: the +/// new root's entry and the old root's entry left behind, which the summary +/// does not verify against the row. +#[cfg(feature = "test-hooks")] #[tokio::test] -async fn test_read_store_ids_lists_the_distinct_ids_of_a_root() { +async fn test_the_summary_counts_the_old_entry_a_stopped_rewrite_leaves() { + use extenddb_storage::backup_catalog::StoreRootCount; + use extenddb_storage_cassandra::test_hooks::{RewriteStop, stop_rewrite}; if crate::helpers::skip_without_cassandra() { return; } let engine = setup_engine().await; let account = TestAccount::new(&engine, "extenddb_ttl_test").await; let account_id = account.account_id.clone(); - let store = unique_store(); - let root = store.store_root.clone(); - let ids = |kind: StoreKind| { - let engine = &engine; - let root = root.clone(); - async move { engine.read_store_ids(kind, &root).await.expect("ids") } + let tag = crate::helpers::unique_test_id(); + let old_root = format!("/srv/backups-old-{tag}"); + let old = (StoreKind::Filesystem, old_root.as_str(), "old-1"); + let to = StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: format!("/srv/backups-new-{tag}"), + store_id: "new-1".to_owned(), }; - assert!(ids(StoreKind::Filesystem).await.is_empty()); - let first = index_row(&account_id, "Music", "a", &store); - let _c1 = engine.claim_backup(&first).await.expect("claim"); - let second = index_row(&account_id, "Music", "b", &store); - let c2 = engine.claim_backup(&second).await.expect("claim"); - assert_eq!( - c2.set_status( - JobStatus::Creating, - JobStatus::Failed, - &StatusFields::default() - ) + let row = row_at(&account_id, "stopped", BackupStatus::Available, old); + assert!(engine.insert_backup_row(&row).await.expect("insert")); + stop_rewrite(&row.backup_arn, RewriteStop::AfterRow); + engine + .rewrite_store(old.0, old.1, &[row.key()], &to) .await - .expect("fail"), - WriteOutcome::Matched - ); - assert_eq!( - ids(StoreKind::Filesystem).await, - ["store-1"], - "two rows of one store, of two statuses, are one id" - ); - let mut other_store = store.clone(); - other_store.store_id = "store-2".to_owned(); - let other = index_row(&account_id, "Songs", "c", &other_store); - let _c3 = engine.claim_backup(&other).await.expect("claim"); - assert_eq!(ids(StoreKind::Filesystem).await, ["store-1", "store-2"]); - assert!( - ids(StoreKind::S3).await.is_empty(), - "another kind at the same root is another store" - ); - assert!( - engine - .read_store_ids(StoreKind::Filesystem, "/elsewhere") - .await - .expect("ids") - .is_empty() - ); + .expect_err("stopped after the transaction"); + let ours: Vec = engine + .summarize_store_roots() + .await + .expect("summary") + .into_iter() + .filter(|c| { + [old_root.as_str(), to.store_root.as_str()].contains(&c.store.store_root.as_str()) + }) + .collect(); + let mut expected = vec![ + StoreRootCount { + store: row.store.clone(), + status: BackupStatus::Available, + rows: 1, + }, + StoreRootCount { + store: to.clone(), + status: BackupStatus::Available, + rows: 1, + }, + ]; + StoreRootCount::sort(&mut expected); + assert_eq!(ours, expected, "the orphan entry counts for the old root"); + let every = under_both(&[&row], &to); + remove_index_rows(&engine, &every.iter().collect::>()).await; } /// `insert_backup_row`: the row lands as given, unowned, with `completed_at` @@ -9130,3 +10286,386 @@ async fn test_a_running_store_backed_restore_keeps_its_backup_in_use_until_activ None ); } + +/// Deleters whose lightweight transaction lands before a `rewrite_store` +/// of their row and whose entry batch lands after it: DeleteBackup's +/// DELETING and the deletion's DELETED each leave an entry under the old +/// root with no row behind it, which the old root's ids and the summary +/// count. `remove_orphan_status_entries` of the old root deletes both, +/// checked against the rows, and leaves the moved rows listed under the new +/// root. A tombstone removal that read its row before the rewrite is +/// conditioned on the store it read, so it leaves the moved row. +#[cfg(feature = "test-hooks")] +#[tokio::test] +async fn test_the_entries_deleters_leave_behind_a_rewrite_are_swept_from_the_old_root() { + use extenddb_storage::backup_catalog::StatusEntrySweep; + use extenddb_storage_cassandra::test_hooks::{ + pause_after_delete_claim, pause_after_deleted_write, pause_before_tombstone_removal, + }; + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = std::sync::Arc::new(setup_engine().await); + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + let tag = crate::helpers::unique_test_id(); + let old_root = format!("/srv/backups-old-{tag}"); + let old = (StoreKind::Filesystem, old_root.as_str(), "old-1"); + let to = StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: format!("/srv/backups-new-{tag}"), + store_id: "new-1".to_owned(), + }; + let old_entry = Some(("filesystem".to_owned(), "old-1".to_owned())); + + // DeleteBackup: its DELETING claim applies, then the move, then its + // entry batch. + let available = row_at(&account_id, "deleting", BackupStatus::Available, old); + assert!(engine.insert_backup_row(&available).await.expect("insert")); + let pause = pause_after_delete_claim(&available.backup_arn); + let start = { + let engine = std::sync::Arc::clone(&engine); + let key = available.key(); + tokio::spawn(async move { engine.start_backup_delete(&key).await }) + }; + pause.reached().await; + assert_eq!( + engine + .rewrite_store(old.0, old.1, &[available.key()], &to) + .await + .expect("rewrite"), + 1 + ); + pause.release(); + assert_eq!( + start.await.expect("delete task").expect("delete"), + DeleteStart::Deleting + ); + assert_eq!( + status_entry( + &engine, + old.1, + BackupStatus::Deleting, + &available.backup_arn + ) + .await, + old_entry, + "the batch wrote the DELETING entry under the old root" + ); + + // The DELETED write: its transaction applies, then the move, then its + // entry batch. + let deleting = row_at(&account_id, "deleted", BackupStatus::Deleting, old); + assert!(engine.insert_backup_row(&deleting).await.expect("insert")); + let pause = pause_after_deleted_write(&deleting.backup_arn); + let finish = { + let engine = std::sync::Arc::clone(&engine); + let key = deleting.key(); + tokio::spawn(async move { engine.set_backup_deleted(&key).await }) + }; + pause.reached().await; + assert_eq!( + engine + .rewrite_store(old.0, old.1, &[deleting.key()], &to) + .await + .expect("rewrite"), + 1 + ); + pause.release(); + assert!(finish.await.expect("task").expect("set DELETED")); + assert_eq!( + status_entry(&engine, old.1, BackupStatus::Deleted, &deleting.backup_arn).await, + old_entry + ); + + // The tombstone removal: read before the move, its transaction after. + let tombstone = row_at(&account_id, "tombstone", BackupStatus::Deleted, old); + assert!(engine.insert_backup_row(&tombstone).await.expect("insert")); + let pause = pause_before_tombstone_removal(&tombstone.backup_arn); + let removal = { + let engine = std::sync::Arc::clone(&engine); + let key = tombstone.key(); + let root = old_root.clone(); + tokio::spawn(async move { + engine + .remove_backup_row(&key, StoreKind::Filesystem, &root) + .await + }) + }; + pause.reached().await; + assert_eq!( + engine + .rewrite_store(old.0, old.1, &[tombstone.key()], &to) + .await + .expect("rewrite"), + 1 + ); + pause.release(); + assert!( + !removal.await.expect("task").expect("removal"), + "the removal names the store it read" + ); + assert!( + listed_with(&engine, BackupStatus::Deleted, &to, &tombstone.backup_arn).await, + "the moved tombstone is the new store's" + ); + + assert_eq!( + engine.read_store_ids(old.0, old.1).await.unwrap(), + ["old-1"], + "the old root's ids count the two entries" + ); + assert!(by_root(&engine, old.0, old.1).await.is_empty()); + assert_eq!( + engine + .remove_orphan_status_entries(old.0, old.1) + .await + .unwrap(), + StatusEntrySweep { + removed: 2, + repaired: 0 + } + ); + assert!( + engine + .read_store_ids(old.0, old.1) + .await + .unwrap() + .is_empty() + ); + let summary: Vec<_> = engine + .summarize_store_roots() + .await + .unwrap() + .into_iter() + .filter(|c| c.store.store_root == old_root) + .collect(); + assert!(summary.is_empty(), "{summary:?}"); + for (status, row) in [ + (BackupStatus::Deleting, &available), + (BackupStatus::Deleted, &deleting), + (BackupStatus::Deleted, &tombstone), + ] { + assert!( + listed_with(&engine, status, &to, &row.backup_arn).await, + "{} is still listed under the new root", + row.backup_arn + ); + } + assert_eq!( + engine + .remove_orphan_status_entries(to.store_kind, &to.store_root) + .await + .unwrap(), + StatusEntrySweep::default(), + "the new root's entries have their rows" + ); + let every = under_both(&[&available, &deleting, &tombstone], &to); + let mut all: Vec = every.clone(); + for row in &every { + for status in [ + BackupStatus::Available, + BackupStatus::Deleting, + BackupStatus::Deleted, + ] { + all.push(BackupRow { + status, + ..row.clone() + }); + } + } + remove_index_rows(&engine, &all.iter().collect::>()).await; +} + +/// A row whose only status entry is under an earlier status: a DeleteBackup +/// that stopped between its applied DELETING transaction and its entry +/// batch (the row DELETING, its entry AVAILABLE), and a job whose AVAILABLE +/// transaction applied and whose entry batch failed (the row AVAILABLE, its +/// entry CREATING). The status-scoped `list_backup_rows` lists neither; +/// `list_backup_rows_by_root` lists each under its own status, and a row +/// with two entries (the `doubled` row's extra DELETED entry) once. +/// `rewrite_store` of what the by-root listing returns moves the DELETING +/// row, after which the old root lists it no more. The sweep of the old +/// root then deletes the moved row's stale entry (its row names another +/// store) and repairs the others: each row's entry under its own status is +/// written and the stale one deleted, so `list_backup_rows` lists them. +#[cfg(feature = "test-hooks")] +#[tokio::test] +async fn test_a_row_whose_entry_is_under_an_earlier_status_is_listed_moved_and_repaired() { + use extenddb_storage::backup_catalog::StatusEntrySweep; + use extenddb_storage_cassandra::test_hooks::pause_after_delete_claim; + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = std::sync::Arc::new(setup_engine().await); + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + let tag = crate::helpers::unique_test_id(); + let old_root = format!("/srv/backups-old-{tag}"); + let old = (StoreKind::Filesystem, old_root.as_str(), "old-1"); + let to = StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: format!("/srv/backups-new-{tag}"), + store_id: "new-1".to_owned(), + }; + let catalog = engine.catalog_keyspace(); + let raw = |cql: String, values: cdrs_tokio::query::QueryValues| { + let engine = std::sync::Arc::clone(&engine); + async move { + engine + .session() + .query_with_values(&cql, values) + .await + .expect("a raw write"); + } + }; + + // DeleteBackup stopped after its DELETING transaction, before its batch. + let deleting = row_at(&account_id, "deleting", BackupStatus::Available, old); + assert!(engine.insert_backup_row(&deleting).await.expect("insert")); + let pause = pause_after_delete_claim(&deleting.backup_arn); + let start = { + let engine = std::sync::Arc::clone(&engine); + let key = deleting.key(); + tokio::spawn(async move { engine.start_backup_delete(&key).await }) + }; + pause.reached().await; + start.abort(); + let _ = start.await; + + // A job's AVAILABLE transaction applied, its entry batch failed. + let job = row_at(&account_id, "job", BackupStatus::Creating, old); + let claim = engine.claim_backup(&job).await.expect("claim"); + raw( + format!( + "UPDATE {catalog}.backups_by_arn SET backup_status = 'AVAILABLE' \ + WHERE account_id = ? AND backup_arn = ?" + ), + cdrs_tokio::query_values!(account_id.as_str(), job.backup_arn.as_str()), + ) + .await; + drop(claim); + + // An AVAILABLE row with an extra DELETED entry beside its own. + let doubled = row_at(&account_id, "doubled", BackupStatus::Available, old); + assert!(engine.insert_backup_row(&doubled).await.expect("insert")); + raw( + format!( + "INSERT INTO {catalog}.backup_index_by_status \ + (store_root, backup_status, backup_arn, account_id, store_kind, store_id) \ + VALUES (?, 'DELETED', ?, ?, 'filesystem', 'old-1')" + ), + cdrs_tokio::query_values!( + old_root.as_str(), + doubled.backup_arn.as_str(), + account_id.as_str() + ), + ) + .await; + + let old_store = deleting.store.clone(); + assert!( + !listed_with( + &engine, + BackupStatus::Deleting, + &old_store, + &deleting.backup_arn + ) + .await + ); + assert!( + !listed_with( + &engine, + BackupStatus::Available, + &old_store, + &job.backup_arn + ) + .await + ); + let listed: Vec = engine + .list_backup_rows_by_root(old.0, old.1, None) + .try_collect() + .await + .expect("listing"); + let mut statuses: Vec<(String, BackupStatus)> = listed + .iter() + .map(|r| (r.backup_arn.clone(), r.status)) + .collect(); + statuses.sort_by(|a, b| a.0.cmp(&b.0)); + let mut expected = vec![ + (deleting.backup_arn.clone(), BackupStatus::Deleting), + (job.backup_arn.clone(), BackupStatus::Available), + (doubled.backup_arn.clone(), BackupStatus::Available), + ]; + expected.sort_by(|a, b| a.0.cmp(&b.0)); + assert_eq!(statuses, expected, "each row once, under its own status"); + let only_deleting: Vec = engine + .list_backup_rows_by_root(old.0, old.1, Some(BackupStatus::Deleting)) + .map_ok(|r| r.backup_arn) + .try_collect() + .await + .expect("listing"); + assert_eq!(only_deleting, std::slice::from_ref(&deleting.backup_arn)); + + // The move of what the by-root listing returns: the DELETING row moves + // (the job and the doubled row are left out of the keys here). + assert_eq!( + engine + .rewrite_store(old.0, old.1, &[deleting.key()], &to) + .await + .expect("rewrite"), + 1 + ); + let mut left = by_root(&engine, old.0, old.1).await; + left.sort(); + let mut expected_left = vec![job.backup_arn.clone(), doubled.backup_arn.clone()]; + expected_left.sort(); + assert_eq!( + left, expected_left, + "the moved row is no longer the old root's" + ); + assert!(listed_with(&engine, BackupStatus::Deleting, &to, &deleting.backup_arn).await); + + assert_eq!( + engine + .remove_orphan_status_entries(old.0, old.1) + .await + .unwrap(), + StatusEntrySweep { + removed: 1, + repaired: 2 + } + ); + assert!( + listed_with( + &engine, + BackupStatus::Available, + &old_store, + &job.backup_arn + ) + .await, + "the job row's entry is under AVAILABLE now" + ); + for (status, arn) in [ + (BackupStatus::Available, &deleting.backup_arn), + (BackupStatus::Creating, &job.backup_arn), + (BackupStatus::Deleted, &doubled.backup_arn), + ] { + assert_eq!(status_entry(&engine, old.1, status, arn).await, None); + } + assert!( + status_entry(&engine, old.1, BackupStatus::Available, &doubled.backup_arn) + .await + .is_some() + ); + assert_eq!( + engine + .remove_orphan_status_entries(old.0, old.1) + .await + .unwrap(), + StatusEntrySweep::default(), + "a second sweep finds nothing" + ); + let every = under_both(&[&deleting, &job, &doubled], &to); + remove_index_rows(&engine, &every.iter().collect::>()).await; +} diff --git a/crates/storage-mongodb/src/backup_catalog.rs b/crates/storage-mongodb/src/backup_catalog.rs index 4dda98fb4..4a8463865 100644 --- a/crates/storage-mongodb/src/backup_catalog.rs +++ b/crates/storage-mongodb/src/backup_catalog.rs @@ -45,7 +45,7 @@ use extenddb_storage::backup_catalog::{ BackupCatalog, BackupKey, BackupRow, BackupStatus, DeleteStart, ForgetRow, Heartbeat, JobClaim, JobRow, JobStatus, MaintenanceClaim, RestoreJobRow, RestoreKind, RestoreRegistered, RestoreRegistrationRow, RestoreTargetKey, StaleClock, StatusFields, StoreIdentity, StoreKind, - TimeBounds, UNOWNED_GENERATION, WriteOutcome, new_generation, + StoreRootCount, TimeBounds, UNOWNED_GENERATION, WriteOutcome, new_generation, }; use extenddb_storage::error::StorageError; use futures::TryStreamExt; @@ -69,6 +69,10 @@ pub(crate) const BACKUP_MAINTENANCE: &str = "backup_maintenance"; pub(crate) const CATALOG_ID: &str = "catalog_id"; /// The counter a row-locking transaction `$inc`s so two of them conflict. pub(crate) const LOCK_SEQ: &str = "lock_seq"; +/// Keys per `update_many` of `rewrite_store`: one statement's `$or` stays +/// far below the 16 MiB command bound, and a root of any size is a few +/// statements. +pub(crate) const REWRITE_CHUNK_KEYS: usize = 1_000; /// The restore job record, a subdocument of a restore target's `tables` /// document, with the field names the job record has on every backend. @@ -946,6 +950,30 @@ impl BackupCatalog for MongoEngine { }) } + fn list_backup_rows_by_root<'a>( + &'a self, + store_kind: StoreKind, + store_root: &'a str, + status: Option, + ) -> BoxStream<'a, Result> { + // The `{store_kind, store_root}` prefix of the store index; the + // status is a residual filter. + rows_stream(async move { + let mut filter = doc! { "store_kind": store_kind.as_str(), "store_root": store_root }; + if let Some(status) = status { + filter.insert("status", status.as_str()); + } + self.backup_index() + .find(filter) + .sort(listing_sort()) + .await + .map_err(internal)? + .try_collect() + .await + .map_err(internal) + }) + } + fn newest_available_backup_row<'a>( &'a self, store: &'a StoreIdentity, @@ -1035,6 +1063,57 @@ impl BackupCatalog for MongoEngine { }) } + fn rewrite_store<'a>( + &'a self, + from_kind: StoreKind, + from_root: &'a str, + keys: &'a [BackupKey], + to: &'a StoreIdentity, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + // One `update_many` per chunk, no transaction: one over every key + // of a large root would meet the transaction size and lifetime + // limits. Each document's update is atomic. This update and a + // transaction that writes the document (a DeleteBackup's + // `start_backup_delete_tx`, a restore's `claim_restore_tx`) are + // serialized: when the update commits after the transaction's + // snapshot and before its write, the transaction conflicts and + // re-runs, reading the row as the update left it; when the + // transaction wrote the document first, the update waits for it + // to end and then applies its filter to the committed document. + // A registration keeps the store its server is configured with. + // The `$inc` of `lock_seq` is a convention, not what serializes + // them: a write that moves the row also advances its lock + // counter. + let mut rewritten = 0; + for chunk in keys.chunks(REWRITE_CHUNK_KEYS) { + let any_key: Vec = chunk.iter().map(key_filter).collect(); + let result = self + .backup_index() + .update_many( + doc! { + "$or": any_key, + "store_kind": from_kind.as_str(), + "store_root": from_root, + "status": { "$ne": BackupStatus::Creating.as_str() }, + }, + doc! { + "$set": { + "store_kind": to.store_kind.as_str(), + "store_root": &to.store_root, + "store_id": &to.store_id, + }, + "$inc": { LOCK_SEQ: 1_i64 }, + }, + ) + .await + .map_err(internal)?; + rewritten += result.modified_count; + } + Ok(rewritten) + }) + } + fn set_backup_deleted<'a>( &'a self, key: &'a BackupKey, @@ -1060,6 +1139,8 @@ impl BackupCatalog for MongoEngine { fn remove_backup_row<'a>( &'a self, key: &'a BackupKey, + store_kind: StoreKind, + store_root: &'a str, ) -> BoxFuture<'a, Result> { Box::pin(async move { let mut filter = key_filter(key); @@ -1067,6 +1148,8 @@ impl BackupCatalog for MongoEngine { "status", doc! { "$in": [BackupStatus::Deleted.as_str(), BackupStatus::Failed.as_str()] }, ); + filter.insert("store_kind", store_kind.as_str()); + filter.insert("store_root", store_root); let result = self .backup_index() .delete_one(filter) @@ -1419,6 +1502,49 @@ impl BackupCatalog for MongoEngine { }) } + fn summarize_store_roots(&self) -> BoxFuture<'_, Result, StorageError>> { + Box::pin(async move { + let groups: Vec = self + .backup_index() + .aggregate([doc! { "$group": { + "_id": { + "store_kind": "$store_kind", + "store_root": "$store_root", + "store_id": "$store_id", + "status": "$status", + }, + "rows": { "$sum": 1_i64 }, + } }]) + .await + .map_err(internal)? + .try_collect() + .await + .map_err(internal)?; + let mut counts = Vec::with_capacity(groups.len()); + for group in groups { + let id = group.get_document("_id").map_err(internal)?; + let rows = match group.get("rows") { + Some(Bson::Int64(n)) => *n, + Some(Bson::Int32(n)) => i64::from(*n), + other => { + return Err(StorageError::Internal(format!( + "a root summary count is not an integer: {other:?}" + ))); + } + }; + counts.push(StoreRootCount::from_columns( + &required_str(id, "store_kind")?, + required_str(id, "store_root")?, + required_str(id, "store_id")?, + &required_str(id, "status")?, + rows, + )?); + } + StoreRootCount::sort(&mut counts); + Ok(counts) + }) + } + fn maintenance_claim<'a>( &'a self, holder: &'a str, diff --git a/crates/storage-mongodb/src/backup_live_tests.rs b/crates/storage-mongodb/src/backup_live_tests.rs index 6bb62bfa8..eb476b45c 100644 --- a/crates/storage-mongodb/src/backup_live_tests.rs +++ b/crates/storage-mongodb/src/backup_live_tests.rs @@ -130,6 +130,15 @@ impl Fixture { self.engine.catalog_db.drop().await.expect("drop catalog"); self.engine.data_db.drop().await.expect("drop data"); } + + /// [`Self::finish`] after a test body run under `catch_unwind`, whether + /// or not it panicked, and then the panic, if any, goes on. + async fn finish_after(self, outcome: std::thread::Result<()>) { + self.finish().await; + if let Err(panic) = outcome { + std::panic::resume_unwind(panic); + } + } } async fn engine_on(catalog: &str, data: &str, liveness: BackupLiveness) -> MongoEngine { @@ -753,6 +762,448 @@ async fn live_read_store_ids_lists_the_distinct_ids_of_a_root() { fx.finish().await; } +/// A row of `row(ACCOUNT, table, name)` with `status`, under the store of +/// `kind`, `root` and `id`. +fn row_at( + table: &str, + name: &str, + status: BackupStatus, + (kind, root, id): (StoreKind, &str, &str), +) -> BackupRow { + let mut row = row(ACCOUNT, table, name); + row.status = status; + row.store = StoreIdentity { + store_kind: kind, + store_root: root.to_owned(), + store_id: id.to_owned(), + }; + row.size_bytes = 10; + row.item_count = 2; + row +} + +/// The ARNs of `rows` in the listings' (`created_at`, `backup_arn`) order. +fn in_listing_order(rows: &[&BackupRow]) -> Vec { + let mut rows = rows.to_vec(); + rows.sort_by(|a, b| { + a.created_at + .total_cmp(&b.created_at) + .then_with(|| a.backup_arn.cmp(&b.backup_arn)) + }); + rows.into_iter().map(|r| r.backup_arn.clone()).collect() +} + +/// `list_backup_rows_by_root`: every row of the kind and root, whatever its +/// id, of every status or of one, whole and in the listings' order; nothing +/// of a root that differs by one byte (a trailing `/`, one character fewer), +/// of an S3 root that differs by its `?endpoint=` suffix, or of the other +/// kind at the same label. +#[tokio::test(flavor = "multi_thread")] +#[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] +async fn live_list_backup_rows_by_root_returns_every_id_and_status_of_the_root_and_nothing_else() { + let fixture = Fixture::new().await; + let outcome = std::panic::AssertUnwindSafe(async { + let fx = &fixture; + let root = "/srv/backups-old"; + let at = |id| (StoreKind::Filesystem, root, id); + let mut of_root = Vec::new(); + for (i, status) in [ + BackupStatus::Available, + BackupStatus::Deleting, + BackupStatus::Deleted, + BackupStatus::Failed, + ] + .into_iter() + .enumerate() + { + let id = if i % 2 == 0 { "store-1" } else { "store-2" }; + of_root.push(row_at("Music", &format!("r{i}"), status, at(id))); + } + // Inserted newest first: the order asserted is the listing's. + for row in of_root.iter().rev() { + assert!(fx.engine.insert_backup_row(row).await.expect("insert")); + } + let creating = row_at("Music", "job", BackupStatus::Creating, at("store-1")); + let claim = fx.engine.claim_backup(&creating).await.expect("claim"); + let slash = format!("{root}/"); + let s3 = "s3://bucket/prefix"; + let s3_endpoint = "s3://bucket/prefix?endpoint=http://127.0.0.1:9200"; + let others = [ + row_at( + "Music", + "slash", + BackupStatus::Available, + (StoreKind::Filesystem, &slash, "store-1"), + ), + row_at( + "Music", + "shorter", + BackupStatus::Available, + (StoreKind::Filesystem, "/srv/backups-ol", "store-1"), + ), + row_at( + "Music", + "kind", + BackupStatus::Available, + (StoreKind::S3, root, "store-1"), + ), + row_at( + "Music", + "s3", + BackupStatus::Available, + (StoreKind::S3, s3, "store-3"), + ), + row_at( + "Music", + "endpoint", + BackupStatus::Available, + (StoreKind::S3, s3_endpoint, "store-3"), + ), + ]; + for row in &others { + assert!(fx.engine.insert_backup_row(row).await.expect("insert")); + } + let listed = |kind: StoreKind, root: String, status: Option| { + let engine = &fx.engine; + async move { + let rows: Vec = engine + .list_backup_rows_by_root(kind, &root, status) + .try_collect() + .await + .expect("listing"); + rows + } + }; + let arns = |rows: Vec| -> Vec { + rows.into_iter().map(|r| r.backup_arn).collect() + }; + let mut all: Vec<&BackupRow> = of_root.iter().collect(); + all.push(&creating); + let whole = listed(StoreKind::Filesystem, root.to_owned(), None).await; + assert_eq!(arns(whole.clone()), in_listing_order(&all)); + for row in &whole { + assert_eq!( + Some(row), + fx.engine + .read_backup_row(&arn_of(row)) + .await + .expect("read") + .as_ref(), + "the listing returns rows whole" + ); + } + for status in BackupStatus::ALL { + let of_status: Vec<&BackupRow> = + all.iter().copied().filter(|r| r.status == status).collect(); + assert_eq!( + arns(listed(StoreKind::Filesystem, root.to_owned(), Some(status)).await), + in_listing_order(&of_status), + "{status}" + ); + } + for (kind, label, row) in [ + (StoreKind::Filesystem, slash.as_str(), &others[0]), + (StoreKind::Filesystem, "/srv/backups-ol", &others[1]), + (StoreKind::S3, root, &others[2]), + (StoreKind::S3, s3, &others[3]), + (StoreKind::S3, s3_endpoint, &others[4]), + ] { + assert_eq!( + arns(listed(kind, label.to_owned(), None).await), + std::slice::from_ref(&row.backup_arn), + "{kind} {label}" + ); + } + assert!( + listed(StoreKind::Filesystem, "/srv/backups".to_owned(), None) + .await + .is_empty() + ); + drop(claim); + }) + .catch_unwind() + .await; + fixture.finish_after(outcome).await; +} + +/// `rewrite_store`: of the keys given, the rows still of the old kind and +/// root and not CREATING get the new store, with every other column left, +/// and are counted; a CREATING row, a row of another root, a row of the +/// other kind at the same label and a key with no row are left and not +/// counted; a second call counts 0; a row another writer moved to a third +/// root before the call is left there. The `$inc` of `lock_seq` is the one +/// other field the rewrite changes. +#[tokio::test(flavor = "multi_thread")] +#[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] +async fn live_rewrite_store_moves_only_the_old_roots_rows_that_are_not_creating() { + let fixture = Fixture::new().await; + let outcome = std::panic::AssertUnwindSafe(async { + let fx = &fixture; + let old = (StoreKind::Filesystem, "/srv/backups-old", "old-1"); + let to = StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: "/srv/backups-new".to_owned(), + store_id: "new-1".to_owned(), + }; + let mut movable = Vec::new(); + for (i, status) in [ + BackupStatus::Available, + BackupStatus::Deleting, + BackupStatus::Deleted, + BackupStatus::Failed, + ] + .into_iter() + .enumerate() + { + movable.push(row_at("Music", &format!("m{i}"), status, old)); + } + let moved_away = row_at("Music", "away", BackupStatus::Available, old); + let other_root = row_at( + "Music", + "other-root", + BackupStatus::Available, + (StoreKind::Filesystem, "/srv/backups-other", "old-1"), + ); + let other_kind = row_at( + "Music", + "other-kind", + BackupStatus::Available, + (StoreKind::S3, "/srv/backups-old", "old-1"), + ); + for row in movable + .iter() + .chain([&moved_away, &other_root, &other_kind]) + { + assert!(fx.engine.insert_backup_row(row).await.expect("insert")); + } + let creating = row_at("Music", "job", BackupStatus::Creating, old); + let claim = fx.engine.claim_backup(&creating).await.expect("claim"); + let stored = |row: &BackupRow| { + let engine = &fx.engine; + let arn = arn_of(row); + async move { + engine + .read_backup_row(&arn) + .await + .expect("read") + .expect("the row exists") + } + }; + let lock_seq = |row: &BackupRow| { + let engine = &fx.engine; + let id = row.backup_id.clone(); + async move { + engine + .catalog_db + .collection::(BACKUP_INDEX) + .find_one(doc! { "backup_id": &id }) + .await + .expect("read") + .expect("the document exists") + .get_i64("lock_seq") + .expect("lock_seq") + } + }; + let mut before = Vec::new(); + for row in &movable { + before.push((stored(row).await, lock_seq(row).await)); + } + // Another writer moves one row to a third root first. + let third = StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: "/srv/backups-third".to_owned(), + store_id: "third-1".to_owned(), + }; + assert_eq!( + fx.engine + .rewrite_store(old.0, old.1, &[moved_away.key()], &third) + .await + .expect("rewrite"), + 1 + ); + let mut ghost = row_at("Music", "ghost", BackupStatus::Available, old).key(); + ghost.backup_id = extenddb_storage::util::new_backup_id(); + let keys: Vec = movable + .iter() + .chain([&creating, &moved_away, &other_root, &other_kind]) + .map(BackupRow::key) + .chain([ghost]) + .collect(); + assert_eq!( + fx.engine + .rewrite_store(old.0, old.1, &keys, &to) + .await + .expect("rewrite"), + 4, + "the four rows of the old root that are not CREATING" + ); + for (row, (before, seq)) in movable.iter().zip(&before) { + assert_eq!( + stored(row).await, + BackupRow { + store: to.clone(), + ..before.clone() + }, + "only the store changed" + ); + assert_eq!( + lock_seq(row).await, + seq + 1, + "the move advances the row's lock counter" + ); + } + let creating_now = stored(&creating).await; + assert_eq!( + (creating_now.status, creating_now.store.store_root.as_str()), + (BackupStatus::Creating, old.1) + ); + assert_eq!(creating_now.owner_generation, claim.generation()); + assert_eq!(stored(&moved_away).await.store, third); + assert_eq!(stored(&other_root).await.store, other_root.store); + assert_eq!(stored(&other_kind).await.store, other_kind.store); + assert_eq!( + fx.engine + .rewrite_store(old.0, old.1, &keys, &to) + .await + .expect("rewrite"), + 0, + "a second call finds nothing left to move" + ); + let listed: Vec = fx + .engine + .list_backup_rows(None, &to) + .try_collect() + .await + .expect("listing"); + let movable_refs: Vec<&BackupRow> = movable.iter().collect(); + assert_eq!( + listed.into_iter().map(|r| r.backup_arn).collect::>(), + in_listing_order(&movable_refs) + ); + assert_eq!( + fx.engine.read_store_ids(old.0, old.1).await.expect("ids"), + ["old-1"], + "the CREATING row keeps the old root" + ); + assert_eq!( + fx.engine + .rewrite_store(old.0, old.1, &[], &to) + .await + .expect("rewrite"), + 0 + ); + drop(claim); + }) + .catch_unwind() + .await; + fixture.finish_after(outcome).await; +} + +/// `rewrite_store` over more keys than one statement takes moves them all, +/// every chunk counted. +#[tokio::test(flavor = "multi_thread")] +#[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] +async fn live_rewrite_store_moves_every_chunk_of_keys() { + use crate::backup_catalog::REWRITE_CHUNK_KEYS; + let fixture = Fixture::new().await; + let outcome = std::panic::AssertUnwindSafe(async { + let fx = &fixture; + let old = (StoreKind::Filesystem, "/srv/backups-old", "old-1"); + let to = StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: "/srv/backups-new".to_owned(), + store_id: "new-1".to_owned(), + }; + let rows = REWRITE_CHUNK_KEYS + 1; + let mut keys = Vec::with_capacity(rows); + for i in 0..rows { + let row = row_at("Music", &format!("r{i}"), BackupStatus::Available, old); + assert!(fx.engine.insert_backup_row(&row).await.expect("insert")); + keys.push(row.key()); + } + assert_eq!( + fx.engine + .rewrite_store(old.0, old.1, &keys, &to) + .await + .expect("rewrite"), + rows as u64 + ); + let left: Vec = fx + .engine + .list_backup_rows_by_root(old.0, old.1, None) + .try_collect() + .await + .expect("listing"); + assert!(left.is_empty(), "{} rows left", left.len()); + }) + .catch_unwind() + .await; + fixture.finish_after(outcome).await; +} + +/// `summarize_store_roots`: one count per kind, root, id and status with a +/// row, in the one order; two ids of one root, two statuses of one id, two +/// roots and the two kinds are each counted apart. +#[tokio::test(flavor = "multi_thread")] +#[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] +async fn live_summarize_store_roots_counts_the_rows_of_each_store_and_status() { + use extenddb_storage::backup_catalog::StoreRootCount; + let fixture = Fixture::new().await; + let outcome = std::panic::AssertUnwindSafe(async { + let fx = &fixture; + assert!( + fx.engine + .summarize_store_roots() + .await + .expect("summary") + .is_empty() + ); + let a1 = (StoreKind::Filesystem, "/srv/a", "id-1"); + let a2 = (StoreKind::Filesystem, "/srv/a", "id-2"); + let b1 = (StoreKind::Filesystem, "/srv/b", "id-1"); + let s3 = (StoreKind::S3, "s3://bucket/a", "id-1"); + let rows = [ + row_at("Music", "1", BackupStatus::Available, a1), + row_at("Music", "2", BackupStatus::Available, a1), + row_at("Music", "3", BackupStatus::Failed, a1), + row_at("Music", "4", BackupStatus::Available, a2), + row_at("Music", "5", BackupStatus::Deleting, b1), + row_at("Music", "7", BackupStatus::Available, b1), + row_at("Music", "6", BackupStatus::Available, s3), + ]; + for row in &rows { + assert!(fx.engine.insert_backup_row(row).await.expect("insert")); + } + let creating = row_at("Music", "job", BackupStatus::Creating, b1); + let claim = fx.engine.claim_backup(&creating).await.expect("claim"); + let line = |(kind, root, id): (StoreKind, &str, &str), status, rows| StoreRootCount { + store: StoreIdentity { + store_kind: kind, + store_root: root.to_owned(), + store_id: id.to_owned(), + }, + status, + rows, + }; + assert_eq!( + fx.engine.summarize_store_roots().await.expect("summary"), + [ + line(a1, BackupStatus::Available, 2), + line(a1, BackupStatus::Failed, 1), + line(a2, BackupStatus::Available, 1), + line(b1, BackupStatus::Creating, 1), + line(b1, BackupStatus::Available, 1), + line(b1, BackupStatus::Deleting, 1), + line(s3, BackupStatus::Available, 1), + ] + ); + drop(claim); + }) + .catch_unwind() + .await; + fixture.finish_after(outcome).await; +} + /// `insert_backup_row`: the row lands as given, unowned (the document /// carries `owner_generation: 0`, which the claim's upsert filter relies on), /// with `completed_at` and `heartbeat_at` on the server's clock; a second @@ -1523,7 +1974,10 @@ async fn live_start_backup_delete_checks_the_status_and_the_in_use_rule_under_th Err(StorageError::Validation(m)) if m.contains("Backup not found") )); assert!( - !fx.engine.remove_backup_row(&key).await.unwrap(), + !fx.engine + .remove_backup_row(&key, row.store.store_kind, &row.store.store_root) + .await + .unwrap(), "DELETING is not a tombstone" ); assert!(fx.engine.set_backup_deleted(&key).await.unwrap()); @@ -1536,8 +1990,82 @@ async fn live_start_backup_delete_checks_the_status_and_the_in_use_rule_under_th .unwrap(); assert_eq!(stored.status, BackupStatus::Deleted); assert!(stored.completed_at.is_some()); - assert!(fx.engine.remove_backup_row(&key).await.unwrap()); - assert!(!fx.engine.remove_backup_row(&key).await.unwrap()); + // A row another writer moved to another store after the caller read it + // is left by a removal that names the store read. + let moved_to = StoreIdentity { + store_kind: row.store.store_kind, + store_root: format!("{}-moved", row.store.store_root), + store_id: "moved-1".to_owned(), + }; + assert_eq!( + fx.engine + .rewrite_store( + row.store.store_kind, + &row.store.store_root, + std::slice::from_ref(&key), + &moved_to + ) + .await + .unwrap(), + 1 + ); + assert!( + !fx.engine + .remove_backup_row(&key, row.store.store_kind, &row.store.store_root) + .await + .unwrap(), + "a row moved to another store is left" + ); + assert_eq!( + fx.engine + .read_backup_row(&arn_of(&row)) + .await + .unwrap() + .map(|r| (r.status, r.store)), + Some((BackupStatus::Deleted, moved_to.clone())) + ); + // The same root under the other kind is another store: a removal that + // names the kind it read leaves the row. + let other_kind = StoreIdentity { + store_kind: if moved_to.store_kind == StoreKind::S3 { + StoreKind::Filesystem + } else { + StoreKind::S3 + }, + ..moved_to.clone() + }; + assert_eq!( + fx.engine + .rewrite_store( + moved_to.store_kind, + &moved_to.store_root, + std::slice::from_ref(&key), + &other_kind + ) + .await + .unwrap(), + 1 + ); + assert!( + !fx.engine + .remove_backup_row(&key, moved_to.store_kind, &moved_to.store_root) + .await + .unwrap(), + "a row moved to the other kind is left" + ); + let moved_to = other_kind; + assert!( + fx.engine + .remove_backup_row(&key, moved_to.store_kind, &moved_to.store_root) + .await + .unwrap() + ); + assert!( + !fx.engine + .remove_backup_row(&key, moved_to.store_kind, &moved_to.store_root) + .await + .unwrap() + ); assert!( fx.engine .read_backup_row(&arn_of(&row)) diff --git a/crates/storage-postgres/src/backup_catalog.rs b/crates/storage-postgres/src/backup_catalog.rs index 83453dff2..d5d8fb455 100644 --- a/crates/storage-postgres/src/backup_catalog.rs +++ b/crates/storage-postgres/src/backup_catalog.rs @@ -48,7 +48,7 @@ use extenddb_storage::backup_catalog::{ BackupCatalog, BackupKey, BackupRow, BackupStatus, DeleteStart, ForgetRow, Heartbeat, JobClaim, JobRow, JobStatus, MaintenanceClaim, RestoreJobRow, RestoreKind, RestoreRegistered, RestoreRegistrationRow, RestoreTargetKey, StaleClock, StatusFields, StoreIdentity, StoreKind, - TimeBounds, UNOWNED_GENERATION, WriteOutcome, new_generation, + StoreRootCount, TimeBounds, UNOWNED_GENERATION, WriteOutcome, new_generation, }; use extenddb_storage::error::StorageError; use futures::future::BoxFuture; @@ -604,6 +604,29 @@ impl BackupCatalog for PostgresEngine { }) } + fn list_backup_rows_by_root<'a>( + &'a self, + store_kind: StoreKind, + store_root: &'a str, + status: Option, + ) -> BoxStream<'a, Result> { + // The `(store_kind, store_root)` prefix of + // `idx_backup_index_store_status`; the status is a filter on it. + rows_stream(async move { + sqlx::query_as(&format!( + "SELECT {ROW_COLUMNS} FROM backup_index \ + WHERE store_kind = $1 AND store_root = $2 \ + AND ($3::TEXT IS NULL OR status = $3) ORDER BY created_at, backup_arn" + )) + .bind(store_kind.as_str()) + .bind(store_root) + .bind(status.map(BackupStatus::as_str)) + .fetch_all(&self.pool) + .await + .map_err(db_err) + }) + } + fn newest_available_backup_row<'a>( &'a self, store: &'a StoreIdentity, @@ -688,6 +711,36 @@ impl BackupCatalog for PostgresEngine { }) } + fn rewrite_store<'a>( + &'a self, + from_kind: StoreKind, + from_root: &'a str, + keys: &'a [BackupKey], + to: &'a StoreIdentity, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + if keys.is_empty() { + return Ok(0); + } + let mut attempt = 0; + loop { + attempt += 1; + match self.rewrite_store_tx(from_kind, from_root, keys, to).await { + Ok(rewritten) => return Ok(rewritten), + Err(e) if attempt < REWRITE_ATTEMPTS && crate::data::lock_not_available(&e) => { + // A row lock held past the timeout (a DeleteBackup's + // or a registration's transaction, or the backend of + // a client that died mid-statement): the + // transaction rolled back, and the next attempt + // waits again after a pause. + tokio::time::sleep(Duration::from_millis(500 * u64::from(attempt))).await; + } + Err(e) => return Err(db_err(e)), + } + } + }) + } + fn set_backup_deleted<'a>( &'a self, key: &'a BackupKey, @@ -710,14 +763,18 @@ impl BackupCatalog for PostgresEngine { fn remove_backup_row<'a>( &'a self, key: &'a BackupKey, + store_kind: StoreKind, + store_root: &'a str, ) -> BoxFuture<'a, Result> { Box::pin(async move { let affected = sqlx::query( "DELETE FROM backup_index WHERE account_id = $1 AND backup_id = $2 \ - AND status IN ('DELETED', 'FAILED')", + AND status IN ('DELETED', 'FAILED') AND store_kind = $3 AND store_root = $4", ) .bind(&key.account_id) .bind(&key.backup_id) + .bind(store_kind.as_str()) + .bind(store_root) .execute(&self.pool) .await .map_err(db_err)? @@ -1129,6 +1186,26 @@ impl BackupCatalog for PostgresEngine { }) } + fn summarize_store_roots(&self) -> BoxFuture<'_, Result, StorageError>> { + Box::pin(async move { + let grouped: Vec<(String, String, String, String, i64)> = sqlx::query_as( + "SELECT store_kind, store_root, store_id, status, count(*) FROM backup_index \ + GROUP BY store_kind, store_root, store_id, status", + ) + .fetch_all(&self.pool) + .await + .map_err(db_err)?; + let mut counts = grouped + .into_iter() + .map(|(kind, root, id, status, rows)| { + StoreRootCount::from_columns(&kind, root, id, &status, rows) + }) + .collect::, _>>()?; + StoreRootCount::sort(&mut counts); + Ok(counts) + }) + } + fn maintenance_claim<'a>( &'a self, holder: &'a str, @@ -1234,7 +1311,127 @@ const MAINTENANCE_LOCK_KEY: &str = "backup_maintenance"; const MAINTENANCE_TAKE_TRIES: u32 = 6; const MAINTENANCE_TAKE_PAUSE: Duration = Duration::from_millis(2); +/// `rewrite_store`'s `lock_timeout`. The rewrite waits on the row locks a +/// DeleteBackup's or a registration's `FOR UPDATE` transaction takes, each +/// a few statements long, so a wait past this is a lock held by something +/// else: the backend of a client that died with its transaction open, or a +/// stalled session. The statement then fails with 55P03, its transaction +/// rolls back, and the rewrite tries again ([`REWRITE_ATTEMPTS`]). +#[doc(hidden)] +pub const REWRITE_LOCK_TIMEOUT: Duration = Duration::from_secs(3); + +/// `rewrite_store`'s `statement_timeout`: how long the one statement may run +/// in all, its lock waits included. A rewrite of 20,000 keys over a catalog +/// of 200,000 rows ran in 0.8 s, and one of 200,000 keys in 3.6 s, so this +/// leaves room for a catalog far larger than that; past it the statement +/// fails, rolls back, and is not tried again. +#[doc(hidden)] +pub const REWRITE_STATEMENT_TIMEOUT: Duration = Duration::from_secs(30); + +/// How many times `rewrite_store` runs its transaction when the statement +/// meets [`REWRITE_LOCK_TIMEOUT`]; the pauses between grow by half a second. +#[doc(hidden)] +pub const REWRITE_ATTEMPTS: u32 = 3; + impl crate::PostgresEngine { + /// One attempt of `rewrite_store`: one statement over every key inside a + /// transaction this call commits, under [`REWRITE_LOCK_TIMEOUT`] and + /// [`REWRITE_STATEMENT_TIMEOUT`]. The row lock the statement takes + /// orders it against a registration's and a delete's `FOR UPDATE`. + /// CREATING rows are excluded; the statement takes no advisory lock. + /// Because the commit is the client's own, a client that dies while the + /// statement waits or runs rewrites nothing: the server rolls the + /// transaction back when it finds the connection closed, at the latest + /// when the statement ends. + async fn rewrite_store_tx( + &self, + from_kind: StoreKind, + from_root: &str, + keys: &[BackupKey], + to: &StoreIdentity, + ) -> Result { + let mut conn = self.pool.acquire().await?; + Self::rewrite_store_on(&mut conn, from_kind, from_root, keys, to).await + } + + /// [`Self::rewrite_store_tx`] on `conn`: the transaction begins and + /// commits there, so a test can read the connection's settings after + /// it ([`Self::rewrite_store_settings_after`]). + async fn rewrite_store_on( + conn: &mut sqlx::PgConnection, + from_kind: StoreKind, + from_root: &str, + keys: &[BackupKey], + to: &StoreIdentity, + ) -> Result { + use sqlx::Connection as _; + let accounts: Vec<&str> = keys.iter().map(|k| k.account_id.as_str()).collect(); + let ids: Vec<&str> = keys.iter().map(|k| k.backup_id.as_str()).collect(); + let mut tx = conn.begin().await?; + for (setting, bound) in [ + ("lock_timeout", REWRITE_LOCK_TIMEOUT), + ("statement_timeout", REWRITE_STATEMENT_TIMEOUT), + ] { + sqlx::query("SELECT set_config($1, $2, true)") + .bind(setting) + .bind(format!("{}ms", bound.as_millis())) + .execute(&mut *tx) + .await?; + } + let rewritten = sqlx::query( + "UPDATE backup_index SET store_kind = $1, store_root = $2, store_id = $3 \ + WHERE (account_id, backup_id) IN \ + (SELECT * FROM unnest($4::TEXT[], $5::TEXT[])) \ + AND store_kind = $6 AND store_root = $7 AND status <> 'CREATING'", + ) + .bind(to.store_kind.as_str()) + .bind(&to.store_root) + .bind(&to.store_id) + .bind(&accounts) + .bind(&ids) + .bind(from_kind.as_str()) + .bind(from_root) + .execute(&mut *tx) + .await? + .rows_affected(); + tx.commit().await?; + Ok(rewritten) + } + + /// One rewrite's transaction on one pooled connection, then `SHOW + /// lock_timeout` and `SHOW statement_timeout` on that connection: for the + /// test that checks the rewrite's bounds end with its transaction and do + /// not stay on the pooled connection. + /// + /// # Errors + /// + /// The rewrite's or a `SHOW`'s error. + #[cfg(feature = "test-hooks")] + #[doc(hidden)] + pub async fn rewrite_store_settings_after( + &self, + from_kind: StoreKind, + from_root: &str, + keys: &[BackupKey], + to: &StoreIdentity, + ) -> Result<(u64, String, String), StorageError> { + let mut conn = self.pool.acquire().await.map_err(db_err)?; + let rewritten = Self::rewrite_store_on(&mut conn, from_kind, from_root, keys, to) + .await + .map_err(db_err)?; + let mut shown = Vec::new(); + for name in ["lock_timeout", "statement_timeout"] { + let value: String = sqlx::query_scalar(&format!("SHOW {name}")) + .fetch_one(&mut *conn) + .await + .map_err(db_err)?; + shown.push(value); + } + let statement = shown.pop().expect("two settings"); + let lock = shown.pop().expect("two settings"); + Ok((rewritten, lock, statement)) + } + /// DeleteBackup's first step, and with `remove` the outright removal of /// `forget_available_backup_row`, as one transaction: lock the row, read /// its status, run the in-use rule, then write DELETING or, with diff --git a/crates/storage-postgres/src/lib.rs b/crates/storage-postgres/src/lib.rs index e916f9cb9..fecab7629 100755 --- a/crates/storage-postgres/src/lib.rs +++ b/crates/storage-postgres/src/lib.rs @@ -43,6 +43,8 @@ mod vector_search; mod worker_store; mod workers; +#[doc(hidden)] +pub use backup_catalog::{REWRITE_ATTEMPTS, REWRITE_LOCK_TIMEOUT, REWRITE_STATEMENT_TIMEOUT}; #[cfg(feature = "test-hooks")] #[doc(hidden)] pub use backup_engine::claim_session_settings; diff --git a/crates/storage-postgres/tests/backup_restore.rs b/crates/storage-postgres/tests/backup_restore.rs index e57347186..b40fb4533 100644 --- a/crates/storage-postgres/tests/backup_restore.rs +++ b/crates/storage-postgres/tests/backup_restore.rs @@ -3616,7 +3616,10 @@ mod catalog_index { Err(StorageError::Validation(m)) if m.contains("Backup not found") )); assert!( - !s.engine.remove_backup_row(&key).await.unwrap(), + !s.engine + .remove_backup_row(&key, row.store.store_kind, &row.store.store_root) + .await + .unwrap(), "DELETING is not a tombstone" ); assert!(s.engine.set_backup_deleted(&key).await.unwrap()); @@ -3629,8 +3632,82 @@ mod catalog_index { .unwrap(); assert_eq!(stored.status, BackupStatus::Deleted); assert!(stored.completed_at.is_some()); - assert!(s.engine.remove_backup_row(&key).await.unwrap()); - assert!(!s.engine.remove_backup_row(&key).await.unwrap()); + // A row another writer moved to another store after the caller read it + // is left by a removal that names the store read. + let moved_to = StoreIdentity { + store_kind: row.store.store_kind, + store_root: format!("{}-moved", row.store.store_root), + store_id: "moved-1".to_owned(), + }; + assert_eq!( + s.engine + .rewrite_store( + row.store.store_kind, + &row.store.store_root, + std::slice::from_ref(&key), + &moved_to + ) + .await + .unwrap(), + 1 + ); + assert!( + !s.engine + .remove_backup_row(&key, row.store.store_kind, &row.store.store_root) + .await + .unwrap(), + "a row moved to another store is left" + ); + assert_eq!( + s.engine + .read_backup_row(&arn_of(&row)) + .await + .unwrap() + .map(|r| (r.status, r.store)), + Some((BackupStatus::Deleted, moved_to.clone())) + ); + // The same root under the other kind is another store: a removal that + // names the kind it read leaves the row. + let other_kind = StoreIdentity { + store_kind: if moved_to.store_kind == StoreKind::S3 { + StoreKind::Filesystem + } else { + StoreKind::S3 + }, + ..moved_to.clone() + }; + assert_eq!( + s.engine + .rewrite_store( + moved_to.store_kind, + &moved_to.store_root, + std::slice::from_ref(&key), + &other_kind + ) + .await + .unwrap(), + 1 + ); + assert!( + !s.engine + .remove_backup_row(&key, moved_to.store_kind, &moved_to.store_root) + .await + .unwrap(), + "a row moved to the other kind is left" + ); + let moved_to = other_kind; + assert!( + s.engine + .remove_backup_row(&key, moved_to.store_kind, &moved_to.store_root) + .await + .unwrap() + ); + assert!( + !s.engine + .remove_backup_row(&key, moved_to.store_kind, &moved_to.store_root) + .await + .unwrap() + ); assert!( s.engine .read_backup_row(&arn_of(&row)) @@ -4042,6 +4119,635 @@ mod catalog_index { s.cleanup().await; } + /// A row of `row(ACCOUNT, table, name)` with `status`, under the store + /// of `kind`, `root` and `id`. + fn row_at( + table: &str, + name: &str, + status: BackupStatus, + (kind, root, id): (StoreKind, &str, &str), + ) -> BackupRow { + let mut row = row(ACCOUNT, table, name); + row.status = status; + row.store = StoreIdentity { + store_kind: kind, + store_root: root.to_owned(), + store_id: id.to_owned(), + }; + row.size_bytes = 10; + row.item_count = 2; + row + } + + /// The ARNs of `rows` in the listings' (`created_at`, `backup_arn`) + /// order. + fn in_listing_order(rows: &[&BackupRow]) -> Vec { + let mut rows = rows.to_vec(); + rows.sort_by(|a, b| { + a.created_at + .total_cmp(&b.created_at) + .then_with(|| a.backup_arn.cmp(&b.backup_arn)) + }); + rows.into_iter().map(|r| r.backup_arn.clone()).collect() + } + + /// `list_backup_rows_by_root`: every row of the kind and root, whatever + /// its id, of every status or of one, whole and in the listings' order; + /// nothing of a root that differs by one byte (a trailing `/`, one + /// character fewer), of an S3 root that differs by its `?endpoint=` + /// suffix, or of the other kind at the same label. + #[tokio::test] + async fn list_backup_rows_by_root_returns_every_id_and_status_of_the_root_and_nothing_else() { + if base_conn().is_none() { + eprintln!( + "SKIP list_backup_rows_by_root_returns_every_id_and_status_of_the_root_and_nothing_else: no PostgreSQL" + ); + return; + } + let s = scratch().await; + let root = "/srv/backups-old"; + let at = |id| (StoreKind::Filesystem, root, id); + let mut of_root = Vec::new(); + for (i, status) in [ + BackupStatus::Available, + BackupStatus::Deleting, + BackupStatus::Deleted, + BackupStatus::Failed, + ] + .into_iter() + .enumerate() + { + let id = if i % 2 == 0 { "store-1" } else { "store-2" }; + of_root.push(row_at("Music", &format!("r{i}"), status, at(id))); + } + // Inserted newest first: the order asserted is the listing's. + for row in of_root.iter().rev() { + assert!(s.engine.insert_backup_row(row).await.expect("insert")); + } + let creating = row_at("Music", "job", BackupStatus::Creating, at("store-1")); + let claim = s.engine.claim_backup(&creating).await.expect("claim"); + let slash = format!("{root}/"); + let s3 = "s3://bucket/prefix"; + let s3_endpoint = "s3://bucket/prefix?endpoint=http://127.0.0.1:9200"; + let others = [ + row_at( + "Music", + "slash", + BackupStatus::Available, + (StoreKind::Filesystem, &slash, "store-1"), + ), + row_at( + "Music", + "shorter", + BackupStatus::Available, + (StoreKind::Filesystem, "/srv/backups-ol", "store-1"), + ), + row_at( + "Music", + "kind", + BackupStatus::Available, + (StoreKind::S3, root, "store-1"), + ), + row_at( + "Music", + "s3", + BackupStatus::Available, + (StoreKind::S3, s3, "store-3"), + ), + row_at( + "Music", + "endpoint", + BackupStatus::Available, + (StoreKind::S3, s3_endpoint, "store-3"), + ), + ]; + for row in &others { + assert!(s.engine.insert_backup_row(row).await.expect("insert")); + } + let listed = |kind: StoreKind, root: String, status: Option| { + let engine = &s.engine; + async move { + let rows: Vec = engine + .list_backup_rows_by_root(kind, &root, status) + .try_collect() + .await + .expect("listing"); + rows + } + }; + let arns = |rows: Vec| -> Vec { + rows.into_iter().map(|r| r.backup_arn).collect() + }; + let mut all: Vec<&BackupRow> = of_root.iter().collect(); + all.push(&creating); + let whole = listed(StoreKind::Filesystem, root.to_owned(), None).await; + assert_eq!(arns(whole.clone()), in_listing_order(&all)); + for row in &whole { + assert_eq!( + Some(row), + s.engine + .read_backup_row(&arn_of(row)) + .await + .expect("read") + .as_ref(), + "the listing returns rows whole" + ); + } + for status in BackupStatus::ALL { + let of_status: Vec<&BackupRow> = + all.iter().copied().filter(|r| r.status == status).collect(); + assert_eq!( + arns(listed(StoreKind::Filesystem, root.to_owned(), Some(status)).await), + in_listing_order(&of_status), + "{status}" + ); + } + for (kind, label, row) in [ + (StoreKind::Filesystem, slash.as_str(), &others[0]), + (StoreKind::Filesystem, "/srv/backups-ol", &others[1]), + (StoreKind::S3, root, &others[2]), + (StoreKind::S3, s3, &others[3]), + (StoreKind::S3, s3_endpoint, &others[4]), + ] { + assert_eq!( + arns(listed(kind, label.to_owned(), None).await), + std::slice::from_ref(&row.backup_arn), + "{kind} {label}" + ); + } + assert!( + listed(StoreKind::Filesystem, "/srv/backups".to_owned(), None) + .await + .is_empty() + ); + drop(claim); + s.cleanup().await; + } + + /// `rewrite_store`: of the keys given, the rows still of the old kind + /// and root and not CREATING get the new store, with every other column + /// left, and are counted; a CREATING row, a row of another root, a row + /// of the other kind at the same label and a key with no row are left + /// and not counted; a second call counts 0; a row another writer moved + /// to a third root before the call is left there. + #[tokio::test] + async fn rewrite_store_moves_only_the_old_roots_rows_that_are_not_creating() { + if base_conn().is_none() { + eprintln!( + "SKIP rewrite_store_moves_only_the_old_roots_rows_that_are_not_creating: no PostgreSQL" + ); + return; + } + let s = scratch().await; + let old = (StoreKind::Filesystem, "/srv/backups-old", "old-1"); + let to = StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: "/srv/backups-new".to_owned(), + store_id: "new-1".to_owned(), + }; + let mut movable = Vec::new(); + for (i, status) in [ + BackupStatus::Available, + BackupStatus::Deleting, + BackupStatus::Deleted, + BackupStatus::Failed, + ] + .into_iter() + .enumerate() + { + movable.push(row_at("Music", &format!("m{i}"), status, old)); + } + let moved_away = row_at("Music", "away", BackupStatus::Available, old); + let other_root = row_at( + "Music", + "other-root", + BackupStatus::Available, + (StoreKind::Filesystem, "/srv/backups-other", "old-1"), + ); + let other_kind = row_at( + "Music", + "other-kind", + BackupStatus::Available, + (StoreKind::S3, "/srv/backups-old", "old-1"), + ); + for row in movable + .iter() + .chain([&moved_away, &other_root, &other_kind]) + { + assert!(s.engine.insert_backup_row(row).await.expect("insert")); + } + let creating = row_at("Music", "job", BackupStatus::Creating, old); + let claim = s.engine.claim_backup(&creating).await.expect("claim"); + let stored = |row: &BackupRow| { + let engine = &s.engine; + let arn = arn_of(row); + async move { + engine + .read_backup_row(&arn) + .await + .expect("read") + .expect("the row exists") + } + }; + let mut before = Vec::new(); + for row in &movable { + before.push(stored(row).await); + } + // Another writer moves one row to a third root first. + let third = StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: "/srv/backups-third".to_owned(), + store_id: "third-1".to_owned(), + }; + assert_eq!( + s.engine + .rewrite_store(old.0, old.1, &[moved_away.key()], &third) + .await + .expect("rewrite"), + 1 + ); + let mut ghost = row_at("Music", "ghost", BackupStatus::Available, old).key(); + ghost.backup_id = extenddb_storage::util::new_backup_id(); + let keys: Vec = movable + .iter() + .chain([&creating, &moved_away, &other_root, &other_kind]) + .map(BackupRow::key) + .chain([ghost]) + .collect(); + assert_eq!( + s.engine + .rewrite_store(old.0, old.1, &keys, &to) + .await + .expect("rewrite"), + 4, + "the four rows of the old root that are not CREATING" + ); + for (row, before) in movable.iter().zip(&before) { + assert_eq!( + stored(row).await, + BackupRow { + store: to.clone(), + ..before.clone() + }, + "only the store changed" + ); + } + let creating_now = stored(&creating).await; + assert_eq!( + (creating_now.status, creating_now.store.store_root.as_str()), + (BackupStatus::Creating, old.1) + ); + assert_eq!(creating_now.owner_generation, claim.generation()); + assert_eq!(stored(&moved_away).await.store, third); + assert_eq!(stored(&other_root).await.store, other_root.store); + assert_eq!(stored(&other_kind).await.store, other_kind.store); + assert_eq!( + s.engine + .rewrite_store(old.0, old.1, &keys, &to) + .await + .expect("rewrite"), + 0, + "a second call finds nothing left to move" + ); + let listed: Vec = s + .engine + .list_backup_rows(None, &to) + .try_collect() + .await + .expect("listing"); + let movable_refs: Vec<&BackupRow> = movable.iter().collect(); + assert_eq!( + listed.into_iter().map(|r| r.backup_arn).collect::>(), + in_listing_order(&movable_refs) + ); + assert_eq!( + s.engine.read_store_ids(old.0, old.1).await.expect("ids"), + ["old-1"], + "the CREATING row keeps the old root" + ); + assert_eq!( + s.engine + .rewrite_store(old.0, old.1, &[], &to) + .await + .expect("rewrite"), + 0 + ); + drop(claim); + s.cleanup().await; + } + + /// The store root of each row of `rows`, as the catalog holds it now. + async fn roots_of(s: &Scratch, rows: &[&BackupRow]) -> Vec { + let mut roots = Vec::new(); + for row in rows { + let root: String = + sqlx::query_scalar("SELECT store_root FROM backup_index WHERE backup_id = $1") + .bind(&row.backup_id) + .fetch_one(&s.catalog) + .await + .expect("read a row's root"); + roots.push(root); + } + roots + } + + /// How many backends of the scratch catalog run the rewrite's + /// statement, waiting on a lock when `waiting`. + async fn rewrites_running(s: &Scratch, waiting: bool) -> i64 { + sqlx::query_scalar( + "SELECT count(*) FROM pg_stat_activity WHERE datname = $1 \ + AND query LIKE 'UPDATE backup_index SET store_kind%' \ + AND state <> 'idle' AND (NOT $2 OR wait_event_type = 'Lock')", + ) + .bind(&s.db_names[0]) + .bind(waiting) + .fetch_one(&s.admin) + .await + .expect("read pg_stat_activity") + } + + /// Hold `FOR UPDATE` on the `backup_index` row of `row` in a + /// transaction of the scratch catalog's pool, until it ends. + async fn hold_row_lock( + s: &Scratch, + row: &BackupRow, + ) -> sqlx::Transaction<'static, sqlx::Postgres> { + let mut tx = s.catalog.begin().await.expect("begin"); + sqlx::query("SELECT 1 FROM backup_index WHERE backup_id = $1 FOR UPDATE") + .bind(&row.backup_id) + .execute(&mut *tx) + .await + .expect("lock the row"); + tx + } + + /// A rewrite whose client dies while its statement waits on a row lock + /// rewrites nothing, even when the lock is granted after the death: the + /// statement runs in a transaction the client commits, and a dead client + /// commits nothing. The client here is an engine on its own runtime, + /// shut down while its statement waits; the lock is released well + /// within `REWRITE_LOCK_TIMEOUT`, so the statement does get its lock. + #[tokio::test] + async fn a_rewrite_whose_client_dies_mid_statement_rewrites_nothing() { + if base_conn().is_none() { + eprintln!( + "SKIP a_rewrite_whose_client_dies_mid_statement_rewrites_nothing: no PostgreSQL" + ); + return; + } + let s = scratch().await; + let old = (StoreKind::Filesystem, "/srv/backups-old", "old-1"); + let to = StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: "/srv/backups-new".to_owned(), + store_id: "new-1".to_owned(), + }; + let rows = [ + row_at("Music", "free", BackupStatus::Available, old), + row_at("Music", "locked", BackupStatus::Available, old), + ]; + for row in &rows { + assert!(s.engine.insert_backup_row(row).await.expect("insert")); + } + let lock = hold_row_lock(&s, &rows[1]).await; + let (stop, stopped) = std::sync::mpsc::channel::<()>(); + let client = { + let url = s.catalog_url.clone(); + let keys: Vec<_> = rows.iter().map(BackupRow::key).collect(); + let to = to.clone(); + std::thread::spawn(move || { + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(1) + .enable_all() + .build() + .expect("a runtime"); + runtime.spawn(async move { + let engine = PostgresEngine::new( + &PostgresConfig { + connection_string: url, + pool_size: 2, + max_item_size_bytes: 400_000, + }, + REGION, + ) + .await + .expect("a second engine"); + let _ = engine.rewrite_store(old.0, old.1, &keys, &to).await; + panic!("the rewrite returned while its row was locked"); + }); + stopped.recv().expect("the stop"); + // Every task and connection of the client goes with it. + runtime.shutdown_timeout(std::time::Duration::from_secs(5)); + }) + }; + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(20); + while rewrites_running(&s, true).await == 0 { + assert!( + std::time::Instant::now() < deadline, + "the rewrite never waited" + ); + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + } + stop.send(()).expect("stop the client"); + client.join().expect("the client thread"); + tokio::time::sleep(std::time::Duration::from_millis(200)).await; + lock.commit().await.expect("release the row lock"); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(20); + while rewrites_running(&s, false).await > 0 { + assert!( + std::time::Instant::now() < deadline, + "the statement never ended" + ); + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + } + assert_eq!( + roots_of(&s, &rows.iter().collect::>()).await, + [old.1, old.1], + "no key is rewritten" + ); + s.cleanup().await; + } + + /// A rewrite that meets a row lock held past `REWRITE_LOCK_TIMEOUT` + /// rolls back and runs again: a lock released after the first timeout + /// lets the second attempt move every key; a lock held past every + /// attempt is an error after `REWRITE_ATTEMPTS` waits, and no key moves, + /// the unlocked one included. + #[tokio::test] + async fn a_rewrite_that_meets_a_held_row_lock_runs_again_a_bounded_number_of_times() { + use extenddb_storage_postgres::{REWRITE_ATTEMPTS, REWRITE_LOCK_TIMEOUT}; + if base_conn().is_none() { + eprintln!( + "SKIP a_rewrite_that_meets_a_held_row_lock_runs_again_a_bounded_number_of_times: no PostgreSQL" + ); + return; + } + let s = scratch().await; + let old = (StoreKind::Filesystem, "/srv/backups-old", "old-1"); + let to = StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: "/srv/backups-new".to_owned(), + store_id: "new-1".to_owned(), + }; + let rows = [ + row_at("Music", "free", BackupStatus::Available, old), + row_at("Music", "locked", BackupStatus::Available, old), + ]; + for row in &rows { + assert!(s.engine.insert_backup_row(row).await.expect("insert")); + } + let keys: Vec<_> = rows.iter().map(BackupRow::key).collect(); + let refs: Vec<&BackupRow> = rows.iter().collect(); + + let lock = hold_row_lock(&s, &rows[1]).await; + let started = std::time::Instant::now(); + let (rewritten, ()) = + tokio::join!(s.engine.rewrite_store(old.0, old.1, &keys, &to), async { + tokio::time::sleep(REWRITE_LOCK_TIMEOUT + std::time::Duration::from_millis(250)) + .await; + lock.commit().await.expect("release the row lock"); + }); + assert_eq!(rewritten.expect("the second attempt"), 2); + assert!( + started.elapsed() > REWRITE_LOCK_TIMEOUT, + "{:?}", + started.elapsed() + ); + assert_eq!(roots_of(&s, &refs).await, [to.store_root.as_str(); 2]); + + let lock = hold_row_lock(&s, &rows[1]).await; + let started = std::time::Instant::now(); + let refused = tokio::time::timeout( + std::time::Duration::from_secs(60), + s.engine.rewrite_store( + to.store_kind, + &to.store_root, + &keys, + &StoreIdentity { + store_kind: old.0, + store_root: old.1.to_owned(), + store_id: old.2.to_owned(), + }, + ), + ) + .await + .expect("the attempts are bounded") + .expect_err("the lock outlasts every attempt"); + // Three attempts with their two pauses (0.5 s, 1 s) take about 10.5 s; + // no pause would end near 9 s, and a fourth attempt near 15 s. + let took = started.elapsed(); + assert!( + took > REWRITE_LOCK_TIMEOUT * REWRITE_ATTEMPTS + std::time::Duration::from_secs(1), + "{took:?}: the attempts paused between them" + ); + assert!( + took < REWRITE_LOCK_TIMEOUT * REWRITE_ATTEMPTS + std::time::Duration::from_secs(3), + "{took:?}: no more than {REWRITE_ATTEMPTS} attempts" + ); + assert!(refused.to_string().contains("lock timeout"), "{refused}"); + assert_eq!( + roots_of(&s, &refs).await, + [to.store_root.as_str(); 2], + "no key moves" + ); + lock.rollback().await.expect("release the row lock"); + + // The bounds are the transaction's own: the connection the rewrite + // ran on reads the server's defaults once it has committed. + let (rewritten, lock_timeout, statement_timeout) = s + .engine + .rewrite_store_settings_after( + to.store_kind, + &to.store_root, + &keys, + &StoreIdentity { + store_kind: old.0, + store_root: old.1.to_owned(), + store_id: old.2.to_owned(), + }, + ) + .await + .expect("a rewrite with no lock held"); + assert_eq!(rewritten, 2); + let mut defaults = Vec::new(); + for name in ["lock_timeout", "statement_timeout"] { + let value: String = sqlx::query_scalar(&format!("SHOW {name}")) + .fetch_one(&s.catalog) + .await + .expect("SHOW on another session"); + defaults.push(value); + } + assert_eq!( + [lock_timeout, statement_timeout], + [defaults[0].clone(), defaults[1].clone()], + "the rewrite's settings stay with its transaction" + ); + s.cleanup().await; + } + + /// `summarize_store_roots`: one count per kind, root, id and status with + /// a row, in the one order; two ids of one root, two statuses of one id, + /// two roots and the two kinds are each counted apart. + #[tokio::test] + async fn summarize_store_roots_counts_the_rows_of_each_store_and_status() { + use extenddb_storage::backup_catalog::StoreRootCount; + if base_conn().is_none() { + eprintln!( + "SKIP summarize_store_roots_counts_the_rows_of_each_store_and_status: no PostgreSQL" + ); + return; + } + let s = scratch().await; + assert!( + s.engine + .summarize_store_roots() + .await + .expect("summary") + .is_empty() + ); + let a1 = (StoreKind::Filesystem, "/srv/a", "id-1"); + let a2 = (StoreKind::Filesystem, "/srv/a", "id-2"); + let b1 = (StoreKind::Filesystem, "/srv/b", "id-1"); + let s3 = (StoreKind::S3, "s3://bucket/a", "id-1"); + let rows = [ + row_at("Music", "1", BackupStatus::Available, a1), + row_at("Music", "2", BackupStatus::Available, a1), + row_at("Music", "3", BackupStatus::Failed, a1), + row_at("Music", "4", BackupStatus::Available, a2), + row_at("Music", "5", BackupStatus::Deleting, b1), + row_at("Music", "7", BackupStatus::Available, b1), + row_at("Music", "6", BackupStatus::Available, s3), + ]; + for row in &rows { + assert!(s.engine.insert_backup_row(row).await.expect("insert")); + } + let creating = row_at("Music", "job", BackupStatus::Creating, b1); + let claim = s.engine.claim_backup(&creating).await.expect("claim"); + let line = |(kind, root, id): (StoreKind, &str, &str), status, rows| StoreRootCount { + store: StoreIdentity { + store_kind: kind, + store_root: root.to_owned(), + store_id: id.to_owned(), + }, + status, + rows, + }; + assert_eq!( + s.engine.summarize_store_roots().await.expect("summary"), + [ + line(a1, BackupStatus::Available, 2), + line(a1, BackupStatus::Failed, 1), + line(a2, BackupStatus::Available, 1), + line(b1, BackupStatus::Creating, 1), + line(b1, BackupStatus::Available, 1), + line(b1, BackupStatus::Deleting, 1), + line(s3, BackupStatus::Available, 1), + ] + ); + drop(claim); + s.cleanup().await; + } + /// `insert_backup_row`: the row lands as given, unowned, with /// `completed_at` and `heartbeat_at` on the database's clock; a second /// insert, an insert over a CREATING row a claim holds, and an insert of @@ -4724,7 +5430,7 @@ mod catalog_index { let (_tx, mut interrupt) = Interrupt::new(); let started = std::time::Instant::now(); assert_eq!( - wait_for_quiet(&s.engine, &store(), &held, wait, &mut interrupt, announce).await, + wait_for_quiet(&s.engine, &[store()], &held, wait, &mut interrupt, announce).await, WaitOutcome::Quiet, "a FAILED restore record is the reconciler's duty, not a running job" ); @@ -4764,7 +5470,7 @@ mod catalog_index { failed.backup_arn )]; assert_eq!( - wait_for_quiet(&s.engine, &store(), &held, wait, &mut interrupt, announce).await, + wait_for_quiet(&s.engine, &[store()], &held, wait, &mut interrupt, announce).await, WaitOutcome::Busy(expected.clone()), "a FAILED row with a live heartbeat holds the wait and is named" ); @@ -4780,7 +5486,7 @@ mod catalog_index { .await .expect("age"); assert_eq!( - wait_for_quiet(&s.engine, &store(), &held, wait, &mut interrupt, announce).await, + wait_for_quiet(&s.engine, &[store()], &held, wait, &mut interrupt, announce).await, WaitOutcome::Quiet, "a FAILED row whose heartbeat has aged is not waited on" ); diff --git a/crates/storage-sqlite/src/backup.rs b/crates/storage-sqlite/src/backup.rs index 5216386c4..9f0300dc1 100644 --- a/crates/storage-sqlite/src/backup.rs +++ b/crates/storage-sqlite/src/backup.rs @@ -4830,7 +4830,10 @@ mod restore_tests { Err(StorageError::Validation(m)) if m.contains("Backup not found") )); assert!( - !engine.remove_backup_row(&key).await.unwrap(), + !engine + .remove_backup_row(&key, row.store.store_kind, &row.store.store_root) + .await + .unwrap(), "DELETING is not a tombstone" ); assert!(engine.set_backup_deleted(&key).await.unwrap()); @@ -4842,8 +4845,82 @@ mod restore_tests { .unwrap(); assert_eq!(stored.status, BackupStatus::Deleted); assert!(stored.completed_at.is_some()); - assert!(engine.remove_backup_row(&key).await.unwrap()); - assert!(!engine.remove_backup_row(&key).await.unwrap()); + // A row another writer moved to another store after the caller read it + // is left by a removal that names the store read. + let moved_to = StoreIdentity { + store_kind: row.store.store_kind, + store_root: format!("{}-moved", row.store.store_root), + store_id: "moved-1".to_owned(), + }; + assert_eq!( + engine + .rewrite_store( + row.store.store_kind, + &row.store.store_root, + std::slice::from_ref(&key), + &moved_to + ) + .await + .unwrap(), + 1 + ); + assert!( + !engine + .remove_backup_row(&key, row.store.store_kind, &row.store.store_root) + .await + .unwrap(), + "a row moved to another store is left" + ); + assert_eq!( + engine + .read_backup_row(&arn_of(&row)) + .await + .unwrap() + .map(|r| (r.status, r.store)), + Some((BackupStatus::Deleted, moved_to.clone())) + ); + // The same root under the other kind is another store: a removal that + // names the kind it read leaves the row. + let other_kind = StoreIdentity { + store_kind: if moved_to.store_kind == StoreKind::S3 { + StoreKind::Filesystem + } else { + StoreKind::S3 + }, + ..moved_to.clone() + }; + assert_eq!( + engine + .rewrite_store( + moved_to.store_kind, + &moved_to.store_root, + std::slice::from_ref(&key), + &other_kind + ) + .await + .unwrap(), + 1 + ); + assert!( + !engine + .remove_backup_row(&key, moved_to.store_kind, &moved_to.store_root) + .await + .unwrap(), + "a row moved to the other kind is left" + ); + let moved_to = other_kind; + assert!( + engine + .remove_backup_row(&key, moved_to.store_kind, &moved_to.store_root) + .await + .unwrap() + ); + assert!( + !engine + .remove_backup_row(&key, moved_to.store_kind, &moved_to.store_root) + .await + .unwrap() + ); assert!( engine .read_backup_row(&arn_of(&row)) @@ -5259,6 +5336,427 @@ mod restore_tests { ); } + /// A row of `row(ACCOUNT, table, name)` with `status`, under the + /// store of `kind`, `root` and `id`. + fn row_at( + table: &str, + name: &str, + status: BackupStatus, + (kind, root, id): (StoreKind, &str, &str), + ) -> BackupRow { + let mut row = row(ACCOUNT, table, name); + row.status = status; + row.store = StoreIdentity { + store_kind: kind, + store_root: root.to_owned(), + store_id: id.to_owned(), + }; + row.size_bytes = 10; + row.item_count = 2; + row + } + + /// The ARNs of `rows` in the listings' (`created_at`, `backup_arn`) + /// order. + fn in_listing_order(rows: &[&BackupRow]) -> Vec { + let mut rows = rows.to_vec(); + rows.sort_by(|a, b| { + a.created_at + .total_cmp(&b.created_at) + .then_with(|| a.backup_arn.cmp(&b.backup_arn)) + }); + rows.into_iter().map(|r| r.backup_arn.clone()).collect() + } + + /// `list_backup_rows_by_root`: every row of the kind and root, + /// whatever its id, of every status or of one, whole and in the + /// listings' order; nothing of a root that differs by one byte (a + /// trailing `/`, one character fewer), of an S3 root that differs by + /// its `?endpoint=` suffix, or of the other kind at the same label. + #[tokio::test] + async fn list_backup_rows_by_root_returns_every_id_and_status_of_the_root_and_nothing_else() + { + let engine = engine().await; + let root = "/srv/backups-old"; + let at = |id| (StoreKind::Filesystem, root, id); + let mut of_root = Vec::new(); + for (i, status) in [ + BackupStatus::Available, + BackupStatus::Deleting, + BackupStatus::Deleted, + BackupStatus::Failed, + ] + .into_iter() + .enumerate() + { + let id = if i % 2 == 0 { "store-1" } else { "store-2" }; + of_root.push(row_at("Music", &format!("r{i}"), status, at(id))); + } + // Inserted newest first: the order asserted is the listing's. + for row in of_root.iter().rev() { + assert!(engine.insert_backup_row(row).await.expect("insert")); + } + let creating = row_at("Music", "job", BackupStatus::Creating, at("store-1")); + let _claim = engine.claim_backup(&creating).await.expect("claim"); + let slash = format!("{root}/"); + let s3 = "s3://bucket/prefix"; + let s3_endpoint = "s3://bucket/prefix?endpoint=http://127.0.0.1:9200"; + let others = [ + row_at( + "Music", + "slash", + BackupStatus::Available, + (StoreKind::Filesystem, &slash, "store-1"), + ), + row_at( + "Music", + "shorter", + BackupStatus::Available, + (StoreKind::Filesystem, "/srv/backups-ol", "store-1"), + ), + row_at( + "Music", + "kind", + BackupStatus::Available, + (StoreKind::S3, root, "store-1"), + ), + row_at( + "Music", + "s3", + BackupStatus::Available, + (StoreKind::S3, s3, "store-3"), + ), + row_at( + "Music", + "endpoint", + BackupStatus::Available, + (StoreKind::S3, s3_endpoint, "store-3"), + ), + ]; + for row in &others { + assert!(engine.insert_backup_row(row).await.expect("insert")); + } + let listed = |kind: StoreKind, root: String, status: Option| { + let engine = &engine; + async move { + let rows: Vec = engine + .list_backup_rows_by_root(kind, &root, status) + .try_collect() + .await + .expect("listing"); + rows + } + }; + let arns = |rows: Vec| -> Vec { + rows.into_iter().map(|r| r.backup_arn).collect() + }; + let mut all: Vec<&BackupRow> = of_root.iter().collect(); + all.push(&creating); + let whole = listed(StoreKind::Filesystem, root.to_owned(), None).await; + assert_eq!(arns(whole.clone()), in_listing_order(&all)); + for row in &whole { + assert_eq!( + Some(row), + engine + .read_backup_row(&arn_of(row)) + .await + .expect("read") + .as_ref(), + "the listing returns rows whole" + ); + } + for status in BackupStatus::ALL { + let of_status: Vec<&BackupRow> = + all.iter().copied().filter(|r| r.status == status).collect(); + assert_eq!( + arns(listed(StoreKind::Filesystem, root.to_owned(), Some(status)).await), + in_listing_order(&of_status), + "{status}" + ); + } + for (kind, label, row) in [ + (StoreKind::Filesystem, slash.as_str(), &others[0]), + (StoreKind::Filesystem, "/srv/backups-ol", &others[1]), + (StoreKind::S3, root, &others[2]), + (StoreKind::S3, s3, &others[3]), + (StoreKind::S3, s3_endpoint, &others[4]), + ] { + assert_eq!( + arns(listed(kind, label.to_owned(), None).await), + std::slice::from_ref(&row.backup_arn), + "{kind} {label}" + ); + } + assert!( + listed(StoreKind::Filesystem, "/srv/backups".to_owned(), None) + .await + .is_empty() + ); + } + + /// `rewrite_store`: of the keys given, the rows still of the old kind + /// and root and not CREATING get the new store, with every other + /// column left, and are counted; a CREATING row, a row of another + /// root, a row of the other kind at the same label and a key with no + /// row are left and not counted; a second call counts 0; a row + /// another writer moved to a third root before the call is left + /// there. + #[tokio::test] + async fn rewrite_store_moves_only_the_old_roots_rows_that_are_not_creating() { + let engine = engine().await; + let old = (StoreKind::Filesystem, "/srv/backups-old", "old-1"); + let to = StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: "/srv/backups-new".to_owned(), + store_id: "new-1".to_owned(), + }; + let mut movable = Vec::new(); + for (i, status) in [ + BackupStatus::Available, + BackupStatus::Deleting, + BackupStatus::Deleted, + BackupStatus::Failed, + ] + .into_iter() + .enumerate() + { + movable.push(row_at("Music", &format!("m{i}"), status, old)); + } + let moved_away = row_at("Music", "away", BackupStatus::Available, old); + let other_root = row_at( + "Music", + "other-root", + BackupStatus::Available, + (StoreKind::Filesystem, "/srv/backups-other", "old-1"), + ); + let other_kind = row_at( + "Music", + "other-kind", + BackupStatus::Available, + (StoreKind::S3, "/srv/backups-old", "old-1"), + ); + for row in movable + .iter() + .chain([&moved_away, &other_root, &other_kind]) + { + assert!(engine.insert_backup_row(row).await.expect("insert")); + } + let creating = row_at("Music", "job", BackupStatus::Creating, old); + let claim = engine.claim_backup(&creating).await.expect("claim"); + let stored = |row: &BackupRow| { + let engine = &engine; + let arn = arn_of(row); + async move { + engine + .read_backup_row(&arn) + .await + .expect("read") + .expect("the row exists") + } + }; + let mut before = Vec::new(); + for row in &movable { + before.push(stored(row).await); + } + // Another writer moves one row to a third root first. + let third = StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: "/srv/backups-third".to_owned(), + store_id: "third-1".to_owned(), + }; + assert_eq!( + engine + .rewrite_store(old.0, old.1, &[moved_away.key()], &third) + .await + .expect("rewrite"), + 1 + ); + let mut ghost = row_at("Music", "ghost", BackupStatus::Available, old).key(); + ghost.backup_id = extenddb_storage::util::new_backup_id(); + let keys: Vec = movable + .iter() + .chain([&creating, &moved_away, &other_root, &other_kind]) + .map(BackupRow::key) + .chain([ghost]) + .collect(); + assert_eq!( + engine + .rewrite_store(old.0, old.1, &keys, &to) + .await + .expect("rewrite"), + 4, + "the four rows of the old root that are not CREATING" + ); + for (row, before) in movable.iter().zip(&before) { + assert_eq!( + stored(row).await, + BackupRow { + store: to.clone(), + ..before.clone() + }, + "only the store changed" + ); + } + let creating_now = stored(&creating).await; + assert_eq!( + (creating_now.status, creating_now.store.store_root.as_str()), + (BackupStatus::Creating, old.1) + ); + assert_eq!(creating_now.owner_generation, claim.generation()); + assert_eq!(stored(&moved_away).await.store, third); + assert_eq!(stored(&other_root).await.store, other_root.store); + assert_eq!(stored(&other_kind).await.store, other_kind.store); + assert_eq!( + engine + .rewrite_store(old.0, old.1, &keys, &to) + .await + .expect("rewrite"), + 0, + "a second call finds nothing left to move" + ); + let listed: Vec = engine + .list_backup_rows(None, &to) + .try_collect() + .await + .expect("listing"); + let movable_refs: Vec<&BackupRow> = movable.iter().collect(); + assert_eq!( + listed.into_iter().map(|r| r.backup_arn).collect::>(), + in_listing_order(&movable_refs) + ); + assert_eq!( + engine.read_store_ids(old.0, old.1).await.expect("ids"), + ["old-1"], + "the CREATING row keeps the old root" + ); + assert_eq!( + engine + .rewrite_store(old.0, old.1, &[], &to) + .await + .expect("rewrite"), + 0 + ); + } + + /// `rewrite_store` moves every key or none: a statement that fails + /// on a later key (a trigger here raises on the last one) rolls back + /// the keys already rewritten in the same call; once the trigger is + /// gone the same call moves them all. + #[tokio::test] + async fn a_rewrite_that_fails_on_a_later_key_moves_no_row() { + let engine = engine().await; + let old = (StoreKind::Filesystem, "/srv/backups-old", "old-1"); + let to = StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: "/srv/backups-new".to_owned(), + store_id: "new-1".to_owned(), + }; + let rows: Vec = ["a", "b", "c"] + .into_iter() + .map(|name| row_at("Music", name, BackupStatus::Available, old)) + .collect(); + for row in &rows { + assert!(engine.insert_backup_row(row).await.expect("insert")); + } + sqlx::query(&format!( + "CREATE TRIGGER refuse_the_last_key BEFORE UPDATE OF store_root ON backup_index \ + WHEN OLD.backup_id = '{}' BEGIN SELECT RAISE(ABORT, 'refused'); END", + rows[2].backup_id + )) + .execute(&engine.pool) + .await + .expect("create the trigger"); + let keys: Vec<_> = rows.iter().map(BackupRow::key).collect(); + let refused = engine + .rewrite_store(old.0, old.1, &keys, &to) + .await + .expect_err("the last key's statement fails"); + assert!(refused.to_string().contains("refused"), "{refused}"); + let roots = || async { + let mut roots = Vec::new(); + for row in &rows { + roots.push( + engine + .read_backup_row(&arn_of(row)) + .await + .expect("read") + .expect("the row exists") + .store + .store_root, + ); + } + roots + }; + assert_eq!(roots().await, [old.1; 3], "no row moved"); + sqlx::query("DROP TRIGGER refuse_the_last_key") + .execute(&engine.pool) + .await + .expect("drop the trigger"); + assert_eq!( + engine + .rewrite_store(old.0, old.1, &keys, &to) + .await + .expect("rewrite"), + 3 + ); + assert_eq!(roots().await, [to.store_root.as_str(); 3]); + } + + /// `summarize_store_roots`: one count per kind, root, id and status + /// with a row, in the one order; two ids of one root, two statuses of + /// one id, two roots and the two kinds are each counted apart. + #[tokio::test] + async fn summarize_store_roots_counts_the_rows_of_each_store_and_status() { + use extenddb_storage::backup_catalog::StoreRootCount; + let engine = engine().await; + assert!( + engine + .summarize_store_roots() + .await + .expect("summary") + .is_empty() + ); + let a1 = (StoreKind::Filesystem, "/srv/a", "id-1"); + let a2 = (StoreKind::Filesystem, "/srv/a", "id-2"); + let b1 = (StoreKind::Filesystem, "/srv/b", "id-1"); + let s3 = (StoreKind::S3, "s3://bucket/a", "id-1"); + let rows = [ + row_at("Music", "1", BackupStatus::Available, a1), + row_at("Music", "2", BackupStatus::Available, a1), + row_at("Music", "3", BackupStatus::Failed, a1), + row_at("Music", "4", BackupStatus::Available, a2), + row_at("Music", "5", BackupStatus::Deleting, b1), + row_at("Music", "7", BackupStatus::Available, b1), + row_at("Music", "6", BackupStatus::Available, s3), + ]; + for row in &rows { + assert!(engine.insert_backup_row(row).await.expect("insert")); + } + let creating = row_at("Music", "job", BackupStatus::Creating, b1); + let _claim = engine.claim_backup(&creating).await.expect("claim"); + let line = |(kind, root, id): (StoreKind, &str, &str), status, rows| StoreRootCount { + store: StoreIdentity { + store_kind: kind, + store_root: root.to_owned(), + store_id: id.to_owned(), + }, + status, + rows, + }; + assert_eq!( + engine.summarize_store_roots().await.expect("summary"), + [ + line(a1, BackupStatus::Available, 2), + line(a1, BackupStatus::Failed, 1), + line(a2, BackupStatus::Available, 1), + line(b1, BackupStatus::Creating, 1), + line(b1, BackupStatus::Available, 1), + line(b1, BackupStatus::Deleting, 1), + line(s3, BackupStatus::Available, 1), + ] + ); + } + /// `insert_backup_row`: the row lands as given, unowned, with /// `completed_at` and `heartbeat_at` on the database's clock; a /// second insert, an insert over a CREATING row a claim holds, and an diff --git a/crates/storage-sqlite/src/backup_catalog.rs b/crates/storage-sqlite/src/backup_catalog.rs index a91a34734..31a77102e 100644 --- a/crates/storage-sqlite/src/backup_catalog.rs +++ b/crates/storage-sqlite/src/backup_catalog.rs @@ -54,7 +54,7 @@ use extenddb_storage::backup_catalog::{ BackupCatalog, BackupKey, BackupRow, BackupStatus, DeleteStart, ForgetRow, Heartbeat, JobClaim, JobRow, JobStatus, MaintenanceClaim, RestoreJobRow, RestoreKind, RestoreRegistered, RestoreRegistrationRow, RestoreTargetKey, StaleClock, StatusFields, StoreIdentity, StoreKind, - TimeBounds, UNOWNED_GENERATION, WriteOutcome, new_generation, + StoreRootCount, TimeBounds, UNOWNED_GENERATION, WriteOutcome, new_generation, }; use extenddb_storage::error::StorageError; use futures::future::BoxFuture; @@ -898,6 +898,29 @@ impl BackupCatalog for SqliteEngine { }) } + fn list_backup_rows_by_root<'a>( + &'a self, + store_kind: StoreKind, + store_root: &'a str, + status: Option, + ) -> BoxStream<'a, Result> { + // `=` on TEXT compares bytes (the BINARY collation the columns + // carry), so a root matches as stored. + rows_stream(async move { + sqlx::query_as(&format!( + "SELECT {ROW_COLUMNS} FROM backup_index \ + WHERE store_kind = ?1 AND store_root = ?2 \ + AND (?3 IS NULL OR status = ?3) ORDER BY created_at, backup_arn" + )) + .bind(store_kind.as_str()) + .bind(store_root) + .bind(status.map(BackupStatus::as_str)) + .fetch_all(&self.pool) + .await + .map_err(db_err) + }) + } + fn newest_available_backup_row<'a>( &'a self, store: &'a StoreIdentity, @@ -984,6 +1007,48 @@ impl BackupCatalog for SqliteEngine { }) } + fn rewrite_store<'a>( + &'a self, + from_kind: StoreKind, + from_root: &'a str, + keys: &'a [BackupKey], + to: &'a StoreIdentity, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + if keys.is_empty() { + return Ok(0); + } + // One transaction under the writer lock: every key or none. + let _writer = self.write_lock.lock().await; + let mut tx = self + .pool + .begin_with("BEGIN IMMEDIATE") + .await + .map_err(db_err)?; + let mut rewritten = 0; + for key in keys { + rewritten += sqlx::query( + "UPDATE backup_index SET store_kind = ?, store_root = ?, store_id = ? \ + WHERE account_id = ? AND backup_id = ? AND store_kind = ? \ + AND store_root = ? AND status <> 'CREATING'", + ) + .bind(to.store_kind.as_str()) + .bind(&to.store_root) + .bind(&to.store_id) + .bind(&key.account_id) + .bind(&key.backup_id) + .bind(from_kind.as_str()) + .bind(from_root) + .execute(&mut *tx) + .await + .map_err(db_err)? + .rows_affected(); + } + tx.commit().await.map_err(db_err)?; + Ok(rewritten) + }) + } + fn set_backup_deleted<'a>( &'a self, key: &'a BackupKey, @@ -1007,15 +1072,19 @@ impl BackupCatalog for SqliteEngine { fn remove_backup_row<'a>( &'a self, key: &'a BackupKey, + store_kind: StoreKind, + store_root: &'a str, ) -> BoxFuture<'a, Result> { Box::pin(async move { let _writer = self.write_lock.lock().await; let affected = sqlx::query( "DELETE FROM backup_index WHERE account_id = ? AND backup_id = ? \ - AND status IN ('DELETED', 'FAILED')", + AND status IN ('DELETED', 'FAILED') AND store_kind = ? AND store_root = ?", ) .bind(&key.account_id) .bind(&key.backup_id) + .bind(store_kind.as_str()) + .bind(store_root) .execute(&self.pool) .await .map_err(db_err)? @@ -1463,6 +1532,26 @@ impl BackupCatalog for SqliteEngine { }) } + fn summarize_store_roots(&self) -> BoxFuture<'_, Result, StorageError>> { + Box::pin(async move { + let grouped: Vec<(String, String, String, String, i64)> = sqlx::query_as( + "SELECT store_kind, store_root, store_id, status, count(*) FROM backup_index \ + GROUP BY store_kind, store_root, store_id, status", + ) + .fetch_all(&self.pool) + .await + .map_err(db_err)?; + let mut counts = grouped + .into_iter() + .map(|(kind, root, id, status, rows)| { + StoreRootCount::from_columns(&kind, root, id, &status, rows) + }) + .collect::, _>>()?; + StoreRootCount::sort(&mut counts); + Ok(counts) + }) + } + fn maintenance_claim<'a>( &'a self, holder: &'a str, diff --git a/crates/storage/src/backup_catalog.rs b/crates/storage/src/backup_catalog.rs index 52772fc7d..bff6e4926 100644 --- a/crates/storage/src/backup_catalog.rs +++ b/crates/storage/src/backup_catalog.rs @@ -153,6 +153,95 @@ pub struct StoreIdentity { pub store_id: String, } +/// What [`BackupCatalog::remove_orphan_status_entries`] did with a root's +/// status entries. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct StatusEntrySweep { + /// Entries deleted: their row is gone or names another store. + pub removed: u64, + /// Entries whose row is of the root under another status, rewritten + /// under the row's status (the stale entry deleted). + pub repaired: u64, +} + +/// One line of [`BackupCatalog::summarize_store_roots`]: the catalog holds +/// `rows` rows of `status` under `store`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StoreRootCount { + pub store: StoreIdentity, + pub status: BackupStatus, + pub rows: u64, +} + +impl StoreRootCount { + /// A count as a grouping query returns it: the `store_kind`, + /// `store_root`, `store_id` and `status` columns and the count. + /// + /// # Errors + /// + /// `Internal` for a kind or status no release writes. + pub fn from_columns( + store_kind: &str, + store_root: String, + store_id: String, + status: &str, + rows: i64, + ) -> Result { + let kind = StoreKind::parse(store_kind).ok_or_else(|| { + StorageError::Internal(format!( + "backup rows of root {store_root:?} carry an unknown store kind {store_kind:?}" + )) + })?; + Ok(Self { + store: StoreIdentity { + store_kind: kind, + store_root, + store_id, + }, + status: BackupStatus::parse(status)?, + rows: u64::try_from(rows).unwrap_or_default(), + }) + } + + /// The order [`BackupCatalog::summarize_store_roots`] returns on every + /// backend: by kind, root and id, compared as bytes, then by status in + /// [`BackupStatus::ALL`]'s order, whatever the database's collation. + pub fn sort(counts: &mut [Self]) { + counts.sort_by(|a, b| { + fn key(c: &StoreRootCount) -> (&str, &[u8], &[u8], Option) { + ( + c.store.store_kind.as_str(), + c.store.store_root.as_bytes(), + c.store.store_id.as_bytes(), + BackupStatus::ALL.iter().position(|s| *s == c.status), + ) + } + key(a).cmp(&key(b)) + }); + } + + /// One count per (store, status) of `rows`, sorted: the tally a backend + /// that cannot group in its query (Cassandra) makes of what it read. + #[must_use] + pub fn tally(rows: impl IntoIterator) -> Vec { + let mut counts: std::collections::HashMap<(StoreIdentity, BackupStatus), u64> = + std::collections::HashMap::new(); + for key in rows { + *counts.entry(key).or_default() += 1; + } + let mut counts: Vec = counts + .into_iter() + .map(|((store, status), rows)| Self { + store, + status, + rows, + }) + .collect(); + Self::sort(&mut counts); + counts + } +} + /// The status column of a backup row. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum BackupStatus { @@ -164,6 +253,16 @@ pub enum BackupStatus { } impl BackupStatus { + /// Every status, in the order a row moves through them (CREATING first, + /// FAILED last): the order a per-status listing or report uses. + pub const ALL: [Self; 5] = [ + Self::Creating, + Self::Available, + Self::Deleting, + Self::Deleted, + Self::Failed, + ]; + /// The value stored in the `status` column. #[must_use] pub const fn as_str(self) -> &'static str { @@ -942,6 +1041,29 @@ pub trait BackupCatalog: Send + Sync { store: &'a StoreIdentity, ) -> BoxStream<'a, Result>; + /// Every row whose `store_kind` and `store_root` equal these, whatever + /// its `store_id`, with `status` or of every status for `None`, in the + /// (`created_at`, `backup_arn`) order [`Self::list_backup_rows`] uses: + /// `backup sync --move-store` and `--forget-store`, which get a root and + /// no id, and the report of the rows of other roots. `store_root` + /// matches as stored, byte for byte: a root that differs by a trailing + /// `/`, or an S3 root that differs by its `?endpoint=` suffix, is + /// another root. The rows are read whole, as `list_backup_rows` reads + /// them. On Cassandra the root's five status partitions are read + /// whatever `status` is, each row behind an entry is re-read once, and + /// it is kept when its kind and root still match (and its status, with + /// `Some(status)`), whatever partition its entry is in: a row whose + /// entry is under an earlier status (a status change whose entry batch + /// failed, or whose writer died between the row and the batch) is + /// listed under its own status. A row with no status entry at all (an + /// insert that stopped before its entries) is not listed. + fn list_backup_rows_by_root<'a>( + &'a self, + store_kind: StoreKind, + store_root: &'a str, + status: Option, + ) -> BoxStream<'a, Result>; + /// The newest AVAILABLE row of `store` (the last in the /// (`created_at`, `backup_arn`) order [`Self::list_backup_rows`] lists /// in), or `None` when the store has none. The store's manifest probe @@ -982,23 +1104,47 @@ pub trait BackupCatalog: Send + Sync { ) -> BoxFuture<'a, Result>; /// DELETING to DELETED, conditional on the row being DELETING; `false` - /// when it was not. + /// when it was not. Besides the deletion's last step, `backup sync + /// --forget-store` and `--forget-backup` remove a DELETING row with this + /// write and then [`Self::remove_backup_row`], under the maintenance + /// claim. No DELETED-prefix or tombstone duty acts on the row between + /// the two, by the order of the listings: a duty that listed its rows + /// before the claim was taken listed this one DELETING, not DELETED, and + /// a duty that lists it DELETED did so after the claim was taken, and + /// reads the claim held after its re-read of the row. A DELETING row of + /// the configured store has its deletion finished first, by the forget + /// itself or by a reconciler under the claim: its bytes were asked to be + /// deleted. A DELETING row of another store keeps its bytes. fn set_backup_deleted<'a>( &'a self, key: &'a BackupKey, ) -> BoxFuture<'a, Result>; - /// Remove a DELETED or FAILED tombstone; `false` when the row is not one. + /// Remove a DELETED or FAILED tombstone that is still of `store_kind` + /// and `store_root`, the store the caller read the row with; `false` + /// when the row is not one, or names another store. The tombstone + /// duty's write, and `backup sync --forget-store`'s and + /// `--forget-backup`'s for a DELETED or FAILED row (and for a DELETING + /// row after [`Self::set_backup_deleted`]). The store condition keeps a + /// duty of a server still configured with an old root from removing a + /// row `rewrite_store` moved after the duty read it: the configured + /// store's own duties then finish that row (on Cassandra the lightweight + /// transaction carries the condition, and a `NotMatched` answers + /// `false`). fn remove_backup_row<'a>( &'a self, key: &'a BackupKey, + store_kind: StoreKind, + store_root: &'a str, ) -> BoxFuture<'a, Result>; /// Remove an AVAILABLE row outright, leaving no DELETING or DELETED /// tombstone, so no deletion duty ever acts on the backup's objects: /// `backup sync --prune-missing`'s write for a row whose manifest the /// configured root does not hold, whose bytes may be under another - /// host's root of the same path. Under the row's lock, as + /// host's root of the same path, and `--forget-backup`'s and + /// `--forget-store`'s for an AVAILABLE row, whose bytes the operator + /// keeps. Under the row's lock, as /// [`Self::start_backup_delete`] takes it, the status check and the /// in-use rule in one step with the removal: a row a CREATING restore /// target reads is left ([`ForgetRow::InUse`]), and so is a row another @@ -1012,12 +1158,75 @@ pub trait BackupCatalog: Send + Sync { key: &'a BackupKey, ) -> BoxFuture<'a, Result>; + /// `backup sync --move-store`'s write: each row of `keys` that is still + /// of `from_kind` and `from_root` and is not CREATING gets `to`'s kind, + /// root and id; every other column is left. Returns how many rows were + /// rewritten. A key with no row, or whose row no longer matches (another + /// writer moved it to another root, or it is CREATING: a job may still + /// write its bytes under the old root), is left and not counted, so a + /// second call with the same keys counts 0 once `to` is another root. + /// A DELETING, DELETED or + /// FAILED row is rewritten like an AVAILABLE one. No claim and no fence + /// are taken: no object is touched. + /// + /// PostgreSQL rewrites every key in one statement and SQLite in one + /// transaction, so a failed call rewrites none. MongoDB rewrites in + /// chunks of keys, each chunk one statement, and Cassandra key by key, + /// so a call that stops leaves the rows split between the two roots: + /// every step is idempotent, and the caller re-lists the old root and + /// runs again for what is left. Cassandra keeps no `backup_id` + /// uniqueness, so the caller, not this method, refuses a move that + /// would give an account two rows of one backup id. + /// + /// What a client that dies in the call (a killed command) leaves: + /// - PostgreSQL: nothing, unless its commit was already sent. The + /// statement runs in a transaction the client commits, under a + /// `lock_timeout` (a lock timeout rolls back and runs again, a bounded + /// number of times) and a `statement_timeout`, so a statement still + /// waiting or running when the client dies commits nothing: the server + /// rolls it back once it finds the connection closed, at the latest + /// when those bounds end it, and no rewrite lands after the + /// maintenance claim's advisory lock went with the client. + /// - SQLite: nothing; the transaction is in the process and rolls back + /// with it. + /// - MongoDB: the chunks before the one in flight are rewritten, and + /// the one in flight in whole or in part (each document's update is + /// atomic, and the server may finish the statement after the client + /// is gone). The claim is a heartbeat, live until it is stale, which + /// outlasts the statement. + /// - Cassandra: the keys before the one in flight are rewritten, and + /// the one in flight is left at one of the points the module doc + /// names (its new entry written, or its row moved with the old entry + /// left); a lightweight transaction in flight may still commit. The + /// claim is a heartbeat, as on MongoDB. + /// + /// In each case the re-run of the move re-lists the old root and moves + /// what is left. + /// + /// # Errors + /// + /// The backend's error for a failed read or write. On Cassandra an + /// answer lost in transit is settled by a serial read of the row: a row + /// that carries `to`'s root counts as rewritten, a row still of the old + /// root is sent again (up to three attempts in all), and an outcome + /// still unknown after the last is an error. + fn rewrite_store<'a>( + &'a self, + from_kind: StoreKind, + from_root: &'a str, + keys: &'a [BackupKey], + to: &'a StoreIdentity, + ) -> BoxFuture<'a, Result>; + /// Remove a CREATING row whose owner is dead: `heartbeat_at` older than /// `stale_after` on the catalog's clock. `false` when the row is not /// CREATING, is gone, or has a live owner. The maintenance command's /// write for a CREATING row its wait saw no live owner for, after it has /// deleted the row's prefix; the row's own job, were it alive, would - /// find its claim lost at its next heartbeat. + /// find its claim lost at its next heartbeat. Also `backup sync + /// --forget-store`'s and `--forget-backup`'s write for a CREATING row, + /// which delete no prefix: `false` with the row still CREATING is a live + /// owner, and they refuse the row. fn remove_dead_creating_backup_row<'a>( &'a self, key: &'a BackupKey, @@ -1251,6 +1460,48 @@ pub trait BackupCatalog: Send + Sync { store_root: &'a str, ) -> BoxFuture<'a, Result, StorageError>>; + /// How many rows the catalog holds per store (kind, root and id) and + /// status, every store and status with at least one row, in + /// [`StoreRootCount::sort`]'s order: plain `backup sync`'s report of the + /// rows of roots other than the configured one, and the roots + /// `--move-store` and `--forget-store` name when given one with no + /// rows. Counts only; no row is read whole. On Cassandra the counts are + /// of the status entries (`backup_index_by_status`, one partition per + /// root and status), not verified against their rows, so an entry a + /// stopped writer left behind counts as [`Self::read_store_ids`] counts + /// it ([`Self::summary_counts_status_entries`]). + fn summarize_store_roots(&self) -> BoxFuture<'_, Result, StorageError>>; + + /// Whether [`Self::summarize_store_roots`] counts status entries rather + /// than rows (Cassandra), so a caller that reports the counts verifies + /// them against the rows' listing. `false` by default: the other + /// backends count rows. + fn summary_counts_status_entries(&self) -> bool { + false + } + + /// The sweep of the status entries under `store_root` whose entry + /// names `store_kind`, each verified against its row as it is read: + /// an entry whose row is gone or names another store is deleted, and + /// one whose row is of the root under another status is written again + /// under the row's status and then deleted (repaired). `backup sync + /// --forget-store`'s sweep of the root it forgets, under the + /// maintenance claim. Only Cassandra keeps status entries beside the + /// rows (`backup_index_by_status`): a deleter whose entry batch lands + /// after a `rewrite_store` of its row leaves an entry with no row of + /// the root behind it, and a status change whose entry batch failed, or + /// whose writer died between its row and its batch, leaves the row's + /// entry under the earlier status. Every other backend has none, which + /// is what the default body says. + fn remove_orphan_status_entries<'a>( + &'a self, + store_kind: StoreKind, + store_root: &'a str, + ) -> BoxFuture<'a, Result> { + let _ = (store_kind, store_root); + Box::pin(async { Ok(StatusEntrySweep::default()) }) + } + /// Take the catalog-level maintenance claim for `holder` (a label naming /// the command and the host it runs on, for the operator who finds it /// held). The claim is one record, `backup_maintenance`, born owned @@ -1820,4 +2071,46 @@ mod tests { has no connection slot left for the job); retry later" ); } + + /// The tally counts each (kind, root, id, status) apart, and the order + /// is the bytes of kind, root and id, then the statuses in the order a + /// row moves through them, not their names' order. + #[test] + fn the_root_tally_counts_each_store_and_status_apart_in_one_order() { + let store = |kind: StoreKind, root: &str, id: &str| StoreIdentity { + store_kind: kind, + store_root: root.to_owned(), + store_id: id.to_owned(), + }; + let a1 = store(StoreKind::Filesystem, "/a", "id-1"); + let a2 = store(StoreKind::Filesystem, "/a", "id-2"); + let lower = store(StoreKind::Filesystem, "/a/b", "id-1"); + let s3 = store(StoreKind::S3, "s3://bucket/p", "id-1"); + let counts = StoreRootCount::tally([ + (s3.clone(), BackupStatus::Available), + (a1.clone(), BackupStatus::Failed), + (a1.clone(), BackupStatus::Available), + (a2.clone(), BackupStatus::Available), + (a1.clone(), BackupStatus::Available), + (lower.clone(), BackupStatus::Creating), + (a1.clone(), BackupStatus::Creating), + ]); + let line = |store: &StoreIdentity, status, rows| StoreRootCount { + store: store.clone(), + status, + rows, + }; + assert_eq!( + counts, + [ + line(&a1, BackupStatus::Creating, 1), + line(&a1, BackupStatus::Available, 2), + line(&a1, BackupStatus::Failed, 1), + line(&a2, BackupStatus::Available, 1), + line(&lower, BackupStatus::Creating, 1), + line(&s3, BackupStatus::Available, 1), + ] + ); + assert!(StoreRootCount::tally([]).is_empty()); + } } diff --git a/docs/design/15-backup-restore.md b/docs/design/15-backup-restore.md index 050c47e51..5a376fb73 100644 --- a/docs/design/15-backup-restore.md +++ b/docs/design/15-backup-restore.md @@ -169,7 +169,7 @@ spool_path = "" # restore download area for an S3 store; d On SQLite (file-backed) a missing `[backup]` section is equivalent to `kind = "filesystem"` with `path` = `backups/` beside the database file; a `validate` failure of that defaulted root is a logged warning that leaves the store unconfigured, not a startup error, so an upgrade never stops a server that never configured backups, while an explicitly configured root that fails `validate` is still a startup error. The warning holds only while the catalog has no backup rows for the defaulted root (`read_store_ids`); once it has, the root has been used and is handled as an explicitly configured one (revision item 29). On PostgreSQL and MongoDB a missing `[backup.store]` leaves the off-host path unconfigured and startup logs that; `kind = "disabled"` leaves it unconfigured on purpose on any backend. Without a configured store, CreateBackup on every backend uses the legacy in-catalog create (`LegacyBackupReader::create`, section 3) and logs a deprecation warning naming `[backup.store]` on each call, until PR 14 removes that path; after it CreateBackup without a configured store returns a ValidationException naming `[backup.store]`. Legacy backups stay readable, listable, restorable and deletable either way. `sqlite-memory` (any `:memory:` database path) has no database file and so no default root: CreateBackup there takes the off-host path only with `[backup.store]` set and the legacy create without it. Cassandra has no `SnapshotEngine`, so it uses the legacy create even with a store configured (section 3); a configured store serves restores of off-host backups into Cassandra and nothing else there. -Memory per running job, which operators size hosts from. A backup's terms, for a row of `r` bytes of item text whose decoded `Item` is `d` bytes (`d` is about 1.2 × `r` for a row of many short strings, #394's worst shape, where every `N` is a 133-character string): the read path holds the page's decoded rows, `d` each, while they wait for the pipeline, at most the rows of the budget PR 8 adds to the PostgreSQL and SQLite streams (`SNAPSHOT_FETCH_BYTES`, 64 MiB, a constant beside `DEFAULT_FETCH_BATCH`, not a `[backup]` key in v1) plus one row, and one row's parse, `r + 2d`, since the deserializer builds an intermediate value before the item; PostgreSQL pulls the page's rows from the connection one at a time and parses each as it arrives, so it holds no page text beyond the row being parsed, and SQLite pulls a row at a time with the driver's worker at most five rows ahead of the pull, `r` each (`row_buffer_size` on the pool, four, plus the row the worker holds while its `send` waits; 160 MiB at the line cap; the default of 50 made the run-ahead, not the page, the largest term on a table of large rows). The pipeline holds the feeder's item, the item channel's items (`ITEM_CHANNEL`, four) and the encoder's item, `d` each; the encoder's line buffer (one line, at most 32 MiB, the line cap) and gzip window; the feeder's measurement of an item over the default `max_item_size_bytes` holds no line (it counts the line's bytes through a writer that stops at the cap), so the terms are the same on an instance that raises the limit, at the largest stored row; the chunk sink's 1 MiB buffer and the chunk channel (four 1 MiB chunks); the upload window (`upload_part_size_bytes * max_concurrent_uploads` on an S3 store, one write buffer on a filesystem store); and the manifest's data-file entries until the manifest is written. No term grows with `data_file_target_bytes` or with the table, except on `sqlite-memory`, whose open copies the whole table (section 3); every term but the budget and the chunks grows with the row, which `max_item_size_bytes` bounds (the line cap bounds the line the job writes and names the item it refuses; the row is read and decoded before the feeder measures it). The sizing rule, per running job, is the terms at the line cap times the allocator's factor, times `max_concurrent_jobs`; the rule is stated at the default `max_item_size_bytes`, where no stored item's line exceeds the cap (27.7 MB is the largest line a 400 KB item expands to, `MAX_LINE_BYTES`'s doc); on an instance that raises it the page's decoded rows, one row's parse and the pipeline's items are at the largest stored row, which only `max_item_size_bytes` bounds (about 68 times the item's size as a line, for #394's worst shape). The terms at the line cap (`r` = 32 MiB, `d` = 1.2 `r`): the page's rows, 3 `d` = 115 MiB; one row's parse, `r` + 2 `d` = 109 MiB; the pipeline's six items, 6 `d` = 230 MiB; the line buffer, 32 MiB; the chunks, 5 MiB; on SQLite the run-ahead, 5 `r` = 160 MiB: about 490 MiB on PostgreSQL and 650 MiB on SQLite, an upper bound, since the run-ahead and a full pipeline are not both at their largest at one instant. The allocator's factor is glibc's: the row's 200,000 short strings are allocated on the tokio worker that parses the row and freed on the blocking thread that encodes it, and with glibc's default per-thread arenas that memory stays in the arenas between pages, so the process's high-water mark grows with the number of large rows the job reads and varies by about 200 MiB between identical runs; under `MALLOC_ARENA_MAX=1` (one arena, set in the server's environment on a host that backs up tables of large items; the admin guide's `max_concurrent_jobs` row carries it and the deployment guide's service unit carries the line commented out with that condition, since one arena serializes the worker threads' allocations on one lock and costs the request path 3.5 to 9.2% on SQLite and about 2% on PostgreSQL (the scans, 2.1%), measured on GetItem, Query and Scan with a Python client, the record) the peak is the terms' and does not grow with the table. Measured on PR 8's 16-core host at 27.7 MB rows (`r` = 26.4 MiB, three rows to a page; the terms' sum 545 MiB on SQLite), 1,000 small rows then 12, 24 and 48 such rows, each series six runs back to back on one SQLite binary (`docs/design/15-backup-restore-bench-pr8.md`; revision item 31): with the default arenas the peak over idle was 466, 588 and 760 MiB on a quiet host and 707, 700 and 1,281 MiB on the final build with the kill-point runs beside it (0.86 to 2.35 of the terms, growing with the rows and varying by hundreds of MiB between series; runs of the PR's first build on a loaded host reached 1,096 MiB at 48 rows; no plateau was measured), and under `MALLOC_ARENA_MAX=1` 481, 452 and 480 MiB and then 452, 480 and 455 MiB (0.83 to 0.88 of the terms, flat from 12 to 48 rows in both series; the SQLite ratios are of the peak over idle), and on PostgreSQL at 12 rows 340 MiB `VmHWM` under one arena against 750 to 788 MiB under the default arenas (0.82 of the terms at that row on PostgreSQL, 413 MiB, which has no run-ahead; that ratio is of the `VmHWM`, the process's whole high-water mark, and about 0.47 over its idle of 144 MiB); so the bound an operator can hold the process to is the terms at the line cap times 1.2 per job under `MALLOC_ARENA_MAX=1`, and under the default arenas a figure that grows with the large rows a job reads, 2.35 of the terms at 48 rows being the largest measured. A one-item channel saved about 50 MiB of the peak and cost a 1M-row backup 75% of its duration, so the channel stays at four. With 200-byte rows every term that grows with the row is under 1 MiB, the chunk channel and the sink hold their 5 MiB whatever the row, and the high-water mark of a 1M-row backup is within 10 MiB of a 100k-row backup's on PostgreSQL and SQLite (the bound the acceptance asserts is 112 MiB: the budget, the line cap and the channels) and the 500k-row backup's within 5 MiB on MongoDB, where the 500k run stands in for the 1M run (its load alone took 27 minutes through the wire). A running restore holds the parsed manifest, the decoder's line (at most `MAX_LINE_BYTES`) and the deserializer's intermediate value for it, the decoded items of `LOAD_CHUNK_BYTES` (32 MiB) of data-file text plus one line per chunk (the chunk closes with the item that reaches the bound) at four chunk positions (two in the channel to `bulk_load`, the one being filled, the one being drained), and the backend's write batch, times `max_concurrent_jobs`; the chunk bound counts the bytes the decoder read for the chunk's items, the unit the backup side budgets in, not the service's item sizing, and a chunk also closes at `LOAD_CHUNK_ITEMS` (1,000) items. The backend's write batch: at most 32 MiB of item text on SQLite and Cassandra (10,000 items when they are smaller), held beside the decoded items of that text, 4 MiB per INSERT statement on PostgreSQL (500 rows when they are smaller; one transaction per file; the item and its text, two copies), 1,000 documents per `insert_many` on MongoDB (the driver splits the call; the documents are held until it returns, bounded in bytes only by the data file, since `bulk_load` is per file). A `bulk_load` is one data file at a time, so the chunks in flight never hold more than one file's items, at most the source's `data_file_target_bytes` plus one line of text. None of these grows with the table: they grow with the item, which `max_item_size_bytes` bounds, and SQLite's `COUNT(*)` after the last file costs the counting connection's page cache, bounded by `cache_size` (32 MiB; 22 to 28 MiB measured at 1M and at 3M rows, the record). The input order of the backup's lines is a term of SQLite's beside these: a key-ordered backup (the server's own, of a hash-only or single-range-key table) appends to the base b-tree's right edge, an unordered one (every MongoDB-sourced backup and every multi-part-key table) dirties pages across the whole tree, and the write connection's page cache plus the allocator's arenas hold the difference (the record's input-order table: 1M items of 200 bytes 87 to 98 MiB key-ordered against 144 to 164 MiB unordered on SQLite, in 27 to 56 s against 66 to 90 s; PostgreSQL 16 MiB either way). A table with secondary indexes adds no term of the restore's: on SQLite it adds the backend's own per-connection page cache for the index b-trees (`cache_size`, 32 MiB), bounded by that setting and growing sub-linearly with the table (the record's index table), on PostgreSQL a few MiB. On SQLite the restore's write batch (`RESTORE_BATCH_ITEMS`, 10,000 items, or `RESTORE_BATCH_BYTES`, 32 MiB of item text) and the decode chunks in front of it grow with the item's size up to the batch bound, whatever the attribute types, so a restore of items of more than a few hundred bytes holds more: 253 MiB over idle for 100k items of about 2.5 KB of strings, 130 MiB under one arena (two runs each), and more for lists of numbers of the same text size, which decode larger (345 to 424 MiB, 184 to 195 MiB under one arena, at 100k items of 128 numbers, with or without a vector index); at 1M items of 128 numbers 552 MiB under the default arenas and 204 MiB under one arena, and 301 to 316 MiB under one arena for 20,000 items of 1,536 numbers (`15-backup-restore-bench-pr11.md`). A PostgreSQL vector index adds 27 to 53 MiB to the restore at 100k items (five pairs of runs with and without it). Measured (`docs/design/15-backup-restore-bench-pr9.md`; revision item 33), `VmHWM` over idle from a fresh start, 24 worst-shape items of 27.7 MB: 848 MiB on SQLite and 778 MiB on PostgreSQL under the default arenas (848 and 1,013 MiB at 12 and 48 items on SQLite: the peak grows with the large items a restore reads, as the backup's does, and no bound was measured), 392 MiB on SQLite and 427 MiB on PostgreSQL under `MALLOC_ARENA_MAX=1`; 1,200 items of 350 KB in one file 270 MiB on SQLite and 156 MiB on PostgreSQL; 1M items of 200 bytes under a composite key with no secondary index, the backup's lines in key order, 89 MiB on SQLite and 16 MiB on PostgreSQL (100k: 38 and 10.7 MiB), restored in 26.8 s and 9.5 s; the same items under the same key with two full-projection GSIs 143 to 225 MiB on SQLite at 100,000 items in three runs and 165 to 172 MiB at 300,000, and 23 to 34 MiB on PostgreSQL at both: the excess on SQLite is the backend's own per-connection page cache (`cache_size`, 32 MiB) holding the index tables' b-tree pages plus the allocator's arenas (about a third less under `MALLOC_ARENA_MAX=1`), not the restore's buffers, which the 32 MiB batch bound holds whatever the indexes; it is bounded by `cache_size` and grows sub-linearly with the table. A restore of a table of large items is of the same order as its backup, not less; the sizing rule is the record's: about 450 MiB per running restore of large items under one arena, times `max_concurrent_jobs`. Each running backup also holds one connection from the PostgreSQL data pool for its duration, and on the separate layout one catalog-pool connection while its snapshot opens (section 3); the backend admits at most half the pool (after the raise below) as open snapshots and refuses more with `LimitExceeded`; a running restore holds one data connection and has no slot of its own after PR 6; and each PostgreSQL job holds one dedicated claim connection outside the pool. `pool_size` sizes both pools, and the rule is `pool_size >= max(MIN_POOL_SIZE, 2 * max_concurrent_jobs + 10)`: twice the jobs, so the half-pool bound admits every job `max_concurrent_jobs` allows, plus ten for requests; 18 at the defaults, under the default `pool_size` of 20. It applies whether or not `[backup.store]` is configured, because legacy restores are jobs too. Startup raises a smaller `pool_size` to that value with a warning naming both numbers, as it already raises a `pool_size` under `MIN_POOL_SIZE` (10) today. On SQLite the snapshot slot (`backup_server_slot_count`, `pool_size / 2 - 1` capped at 16, shared by the in-catalog create and the snapshot open) stays the backend's admission bound, as PostgreSQL's half-pool slot does, and a server raises `[storage.sqlite] pool_size` to `2 * (max_concurrent_jobs + 1)` (at most 34) with a warning naming both numbers so the slot refuses no job the permit admitted; 10 at the defaults, the default pool; above 16 jobs the slot refuses the seventeenth. `backup sync`, which is not a job, keeps of the configured root's rows (every status) each row's status, store id and table and the key of each AVAILABLE row of the configured store id, dropping the rest of each row as it is read, and holds the objects of the one prefix the listing is in with their sizes and the prefix of each manifest it listed; every backend's `list_backup_rows` collects its whole result before it yields the first row, so the read of the rows is sync's largest term on a catalog with many rows of the root (tombstones within their retention included): about 22.5 KiB of peak RSS per row of a table with five GSIs (1,452 bytes of `source_table_details_json`; 610 MiB at 25,000 rows and 1,144 MiB at 50,000, against 47 MiB with none, and 148 MiB at 50,000 rows whose JSON is `{}`, so nearly all of it is the parsed description), measured on a SQLite debug build on a 16-core x86_64 Linux host under glibc's default arenas, the same peak `backup list` reaches over those rows; the filesystem store builds its listing of the whole root in one `Vec` before it yields the first key (about 72 bytes plus the key per object, the store trait's doc), and the S3 store lists a page of 1,000 keys at a time. Measured on a filesystem store with 100 objects per prefix and 60-byte keys: 45 MiB of peak RSS for an empty root, 66 MiB for 100,000 objects and 98 MiB for 300,000, about 190 bytes per object, nearly all of it the store's listing (revision item 34). The report keeps an entry per prefix for the whole run (it lists every prefix the catalog already has a row for among the skipped ones), so at one object per prefix the per-prefix terms double that: about 350 bytes per prefix, 62 MiB of peak RSS for 50,000 such prefixes and 112 MiB for 200,000 against 45 MiB for an empty root (and about 165 bytes per object at 100 objects per prefix), measured on a SQLite debug build of PR 10 with a filesystem root and no rows. On an in-memory SQLite server that cost is paid in the serving process at every start. +Memory per running job, which operators size hosts from. A backup's terms, for a row of `r` bytes of item text whose decoded `Item` is `d` bytes (`d` is about 1.2 × `r` for a row of many short strings, #394's worst shape, where every `N` is a 133-character string): the read path holds the page's decoded rows, `d` each, while they wait for the pipeline, at most the rows of the budget PR 8 adds to the PostgreSQL and SQLite streams (`SNAPSHOT_FETCH_BYTES`, 64 MiB, a constant beside `DEFAULT_FETCH_BATCH`, not a `[backup]` key in v1) plus one row, and one row's parse, `r + 2d`, since the deserializer builds an intermediate value before the item; PostgreSQL pulls the page's rows from the connection one at a time and parses each as it arrives, so it holds no page text beyond the row being parsed, and SQLite pulls a row at a time with the driver's worker at most five rows ahead of the pull, `r` each (`row_buffer_size` on the pool, four, plus the row the worker holds while its `send` waits; 160 MiB at the line cap; the default of 50 made the run-ahead, not the page, the largest term on a table of large rows). The pipeline holds the feeder's item, the item channel's items (`ITEM_CHANNEL`, four) and the encoder's item, `d` each; the encoder's line buffer (one line, at most 32 MiB, the line cap) and gzip window; the feeder's measurement of an item over the default `max_item_size_bytes` holds no line (it counts the line's bytes through a writer that stops at the cap), so the terms are the same on an instance that raises the limit, at the largest stored row; the chunk sink's 1 MiB buffer and the chunk channel (four 1 MiB chunks); the upload window (`upload_part_size_bytes * max_concurrent_uploads` on an S3 store, one write buffer on a filesystem store); and the manifest's data-file entries until the manifest is written. No term grows with `data_file_target_bytes` or with the table, except on `sqlite-memory`, whose open copies the whole table (section 3); every term but the budget and the chunks grows with the row, which `max_item_size_bytes` bounds (the line cap bounds the line the job writes and names the item it refuses; the row is read and decoded before the feeder measures it). The sizing rule, per running job, is the terms at the line cap times the allocator's factor, times `max_concurrent_jobs`; the rule is stated at the default `max_item_size_bytes`, where no stored item's line exceeds the cap (27.7 MB is the largest line a 400 KB item expands to, `MAX_LINE_BYTES`'s doc); on an instance that raises it the page's decoded rows, one row's parse and the pipeline's items are at the largest stored row, which only `max_item_size_bytes` bounds (about 68 times the item's size as a line, for #394's worst shape). The terms at the line cap (`r` = 32 MiB, `d` = 1.2 `r`): the page's rows, 3 `d` = 115 MiB; one row's parse, `r` + 2 `d` = 109 MiB; the pipeline's six items, 6 `d` = 230 MiB; the line buffer, 32 MiB; the chunks, 5 MiB; on SQLite the run-ahead, 5 `r` = 160 MiB: about 490 MiB on PostgreSQL and 650 MiB on SQLite, an upper bound, since the run-ahead and a full pipeline are not both at their largest at one instant. The allocator's factor is glibc's: the row's 200,000 short strings are allocated on the tokio worker that parses the row and freed on the blocking thread that encodes it, and with glibc's default per-thread arenas that memory stays in the arenas between pages, so the process's high-water mark grows with the number of large rows the job reads and varies by about 200 MiB between identical runs; under `MALLOC_ARENA_MAX=1` (one arena, set in the server's environment on a host that backs up tables of large items; the admin guide's `max_concurrent_jobs` row carries it and the deployment guide's service unit carries the line commented out with that condition, since one arena serializes the worker threads' allocations on one lock and costs the request path 3.5 to 9.2% on SQLite and about 2% on PostgreSQL (the scans, 2.1%), measured on GetItem, Query and Scan with a Python client, the record) the peak is the terms' and does not grow with the table. Measured on PR 8's 16-core host at 27.7 MB rows (`r` = 26.4 MiB, three rows to a page; the terms' sum 545 MiB on SQLite), 1,000 small rows then 12, 24 and 48 such rows, each series six runs back to back on one SQLite binary (`docs/design/15-backup-restore-bench-pr8.md`; revision item 31): with the default arenas the peak over idle was 466, 588 and 760 MiB on a quiet host and 707, 700 and 1,281 MiB on the final build with the kill-point runs beside it (0.86 to 2.35 of the terms, growing with the rows and varying by hundreds of MiB between series; runs of the PR's first build on a loaded host reached 1,096 MiB at 48 rows; no plateau was measured), and under `MALLOC_ARENA_MAX=1` 481, 452 and 480 MiB and then 452, 480 and 455 MiB (0.83 to 0.88 of the terms, flat from 12 to 48 rows in both series; the SQLite ratios are of the peak over idle), and on PostgreSQL at 12 rows 340 MiB `VmHWM` under one arena against 750 to 788 MiB under the default arenas (0.82 of the terms at that row on PostgreSQL, 413 MiB, which has no run-ahead; that ratio is of the `VmHWM`, the process's whole high-water mark, and about 0.47 over its idle of 144 MiB); so the bound an operator can hold the process to is the terms at the line cap times 1.2 per job under `MALLOC_ARENA_MAX=1`, and under the default arenas a figure that grows with the large rows a job reads, 2.35 of the terms at 48 rows being the largest measured. A one-item channel saved about 50 MiB of the peak and cost a 1M-row backup 75% of its duration, so the channel stays at four. With 200-byte rows every term that grows with the row is under 1 MiB, the chunk channel and the sink hold their 5 MiB whatever the row, and the high-water mark of a 1M-row backup is within 10 MiB of a 100k-row backup's on PostgreSQL and SQLite (the bound the acceptance asserts is 112 MiB: the budget, the line cap and the channels) and the 500k-row backup's within 5 MiB on MongoDB, where the 500k run stands in for the 1M run (its load alone took 27 minutes through the wire). A running restore holds the parsed manifest, the decoder's line (at most `MAX_LINE_BYTES`) and the deserializer's intermediate value for it, the decoded items of `LOAD_CHUNK_BYTES` (32 MiB) of data-file text plus one line per chunk (the chunk closes with the item that reaches the bound) at four chunk positions (two in the channel to `bulk_load`, the one being filled, the one being drained), and the backend's write batch, times `max_concurrent_jobs`; the chunk bound counts the bytes the decoder read for the chunk's items, the unit the backup side budgets in, not the service's item sizing, and a chunk also closes at `LOAD_CHUNK_ITEMS` (1,000) items. The backend's write batch: at most 32 MiB of item text on SQLite and Cassandra (10,000 items when they are smaller), held beside the decoded items of that text, 4 MiB per INSERT statement on PostgreSQL (500 rows when they are smaller; one transaction per file; the item and its text, two copies), 1,000 documents per `insert_many` on MongoDB (the driver splits the call; the documents are held until it returns, bounded in bytes only by the data file, since `bulk_load` is per file). A `bulk_load` is one data file at a time, so the chunks in flight never hold more than one file's items, at most the source's `data_file_target_bytes` plus one line of text. None of these grows with the table: they grow with the item, which `max_item_size_bytes` bounds, and SQLite's `COUNT(*)` after the last file costs the counting connection's page cache, bounded by `cache_size` (32 MiB; 22 to 28 MiB measured at 1M and at 3M rows, the record). The input order of the backup's lines is a term of SQLite's beside these: a key-ordered backup (the server's own, of a hash-only or single-range-key table) appends to the base b-tree's right edge, an unordered one (every MongoDB-sourced backup and every multi-part-key table) dirties pages across the whole tree, and the write connection's page cache plus the allocator's arenas hold the difference (the record's input-order table: 1M items of 200 bytes 87 to 98 MiB key-ordered against 144 to 164 MiB unordered on SQLite, in 27 to 56 s against 66 to 90 s; PostgreSQL 16 MiB either way). A table with secondary indexes adds no term of the restore's: on SQLite it adds the backend's own per-connection page cache for the index b-trees (`cache_size`, 32 MiB), bounded by that setting and growing sub-linearly with the table (the record's index table), on PostgreSQL a few MiB. On SQLite the restore's write batch (`RESTORE_BATCH_ITEMS`, 10,000 items, or `RESTORE_BATCH_BYTES`, 32 MiB of item text) and the decode chunks in front of it grow with the item's size up to the batch bound, whatever the attribute types, so a restore of items of more than a few hundred bytes holds more: 253 MiB over idle for 100k items of about 2.5 KB of strings, 130 MiB under one arena (two runs each), and more for lists of numbers of the same text size, which decode larger (345 to 424 MiB, 184 to 195 MiB under one arena, at 100k items of 128 numbers, with or without a vector index); at 1M items of 128 numbers 552 MiB under the default arenas and 204 MiB under one arena, and 301 to 316 MiB under one arena for 20,000 items of 1,536 numbers (`15-backup-restore-bench-pr11.md`). A PostgreSQL vector index adds 27 to 53 MiB to the restore at 100k items (five pairs of runs with and without it). Measured (`docs/design/15-backup-restore-bench-pr9.md`; revision item 33), `VmHWM` over idle from a fresh start, 24 worst-shape items of 27.7 MB: 848 MiB on SQLite and 778 MiB on PostgreSQL under the default arenas (848 and 1,013 MiB at 12 and 48 items on SQLite: the peak grows with the large items a restore reads, as the backup's does, and no bound was measured), 392 MiB on SQLite and 427 MiB on PostgreSQL under `MALLOC_ARENA_MAX=1`; 1,200 items of 350 KB in one file 270 MiB on SQLite and 156 MiB on PostgreSQL; 1M items of 200 bytes under a composite key with no secondary index, the backup's lines in key order, 89 MiB on SQLite and 16 MiB on PostgreSQL (100k: 38 and 10.7 MiB), restored in 26.8 s and 9.5 s; the same items under the same key with two full-projection GSIs 143 to 225 MiB on SQLite at 100,000 items in three runs and 165 to 172 MiB at 300,000, and 23 to 34 MiB on PostgreSQL at both: the excess on SQLite is the backend's own per-connection page cache (`cache_size`, 32 MiB) holding the index tables' b-tree pages plus the allocator's arenas (about a third less under `MALLOC_ARENA_MAX=1`), not the restore's buffers, which the 32 MiB batch bound holds whatever the indexes; it is bounded by `cache_size` and grows sub-linearly with the table. A restore of a table of large items is of the same order as its backup, not less; the sizing rule is the record's: about 450 MiB per running restore of large items under one arena, times `max_concurrent_jobs`. Each running backup also holds one connection from the PostgreSQL data pool for its duration, and on the separate layout one catalog-pool connection while its snapshot opens (section 3); the backend admits at most half the pool (after the raise below) as open snapshots and refuses more with `LimitExceeded`; a running restore holds one data connection and has no slot of its own after PR 6; and each PostgreSQL job holds one dedicated claim connection outside the pool. `pool_size` sizes both pools, and the rule is `pool_size >= max(MIN_POOL_SIZE, 2 * max_concurrent_jobs + 10)`: twice the jobs, so the half-pool bound admits every job `max_concurrent_jobs` allows, plus ten for requests; 18 at the defaults, under the default `pool_size` of 20. It applies whether or not `[backup.store]` is configured, because legacy restores are jobs too. Startup raises a smaller `pool_size` to that value with a warning naming both numbers, as it already raises a `pool_size` under `MIN_POOL_SIZE` (10) today. On SQLite the snapshot slot (`backup_server_slot_count`, `pool_size / 2 - 1` capped at 16, shared by the in-catalog create and the snapshot open) stays the backend's admission bound, as PostgreSQL's half-pool slot does, and a server raises `[storage.sqlite] pool_size` to `2 * (max_concurrent_jobs + 1)` (at most 34) with a warning naming both numbers so the slot refuses no job the permit admitted; 10 at the defaults, the default pool; above 16 jobs the slot refuses the seventeenth. `backup sync`, which is not a job, keeps of the configured root's rows (every status) each row's status, store id and table and the key of each AVAILABLE row of the configured store id, dropping the rest of each row as it is read, and holds the objects of the one prefix the listing is in with their sizes and the prefix of each manifest it listed; every backend's `list_backup_rows` collects its whole result before it yields the first row, so the read of the rows is sync's largest term on a catalog with many rows of the root (tombstones within their retention included): about 22.5 KiB of peak RSS per row of a table with five GSIs (1,452 bytes of `source_table_details_json`; 610 MiB at 25,000 rows and 1,144 MiB at 50,000, against 47 MiB with none, and 148 MiB at 50,000 rows whose JSON is `{}`, so nearly all of it is the parsed description), measured on a SQLite debug build on a 16-core x86_64 Linux host under glibc's default arenas, the same peak `backup list` reaches over those rows; `backup sync --move-store` keeps the old root's rows without their parsed description and the configured root's (account, backup id) pairs, and reads one manifest and one prefix listing at a time, so its peak is that same read of the rows, which grows with the description: 569 MiB at 20,000 rows of a table with five GSIs (the same 1,452-byte description) on a SQLite debug build and 500 MiB on a PostgreSQL debug build, against 51 and 49 MiB at 50 rows, 37 s each with filesystem roots on a local disk, on that host (revision item 36); the filesystem store builds its listing of the whole root in one `Vec` before it yields the first key (about 72 bytes plus the key per object, the store trait's doc), and the S3 store lists a page of 1,000 keys at a time. Measured on a filesystem store with 100 objects per prefix and 60-byte keys: 45 MiB of peak RSS for an empty root, 66 MiB for 100,000 objects and 98 MiB for 300,000, about 190 bytes per object, nearly all of it the store's listing (revision item 34). The report keeps an entry per prefix for the whole run (it lists every prefix the catalog already has a row for among the skipped ones), so at one object per prefix the per-prefix terms double that: about 350 bytes per prefix, 62 MiB of peak RSS for 50,000 such prefixes and 112 MiB for 200,000 against 45 MiB for an empty root (and about 165 bytes per object at 100 objects per prefix), measured on a SQLite debug build of PR 10 with a filesystem root and no rows. On an in-memory SQLite server that cost is paid in the serving process at every start. A configuration error that needs no request to the store (a part size, prefix or key-length problem, an insecure endpoint, a root that fails the owner-only rule or whose ancestors others can write to, a filesystem root that is or contains the database directory, a spool rule) is a startup error naming the path or bucket, except for the defaulted SQLite root above while the catalog has no rows for it, where it and a failure of the root's identity check are a warning that leaves the store unconfigured. A filesystem root's `validate` also writes a probe file, a request to the store: an I/O failure there or in resolving the root is a store condition that degrades the store (`failed validation`), as the same condition one reconciler cycle later does, not a startup error. A filesystem store's `validate` and identity check run before the listener, bounded by a timeout, so a mount that stops answering after the root is opened leaves the server starting with the store `checking` (a root that does not answer at startup holds the server before the listener, as the database directory would); an S3 store's are requests to the endpoint and run after the listener is up, in the reconciler's first cycle, with the store `checking` until the first result reports, so a stalled endpoint never holds the data plane (revision item 29). Every later check runs under the same bound (5 s for a filesystem root; 30 s for a bucket, under the SDK's per-attempt timeout so no retry outlives the check) and against the shutdown token: a store that has stopped answering costs a reconciler cycle its bound and nothing more, the cycle's other duties run, the check is recorded as a transport failure, and neither the drain at shutdown nor the process's exit waits on it (a filesystem root's abandoned call stays parked on a thread until the mount answers, no later check starts while it is, the root's calls run one at a time so that calls started before the first reached its bound wait for it under the bound and then have the bound for their own call, rather than parking a thread each, and the binary ends its runtime with a 5 s bound). A transport failure (no region, no credentials, an unreachable endpoint, a check that did not finish within its bound), a missing or denied bucket found after the listener, and a failure of section 4's identity check or manifest probe, do not stop startup, because a store problem must not take the data plane out of rotation (section 9): the failure is logged with its cause, the store component is reported unhealthy with the problem's category as its reason, the operations that touch the store refuse with a ValidationException that names the store's kind and the category and not its root (CreateBackup on PostgreSQL, SQLite and MongoDB, and RestoreTableFromBackup and DeleteBackup of an off-host backup; Describe and List, legacy backups and the in-catalog CreateBackup are unaffected), the reconciler acts on no row of the store (its legacy duty, which touches no store, still runs) and resumes no restore, and the check runs again each reconciler cycle until it passes. @@ -307,7 +307,8 @@ pub trait BackupCatalog: Send + Sync { // DeleteBackup (5.2): the in-use check and the AVAILABLE-to-DELETING write in one transaction under the row's // Deleting | InUse(target name) | Status(the row's status when it is not AVAILABLE); PR 6 fn set_backup_deleted(&self, key: &BackupKey) -> BoxFuture>; // DELETING to DELETED, conditional; PR 6 - fn remove_backup_row(&self, key: &BackupKey) -> BoxFuture>; + fn remove_backup_row(&self, key: &BackupKey, store_kind: StoreKind, store_root: &str) -> BoxFuture>; + // a DELETED or FAILED tombstone still of the store the caller read it with (revision item 36) fn backup_in_use(&self, key: &BackupKey) -> BoxFuture, StorageError>>; // the name of a CREATING restore target registered from the backup, if any: the in-use rule of 5.2 that // 5.3 step 1 applies before the permit (on Cassandra read from `active_restores`, section 4); PR 6 @@ -445,10 +446,10 @@ Kill points (testing summary): every restore kill point, the fenced-out restore ### PR 10: `backup sync`, its options, `remove-target`, `migrate-legacy` -Goal: the catalog-loss recovery and the maintenance commands of section 4. As implemented (revision item 34) the section ships as three PRs: #746 plain `backup sync`, `--prune-missing`, `--adopt-store`, the `sqlite-memory` startup sync and `insert_backup_row`, with the acceptance cells those carry; #820 `--move-store`, `--forget-store` and `--forget-backup` with `list_backup_rows_by_root` and `rewrite_store`, and the report of the rows of other roots, with their cells; #821 `migrate-legacy` with its cells; `remove-target` shipped with PR 9 (item 33). Its line names the interfaces and carries the acceptance: +Goal: the catalog-loss recovery and the maintenance commands of section 4. As implemented (revision item 34) the section ships as three PRs: #746 plain `backup sync`, `--prune-missing`, `--adopt-store`, the `sqlite-memory` startup sync and `insert_backup_row`, with the acceptance cells those carry; #820 `--move-store`, `--forget-store` and `--forget-backup` with `list_backup_rows_by_root` and `rewrite_store`, and the report of the rows of other roots, with their cells (as implemented, revision item 36: `rewrite_store` takes the old kind and root, and a third method, `summarize_store_roots`, gives the report its stores); #821 `migrate-legacy` with its cells; `remove-target` shipped with PR 9 (item 33). Its line names the interfaces and carries the acceptance: - PR 10: `extenddb backup sync` per section 4 with store-root scoping, the store id, the maintenance claim with its printed wait bound, tombstone and identity checks, the row check before the manifest read and the insert-if-absent, `--prune-missing`, `--forget-store`, `--forget-backup`, `--move-store` (with `list_backup_rows_by_root` and `rewrite_store`), `--adopt-store`, the SQLite serve lock on every command that writes (`extenddb backup remove-target --account
`, section 4, shipped with PR 9 rather than here, item 33: a crash before the first progress record on MongoDB or Cassandra leaves a `target_not_empty` target that no other command removes) and the dead-owner resolutions such a command runs first, and `extenddb backup migrate-legacy |--all` with its re-read, validation and count checks and its refusals (section 3). Acceptance: drop the catalog rows, run sync, Describe and List and Restore work; a prefix without a manifest, with a bad sidecar, with a mismatched ARN, under `changelog/`, or with a CREATING, DELETING or FAILED row is not adopted; rows for another store root are reported and untouched, and the reconciler deletes none of their prefixes; after a store tree is copied to a new root, `--move-store` re-points its rows and Describe and Restore work against the new root; `--move-store` resolves the old root's CREATING and DELETING rows, stops on a row whose data files were not all copied, and refuses a DELETING, DELETED or FAILED row whose copied prefix holds a verified manifest; `--forget-backup` refuses a CREATING row with a live owner and a backup from which a restore target is CREATING (a RUNNING restore's target, a FAILED restore's target not yet removed, and on MongoDB a DONE restore's target awaiting the backfill), and says so and does nothing for a legacy ARN; a second maintenance command while one holds the claim fails at once; on SQLite every writing command refuses to run beside a server; `migrate-legacy` on a backup the PR 0 binary created produces a prefix Describe, List and Restore serve under the original ARN, and the legacy rows are gone; with the volume unmounted (an empty root, no id or another id) the store is unhealthy, sync without the option that resolves it refuses, and no row changes; `--adopt-store` of a root another catalog owns makes this catalog the owner, and the other catalog finds the store unhealthy at its next check; with host-local roots on two instances of one catalog no reconciler on either deletes a prefix because its own root lacks the bytes, and `--prune-missing` on the instance that lacks them removes the row and deletes nothing: the backup is BackupNotFound on every instance until a sync on the instance that holds the bytes adopts it again (revision item 34); `migrate-legacy` refuses a backup with a 13-digit id, one holding a value #618 let in and a pre-0.0.4 SQLite backup, keeps their legacy rows and names the reason; without `--prune-missing` a row whose manifest is gone is reported and stays AVAILABLE; after `--forget-store` and the store's return, sync adopts its prefixes; a CreateBackup started on another instance during `--move-store` either finishes before the rewrite or is refused; after `--move-store A` stops on a row with one data file missing under the configured root and `--forget-backup` lets the move finish, sync reports that prefix as incomplete and does not adopt it, and adopts it once the file is copied; a maintenance command started while a DELETING row and a RUNNING restore with a dead owner exist proceeds once the reconciler, which runs those two duties under the claim, has finished the one and failed the other. Depends on PR 6, 8. -Invariants honoured: 4(c) (store identity, scoping, adoption only through `--adopt-store` for a durable catalog and, for an in-memory one, at its startup from another in-memory catalog), 4(f) (every writing command takes the maintenance claim), 4(g) (what sync adopts and never adopts; the opt-ins), 4(d) (the ARN checked against the path and `source_table`), and 4(a)'s rule that no deletion duty acts on the bytes of a row the operator forgot. +Invariants honoured: 4(c) (store identity, scoping, adoption only through `--adopt-store` for a durable catalog and, for an in-memory one, at its startup from another in-memory catalog), 4(f) (every writing command takes the maintenance claim), 4(g) (what sync adopts and never adopts; the opt-ins), 4(d) (the ARN checked against the path and `source_table`), and 4(a)'s rule for the bytes of a row the operator forgot: no deletion duty starts on them once the maintenance claim is held, and a deletion begun before it ends by its deadline, the step in flight at it excepted, for a server process that keeps running (revision item 36). Interfaces it adds: ```rust fn insert_backup_row(&self, row: &BackupRow) -> BoxFuture>; // sync's unowned insert-if-absent @@ -456,10 +457,23 @@ Interfaces it adds: // a root's rows whatever their `store_id`: `--forget-store` and `--move-store` (section 4), which get a root and no id; PR 10 // on Cassandra this and `read_store_ids` read `backup_index_by_status` one partition per status for the root; // migration 6's entry carries `store_kind` and `store_id`, so `read_store_ids` needs no second read, and this - // listing reads each row's `backups_by_arn` row for the rest of the row - fn rewrite_store(&self, keys: &[BackupKey], store: &StoreIdentity) -> BoxFuture>; // `--move-store`, PR 10 + // listing reads each row's `backups_by_arn` row once for the rest of the row, keeping it whatever status + // partition its entry is in (revision item 36) + fn rewrite_store(&self, from_kind: StoreKind, from_root: &str, keys: &[BackupKey], to: &StoreIdentity) -> BoxFuture>; + // `--move-store`, PR 10 (#820, revision item 36): each row of `keys` still of `from_kind` and `from_root` and not + // CREATING gets `to`'s kind, root and id; the count of rows rewritten. On Cassandra per key: the new root's + // status entry, the lightweight transaction on the row, then the old entry deleted + fn summarize_store_roots(&self) -> BoxFuture, StorageError>>; + // the rows per store (kind, root, id) and status, for sync's report of other stores; on Cassandra a count + // of `backup_index_by_status` entries, which the report checks against each store's listing + // (`summary_counts_status_entries`); PR 10 (#820, revision item 36) + fn remove_orphan_status_entries(&self, store_kind: StoreKind, store_root: &str) -> BoxFuture>; + // `--forget-store`'s sweep of the root's status entries under the claim: one whose row is gone or names + // another store is deleted, one whose row is of the root under another status is written again under the + // row's status (both counted); a default of zero for the backends without status entries; PR 10 (#820, + // revision item 36) ``` -`backup sync` and its options, each with its contract: Sync lists the root (every object, since the trait has no delimiter listing; the store lists in key order, so sync holds one prefix's objects at a time and the prefix of each manifest it listed, beside what it keeps of the root's rows, whose read is its larger cost (section 7), the reserved `changelog` and `exports/` components skipped; a filesystem store builds its own listing of the root first, section 7), checks each prefix against the catalog as the listing reaches it, reads the manifest and sidecar of the prefixes that have no row, and inserts an AVAILABLE row for each verified manifest whose data files the listing contains, every `data_files` key present with its `bytes`, with an insert-if-absent (`BackupCatalog::insert_backup_row`): a CreateBackup whose row landed between the check and the insert keeps its row and sync skips the prefix. Before adopting, sync parses the manifest's `backup_arn` with the one `BackupArn` type in `extenddb-core` and requires the parsed account, table and backup id to equal the listed path components and the `source_table` identity; a mismatch is logged and skipped. An ARN of another region is adopted and logged (requests resolve it by the ARN as written); one of another partition than `aws` is skipped, since every request refuses such an ARN before it reads a row (revision item 34). `--move-store ` re-points the rows of that root to the configured store. It first resolves that root's CREATING and DELETING rows itself, against the configured store: a CREATING row becomes AVAILABLE if a verified manifest is there and FAILED otherwise, and a DELETING row whose prefix there holds no manifest has its deletion finished in the configured store. For each AVAILABLE row it compares the configured store's listing of the row's prefix with the manifest's `data_files` (every key present, with its `bytes`) as well as checking the verified manifest, and stops on a row whose copy is incomplete, naming it; `--forget-backup ` deletes one row's catalog entry so the move can go ahead for the rest; it runs under the maintenance claim like the other commands, refuses a CREATING row with a live owner (the job's next conditional write would match nothing and its manifest would be adopted by the next sync) and a backup in use, one from which a restore target is CREATING, and for a legacy ARN, which has no new-format row, says so and does nothing. One rule covers the three tombstone states: the move refuses to carry a DELETING, DELETED or FAILED row whose prefix in the configured store holds a verified manifest, and names it (that prefix is a backup the operator copied before or during its deletion, not a leftover; `--forget-backup` on the row is the way out). Then it rewrites `store_kind`, `store_root` and `store_id`; the reconciler then cleans the leftovers of the moved tombstones in the configured store. `--move-store` and `--forget-store` match `store_root` exactly as stored, the `?endpoint=` suffix of an S3 root included. A root whose file names another catalog stays unhealthy until an operator adopts it with `backup sync --adopt-store`, which takes the maintenance claim, rewrites the file's owner to this catalog and then syncs, recording the file's store id on the rows it adopts (the `backup_index` row PR 9b's matrix seeds by hand is the row this adoption writes); the previous owner acts on the root until a later check that can read its catalog finds the store unhealthy, at the earliest one reconciler interval after the cycle in progress ends, unless its deletion fence (section 4(c)) refuses a deletion or a CreateBackup first, so every server of the previous catalog is stopped before the adoption. On `sqlite-memory` the catalog, and with it the `catalog_id`, is new at every start, so the server runs a plain `backup sync` itself at startup when a store is configured, which adopts a root whose file names a `sqlite-memory` catalog as `--adopt-store` does, rewriting the file's owner to the new `catalog_id` before it syncs, so the root has one owner at a time there too once the previous process has exited (a new in-memory server is started only after the old one has exited; one started beside it adopts the root while the old one still acts on it until a later check or its fence); a root owned by any other catalog is unhealthy there as anywhere, and the backup CLI commands do not exist there (no file to open), so that startup sync is the only catalog-loss recovery there. +`backup sync` and its options, each with its contract: Sync lists the root (every object, since the trait has no delimiter listing; the store lists in key order, so sync holds one prefix's objects at a time and the prefix of each manifest it listed, beside what it keeps of the root's rows, whose read is its larger cost (section 7), the reserved `changelog` and `exports/` components skipped; a filesystem store builds its own listing of the root first, section 7), checks each prefix against the catalog as the listing reaches it, reads the manifest and sidecar of the prefixes that have no row, and inserts an AVAILABLE row for each verified manifest whose data files the listing contains, every `data_files` key present with its `bytes`, with an insert-if-absent (`BackupCatalog::insert_backup_row`): a CreateBackup whose row landed between the check and the insert keeps its row and sync skips the prefix. Before adopting, sync parses the manifest's `backup_arn` with the one `BackupArn` type in `extenddb-core` and requires the parsed account, table and backup id to equal the listed path components and the `source_table` identity; a mismatch is logged and skipped. An ARN of another region is adopted and logged (requests resolve it by the ARN as written); one of another partition than `aws` is skipped, since every request refuses such an ARN before it reads a row (revision item 34). `--move-store ` re-points the rows of that root to the configured store. It needs the configured store healthy, refuses the configured root and a root with no rows, and its wait covers the jobs of every store id the catalog records for the old root as well as the configured store's (revision item 36). It first resolves that root's CREATING and DELETING rows itself, against the configured store: a CREATING row whose owner is dead becomes AVAILABLE if a verified manifest is there, and otherwise stops the move with no status written, since a copy without the complete backup says nothing about the old root's prefix (revision item 36), and a DELETING row whose prefix there holds no verified manifest has its deletion finished in the configured store (a manifest that fails verification is a leftover the deletion removes). For each AVAILABLE row it compares the configured store's listing of the row's prefix with the manifest's `data_files` (every key present, with its `bytes`) as well as checking the verified manifest and that its size and item total are the row's, and stops on a row whose copy is incomplete, naming it (every row that stops the move is named in one run, and none moves while any does; a row that would give an account two rows of one backup id stops it too); `--forget-backup ` deletes one row's catalog entry so the move can go ahead for the rest; it runs under the maintenance claim like the other commands, refuses a CREATING row with a live owner (the job's next conditional write would match nothing and its manifest would be adopted by the next sync), a FAILED row whose heartbeat is within `stale_job_after_secs` (its job may still write under the prefix) and a backup in use, one from which a restore target is CREATING, and for a legacy ARN, which has no new-format row, says so and does nothing. One rule covers the three tombstone states: the move refuses to carry a DELETING, DELETED or FAILED row whose prefix in the configured store holds a verified manifest, and names it (that prefix is a backup the operator copied before or during its deletion, not a leftover; `--forget-backup` on the row is the way out). Then, after one re-read of the configured root's identity file (the deletion fence's `check_catalog`), it rewrites `store_kind`, `store_root` and `store_id` of the rows still of the old root and not CREATING, and lists the old root again, reporting a row still there; the reconciler then cleans the leftovers of the moved tombstones in the configured store. `--move-store` and `--forget-store` match `store_root` exactly as stored, the `?endpoint=` suffix of an S3 root included. A root whose file names another catalog stays unhealthy until an operator adopts it with `backup sync --adopt-store`, which takes the maintenance claim, rewrites the file's owner to this catalog and then syncs, recording the file's store id on the rows it adopts (the `backup_index` row PR 9b's matrix seeds by hand is the row this adoption writes); the previous owner acts on the root until a later check that can read its catalog finds the store unhealthy, at the earliest one reconciler interval after the cycle in progress ends, unless its deletion fence (section 4(c)) refuses a deletion or a CreateBackup first, so every server of the previous catalog is stopped before the adoption. On `sqlite-memory` the catalog, and with it the `catalog_id`, is new at every start, so the server runs a plain `backup sync` itself at startup when a store is configured, which adopts a root whose file names a `sqlite-memory` catalog as `--adopt-store` does, rewriting the file's owner to the new `catalog_id` before it syncs, so the root has one owner at a time there too once the previous process has exited (a new in-memory server is started only after the old one has exited; one started beside it adopts the root while the old one still acts on it until a later check or its fence); a root owned by any other catalog is unhealthy there as anywhere, and the backup CLI commands do not exist there (no file to open), so that startup sync is the only catalog-loss recovery there. `migrate-legacy`: PR 10 adds `extenddb backup migrate-legacy |--all`, which streams a legacy backup through the frozen reader's item stream, checking each item with the backup job's item validator (5.1 step 4), into `DataFileEncoder` and the configured store under the legacy backup id, so the ARN survives, with a manifest whose `snapshot.kind` says it is a copy of an in-catalog backup (`legacy_in_catalog`, no marker). Before it inserts the AVAILABLE row or deletes anything it re-reads the manifest it wrote through `ExtenddbManifest::from_json_bytes` and checks that the manifest's item count equals the legacy backup's; only then does it insert the row and delete the legacy rows through the reader's `delete`. It refuses a backup, keeping its legacy rows, deleting the prefix it wrote and naming the reason, when the id carries no instant (the 13-digit and bare shapes, which the manifest parser refuses, section 4), when the validator refuses an item (a value #618 let in, which main's legacy restore copies as stored), and when it cannot state the definition (a pre-0.0.4 SQLite backup recorded no vector state, so `vector_indexes` would be a guess); the way out for such a backup is to restore it through the legacy reader and take a new backup of the restored table, which has a new ARN (section 10). The maintenance claim per backend and the SQLite serve lock (4(f) states the rules): On PostgreSQL the claim is an advisory lock keyed `hashtextextended('backup_maintenance', RESTORE_LOCK_SEED)` (`MAINTENANCE_LOCK_KEY`), held exclusively on the command's dedicated connection, with the `backup_maintenance` row carrying the holder, the generation and the heartbeat; a job tests the lock with `pg_try_advisory_xact_lock_shared` in one autocommit statement, so the shared lock ends with the statement, and then reads the row's heartbeat age, so a holder whose session ended is still seen held until its heartbeat is older than `stale_job_after_secs`; a command's take tries the exclusive lock a few times over about 10 ms on its connection, so a job's one-statement shared lock does not refuse it. On MongoDB it is a document in a `backup_maintenance` collection, claimed and heartbeated like a job row (5.1 step 3) and held while its heartbeat is younger than `stale_job_after_secs`; on Cassandra a `backup_maintenance` row claimed by lightweight transaction with a coordinator-clock heartbeat, read by jobs at `LOCAL_SERIAL`. On SQLite every backup CLI command that writes the catalog takes the database's serve lock, as `destroy` and `migrate` do, so the server must be stopped first and no other claim is needed; with the serve lock held every job owner is dead, and so is the holder of any maintenance claim row, which such a command takes over whatever its heartbeat says; before the wait it runs the reconciler's dead-owner resolutions itself, as the claim's holder (the cycle takes the held branch without reading the claim back through the zero window) (a CREATING backup row to AVAILABLE or FAILED, a RUNNING restore to FAILED with its target removed, a DELETING row finished) and then waits and removes CREATING rows with a zero stale window, so a job the stop interrupted never makes it wait or refuse whatever its heartbeat says (a RUNNING `manifest` restore, which PR 9a introduces, is the exception revision item 29 states), and a server starting under the serve lock removes a maintenance claim a stopped command left; `backup list` is a read and runs beside a server. @@ -595,3 +609,5 @@ Changed after a review of the draft against main `9f43d03` and PRs #384, #335, # 34. (2026-11-04, PR 10 as implemented, #746.) PR 10 is split in three: #746 ships plain `backup sync`, `--prune-missing`, `--adopt-store`, the `sqlite-memory` startup sync and `BackupCatalog::insert_backup_row`; #820 carries `--move-store`, `--forget-store` and `--forget-backup` with `list_backup_rows_by_root` and `rewrite_store`, which rewrite or delete rows of other stores, and the full report of the rows of other roots; #821 carries `migrate-legacy`, whose data path (the frozen reader, `DataFileEncoder`, the item validator) shares nothing with sync but the insert; `remove-target` shipped with PR 9 (item 33). The PR 10 section's line, the PR 9b paragraph (the matrix now adopts with `backup sync`) and the doc-row table's `migrate-legacy` row say so (PR 9b, PR 10, section 10). Neither the Cassandra sweep of orphan `backup_index_by_status` entries nor the MongoDB sweep of the data collections a failed restore-target removal leaves is in any of the three: the code comments that gave them to `backup sync` now say no sweep removes them yet, and the MongoDB error line names the collections for an operator to drop. `insert_backup_row(row) -> bool` inserts the row as given, unowned (`owner_generation` 0), with `completed_at` and `heartbeat_at` on the database's clock, and answers `false` when a row with its key exists: PostgreSQL `INSERT ... ON CONFLICT DO NOTHING` with no conflict target, so both unique indexes (the key and `backup_id` alone) refuse it, on the pool and with no advisory lock; SQLite `INSERT ... ON CONFLICT DO NOTHING` under the write lock and `BEGIN IMMEDIATE`, without a job slot; MongoDB the claim's pipeline upsert filtered on `owner_generation: {$exists: false}` with the row's status, `owner_generation: 0`, and E11000 (the unique `backup_id`) meaning a row exists; Cassandra the `backups_by_arn` lightweight transaction `IF NOT EXISTS`, then the LOCAL_QUORUM batch of the two list rows and the `backup_index_by_status` entry under the row's status (the batch `claim_backup` wrote inline, now shared), an unknown outcome or a `NotMatched` answer re-read at `LOCAL_SERIAL` and settled as inserted when the row read is this one (its entries then written, which completes an insert that stopped between its two writes), as existing when it is another; no row after a lost answer is sent once more, and no row after `NotMatched` answers `false` (a legacy in-catalog backup holds the ARN); a failed batch of the entries is sent once more before the error. The row is AVAILABLE from the transaction on, so a DeleteBackup between the transaction and the batch moves a status entry the batch has not written yet, and the batch then writes an AVAILABLE entry no later write removes; every reader of the status table re-reads the row and skips it, one wasted read each, until a sweep of orphan entries exists. Cassandra has no `backup_id` uniqueness, so its insert-if-absent is per ARN spelling: the same id under another region is another row there, as the trait doc says, and sync closes the gap itself by reading the row under both spellings of the manifest's ARN (this server's region and the manifest's) before the insert and skipping a backup the other spelling has a row for, while a row under the manifest's own spelling is left to the insert (which answers `false` for it or, on Cassandra, completes its own torn insert), so through sync no account holds two rows of one backup id under the manifest's account, table and id on any backend (section 3, 4); on Cassandra a row of that id under another table that the rows read did not return (a row of another root, or one whose list entries are missing) is not seen by that read, and within one run the row check keeps a second prefix of an id the run adopted out. Sync reads the configured root's rows first, every status and every store id the catalog records for the root (`read_store_ids`, then `list_backup_rows` per id), keeping of each only the status, store id and table by account and backup id and the key of each AVAILABLE row of the configured store id (the backends collect the rows whole first, so the read peaks at about 22.5 KiB per row of a five-GSI table, section 7), and checks each listed prefix against that set before it reads anything under the prefix; the sync contract's "checks each prefix against the catalog as the listing reaches it" holds with the rows read once up front, and the insert-if-absent then decides against a CreateBackup that landed after the read and against rows of other roots. The listing is consumed as it streams: the store lists in key order, so a prefix is complete once the listing passes it, and sync holds that prefix's objects and the prefix of each manifest it listed; the filesystem store's own listing is the larger cost, measured in section 7. A listing that fails part-way keeps the rows inserted before it, does not read the prefix it was in, and makes no missing-manifest report. An insert that answers `false` is re-read to say which row it met: one that landed after the read, one of another store, account or table holding the backup id, or on Cassandra a row whose list entries are missing (a CreateBackup's claim that stopped between its two writes; sync's own insert that stopped there is completed by the next sync of the prefix). The prefix check is the dead-owner duty's: `reconciler::verify_prefix` reads the manifest and sidecar within their bounds, checks the digest and the reader's acceptance and every listed data file at its size from the listing sync already holds; the dead-owner probe keeps its comparison of the whole ARN, and sync compares the manifest's account, table and backup id with the path, adopts an ARN of another region, logging it (requests resolve it by the ARN as written), and skips one of another partition than `aws`, since every request refuses such an ARN before it reads a row. `layout.rs` gains `EXPORTS_COMPONENT`, `CHANGELOG_COMPONENT`, `classify_store_key` and `parse_store_prefix`: keys outside every backup prefix are passed over, the reserved components and a third component of another id shape (only the 14-digit shape carries a manifest) are reported as skipped and never read. The adopted row carries the manifest's name, ARN, size and item total and `source_table`, the id's instant as `created_at`, and the configured store's identity (section 4, PR 10). Rows of other roots are reported in part: sync prints the prefixes whose backup id a row of another root holds (the read under the ARN's spellings, or the insert's re-read). The rows of the configured root under another store id are not a case sync meets: a store whose rows carry another id than its file, or more than one id, is unhealthy, and sync refuses an unhealthy store. A listing of every root's rows needs `list_backup_rows_by_root`, which is #820's, so until then 4(g)'s "rows for another store are reported" holds for that one case (4(g)). Every sync takes the maintenance claim where it runs beside servers (PostgreSQL, MongoDB, Cassandra), plain sync with no wait for running jobs: its insert-if-absent orders it against a CreateBackup, and the claim orders it against `purge`, whose `purge_row` removes a dead CREATING row before it deletes the prefix, which can hold a complete manifest (a job that died after its manifest); a sync between the two steps would adopt bytes purge then deletes. While the claim is held the jobs refuse, as during purge (4(f)), and a sync killed while it holds the claim keeps the jobs of every instance refused until its heartbeat is `stale_job_after_secs` old. The restore's line for a row whose prefix lacks its sidecar says to put the prefix back; only a missing manifest names `--prune-missing`, the one case it removes. On SQLite it opens the catalog as `serve` does, under the serve lock (exit 6 beside a server, the refusal purge gives), after the dead-owner resolutions purge runs, so a DELETING row a stopped server left is finished and a dead CREATING row resolved before sync reads the rows. `--prune-missing` and `--adopt-store` also wait for running jobs as purge does; every sync installs purge's signal handling, checks it before each prefix and exits 1 "stopped" on a signal, releasing the claim it holds and its serving registration. `--prune-missing` removes each AVAILABLE row whose manifest is gone outright, through `BackupCatalog::forget_available_backup_row` (added here on the four backends: under the row's lock as `start_backup_delete` takes it, the status check and the in-use rule with a delete of the row, and on Cassandra its index entries after it), and deletes nothing in the store. The manifest probe sees only the newest AVAILABLE row's manifest, so a host-local root that holds the newest backup passes it while it lacks older ones; with no tombstone left, no deletion duty on the host that holds those bytes acts on them, but the row is gone for every instance: the backup is BackupNotFound on every instance until a sync on the host that holds the bytes adopts it again, which is the host-local acceptance line as amended here. The objects a removed row's prefix still holds stay until an operator deletes the prefix by hand: `purge` deletes only the prefixes of rows, and every later sync reports the prefix as not a complete backup. `--prune-missing` exits 1 when it leaves a row a later run may remove (a store or catalog call failed, the manifest's `head` included, which keeps the row; or a restore is reading the backup, "run the command again once the restore ends") and 0 when the rows it leaves have their manifest back or were moved by another writer. An unhealthy or `checking` store refuses plain sync and `--prune-missing`, exit 1, naming the problem and changing no row. Sync exits 1 when a prefix was skipped for a store read or catalog call that failed (the run is incomplete; the next run adopts it), and a startup sync that met one stays pending, its skips logged at WARN. So when the newest AVAILABLE row's own manifest is gone the store is unhealthy and neither command runs; the way out in this release is to put the prefix back (#820's `--forget-backup` is the other). The serving registration is named `extenddb backup sync`, and the catalog refusals exit 6 and 3 as the other commands' do (4(f), PR 10). `--adopt-store` is `store_identity::adopt`: with the claim held it rewrites `extenddb-store.json` with a plain `put`, the file's `catalog_id` and `backend` changed and its `store_id` and `created_at_ms` kept, reads it back, and syncs with the adopted identity; it logs an `extenddb::audit` intent line before the put and the audit line after the re-read with its outcome (adopted, or race lost), so a put whose caller stopped waiting is on record too. The re-read detects another adopter's put that landed between this put and the re-read (the store trait has no conditional replace); the command reads the file once more after the sync and fails naming the owner when it changed, which detects one that landed during the sync; a put after both reads is found by this catalog's next check. It refuses when this catalog's rows for the root carry another store id, and on a root this catalog owns it says so and syncs. A later check of the previous owner that can read its catalog finds the file naming another catalog, at the earliest one reconciler interval after its cycle in progress ends; until then it acts on the root, except where its deletion fence stops it. `StoreFence` re-reads the file (one `get` through the store's bounded calls, under its check bound and its one call slot on a filesystem root) right before each `delete_prefix` of a backup's objects (`finish_backup_delete`, which DeleteBackup and the reconciler's DELETING duty run, the backup job's failure path, the FAILED-prefix and DELETED-prefix duties, purge), before DeleteBackup's DELETING write and before purge's removal of a dead CREATING row or its DELETING write, before the reconciler removes a tombstone, and before a CreateBackup's row and first put; when the file names another catalog it refuses, records `ForeignCatalog` so the store is unhealthy at once (and a file naming this catalog under another store id than the store's check settled, a root replaced by another root of the catalog, is refused and recorded as `IdMismatch`; DeleteBackup and CreateBackup answer the unhealthy store's `ValidationException`, and the cycle re-tests the store's health after each store duty and runs none after the refusal, the dead-owner duty included), and a file it cannot read, or a root with no file, refuses the call and leaves the health to the next check, except at DeleteBackup's read before the DELETING write, where it is a store failure: the request writes DELETING and answers DELETED, as for a deletion the store does not answer (revision item 30), and the read before the deletion refuses it. A refusal before the row is changed leaves it as read, so DescribeBackup does not answer a refused DeleteBackup with DELETED; one at the read before the deletion (an adoption between the two reads) leaves the row DELETING with nothing deleted and DeleteBackup answering DELETED (revision item 30), which this catalog's reconciler finishes only if this catalog owns the root again. The restore deletes nothing in the store, so it has no fence, and neither has `--prune-missing`'s removal of a row, which deletes no object. A write already past the fence's read lands, so every server of the catalog the file names is stopped before an adoption, and a new in-memory server is started only after the old one has exited; the admin guide, the refusal and the command's output say so (4(c)). For `backup sync` a defaulted SQLite root whose file names another catalog while this catalog has no rows for it is configured and unhealthy (`Caller::SyncCommand`) instead of the warning that leaves it unconfigured (item 29), so `--adopt-store` can adopt the root a catalog re-created by `destroy` and `init` finds; the server and purge keep the warning (4(c), section 7, PR 7). The identity file gains `ephemeral_catalog` (`#[serde(default)]`, so the format stays version 1 both ways), written `true` by a server whose SQLite database is in memory (`database_file()` names no file: the state, not the `sqlite-memory` cargo feature) and `false` by every other writer, an adoption included. The adoption of a root an in-memory catalog owned is part of `store_identity::establish` when this catalog is itself in memory and the file carries the marker, and never of `recheck`, so two in-memory servers on one root do not take it from each other each cycle; a root whose file names a durable catalog stays unhealthy for an in-memory server, as 4(c) requires. The startup sync runs on every start of an in-memory catalog with a store configured, not only after an adoption (on a fresh root it is one listing): for a filesystem store in `serve` before the routes serve; for an S3 store, and for a startup sync not finished, in the reconciler's cycle after a healthy check. Every run is bounded at 60 s (`sync::STARTUP_SYNC_TIMEOUT`), in `serve` and in the cycle, where the duties after it wait at most that bound; a run that stops before it listed the whole root, reaches the bound (the rows it inserted stay, but every run lists the root from its first key, so the sync completes only while listing the whole root fits in the bound; past it the sync stays pending and is retried every cycle), or skipped a prefix for a store or catalog error stays pending. On a filesystem store a run abandoned at the bound does not stop its listing (the walk has no cancel flag): it runs to its end on the blocking pool, holding the listing it builds, and the next cycle starts another walk; and `serve` answers nothing, `/health` included, until the startup sync before the routes ends, while a signal in that time ends the process at once. Until a run finishes, requests for the earlier backups answer `BackupNotFoundException`, `/health` possibly already healthy; the log line that reports the adopted count ends that window, and for a prefix that line reports as having met a store or catalog error, a later cycle's line does. It takes no claim: the process is the catalog's only user (4(c), PR 7, PR 10). Every `extenddb backup` command refuses an in-memory SQLite database before it opens a catalog, exit 1, with a sentence saying why (a location that does not parse is reported with its parse error instead, as `serve` reports it): it would open a new, empty catalog that ends with the command, and a writing command would hand an adopted root to a catalog nothing can open again; this is the sync contract's "the backup CLI commands do not exist there" (PR 10). The matrix (`devtools/backup-restore-matrix`) copies the source's prefix under the target's root and runs `extenddb backup sync` on the target (on SQLite with the target's server stopped) in place of the hand-written row, so a cell proves the adoption and the restore; `--adopt-store` points the target's store at the source's root, adopts it and checks that the source's server then reports the store owned by another catalog, and the nightly runs that cell from SQLite to PostgreSQL and back (PR 9b). 4(c), 4(e), 4(f), 4(g), decision 6, section 7's memory paragraph and the PR 10 contract are edited with this item. 35. (2026-11-05, PR 11 as implemented, #747.) The CI filters: `--filter "not import_export"` came with the PostgreSQL integration job's first version (687e46ff) and was copied into the SQLite job (556c501e); neither commit gives a reason. A foreground server wrote a PID file in every tree up to e0d6a0af, which made foreground mode write none by default and added `--write-pid-file` to the job's start in the same commit; 556c501e came after e0d6a0af, and its SQLite job passes the flag. So the restart `devtools/run-tests` makes for the import and export tests could stop the job's server in every tree, and the filter was not working around it. The MongoDB and Cassandra jobs already run `tests/test_import_export.py` unfiltered (`--pytest --comprehensive --parallel`). The expression selects out the ten tests of that file and nothing else (none of the comprehensive suite, the CLI lifecycle list or the backup-store step); run the job's way on release builds of this tree they passed 10 of 10 on SQLite and on PostgreSQL, so both filters are dropped and no narrower one is kept. The open cells (testing summary) are measured in `15-backup-restore-bench-pr11.md`, in `-pr9.md`'s method, none left open; the Cassandra cells ran on a single 4.1 node of their own with the Cassandra integration workflow's service settings (`PasswordAuthenticator`, a 1 GiB heap), since a node running `AllowAllAuthenticator` cannot hold the role `extenddb init` creates. Later runs on the same host with other harnesses, on release builds of this change's sources (04c29155, an earlier version of this commit with the same Rust sources, and 30553b38, whose Rust sources differ only in the help text of two `init` options), reproduce the cells in kind and are in the record, with a control of items of 2.5 KB strings run on this commit's sources. Findings beyond the figures: SQLite's vector restore under glibc's default arenas is higher at 1M than at 100k (552 against 354 MiB over idle; the 1M run levelled off after about 150 s, and no other size was measured) and is 195 and 204 MiB under `MALLOC_ARENA_MAX=1` (flat within about 25 MiB over the measured runs), the allocator's term section 7 describes for large items; that memory is the restore's write batch (up to 10,000 items or 32 MiB of item text) and decode chunks, not the vector index, and grows with the item's size up to the batch bound whatever the attribute types (100k items of about 2.5 KB of strings held 253 MiB, 130 MiB under one arena), lists of numbers decoding larger at the same text size (345 to 424 MiB and 184 to 195 MiB at 100k items of 128 numbers, with or without the index; 20,000 items of 1,536 numbers 669 MiB and 301 to 316 MiB under one arena), while on PostgreSQL the index adds 27 to 53 MiB (five pairs of runs with and without it, each from one harness; a pair whose two runs came from different harnesses is left out); one arena cost the 1M restore 32% in duration and two 100k restores running at once 131% (57.5 against 24.9 s); the vector index is about 20% of the 100k restore's duration on SQLite and 33 to 47% on PostgreSQL (the same pairs); MongoDB's restore becomes `ACTIVE` at the backfill worker's next cycle after the load (`GSI_BACKFILL_INTERVAL`, 5 s, which the restore's end does not wake), so its duration is the load's plus up to 5 s, 3.9 to 8.9 s at 100k over five runs; the peaks of one cell varied by up to about 20% between runs; and a whole-table `SearchVectors` on SQLite is the exact scan of `crates/storage-sqlite/src/vector_search.rs` (5.5 to 15.8 s over 1M 128-dimension vectors in seven runs, 0.24 s on PostgreSQL). `docs/manuals/14-backup-and-restore.md` is the operator's account derived from this document, which remains the record (section 10's first row): the strategy, RPO and RTO per store kind with the three records' figures and their host class, the configuration (pointing at the admin guide's rows and the store crate), securing the store, what a running backup costs each database, the restore drill, catalog-loss recovery with `backup sync`, `--adopt-store` and the in-memory startup sync, legacy backups (with `migrate-legacy`, #821, not shipped, the restore-and-back-up-again path), what a restore preserves, and ImportTable and export interchange with Amazon DynamoDB (a store-prefix import is #748). Each PR 11 acceptance statement is in it. Figures measured for it on the same host class: a 25 s PostgreSQL backup beside 12,156 overwrites of 200-byte items in another table grew that table from 0.4 to 4.4 MiB, its dead row versions "not yet removable" by a `VACUUM` during the backup (0.5 MiB for the same writer with no backup); on SQLite the WAL grows by the pages each write changes, which depends on the indexes: a 22.8 s backup beside 15,472 overwrites of a table with no secondary index grew the `-wal` file from 4.0 to 61.1 MiB (about 3.9 KB per write), and 13,996 overwrites of a table with two GSIs during a 25.6 s backup from 4.1 to 1,294 MiB (82 to 97 KB per write over four runs, about 20 to 24 pages: an asynchronous GSI write goes through `gsi_pending` and is applied in later transactions); the same writers with no backup left it at 4.0 and 4.1 MiB, and the file keeps its grown size after the backup and after a stop. The record's "Figures measured for the manual" section carries these runs' method (the writer, the relation and file measured, the `VACUUM (VERBOSE)` 10 s in), build and load, with their reproductions on 04c29155. Its process-level statements were checked against release builds of this tree on SQLite and PostgreSQL: the root placement refusals and their texts and an absent root's, the drill, a re-created catalog's `owned by another catalog` health with `[backup.store]` configured and, on file-backed SQLite with the default root, the logged warning, no `backup_store` component and in-catalog `CreateBackup` until the adoption, the accounts re-created with `manage create-account --account-id` and the old key pairs kept with `manage import-access-key`, `--adopt-store` exiting 6 beside a SQLite server and adopting once it stopped, the in-memory startup sync line and, on the `sqlite-memory,dev-mode` build, the adopted backups answering `AccessDeniedException` until their account was re-created with its old id, the in-catalog create's warning and the restore of its backup through a store-configured server, an S3 store at moto with the default spool, `backup sync` exiting 6 beside a SQLite server, ImportTable refusing a gzip file and a `FileSource.Path` through a symbolic link and importing a decompressed backup data file, the export's `FilePath` and `S3Prefix`, and on PostgreSQL the claim session found by its lock key (`hashtextextended('backup:', RESTORE_LOCK_SEED)` in the catalog database, distinct from the `extenddb serve` registration session) whose termination stopped the job at its next heartbeat and left the row to the reconciler, which failed it once stale. Checked on release builds of 04c29155 (this change's Rust sources) as well: an off-host copy with loose modes refused by `backup sync` (exit 1, naming the directory) and its restore failing, the store unhealthy only when the copy is the newest backup; the registration session's termination failing every store-backed backup the server ran (reason `shutdown`); `--prune-missing` waiting on a backup that failed within `stale_job_after_secs`; a catalog database restored alone beside a newer data database (tables created since absent and empty when created again, tables deleted since listed and answering `ResourceNotFoundException`); both symbolic-link refusals of `FileSource.Path`; an on-demand table's GSIs reported with 0 and 0 throughput in `SourceTableFeatureDetails`; and the PostgreSQL pair dumped with the server stopped under `umask 077` (files 0600) and restored into new databases created with `createdb -O` by the admin role, after which `CreateTable` and `PutItem` of a new table work, while a restore over the existing databases restores nothing and one into databases the admin role owns breaks `CreateTable` ("permission denied for schema public"). On release builds of this commit's sources, on PostgreSQL 16 containers: the entry's procedure as it now stands (an older `pg_dump` refused, "aborting because of server version mismatch", exit 1, leaving an empty file, which `psql -v ON_ERROR_STOP=1` restores with exit 0, after which the server refuses the catalog as not initialized; both dumps exit 0 and non-empty; the old databases renamed aside, the restored server reading the table and its GSI and creating a table, then the old databases dropped), and on a new, empty server `createdb -O` refused until the application role was created with the dump's password, the restore, and the server, whose catalog still named the old server's address, connecting its data pool there: it put a new table's items into the old server's data database while that server ran, failed to start once it was gone, and served the restored table, with `CreateTable` and `PutItem` landing on the new server, once `data_database_connection_string` was set to the new address; and an access key imported through the management API from a request body read from a file. A catalog restored from a dump whose newest `AVAILABLE` backup was deleted after the dump keeps its store unhealthy, and this release has no command that clears that without a copy of the prefix (`--prune-missing` refuses while the store is unhealthy); the manual says to take the dump after deletions. The admin guide's Data Recovery entry named the PostgreSQL data database `extenddb_catalog_data`; `init` names it after the catalog without its `_catalog` suffix (`extenddb` by default) and records it as `data_database_name` (the connection string as `data_database_connection_string`), and the entry, the architecture, design, quickstart and security-model guides, `AGENTS.md`, the troubleshooting guide and the `extenddb` skill's runtime-symptoms reference (the `pg_stat_activity` query), the technical-debt register's A-1 entry, the storage-extension guide's database list and `init --help` now say so, with each backend's databases; the quickstart's `init` table gives `--pg-user` (the admin role, `$USER` then `postgres`), `--pg-pass`, `--extenddb-user` and `--extenddb-pass` with their defaults, and its sample lines show migration 7; a catalog name without the suffix needs `--data-db`, since `init` otherwise derives the catalog's own name, creates it once and fails on the second create, leaving an empty database (not changed here). On PostgreSQL the entry, the security model, the deployment guide's checklist and the manual treat the catalog and data databases as one copy (both dumps with the server stopped, or a base backup), with `umask 077` before the dumps, since the catalog dump holds the encryption key and the application role's password, and the restore goes into new, empty databases owned by the application role, with the old databases renamed aside until the restored server has read every table, the `pg_dump` of the server's major version and both dump files checked, and on a new PostgreSQL server the application role created first and, at another address, the recorded data-database address set; the entry's SQLite row gives the restore (the current `-wal` and `-shm` removed first), and the upgrade manual's dumps take `umask 077` and its rollback restores the pair; the manual's catalog-loss section says what a catalog older than the tables gives and that tables backed up since restore once `backup sync` lists their backups. ADR-0019 and the RFC section are superseded in part, not whole: CreateBackup on Cassandra still writes their in-catalog format, before it returns (section 3). ADR-0019 changes only its Status line, as `docs/adr/README.md` allows, to the "Superseded in part by [document]" form the README's process now names for a decision a later design replaces in part, and section 10's closing paragraph states what that Status covers, including that the reconciler now removes an interrupted in-catalog create; the RFC's section carries a note, and its section 4 and Known Gaps lines on synchronous operations and on restored indexes carry notes. Edited with this item: section 7's memory paragraph (the restore-memory and vector-index sentences), section 10's table row and closing paragraph, the PR 11 line and the testing summary's performance paragraph. + +36. (2026-11-06, PR 10b as implemented, #820.) #820 ships `backup sync --move-store `, `--forget-store ` and `--forget-backup ` and plain sync's report of the rows of other stores, with three `BackupCatalog` methods on the four backends: `list_backup_rows_by_root(kind, root, status)` as the interfaces block had it; `rewrite_store(from_kind, from_root, keys, to)`, which rewrites only a row still of the old kind and root and not CREATING and counts the rows it rewrote, so a row another writer moved, or the CREATING row of a server still on the old root, is left rather than moved without its bytes (PostgreSQL one `UPDATE` over `unnest` of the keys in a transaction the client commits, under `SET LOCAL lock_timeout` 3 s and `statement_timeout` 30 s with up to three attempts on a lock timeout, so the statement of a command killed while it waits on a row lock commits nothing once the claim is gone; SQLite one `BEGIN IMMEDIATE` transaction, every key or none; MongoDB `update_many` in chunks of 1,000 keys that also `$inc` `lock_seq`, a convention: the update and a transaction that writes the document are serialized (when the update commits after the transaction's snapshot and before its write, the transaction re-runs and reads the row as the update left it; when the transaction wrote first, the update waits for it and applies its filter to the committed document), and a registration keeps the store its server is configured with; Cassandra per key; the trait doc says what a client's death in the call leaves on each); and `summarize_store_roots()`, the rows per store (kind, root, id) and status, which the design did not have: `list_backup_rows_by_root` takes a root, and nothing listed the roots (PostgreSQL and SQLite one `GROUP BY`, MongoDB a `$group`, Cassandra the status table's partitions with their entries counted and not re-read, which the report of other stores then checks against the root's listing kept to each store's id, printing what no row of its status stands behind as status entries). On Cassandra `list_backup_rows_by_root` reads the entries of all five status partitions of the root whatever status it is asked for, dedupes them by (account, ARN), reads each row once and keeps it when its kind and root match (and its status, when one is asked for), so a row whose only entry is under an earlier status (a status change whose entry batch failed, which is logged and not sent again, or whose writer died between its row and its batch) is listed under its own status by the move, the forget options and the report of other stores. A partial rewrite (between MongoDB chunks, between Cassandra keys) leaves rows under both roots; every step is idempotent and the command exits 1 naming what is left. On Cassandra `rewrite_store` writes the new root's status entry before the row's lightweight transaction (`IF store_kind = ? AND store_root = ? AND backup_status = `) and deletes the old entry after it, the one exception to the module doc's row-first rule: a stop after the new entry leaves only an entry every reader skips, where the row first would leave a moved row that no listing of the configured store returns; an unknown outcome is settled by a `LOCAL_SERIAL` read, and `NotMatched` reads the row at `LOCAL_SERIAL` first, keeps the new entry when the row already names `to` (this call's write), and otherwise removes it and reads again, up to three attempts. `start_backup_delete`'s and `set_backup_deleted`'s lightweight transactions carry the store kind and root they read, so a DeleteBackup of a server on the old root that read the row before the rewrite does not apply, reads the row again and moves its status entry under the root the row now names: the moved row is never left without its own entry. The report of other stores reads `summarize_store_roots` and names each CREATING and DELETING row through `list_backup_rows` of that store's exact identity (on Cassandra through `list_backup_rows_by_root` of its root, kept to the store's id); a store of the configured root under another id counts as another store; it is printed after every sync that listed the root to its end, skipped when the run stopped or the listing failed, and a failed read of it exits 1. So 4(g)'s "rows for another store are reported" now holds for every such row, where item 34 had it for the prefixes whose backup id such a row holds (section 4, PR 10). `--move-store` needs the configured store healthy, refuses the configured root and a root with no rows (printing the stores the catalog holds rows for; a root matches as stored, byte for byte), takes the maintenance claim on every backend, SQLite included under its serve lock, and waits over the configured store and every store id the catalog records for the old root (`maintenance::wait_for_quiet` takes a slice of identities), since a CreateBackup of a server still on the old root has its row there. Each row of the old root is checked against the configured store as the PR 10 contract says, read thus: a CREATING row goes through the reconciler's per-row resolution (`resolve_dead_creating_row`, split out of `resolve_dead_creating_rows` so both callers share one body) with the configured store's calls and `OnIncomplete::Leave`: it is written AVAILABLE when the copy holds its complete backup, and any other probe result stops the move with no status written (the takeover's generation is the one change), since a copy without the complete backup proves nothing about the old root's prefix (a copy's order is arbitrary) and a FAILED row would have the configured store's reconciler delete the copy and a server still on the old root the original; the way out is a reconciler on the old root (a server, or on SQLite plain sync, configured with it), which resolves the row there, or `--forget-backup`; the PR 10 contract's sentence is edited with this item; a DELETING row whose prefix there holds no verified manifest is finished with `finish_backup_delete` behind the fence, and stays DELETED when the move then stops; an AVAILABLE row is probed by `probe_manifest` (now `pub(super)`) and its manifest's size and item total must be the row's, so another backup under the same ARN is reported, not moved; the old root's rows and the configured root's are grouped by (account, backup id) before any row is checked, and every old-root row of a group of more than one (with a row of the configured root, or among the moved rows, which only Cassandra can hold) stops the move, so no such row gets a resolution or a finished deletion (on Cassandra two spellings of an ARN share one prefix), and the configured root's rows are read before anything changes, so a failed read stops the move with nothing changed. Every stopping row is named in one pass and none moves while any does; with none, `StoreFence::check_catalog` re-reads the identity file once before `rewrite_store` (a refusal records the store unhealthy, as the fence does everywhere), the old root is listed again and a row still there is reported with exit 1 (a key counts as moved only when the configured store's listing then holds its row, and a row neither root lists, removed by another writer during the move, is reported as gone; a re-list that fails after a rewrite that counted every key takes every key as moved), and a plain sync of the configured root follows, in whose report of other stores a row of the old root (an insert that landed during that sync) is named and exits 1; a move that stops runs no sync. One `extenddb::audit` line is logged per row moved, resolved or finished (section 4, 5.2, PR 10). The removals of `--forget-store` and `--forget-backup` delete no object and read no store, so they run on a store in any state and with none configured (`kind = "disabled"`), under the claim on every backend, with the wait over the configured store when its identity is known and otherwise every store id the catalog records for the configured root (and the root's ids for `--forget-store`; with no store the wait covers the RUNNING restores). They open the store writing nothing (described as a read leaves it, so no default root is created and no `validate` probe file written; the identity file read, never created; when it agrees with the rows, `check_configured_store_read_only` runs once and is recorded), and on SQLite the command's own pass over what the stopped server left runs without the store's check (`ReconcilerState::for_maintenance_holder_without_store_check`). Each status is removed with a method that exists, no fourth one: AVAILABLE by `forget_available_backup_row` with its in-use rule, DELETED and FAILED by `remove_backup_row`, CREATING by `remove_dead_creating_backup_row` on the catalog's clock, DELETING by `set_backup_deleted` then `remove_backup_row` (no duty meets the DELETED state between them, by the order of the listings: one that listed rows before the claim listed this one DELETING, and one that lists it DELETED did so after the claim was taken and reads the claim held); a removal that meets another status reads the row once more and applies that status's removal. `remove_backup_row` takes the store kind and root the caller read and removes only a row still of them (Cassandra's lightweight transaction `IF backup_status = ? AND store_kind = ? AND store_root = ?`, `NotMatched` answering `false`), so a tombstone duty of a server on the old root does not remove a row the move rewrote after the duty read it. A DELETING row of the configured store is the exception to "delete no object": after the wait for the duties (so a stop during the wait leaves the row DELETING and its objects untouched), the forget finishes its deletion, behind the fence (`finish_backup_delete`, the deletion its DeleteBackup asked for), which needs the store's check to have passed, and otherwise refuses the row; when the check failed on an identity mismatch that the row's own store id takes part in (`IdMismatch`, `MultipleRowIds`: the backup manual's copied identity file), the refusal says so and names the way out the manual gives, the same command with `[backup.store] kind = "disabled"`, whose generic DELETING path removes the row and keeps its bytes, then the move's sync, which adopts the backup again, and a DeleteBackup of it; on SQLite the command's own pass may finish it first, and the removal of its DELETED row says so; a DELETING row of another store keeps its bytes, and a sync of that root after its return adopts the backup again. While the configured root has no identity file and the catalog holds rows for it (`FileMissingWithRows`: an unmounted volume), every row of that root is refused, and so on SQLite is every row of the defaulted root found missing while the catalog holds rows for it (moved away, or a symbolic link to a volume that is gone: `configure_described` reads `read_store_ids` for the path, its canonical parent joined with its name, and a dangling link's target the same two ways, and answers `NotConfigured::DefaultRootMissingWithRows`, which the command's header prints in place of "no server has run with this database file"), since a server's check rewrites a missing file at a root with no rows and reports the store healthy. Both wait for running jobs as `purge` does, refusing when one still runs at the wait's end; a CREATING or FAILED row whose heartbeat, written after the wait, is within the owner window, or that is of a store the wait does not cover, is left, as is an in-use row (whose text names `extenddb backup remove-target --account
` for a failed restore's target), and the command exits 1. `--forget-backup` repeats (in order, stopping at the first ARN not removed; an ARN that no row, under either spelling, and no in-catalog backup carries is printed as having nothing to remove and counts as done, so a command line can be run again after a stop; every ARN's form is checked before the claim is taken), takes the ARN as the row carries it and names a row under the other spelling without removing it, reports an earlier release's in-catalog backup as one (exit 1, nothing changed), and then, when a store is configured, runs the store's check once and prints it, naming the next ARN whose manifest is missing; that check writes nothing (`check_configured_store_read_only`: the identity read without the create and without `validate`, the id compared with the one the command opened the store with, the manifest probe), and is not run when the root had no identity file as the command opened it. That is the remedy for a restored catalog whose newest AVAILABLE backup was deleted after the catalog's backup, which item 34 left with "put the prefix back"; `--prune-missing` keeps refusing on `ManifestMissing`, the host-local detector, and plain sync's refusal there names `--forget-backup` with the ARN. After its removals `--forget-store` sweeps, under the claim, the root's status entries, each checked against its row at `LOCAL_SERIAL` (`remove_orphan_status_entries`, answering `StatusEntrySweep { removed, repaired }`): an entry whose row is gone or names another store is deleted (on Cassandra a deleter whose lightweight transaction lands before a rewrite of its row and whose entry batch lands after it leaves one under the old root), and one whose row is of the root under another status is written again under the row's status and the stale one deleted; it prints both counts, then the store ids the catalog still records for the root, without changing the exit (section 4, PR 10). The reconciler's `Duty::fenced` re-reads the row after the fence and skips one that is gone or whose status or store changed, then reads the maintenance claim again and starts nothing while it is held, two catalog reads per deletion; each FAILED-prefix and DELETED-prefix deletion and each tombstone removal that read the claim free must end by its deadline, `DUTY_DELETION_BOUND` (30 s, the S3 store's per-call bound, under the SDK's 60 s attempt timeout) after that read was sent: a read answered after the deadline starts nothing, the prefix deletion runs through the store's new `delete_prefix_before(prefix, deadline) -> PrefixDeletion { deleted, finished }` (the filesystem walk checks the deadline beside its cancel flag before each removal, the S3 loop before each `ListObjectsV2`, `DeleteObjects`, `ListMultipartUploads` and `AbortMultipartUpload`; the trait's default checks once, for a one-step store), and the call is also dropped at the deadline; past it the duty stops for the cycle, the row left for the next. Beside servers the forget options wait `FORGET_WAIT`, the bound plus one 30 s store call, from the claim before their first FAILED or DELETED removal and print the wait; on SQLite, under the serve lock, they do not wait. So what a deletion duty can still do to the bytes of a row a forget removes (4(a)) is the step in flight at its deadline, one removal on a filesystem root or one S3 request (up to 1,000 keys); the bound, like every lease in the design, assumes the server's process keeps running, and one paused past the wait between its last check and a call already sent (SIGSTOP, a frozen cgroup, a VM pause) runs that call when it resumes. A marker written per deletion, which the forget's removal would be conditioned on, is the full remedy, and is not taken: it adds a catalog write to every deletion for a case outside every other lease. The store-backed CreateBackup reads the claim again immediately before `claim_backup`, and the store-backed restore immediately before its registration as well as after it, so a claim taken before that read refuses the job with no row or record (`LimitExceededException`); what remains is the window from that read to the insert, however long the insert takes: a CreateBackup of a server still on the old root whose insert lands during the move's wait is listed by it, one that lands before the rewrite is left CREATING under the old root and reported by the re-list (the rewrite skips CREATING rows), one that lands during the move's sync of the configured root is named by that sync's closing report of other stores if it lands before it (exit 1), and one that lands after only by a later plain sync; the backup manual says to check plain sync's report of other stores before deleting the old root. `OTHER_STORE` now names the move ("or copy its tree under the configured root and move the backup's rows to the configured store with `extenddb backup sync --move-store `, and restore again"), as 5.3 step 7 says, and the reconciler's module doc with it. The three options conflict with each other, with `--adopt-store` and with `--prune-missing`. Memory: the move keeps the old root's rows without their parsed description, and its peak is the backends' whole read of those rows, 569 MiB at 20,000 rows of a five-GSI table on a SQLite debug build and 500 MiB on a PostgreSQL one, about 27 and 23 KiB per row above about 50 MiB, at 1,452 bytes of table description per row and growing with it (section 7). The PR 10 section's split line, interfaces block and sync contract, and section 7, are edited with this item; 4(c), 4(f), 4(g), 5.3 step 7, PR 8's claim-held paragraph and the PR 10 acceptance and transition lines state what was built and stay as written. The backup manual's procedure copies the old tree without its `extenddb-store.json` (the restarted servers gave the new root its own; a copied file turns every running server's store unhealthy until a restart, and, when a backup was taken under the new root meanwhile, needs that backup's row forgotten before the move, whose sync adopts it again). Not in #820: `migrate-legacy` (#821), a sweep of Cassandra's orphan status entries beyond `--forget-store`'s root, the same all-partition read for Cassandra's store-scoped `list_backup_rows`, which the reconciler reads (a DELETING row of the configured store whose entry batch failed is not finished by its reconciler until the entry is repaired; `--forget-backup` of its ARN finishes and removes it), and any detection of a server left configured with the old root, which the backup manual's procedure reconfigures first. diff --git a/docs/differences-from-dynamodb.md b/docs/differences-from-dynamodb.md index 1f41a44a8..49e3b96a5 100755 --- a/docs/differences-from-dynamodb.md +++ b/docs/differences-from-dynamodb.md @@ -124,7 +124,7 @@ Behaviour stated as measured, and every number below, was observed with release | Concurrent restores of one backup | A second `RestoreTableFromBackup` of a backup while a restore from it is still `CREATING` is `BackupInUseException` (measured) | The same on all four backends: the second restore is refused with `BackupInUseException` and creates nothing; the check runs inside the transaction that registers the restore (PostgreSQL, with the backup row locked `FOR UPDATE`; SQLite), inside the claim on the backup document followed by a check for a `CREATING` target of the backup (MongoDB), or inside the lightweight transaction on the backup row (Cassandra), so two simultaneous requests cannot both pass. For a store-backed restore the registration takes the same lock on the `backup_index` row instead of the legacy `backups` row (PostgreSQL `SELECT ... FOR UPDATE`, SQLite the `BEGIN IMMEDIATE` write transaction, MongoDB a `$inc lock_seq` on the `backup_index` document, Cassandra the `active_restores` lightweight transaction on the backup's partition), the same ordering guarantee, so a legacy restore and a store-backed restore of two different backups never contend and a `DeleteBackup` of a store-backed backup meets only store-backed registrations. Two concurrent registrations of one backup leave one restore target on Cassandra as on MongoDB: the registration's lightweight transaction (`IF backup_status = 'AVAILABLE' AND active_restores = null`) is linearized on the backup's partition, so the second finds `active_restores` non-null and is `BackupInUseException`; the in-use pre-read is advisory *(storage-level tests on both)*. Once the first target is `ACTIVE` the backup restores again; on MongoDB the check reads majority-committed data (at the default transaction read concern) while `DescribeTable` reads at `local`, so a `RestoreTableFromBackup` or `DeleteBackup` issued within the majority-commit lag after `DescribeTable` first shows the target `ACTIVE` can still be `BackupInUseException` and succeeds when retried (the lag is the journal flush plus any concurrent batch write still holding an earlier oplog slot; measured at 7 to 12 ms, once in about 2,000 observations, with a 3,000-item insert running beside the write). A restore of a backup whose `DeleteBackup` is in progress answers `BackupNotFoundException` on every backend (the service's answer was not measured), by two mechanisms: on MongoDB and Cassandra the delete marks the backup `DELETING` before it drops the data, and the restore finds no `AVAILABLE` backup; on PostgreSQL and SQLite the restore waits for the delete's transaction and then finds no backup. Earlier builds accepted concurrent restores of one backup on PostgreSQL, SQLite, and Cassandra, and on MongoDB after a 2 s retry | | Restore interrupted by the server stopping | Managed by the service | The copy runs in the server process, so a stopped process leaves its target `CREATING` with a RUNNING restore job record whose heartbeat stops. The engine's reconciler, once at startup and then every `backup.reconcile_interval_secs` (default 60 s), marks the record FAILED and removes the target once the heartbeat is older than `backup.stale_job_after_secs` (default 300 s) on the database's clock, on every backend alike, logs the removal with the table name, and then lets `DeleteBackup` and a new restore into the name succeed *(from the code)*. On SQLite a restart ages every record of the stopped process at once (one process serves the file), so the first cycle removes it without the stale window. Until then `DeleteTable` and `DeleteBackup` are refused as during a running restore. A store-backed restore (one run from a `[backup.store]`) is instead resumed, not discarded: its owner's death leaves a RUNNING restore job record with a stale heartbeat and the target `CREATING`, and the reconciler of any instance with the configured store takes the record over under a fresh owner generation once the heartbeat is stale (on SQLite at once after a restart, by the record's slot; on PostgreSQL once the owner's claim session has ended as well), stamping a fresh heartbeat in the same write so no second instance takes it too, and resumes the load from its recorded progress (per data file on PostgreSQL, per batch on SQLite, MongoDB and Cassandra), re-downloading to its own spool when the dead owner's `spool_host` is not this host's; the target reaches `ACTIVE` with the backup's full item count, its secondary indexes complete: a resumed Cassandra restore re-applies its last batch as idempotent upserts that rewrite the same base and index rows (the write path rewrites an unchanged item's GSI and LSI rows rather than deleting them, #424). A resume that finds no recorded progress and a target holding more items than the backup's total is `FAILED` with reason `target_not_empty` and the target is left in place: the rule protects a catalog restored from a dump that names a restore which finished and was written to since. A target holding no more than the total with no recorded progress is a first data file applied in part: on Cassandra, which records a batch's progress after its writes, an ordinary crash before the first progress record (the first 10,000 items or 32 MiB) leaves written items and no progress, and the resume re-applies the first file with idempotent upserts and reaches `ACTIVE` with the exact count *(from the code; the storage and engine cells)*. MongoDB records each batch's documents and its progress in one transaction, so a mid-file crash leaves no items the record does not account for and resumes to `ACTIVE`; only a crash before the first batch commits leaves an empty target, which resumes from the start. A record that one instance's reconciler has taken over five times without the restore ending (a resume whose final count or progress read fails every time, or that stops with the record RUNNING) is failed on the next takeover with reason `owner dead: taken over 5 times by one instance without the restore ending` and its target removed, so a record that can never finish does not hold its name and its backup for as long as the server runs; the count is per instance and a restart starts it over *(from the code)*. The target stays `CREATING` with a `FAILED` record: `DeleteTable` on it is `ResourceInUseException`, `DeleteBackup` of its backup and a second restore of that backup are `BackupInUseException`, and the reconciler does not remove it; `extenddb backup remove-target --account
` (the admin guide's `backup` section) removes the target and its record, after which `DeleteBackup` and a new restore into the name succeed. A legacy (in-catalog) restore is still failed and its target removed. Measured *(binary-level, this release)*: SIGTERM during a store-backed restore left the record RUNNING and the restarted server resumed it to `ACTIVE` with every item (on PostgreSQL after the 45 s stale window, on SQLite at once); a SIGKILL at each restore kill point left the target to finish with the exact count (the base table's and the GSI's) or, for a failing restore, gone, except at the first reach of the Cassandra point that pauses before a batch's first progress record: on Cassandra (killed after the first batch's 10,000 upserts, before their record) the resume of that build answered `target_not_empty` and left the target `CREATING`, and `extenddb backup remove-target` then removed it, after which `DescribeTable` was `ResourceNotFoundException` and `DeleteBackup` succeeded; this release resumes that case with upserts to `ACTIVE` *(from the code; the nightly kill-point job runs it)*. On MongoDB a SIGKILL after a file's `insert_many` leaves the batch's items and its progress committed together, so the resume continues to `ACTIVE` rather than `target_not_empty`. Not covered *(from the code)*: on MongoDB, a target whose base copy had finished has its record DONE and is left alone by the reconciler; the index backfill worker finishes it after a restart, and if that backfill fails on every attempt the worker retries it every 5 s with a warning and never gives up, so the target stays `CREATING` (and `DeleteTable` and `DeleteBackup` stay refused) until the cause is fixed or the target's catalog document is removed by hand (no failing case was found to reproduce this). A target left `CREATING` by a build before this release is backfilled by the catalog migration as a RUNNING record with a heartbeat at the epoch, so the first cycle removes it. Heartbeats and the stale window are measured on the database server's clock, so the application hosts' clocks play no part. The same reconciler resolves a store-backed `CreateBackup` the server stopped (one reconciler with one duty list, in place of per-backend sweeps): a `CREATING` backup row whose owner is dead (on PostgreSQL and MongoDB a heartbeat older than `stale_job_after_secs`, on PostgreSQL once the owner's claim session has ended as well; on SQLite, where one process serves the file, every `CREATING` row after a restart and, at runtime, a row whose in-process slot no task holds, so a job task that died is resolved by the next cycle without a restart) is taken over under a fresh owner generation and becomes `AVAILABLE` when a complete, verified manifest is under its prefix in the configured store (the process died between the manifest write and the status write) and `FAILED` otherwise, never continuing the upload; the objects of a `FAILED` backup are deleted once its heartbeat is older than `stale_job_after_secs`; a `DELETING` backup's deletion is finished; a `DELETED` backup's prefix that still lists objects is swept; and `FAILED` and `DELETED` rows are removed once their prefix is empty and their `completed_at` is older than `failed_row_retention_secs` (7 days). Measured *(binary-level, this release)*: SIGKILL between the data file and the manifest left the row `CREATING` and the data file under the prefix, and after the restart the row was `FAILED` in the first cycle on SQLite and 46 s after the kill on PostgreSQL (`stale_job_after_secs = 45`), with the prefix deleted in the same cycle on PostgreSQL (the stale heartbeat is what made the owner dead) and once the dead owner's heartbeat was stale on SQLite, 45 s after the kill (the slot made the owner dead at once; the prefix waits for the heartbeat clock on every backend); the same at a kill between the three documents and the manifest, which leaves the documents under the prefix and no manifest; SIGKILL between the manifest write and the status write left the five objects, and the row was `AVAILABLE` with the manifest's size and count after the restart (SQLite at once, PostgreSQL after 48 s); SIGTERM during an upload left the row `FAILED` with reason `shutdown` and an empty prefix. These duties run only while the configured store is healthy, and under a maintenance claim only the dead-owner and `DELETING` duties run. A `FAILED` backup is `BackupNotFound` to clients; `extenddb backup list --failed` prints it with its reason | | Point-in-time recovery | 35-day window: `UpdateContinuousBackups` enables continuous backups, and `RestoreTableToPointInTime` restores to any second in the window | Not supported; use on-demand backups (`CreateBackup` and `RestoreTableFromBackup`) instead. `DescribeContinuousBackups` reports `ContinuousBackupsStatus: ENABLED` (the service reports this unconditionally) with `PointInTimeRecoveryStatus: DISABLED` and no restorable-time window. `UpdateContinuousBackups` with `PointInTimeRecoveryEnabled: true` returns `ContinuousBackupsUnavailableException`; with `false` it returns the same description. `RestoreTableToPointInTime` checks the source table, then returns `PointInTimeRecoveryUnavailableException` ("Point in time recovery is not enabled for table ''"), the exception the service models on that operation. A table that does not exist is `TableNotFoundException` ("Table not found: ") on all three operations, as on the service. Same on all four backends | -| Where backups live | Managed storage, independent of the table | With a configured `[backup.store]` (a filesystem root, which is the default beside a SQLite database file, or an S3 bucket and prefix) on PostgreSQL, SQLite and MongoDB: under the store's root, at `/
//`, as gzip data files of DynamoDB JSON lines (`data/000001.json.gz`, ...) and the manifest documents (`extenddb-manifest.json` last, the commit record), with a `backup_index` row in the catalog as the index; a lost database loses the rows, not the bytes (`extenddb backup sync` gives a new catalog a row for each backup under the root, and `--adopt-store` first makes it the owner of a root another catalog owned). Without a store, and on Cassandra always, inside the database the server already uses, so a lost database loses its backups too: on PostgreSQL in the catalog database (`backups`, `backup_items`, `backup_definitions`, `table_restores`), on SQLite in the one database file, on MongoDB as a `backups` metadata collection in the catalog database and one data collection per backup in the data database, on Cassandra as backup metadata in the catalog keyspace and item payloads in the account's keyspace; take database-level backups for off-host copies there | +| Where backups live | Managed storage, independent of the table | With a configured `[backup.store]` (a filesystem root, which is the default beside a SQLite database file, or an S3 bucket and prefix) on PostgreSQL, SQLite and MongoDB: under the store's root, at `/
//`, as gzip data files of DynamoDB JSON lines (`data/000001.json.gz`, ...) and the manifest documents (`extenddb-manifest.json` last, the commit record), with a `backup_index` row in the catalog as the index; a lost database loses the rows, not the bytes (`extenddb backup sync` gives a new catalog a row for each backup under the root, and `--adopt-store` first makes it the owner of a root another catalog owned; a root copied to a new place keeps its backups' ARNs once `--move-store` re-points their rows). Without a store, and on Cassandra always, inside the database the server already uses, so a lost database loses its backups too: on PostgreSQL in the catalog database (`backups`, `backup_items`, `backup_definitions`, `table_restores`), on SQLite in the one database file, on MongoDB as a `backups` metadata collection in the catalog database and one data collection per backup in the data database, on Cassandra as backup metadata in the catalog keyspace and item payloads in the account's keyspace; take database-level backups for off-host copies there | | Backup creation | `CreateBackup` returns while the backup is `CREATING`; `ListBackups` and `DescribeBackup` expose that state until it becomes `AVAILABLE`. `BackupName` and `TableName` are checked as the model says: length 3 to 255 and `[a-zA-Z0-9_.-]+`, with "1 validation error detected: Value '' at 'backupName' failed to satisfy constraint: " texts, joined under "2 validation errors detected:" when both fail (measured). `TableName` may be the table's ARN (measured). A table that does not exist or is still `CREATING` is `TableNotFoundException` "Table not found: " (measured) | With a configured `[backup.store]` on PostgreSQL, SQLite and MongoDB, `CreateBackup` takes the table's snapshot and writes the catalog row in the request and returns the backup `CREATING`; a job then uploads it and the backup becomes `AVAILABLE`, which `DescribeBackup` and `ListBackups` show as on the service (`BackupSizeBytes` is 0 while `CREATING` and the compressed size once `AVAILABLE`; `ItemCount` and `TableSizeBytes` in `SourceTableDetails` are the table's figures as last counted by the stats pass while `CREATING`, which is 0 and 0 for a table the stats pass has not reached yet, such as one loaded seconds before CreateBackup, and the backup's exact figures once `AVAILABLE`: the items written and the sum of their DynamoDB sizes; the service reported 0 and 0 in every fresh-table backup measured). `CREATING` lasts the whole upload (the durations below), and while it lasts `DeleteBackup` and `RestoreTableFromBackup` of the backup answer `BackupInUseException` "Backup is being created: " on all three backends; the 16 service backups measured were `AVAILABLE` 12 to 90 ms after `CreateBackup` answered, so a client that deletes or restores a backup right after creating it polls `DescribeBackup` first here. The backup is the table as it stood when its snapshot opened, which is inside the request but after the request's checks (the table's state, the store's health, the maintenance claim) and after any wait for a job permit, up to 30 s (below): every write acknowledged before the open is in the backup and none acknowledged after it, so a `PutItem` acknowledged while the request waited for a permit is in the backup (measured: a write acknowledged 1.01 s after a `CreateBackup` that waited for a permit was in it, 2 of 2 runs), and `BackupCreationDateTime` is the open's millisecond. Measured on the service: a restored backup contained every write acknowledged before `CreateBackup` was sent and the 18 writes acknowledged in the 0.1 s up to `BackupCreationDateTime`, and none of the 11,754 acknowledged in the 75 s that followed, so the service takes the table at an instant inside the request as well. A backup the job cannot finish (a store failure, an item no restore can load, a shutdown) is `FAILED`, which is `BackupNotFound` to clients, and its objects are deleted (the restore-interruption row). A `CreateBackup` still waiting in its snapshot open or row insert when the server stops writes no row; it is answered `ServiceUnavailable` "The server is shutting down. Retry the request." only after a registration loss, where the jobs' token is cancelled while the listener still serves, since on SIGTERM the listener's 5 s connection drain closes the connection before that token is cancelled (measured: the connection closed 8.2 s after the request was sent). Measured *(this release, 16-core host, filesystem store, the record in `docs/design/15-backup-restore-bench-pr8.md`)*: 100,000 items of a composite-key table with two GSIs `AVAILABLE` in 1.85 s on PostgreSQL and 1.44 s on SQLite, 1,000,000 in 36.1 s and 12.45 s. Without a store, and on Cassandra always, `CreateBackup` copies the items before it returns and returns the backup `AVAILABLE`. On PostgreSQL, MongoDB, and Cassandra a concurrent `ListBackups` never saw an in-catalog backup `CREATING` *(earlier build)*; on SQLite the backup row is visible as `CREATING` to other requests while the copy runs. Measured *(earlier build, in-catalog)*: 1.3 s for 60,000 items on PostgreSQL, 1.9 s for 60,000 on SQLite, 0.2 s for 20,000 on MongoDB, 5.1 s for 20,000 on Cassandra. MongoDB's in-catalog backups report `BackupSizeBytes` 0. The name checks are the service's, with its texts (the length clause before the pattern clause when both fail; the service's order was not captured), counting characters, with `TableName` taking the deployment's table-name limits; before them a 200 KB name was stored and returned in every `ListBackups` page (a 500 on Cassandra) and a NUL byte was a 500 on PostgreSQL. `TableName` as an ARN (`arn:aws:dynamodb:::table/`) is accepted on all four backends: an ARN whose region is not the server's (`server.region`, compared as written, so `US-EAST-1` differs from `us-east-1`) is `ValidationException` "1 validation error detected: Invalid AWS region in ''" (the service's answer, measured on DynamoDB twice; an earlier build resolved it to the server region's table), another account's is refused with the service's answers (see "A resource ARN in another account"), the account checked before the region, so another account's ARN in another region is `AccessDeniedException` (measured on DynamoDB for CreateBackup and ListBackups, twice each); an SDK built from the model refuses the ARN form client-side, so only a hand-built request sends it. `TableNotFoundException` "Table not found: " for a missing or `CREATING` table on all four backends, built from the request's name (the backends' own payloads differ). Without a store, and on Cassandra, a table the deployment can create is backed up whatever its name, as the service does (measured on DynamoDB: a table named `...` answered `ContinuousBackupsUnavailableException` "Backups are being enabled for the table: .... Please retry later" 13.2 s after `CreateTable` and was backed up at 33.7 s): the in-catalog format writes no store prefix. With a store, a name over 255 bytes (reachable only with `[limits] max_table_name_length` raised) or made only of dots fits no store prefix, so the store-backed create refuses it with `ValidationException` before it opens a snapshot or writes a row ("TableName cannot be backed up: a backup ARN cannot be formed for it: invalid backup ARN: table name of N bytes is longer than 255" / "... invalid backup ARN: table name \"...\" is only dots"). Every table under a `server.region` outside `[a-z0-9-]` is refused (the `BackupArn` row); the checks run after the table's ACTIVE probe and before the job permit, so the refusal takes no slot. The permit is the last check: a request that finds every `[backup] max_concurrent_jobs` slot held (default 4) waits up to 30 s for one and is then `LimitExceededException` ("Too many backup and restore jobs are running on this server (at most N at once, the [backup] max_concurrent_jobs bound); retry later"), where the service accepted six `CreateBackup` calls sent together (measured on DynamoDB, two runs) | | `DeleteBackup` response | The full `DescribeBackup` description with `BackupStatus` `DELETED`, including `SourceTableDetails.ProvisionedThroughput` (a required member) and `SourceTableFeatureDetails` (measured) | The same on all four backends: the engine reads the backup's recorded definition before the delete and fills the description as `DescribeBackup` does, with `BackupStatus` `DELETED`. A backup whose definition this build cannot read or that does not parse is still deleted and described without feature details (logged at warn), which means a damaged `PROVISIONED` backup is described with on-demand `ProvisionedThroughput` `{0, 0}`: ExtendDB's choice, deletability over the throughput member of a row that is being removed, where `DescribeBackup` on the same row is an `InternalServerError`; a failure to reach the database fails the request and deletes nothing. | | A backup maintenance command holding the catalog's claim | Not applicable: the service has no maintenance command | While `extenddb backup purge` holds the catalog-level maintenance claim, `CreateBackup` and `RestoreTableFromBackup` on every instance of the catalog answer HTTP 400 `LimitExceededException` "A backup maintenance command holds this catalog's maintenance claim; backup and restore jobs are refused until it finishes", after every other check of the request (a request refused on its own terms, a bad table name or override, gets its own error) and before the job permit. The refusal ends when the command releases the claim, or when a command that died goes stale after `[backup] stale_job_after_secs`. | diff --git a/docs/manuals/05-admin-guide.md b/docs/manuals/05-admin-guide.md index 18bc2c1d9..10fa5dfa4 100755 --- a/docs/manuals/05-admin-guide.md +++ b/docs/manuals/05-admin-guide.md @@ -171,11 +171,11 @@ When the section is absent, the backend decides: on file-backed SQLite it is a f Startup opens the configured store and, for a filesystem store, validates it and checks its identity before the listener is up (bounded by 5 s: a mount that stops answering after the root is opened leaves the server starting with the store `checking`; a root that does not answer at startup holds the server before the listener, as the database directory would, with the port bound and silent); an S3 store's validation and identity check are requests to the endpoint, and run after the listener is up, in the reconciler's first cycle, so an endpoint that stalls never holds the data plane down (the component is `checking` until the first result, and the operations that touch the store refuse meanwhile as they do while it is unhealthy). A configuration error, every one decided without a request to the store, stops startup with the key, path or bucket in the message: a root that does not exist or fails the owner-only rule, a filesystem root another user can reach, a filesystem root that is or contains the database directory, a bucket name the SDK routes to S3 Express, a bad part size or prefix, an insecure endpoint, an S3 store without `spool_path` on a backend with no default, a `spool_path` that breaks its placement rule, that another live instance holds or that exists with a mode other than `0700`. A transport failure (no region, no credentials, an endpoint that refuses connections or does not answer within the check's bound), a missing or denied bucket, an I/O failure of a filesystem root's probe write or resolution (a full or read-only disk, a stale network handle), or a failed identity check leaves the server running with the store unhealthy: the problem is logged once at error, `/health` reports the `backup_store` component unhealthy (status `degraded`), CreateBackup (where it writes to the store) and the restore and delete of a store-backed backup are refused with a `ValidationException` that names the store's kind and the problem's category and not its root, the reconciler acts on no row of the store, and the check runs again each `reconcile_interval_secs` until it passes, at which point the server logs `is healthy again` and resumes without a restart. Each check's store calls are bounded (5 s for a filesystem root, 30 s for a bucket); the catalog's own reads (the catalog id, the rows' store ids, the newest AVAILABLE row) run beside the calls and not under the bound, so a slow catalog costs the check time and never marks the store unhealthy. A store that has stopped answering costs a reconciler cycle that bound and nothing more, the cycle's other duties run as before, and the abandoned check is logged; on a filesystem root the abandoned check's call stays parked on one thread until the mount answers, no later cycle starts a check while it is (logged each cycle, the store staying `unreachable`), a root's calls run one at a time so that calls started before the first reached its bound (two `DeleteBackup` requests together, a `DeleteBackup` beside a check) wait for it under the bound, then have the bound for their own call (a slow call behind another finishes, where one bound over both would abandon it), and park no thread of their own, and `stop` or SIGTERM still ends the server, the runtime giving that thread 5 s before the process exits without it (after the job wait of up to 30 s when a backup job is running; see Stopping). The deletions `DeleteBackup` and the reconciler run against the store go through the same bound and rule: `DeleteBackup` answers the description with `BackupStatus` `DELETED` once its DELETING write has committed, and a deletion the store does not answer within the bound, or that leaves an object under the prefix, leaves the row DELETING for the reconciler and is logged, never an error to the client. A default SQLite root whose startup checks did not finish starts `checking` with its rule kept: a first check that fails while the catalog has no rows for it is the same warning and leaves the store unconfigured. CreateBackup on Cassandra is not refused by an unhealthy store: its in-catalog create touches no store. -The store identity. The root carries `extenddb-store.json`: a random store id and the id of the catalog that owns the root. Startup creates it when the root has no file and the catalog has no rows for the root, and otherwise reads it and compares: a file naming another catalog, a store id that differs from the one the catalog's rows for the root carry, a root with no file while rows exist, or rows carrying more than one id each mark the store unhealthy. Instances that share a catalog must share one store root (one S3 bucket and prefix, or one shared mount): host-local roots configured at the same path give each instance different bytes under one root label, so each check also `head`s the manifest of the newest AVAILABLE backup and marks the store unhealthy when it is absent, which is what a second instance with a local copy of the directory sees. The file is left by `backup purge` and by every deletion; remove it only when retiring the root, or after `extenddb destroy`: a catalog re-created with `init` has a new id, and the old root's file names the old one, so the re-initialised server reports the store owned by another catalog until `extenddb backup sync --adopt-store` makes the new catalog its owner (below), the file is removed or a new root configured (on the default SQLite root, that is the logged warning above for the server, which keeps CreateBackup on the in-catalog path, while `backup sync` handles that root as configured, so `--adopt-store` adopts it). The default root, `backups` beside the database file, is shared by every database file in that directory: `--adopt-store` on it takes the root from whichever catalog wrote its file, so give each database its own `[backup.store]` when several share a directory. A server on an in-memory SQLite database writes the file with `ephemeral_catalog: true`; at its next start it adopts a root whose file names another catalog and carries that marker, and no other, so it never takes a durable catalog's root, and then syncs the root's backups into its new catalog (the `path` row above: until that sync's log line, the earlier backups answer `BackupNotFoundException`). A catalog whose root another catalog adopted (by `--adopt-store`, or a second in-memory server's start) keeps acting on the root until a later check that can read the catalog finds it owned by another catalog, at the earliest one reconciler interval after the cycle in progress ends (a check whose catalog read fails leaves the health as it was), with one exception, its deletion fence: right before it deletes a backup's objects (DeleteBackup, the reconciler's deletions, a failed backup job's cleanup, `purge`), removes a FAILED or DELETED row and writes a CreateBackup's first object, and also before DeleteBackup and `purge` change the row, a server or `purge` re-reads `extenddb-store.json`, and when the file names another catalog, names this catalog under another store id than the one the store's check settled (a root replaced by another root of this catalog), or the root has none, it deletes nothing and refuses the request; a file naming another catalog also marks the store unhealthy (`owned by another catalog`) at once, and one of another store id `identity mismatch`. DeleteBackup takes a file it cannot read, or a root with no file, at the read before the row is changed as it takes a store that does not answer: it writes DELETING and answers `DELETED`, the read before the deletion refuses it, and the row stays DELETING for the reconciler, whose deletion runs the fence again. `--prune-missing`'s removal of a row is not fenced: it deletes no object. A refusal at the read before the row is changed (the file names another catalog or another store id) leaves the row as it was, so DescribeBackup still answers AVAILABLE. One at the read before the deletion (an adoption landed between the two) leaves an AVAILABLE row DELETING with nothing deleted (DeleteBackup and DescribeBackup there answer `DELETED`), which this catalog's reconciler finishes only if this catalog owns the root again, and a dead CREATING row that `purge` already removed has its prefix kept. The read narrows the window to the gap between it and the deletion; on S3 that gap is a listing and a `DeleteObjects`, each under the SDK's retries, so seconds when the store throttles. A check that read the file before the adoption and ends after a fence's refusal records the store healthy again until the next cycle's check: each deletion stays fenced in that time, but a restore is accepted. A backup job of the previous owner that fails after the adoption leaves its partial prefix (no manifest, and no row of the new owner) under the root, which no duty or command of the new owner removes. A write that has already passed that read still lands, so stop every server of the catalog a file names before adopting its root. While the catalog has no AVAILABLE backup for a root, two host-local directories at one path cannot be told apart (each instance creates its own file and id, and a file copied from one directory to the other gives both one id), and the manifest probe has nothing to probe: a backup one instance completed under its own directory looks, to the other instance's reconciler, like a dead owner's row with no manifest, which it marks FAILED, and the first instance's restart then deletes the only copy. Point every instance at one shared root before the first backup; copying `extenddb-store.json` between directories defeats the check rather than satisfying it. The identity file is read up to 64 KiB; a larger object under its key is reported unreadable. +The store identity. The root carries `extenddb-store.json`: a random store id and the id of the catalog that owns the root. Startup creates it when the root has no file and the catalog has no rows for the root, and otherwise reads it and compares: a file naming another catalog, a store id that differs from the one the catalog's rows for the root carry, a root with no file while rows exist, or rows carrying more than one id each mark the store unhealthy. Instances that share a catalog must share one store root (one S3 bucket and prefix, or one shared mount): host-local roots configured at the same path give each instance different bytes under one root label, so each check also `head`s the manifest of the newest AVAILABLE backup and marks the store unhealthy when it is absent, which is what a second instance with a local copy of the directory sees. The file is left by `backup purge` and by every deletion; remove it only when retiring the root, or after `extenddb destroy`: a catalog re-created with `init` has a new id, and the old root's file names the old one, so the re-initialised server reports the store owned by another catalog until `extenddb backup sync --adopt-store` makes the new catalog its owner (below), the file is removed or a new root configured (on the default SQLite root, that is the logged warning above for the server, which keeps CreateBackup on the in-catalog path, while `backup sync` handles that root as configured, so `--adopt-store` adopts it). A root that has to change (a new volume, a new bucket) keeps its backups' rows when its tree is copied under the new root and `extenddb backup sync --move-store ` re-points them (below). The default root, `backups` beside the database file, is shared by every database file in that directory: `--adopt-store` on it takes the root from whichever catalog wrote its file, so give each database its own `[backup.store]` when several share a directory. A server on an in-memory SQLite database writes the file with `ephemeral_catalog: true`; at its next start it adopts a root whose file names another catalog and carries that marker, and no other, so it never takes a durable catalog's root, and then syncs the root's backups into its new catalog (the `path` row above: until that sync's log line, the earlier backups answer `BackupNotFoundException`). A catalog whose root another catalog adopted (by `--adopt-store`, or a second in-memory server's start) keeps acting on the root until a later check that can read the catalog finds it owned by another catalog, at the earliest one reconciler interval after the cycle in progress ends (a check whose catalog read fails leaves the health as it was), with one exception, its deletion fence: right before it deletes a backup's objects (DeleteBackup, the reconciler's deletions, a failed backup job's cleanup, `purge`), removes a FAILED or DELETED row and writes a CreateBackup's first object, and also before DeleteBackup and `purge` change the row, a server or `purge` re-reads `extenddb-store.json`, and when the file names another catalog, names this catalog under another store id than the one the store's check settled (a root replaced by another root of this catalog), or the root has none, it deletes nothing and refuses the request; a file naming another catalog also marks the store unhealthy (`owned by another catalog`) at once, and one of another store id `identity mismatch`. DeleteBackup takes a file it cannot read, or a root with no file, at the read before the row is changed as it takes a store that does not answer: it writes DELETING and answers `DELETED`, the read before the deletion refuses it, and the row stays DELETING for the reconciler, whose deletion runs the fence again. `--prune-missing`'s removal of a row is not fenced: it deletes no object. A refusal at the read before the row is changed (the file names another catalog or another store id) leaves the row as it was, so DescribeBackup still answers AVAILABLE. One at the read before the deletion (an adoption landed between the two) leaves an AVAILABLE row DELETING with nothing deleted (DeleteBackup and DescribeBackup there answer `DELETED`), which this catalog's reconciler finishes only if this catalog owns the root again, and a dead CREATING row that `purge` already removed has its prefix kept. The read narrows the window to the gap between it and the deletion; on S3 that gap is a listing and a `DeleteObjects`, each under the SDK's retries, so seconds when the store throttles. A check that read the file before the adoption and ends after a fence's refusal records the store healthy again until the next cycle's check: each deletion stays fenced in that time, but a restore is accepted. A backup job of the previous owner that fails after the adoption leaves its partial prefix (no manifest, and no row of the new owner) under the root, which no duty or command of the new owner removes. A write that has already passed that read still lands, so stop every server of the catalog a file names before adopting its root. While the catalog has no AVAILABLE backup for a root, two host-local directories at one path cannot be told apart (each instance creates its own file and id, and a file copied from one directory to the other gives both one id), and the manifest probe has nothing to probe: a backup one instance completed under its own directory looks, to the other instance's reconciler, like a dead owner's row with no manifest, which it marks FAILED, and the first instance's restart then deletes the only copy. Point every instance at one shared root before the first backup; copying `extenddb-store.json` between directories defeats the check rather than satisfying it. The identity file is read up to 64 KiB; a larger object under its key is reported unreadable. `extenddb destroy` removes the databases and leaves the store untouched: it is the copy that survives destroy. `extenddb backup list [--failed]` prints the catalog's rows for the configured store's root, every store id they carry, with each row's `store_root` and `store_id`; it is a read that runs beside a server (on SQLite it takes no serve lock), reads the identity file without creating one, and runs on an unhealthy store, naming the problem in its header. It lists backups only, never restore jobs: a restore's record is the job's log line, not a `backup list` row. `extenddb backup purge --confirm` deletes every backup of the configured store, each backup's objects (its own prefix, never a whole account prefix, so a prefix the catalog has no row for survives) and its catalog row outright, leaving no tombstone, and leaves `extenddb-store.json` and the `exports/` component; it needs the store healthy. It takes the catalog-level maintenance claim first and heartbeats it for the command's whole run, waits up to `2 * stale_job_after_secs` until no store-backed CreateBackup is running on any instance (a CREATING or FAILED row with a heartbeat within `stale_job_after_secs` on the catalog database's clock) and no restore is RUNNING, refuses and releases the claim if they have not finished, deletes a CREATING row the wait saw no live owner for as it deletes any other row, finishes any DeleteBackup left in progress as part of the purge (a backup a DeleteBackup finishes while the purge runs is counted as deleted, not left), and logs each deletion under the `extenddb::audit` target with the account and the ARN. While the claim is held, CreateBackup and RestoreTableFromBackup on every instance answer `LimitExceededException` saying that a backup maintenance command holds the catalog's maintenance claim (the message does not say which command; the second command's refusal and the claim's row do), and a second maintenance command fails at once naming the holder. Without `--confirm` it prints the plan, changes nothing and exits 0. `--config` is accepted before or after the subcommand. None of the backup commands runs the startup recovery a server runs (on PostgreSQL the vector index rebuild, the control-plane recovery and the GSI queue workers; on MongoDB the key-repair pass; on Cassandra the control-plane transitions; on SQLite the interrupted-restore and index passes): that work belongs to the servers, and a command that ran it could exit with it half done. RUNNING restores are read by status, with no age, so a restore whose record was stamped this instant holds the wait too. On SQLite `purge --confirm` opens the database as `serve` does and so takes the serve lock: stop the server first (while one runs on the file the command refuses with exit 6, as a second `serve` does); under that lock it first resolves what the stopped server left (a RUNNING restore is failed and its target removed, the CREATING target of a FAILED restore removed, an in-catalog CREATING backup removed, a DELETING row finished) and deletes a CREATING or FAILED row whatever its heartbeat says, a heartbeat stamped ahead of the clock by a backward clock step included (the server that wrote it is stopped, so the zero window the command passes selects by status alone), so such a job never makes it wait or refuse. That includes a RUNNING restore from a backup store, which a server would resume: under the serve lock the command fails it and removes its target, as for an in-catalog restore (a resume is a job, which the maintenance claim excludes), except one that recorded no progress while its target holds items, which is failed `target_not_empty` and its target left for `extenddb backup remove-target`. A server starting on the file removes a claim a killed purge left, so no job is refused for the stale window, as a purge run again after the killed one takes its claim over. On the other backends it runs beside live servers, and a claim whose holder died expires after `stale_job_after_secs`. On PostgreSQL, MongoDB and Cassandra the backup commands (`list`, `purge`, and `sync` and `remove-target` below) open the catalog as a server does and hold a serving registration for their run, named after the command (`extenddb backup list`, `extenddb backup purge`, `extenddb backup sync`, `extenddb backup remove-target`: on PostgreSQL the session's `application_name`, on MongoDB the `command` field of its `extenddb_servers` document, on Cassandra the `release` column of its `extenddb_servers` row, after the release and ` as `), and release it when they exit: `extenddb migrate --yes` waits for them as for a server and names them while it waits, and while a migration runs they refuse at once (exit 6). On a catalog `serve` would refuse they exit with `serve`'s code and message: 3 with migrations pending (with the `extenddb migrate --yes` advice), 4 on a catalog a later release migrated with a breaking migration, and 5 on one that needs attention (the upgrade manual's [catalog status](07-upgrade-manual.md#the-catalog-status) table). On every backend Ctrl-C or `SIGTERM` makes the command release what it holds and exit: the serving registration after the open, if the signal came during it; the claim at once while it waits, and before its next backup while it deletes (the release itself is bounded by 10 s, past which the claim is left to the stale rule), so an interrupted purge leaves no half-deleted backup and no claim to wait out; run it again for the rest. The dry run on an unhealthy store lists the rows and says that `--confirm` refuses until the store's check passes. -`extenddb backup sync [--adopt-store] [--prune-missing]` rebuilds the catalog's rows from the configured store: after a lost or re-created catalog, or after a backup's prefix was copied under the root from another deployment or another backend. It reads the catalog's rows for the root (of every status and every store id they carry), then lists the whole root and handles each `/
//` prefix as the listing passes it, checking it against those rows before it reads anything under it. Sync holds one prefix's object list at a time and the prefix of each manifest it saw; a filesystem store builds its listing of the whole root in memory first (about 72 bytes plus the key per object), an S3 store lists a page at a time. The report keeps an entry per prefix for the whole run, and it lists every prefix the catalog already has a row for among the skipped ones, so on a root of small backups the per-prefix term dominates: about 350 bytes per prefix at one object per prefix (62 MiB of peak RSS at 50,000 such prefixes and 112 MiB at 200,000, against 45 MiB for an empty root; about 165 bytes per object at 100 objects per prefix), measured on a SQLite debug build with a filesystem root and no rows. Of the catalog's rows it keeps each row's status, store id and table, and the key of each AVAILABLE row of the configured store, but every backend reads the rows whole before sync sees the first, so the read is the larger cost and grows with the rows of the root, tombstones within their retention included: about 22.5 KiB per row of a table with five GSIs (1,452 bytes of source table description per row; 610 MiB of peak RSS at 25,000 rows and 1,144 MiB at 50,000, against 47 MiB with none; 148 MiB at 50,000 rows whose description is `{}`), measured on a SQLite debug build on a 16-core x86_64 Linux host with glibc's default arenas. `backup list` peaks the same over those rows. A prefix with no row gets an AVAILABLE row under the configured store's identity when its manifest and sidecar verify, every data file the manifest lists is under the prefix at its recorded size, and the manifest's ARN names the prefix's account, table and backup id; an ARN of another region is adopted and logged (requests find it by its ARN as written), and one of another partition than `aws` is skipped, since every request refuses such an ARN. The row carries the manifest's backup name, size, item count and source table, and the backup id's instant as its creation time. It never adopts a prefix with a row of any status (CREATING, DELETING, DELETED, FAILED or AVAILABLE), one whose manifest, sidecar or data files are missing or do not match, one whose manifest names another backup, or anything under `exports/` or a table's `changelog/`, and prints each with its reason. Before the insert it reads the row under both spellings of the ARN (this server's region and the manifest's), and the insert happens only if no row exists: a CreateBackup whose row landed after sync read the rows keeps its row, as does a row of another root, account or table holding the backup id, and sync changes no existing row. It also prints the AVAILABLE rows whose manifest the listing does not hold. It exits 0 once it ran, skipped prefixes included, and 1 when a prefix was skipped because a store read or a catalog call failed (the run is incomplete; a second run adopts it), when the listing failed part-way (the backups adopted before the failure keep their rows), when a signal stopped it, or when `--prune-missing` left a row it may remove on a later run (a store or catalog call failed, its manifest's `head` included, or a restore is reading the backup: run the command again once the restore ends). A row `--prune-missing` leaves because its manifest is back or another writer moved it is reported, and the exit stays 0: +`extenddb backup sync [--adopt-store] [--prune-missing]` (or with one of `--move-store `, `--forget-store ` and `--forget-backup `, below) rebuilds the catalog's rows from the configured store: after a lost or re-created catalog, or after a backup's prefix was copied under the root from another deployment or another backend. It reads the catalog's rows for the root (of every status and every store id they carry), then lists the whole root and handles each `/
//` prefix as the listing passes it, checking it against those rows before it reads anything under it. Sync holds one prefix's object list at a time and the prefix of each manifest it saw; a filesystem store builds its listing of the whole root in memory first (about 72 bytes plus the key per object), an S3 store lists a page at a time. The report keeps an entry per prefix for the whole run, and it lists every prefix the catalog already has a row for among the skipped ones, so on a root of small backups the per-prefix term dominates: about 350 bytes per prefix at one object per prefix (62 MiB of peak RSS at 50,000 such prefixes and 112 MiB at 200,000, against 45 MiB for an empty root; about 165 bytes per object at 100 objects per prefix), measured on a SQLite debug build with a filesystem root and no rows. Of the catalog's rows it keeps each row's status, store id and table, and the key of each AVAILABLE row of the configured store, but every backend reads the rows whole before sync sees the first, so the read is the larger cost and grows with the rows of the root, tombstones within their retention included: about 22.5 KiB per row of a table with five GSIs (1,452 bytes of source table description per row; 610 MiB of peak RSS at 25,000 rows and 1,144 MiB at 50,000, against 47 MiB with none; 148 MiB at 50,000 rows whose description is `{}`), measured on a SQLite debug build on a 16-core x86_64 Linux host with glibc's default arenas. `backup list` peaks the same over those rows. A prefix with no row gets an AVAILABLE row under the configured store's identity when its manifest and sidecar verify, every data file the manifest lists is under the prefix at its recorded size, and the manifest's ARN names the prefix's account, table and backup id; an ARN of another region is adopted and logged (requests find it by its ARN as written), and one of another partition than `aws` is skipped, since every request refuses such an ARN. The row carries the manifest's backup name, size, item count and source table, and the backup id's instant as its creation time. It never adopts a prefix with a row of any status (CREATING, DELETING, DELETED, FAILED or AVAILABLE), one whose manifest, sidecar or data files are missing or do not match, one whose manifest names another backup, or anything under `exports/` or a table's `changelog/`, and prints each with its reason. Before the insert it reads the row under both spellings of the ARN (this server's region and the manifest's), and the insert happens only if no row exists: a CreateBackup whose row landed after sync read the rows keeps its row, as does a row of another root, account or table holding the backup id, and sync changes no existing row. It also prints the AVAILABLE rows whose manifest the listing does not hold, and, once the listing has reached its end, the rows of the other stores (below). It exits 0 once it ran, skipped prefixes included, and 1 when a prefix was skipped because a store read or a catalog call failed (the run is incomplete; a second run adopts it), when the listing failed part-way (the backups adopted before the failure keep their rows), when a signal stopped it, when the rows of the other stores could not be read, or when `--prune-missing` left a row it may remove on a later run (a store or catalog call failed, its manifest's `head` included, or a restore is reading the backup: run the command again once the restore ends). A row `--prune-missing` leaves because its manifest is back or another writer moved it is reported, and the exit stays 0: ``` === extenddb backup sync === @@ -190,10 +190,32 @@ Store: filesystem (store id ) --- AVAILABLE rows whose manifest is not in the store (the rows were AVAILABLE when the catalog was read): (none) +--- Rows of other stores (left as they are; sync reads and changes none of them): + filesystem /srv/backups/old (store id ): 41 AVAILABLE, 1 DELETING, 2 DELETED + DELETING (account ) + (to keep a root's backups, copy its tree under the configured root and run extenddb backup sync --move-store ; to drop its rows, extenddb backup sync --forget-store ) + === extenddb backup sync complete: 1 backup(s) adopted, 1 prefix(es) skipped, 0 AVAILABLE row(s) whose manifest is gone === ``` -Sync needs the store healthy: on an unhealthy store it refuses, exit 1, naming the problem and changing no row. That covers an empty root under existing rows, a root whose file carries another store id and a root without the newest AVAILABLE row's manifest, which is what an unmounted volume shows. When the newest AVAILABLE row's own manifest is gone, the store is unhealthy for the same reason, so plain sync and `--prune-missing` both refuse; put the prefix (or its manifest) back and run the command again. On PostgreSQL, MongoDB and Cassandra every sync takes the maintenance claim and heartbeats it, so no `purge` runs between its read of the rows and its insert; plain sync waits for no job, and while it holds the claim CreateBackup and RestoreTableFromBackup are refused, as during `purge`. A sync killed while it holds the claim (`SIGKILL`, a lost host) leaves it held until its heartbeat is `stale_job_after_secs` old (300 s by default): until then every instance refuses CreateBackup and RestoreTableFromBackup. On PostgreSQL the claim is a session's advisory lock, so the next maintenance command (the sync or `purge` run again) takes a killed command's claim over at once and ends that window; MongoDB and Cassandra take it over only once it is stale. `--prune-missing` and `--adopt-store` also wait for running jobs as `purge` does. A first Ctrl-C or `SIGTERM` stops any sync before its next prefix (or the wait at once), releases the claim it holds and exits 1; the backups adopted before the stop keep their rows. `--prune-missing` removes each AVAILABLE row whose manifest is gone outright, one line per row, and deletes nothing in the store: no DELETING or DELETED row is left for a deletion duty to act on, so on host-local roots (each instance's root at one path holding other bytes) a prune on the instance that lacks a backup's bytes removes its row for every instance of the catalog and leaves the bytes on the other host: the backup answers `BackupNotFoundException` on every instance, the one holding the bytes included, until a sync on that instance adopts them again. A row a restore is reading, or one another writer moved meanwhile, is reported and left (the first exits 1). The objects a removed row's prefix still holds stay until an operator deletes the prefix by hand: `purge` deletes only the prefixes of rows, and every later sync reports the prefix as not a complete backup. `--adopt-store` is the way a durable catalog takes a root whose `extenddb-store.json` names another catalog (plain sync refuses such a root, naming the option; a server on an in-memory database adopts, at startup, a root another in-memory catalog wrote, as the identity paragraph says): under the claim it logs its intent under `extenddb::audit`, rewrites the file's catalog id and backend, keeping the store id and creation time, reads the file back and logs the outcome (adopted, or the race lost when another adopter's write landed between the two), syncs, and then reads the file once more and fails, naming the owner, when another catalog adopted the root during the sync. A write that lands after both reads is found by this catalog's next check. It refuses when this catalog's rows for the root carry another store id, and on a root this catalog already owns it says so and syncs. Stop every server of the catalog the file names before running it: the previous owner finds the store owned by another catalog only at a later check that can read its catalog, and until then its deletion fence (the identity paragraph) refuses each deletion and CreateBackup once it re-reads the file, but a write already past that read lands. A second maintenance command while one holds the claim fails at once. On SQLite every form takes the serve lock: stop the server first (beside a running server it exits 6); under the lock it first resolves what the stopped server left, as `purge` does. The rows of other roots are reported in part: a prefix whose backup id a row of another root holds is skipped naming that root; rows of other roots as such are not listed. +Sync needs the store healthy: on an unhealthy store it refuses, exit 1, naming the problem and changing no row. That covers an empty root under existing rows, a root whose file carries another store id and a root without the newest AVAILABLE row's manifest, which is what an unmounted volume shows. When the newest AVAILABLE row's own manifest is gone, the store is unhealthy for the same reason, so plain sync and `--prune-missing` both refuse; put the prefix (or its manifest) back and run the command again, or, when that backup is gone for good, remove its row with `--forget-backup ` (below), which runs on the unhealthy store. On PostgreSQL, MongoDB and Cassandra every sync takes the maintenance claim and heartbeats it, so no `purge` runs between its read of the rows and its insert; plain sync waits for no job, and while it holds the claim CreateBackup and RestoreTableFromBackup are refused, as during `purge`. A sync killed while it holds the claim (`SIGKILL`, a lost host) leaves it held until its heartbeat is `stale_job_after_secs` old (300 s by default): until then every instance refuses CreateBackup and RestoreTableFromBackup. On PostgreSQL the claim is a session's advisory lock, so the next maintenance command (the sync or `purge` run again) takes a killed command's claim over at once and ends that window; MongoDB and Cassandra take it over only once it is stale. `--prune-missing` and `--adopt-store` also wait for running jobs as `purge` does. A first Ctrl-C or `SIGTERM` stops any sync before its next prefix (or the wait at once), releases the claim it holds and exits 1; the backups adopted before the stop keep their rows. `--prune-missing` removes each AVAILABLE row whose manifest is gone outright, one line per row, and deletes nothing in the store: no DELETING or DELETED row is left for a deletion duty to act on, so on host-local roots (each instance's root at one path holding other bytes) a prune on the instance that lacks a backup's bytes removes its row for every instance of the catalog and leaves the bytes on the other host: the backup answers `BackupNotFoundException` on every instance, the one holding the bytes included, until a sync on that instance adopts them again. A row a restore is reading, or one another writer moved meanwhile, is reported and left (the first exits 1). The objects a removed row's prefix still holds stay until an operator deletes the prefix by hand: `purge` deletes only the prefixes of rows, and every later sync reports the prefix as not a complete backup. `--adopt-store` is the way a durable catalog takes a root whose `extenddb-store.json` names another catalog (plain sync refuses such a root, naming the option; a server on an in-memory database adopts, at startup, a root another in-memory catalog wrote, as the identity paragraph says): under the claim it logs its intent under `extenddb::audit`, rewrites the file's catalog id and backend, keeping the store id and creation time, reads the file back and logs the outcome (adopted, or the race lost when another adopter's write landed between the two), syncs, and then reads the file once more and fails, naming the owner, when another catalog adopted the root during the sync. A write that lands after both reads is found by this catalog's next check. It refuses when this catalog's rows for the root carry another store id, and on a root this catalog already owns it says so and syncs. Stop every server of the catalog the file names before running it: the previous owner finds the store owned by another catalog only at a later check that can read its catalog, and until then its deletion fence (the identity paragraph) refuses each deletion and CreateBackup once it re-reads the file, but a write already past that read lands. A second maintenance command while one holds the claim fails at once. On SQLite every form takes the serve lock: stop the server first (beside a running server it exits 6); under the lock it first resolves what the stopped server left, as `purge` does. Sync never changes a row of another store (another kind or root, or the configured root under another store id): a prefix whose backup id such a row holds is skipped naming that row's status and store, and the report's section of other stores, printed once the listing has reached its end and skipped when the run stopped, gives each other store with its rows per status and each of its CREATING and DELETING rows by ARN (the rows the reconciler of the configured store leaves alone), then the two options that act on them; on Cassandra the summary counts status entries, which a write that stopped part-way, a status change whose entry batch failed, or a deleter whose entry batch landed after a move of its row, can leave with no row of their status behind them, so each count is checked against the root's listing (which finds a row whatever status its entry is under, and counts it under its own) and what no row of its status stands behind is printed as status entries (with no row of the root left, `--forget-store ` removes them). + +`extenddb backup sync --move-store ` re-points the rows of a root whose tree was copied under the configured root, so the backups keep their ARNs (the backup manual's [Moving a store to a new root](14-backup-and-restore.md#moving-a-store-to-a-new-root) gives the procedure: reconfigure every server first, copy the tree without the old root's `extenddb-store.json`, which would turn every running server's store unhealthy, and finish the copy before the move). The old root is given as the catalog stores it and as the report of other stores prints it, and matches byte for byte (a trailing `/`, or an S3 root's `?endpoint=` suffix, makes another root); a root with no rows is refused, printing the stores the catalog holds rows for, and so is the configured root. It needs the configured store healthy, takes the maintenance claim on every backend (on SQLite under the serve lock as well) and waits, as `purge` does, for the running jobs of the configured store and of every store id the catalog records for the old root. It then checks each row of the old root against the configured store: a CREATING row whose owner is dead is resolved AVAILABLE when the copy holds its complete backup, and otherwise stops the move with no status written (the copy proves nothing about the old root's prefix, and a FAILED row would have the reconciler delete the copy: a reconciler on the old root resolves it there, or `--forget-backup` removes it), and one with a live owner stops the move; a DELETING row whose copied prefix holds no verified manifest has its deletion finished in the configured store, behind the deletion fence; a DELETING, DELETED or FAILED row whose copied prefix holds a verified manifest stops the move, naming `--forget-backup`, since moving the tombstone would have the reconciler delete a complete backup; an AVAILABLE row needs its prefix complete under the configured root (the manifest and its sidecar, the row's ARN, every data file at its recorded size, the row's size and item count); every row of the old root that would give an account two rows of one backup id (with a row of the configured root, or, on Cassandra, with another spelling of its ARN among the moved rows) stops it, all of them set aside before any row is checked, so none of them is resolved or has its deletion finished. Every row that stops the move is printed with its reason in one run, and while any does no row moves and the command exits 1 (the rows resolved AVAILABLE and the finished deletions keep their new status; a second run sees them). Otherwise the identity file is re-read once, so no row moves into a root another catalog adopted since the check, the rows' store is rewritten (only rows still of the old root and not CREATING), each logged under `extenddb::audit`, the old root is listed again (a row still there, as a CreateBackup of a server left on the old root leaves it, exits 1: its prefix was written after the copy, so reconfigure that server, copy the prefix under the new root or remove the row with `--forget-backup`, and run the move again; a row is counted as moved only when the configured store's listing holds it, and one gone from both is reported as gone), and a plain sync of the configured root follows with its report; a row of the old root in that report's section of other stores is named and the command exits 1 ("a row landed under the old root during the move"). A CreateBackup reads the maintenance claim once more right before its row's insert and a restore right before its registration, so one whose read comes after the move took the claim lands no row; one whose read came before lands its row whenever its insert does, from that read on: during the move's wait (which lists it), before the rewrite (which skips CREATING rows, and the re-list reports it), during the move's sync (its closing report names it if the insert lands before that report), or after the command (only a later plain sync reports it). Before deleting the old root, run plain sync and check that its report of other stores names no row of it. Nothing under the old root is read or changed, and no object is copied; the reconciler then cleans the leftovers of the moved FAILED and DELETED rows under the configured root. Its peak memory is the read of the old root's rows, as for sync, about 23 KiB per row on PostgreSQL and 27 on SQLite, above about 50 MiB, at 1,452 bytes of table description per row (the figure grows with the description): 569 MiB at 20,000 rows of a table with five GSIs on a SQLite debug build and 500 MiB on a PostgreSQL debug build (51 and 49 MiB at 50 rows), 37 s each, on the 16-core host above with filesystem roots on a local disk. + +`--forget-store ` removes every row of a root other than the configured one outright, of any status, leaving no DELETED tombstone; if the root is configured again later, sync adopts its backups again. `--forget-backup ` does the same for one backup of any status and any store, and can be given more than once (the ARNs are taken in order, up to the first that is refused; an ARN no row and no in-catalog backup carries is reported as having nothing to remove and counts as done, so a command line can be run again after a stop; every ARN's form is checked before the claim is taken). The removals delete no object and read no store, so both run on a store in any state and with no store configured (`kind = "disabled"` included); both take the maintenance claim on every backend and wait for running jobs as `purge` does (over the configured root's store ids too when its identity is not known). They open the store writing nothing to it: the identity file is read, never created, and when it agrees with the rows the store's check runs once without writing (no `validate` probe file). A reconciler's FAILED-prefix, DELETED-prefix and tombstone duties read the claim again right before each deletion and start none while it is held; each deletion that read it free ends 30 s after that read was sent (a read answered later starts nothing, and the store checks the deadline before each removal, or each request on S3), and on PostgreSQL, MongoDB and Cassandra the command waits 60 s from taking the claim (the 30 s and one store call's 30 s) before its first removal of a FAILED or DELETED row, printing the wait; on SQLite the serve lock excludes every server and nothing waits. What a duty can still do to a removed row's bytes is the one removal, or one S3 request (up to 1,000 keys), in flight at its deadline, and only a server process paused past the wait (`SIGSTOP`, a frozen container, a VM pause) between its last check and that step runs it after the removal: the bound, like every lease of the design, assumes the server keeps running. A DELETING row of the configured store is the exception: before removing it the command waits as for a DELETED row, then finishes its deletion behind the deletion fence, the deletion its DeleteBackup asked for, which needs the store's check to have passed (a stop during the wait leaves the row DELETING and its objects untouched); with the store unhealthy or its id unknown the row is refused ("let a server finish the DeleteBackup, or run again once the store's check passes"). When the check failed on an identity mismatch that the row's own store id takes part in (the backup manual's copied `extenddb-store.json`), the refusal says so and names the way out: run the same command with `[backup.store] kind = "disabled"`, which removes the row and keeps its bytes, then, with the store configured again, let a sync (the move's own, in the manual's procedure) adopt the backup once the check passes, and DeleteBackup it again. The per-row line says when the command deleted the objects (on SQLite the command's own pass over what the stopped server left may finish it first, under the claim, and the line says so too). A DELETING row of another store keeps its bytes, and a sync of that root after its return adopts the backup again. Because they wait for running jobs first, as `purge` does, a CREATING or FAILED row whose job is live holds the wait, and the command refuses if it is still live when the wait ends; a row whose heartbeat, written after the wait, is within `stale_job_after_secs` (its job perhaps still writing under the prefix), or one of a store the wait does not cover (a `--forget-backup` of another store's row), is left, as is a row a restore target is being restored from (as DeleteBackup refuses it: wait for the restore to end, or, once it failed or its owner is dead, remove the target with `extenddb backup remove-target --account
`), and the command exits 1 naming each. `--forget-store` matches the root as `--move-store` does and refuses the configured root (`purge` and `--prune-missing` are the configured store's removals); on Cassandra it then sweeps the root's status entries under the claim, each checked against its row: one whose row is gone or names another store (a deleter whose entry batch landed after a move of its row, or the earlier-status entry of a row removed above) is deleted, and one whose row is of the root under another status (a status change whose entry batch failed, or whose writer died before it) is written again under the row's status; it reports both counts, and then the store ids the catalog still records for the root (entries written after the root was listed, by a writer still configured with it). `--forget-backup` takes the ARN as the row carries it: a row under another spelling of the ARN (another region) is named and left, and an earlier release's in-catalog backup is reported as one with nothing changed (DeleteBackup removes it). While the configured root has no `extenddb-store.json` and the catalog holds rows for it (an unmounted volume, say), or, on SQLite, the default root is missing while the catalog holds rows for it (moved away, or a symbolic link to a volume that is gone; the command's header then says so), `--forget-backup` refuses every row of that root and changes nothing for it ("mount it, or point `[backup.store]` at the root that holds these backups, then run again"): a running server's check creates the file at a root the catalog has no rows for and reports the store healthy, so removing the root's last rows would have the next CreateBackup write to the empty mount point. After `--forget-backup`, when a store is configured, the store's check runs once, writing nothing to the root (no identity file is created; when the root had no `extenddb-store.json` as the command opened it, the check is not run and the command says so), and its outcome is printed: the check passing, or the problem it finds, which for a missing manifest names the next backup and the command for it; the exit stays 0 once the rows were removed. That is the way out of the unhealthy store a restored catalog is left with when its newest AVAILABLE backup was deleted after the catalog's backup: remove that row, and the next one the check names, then run `--prune-missing`; each running server finds the store healthy at its own next check. The three options conflict with each other, with `--adopt-store` and with `--prune-missing` (exit 2), and each row they change is logged under `extenddb::audit`: + +``` +=== extenddb backup sync === +Store: filesystem (unhealthy: ; the removals read no store; the check after them does) +Took the maintenance claim as "extenddb backup sync on (pid )". + +--- --forget-backup (rows removed outright; no object is deleted, but a DELETING row of the configured store has its deletion finished first): + : removed: the row (AVAILABLE) is gone, and no object was deleted + +=== extenddb backup sync --forget-backup complete: 1 row(s) removed, 0 not === + +The configured store's check passes (store id ); each server finds it so at its next reconciler cycle. +``` Every `extenddb backup` command refuses an in-memory SQLite database (`[storage.sqlite] path = ":memory:"`, or no path on a `sqlite-memory` build) before it opens the catalog, exit 1: it would open a new, empty catalog that ends with the command. A server on such a database syncs with its store at every start instead (the `path` row above). diff --git a/docs/manuals/11-deployment-guide.md b/docs/manuals/11-deployment-guide.md index 35b506768..c217a2ae9 100755 --- a/docs/manuals/11-deployment-guide.md +++ b/docs/manuals/11-deployment-guide.md @@ -295,7 +295,7 @@ Run one extenddb instance per catalog. Multiple instances behind a load balancer - Credentials, policies, and table metadata are cached in each process (see the cache TTLs in the admin guide). Nothing invalidates a cache on another instance, so a key revoked or a policy changed through one instance stays in force on the others until the TTL expires. - Every instance runs every background worker (TTL expiry, GSI backfill, stream retention, control-plane transitions). The workers are safe to run twice, but the work is duplicated. - `/health` is a liveness check that does not query the database, so a load balancer cannot use it to take an instance with a broken database connection out of rotation. It does report the backup store: `components.backup_store` is `unhealthy` and `status` is `degraded` while the configured store cannot be reached or fails its identity check, with HTTP 200 kept, so a store outage never drains a replica. -- Instances that share a catalog must share one backup store root (`[backup.store]`: one S3 bucket and prefix, or one shared mount that supports hard links, which the store's conditional create needs; some CIFS and FUSE mounts refuse `link(2)` with `EPERM`, and the server then reports the store as having failed validation, with the `link` error in the log line). A host-local directory configured at the same path on each host is not one root: once the catalog has rows for the root, the second instance finds the root's identity file missing or the newest backup's manifest absent and starts with the store unhealthy. While the catalog has no rows for the root (a fresh deployment, or one whose backups were purged), nothing tells the two directories apart: each instance creates its own identity file and store id and both start healthy, and the first backups then split the rows between two ids, which the next check reports as an identity mismatch that only `backup purge` resolves. Point every instance at one root before the first backup ([Backup and Restore manual](14-backup-and-restore.md#configuration)). Each instance on a host needs its own `spool_path`; the spool's `.lock` refuses a second instance. An existing spool directory must be owned by the server's user with mode `0700`, as the root must; a spool created `0755` stops startup naming it. +- Instances that share a catalog must share one backup store root (`[backup.store]`: one S3 bucket and prefix, or one shared mount that supports hard links, which the store's conditional create needs; some CIFS and FUSE mounts refuse `link(2)` with `EPERM`, and the server then reports the store as having failed validation, with the `link` error in the log line). A host-local directory configured at the same path on each host is not one root: once the catalog has rows for the root, the second instance finds the root's identity file missing or the newest backup's manifest absent and starts with the store unhealthy. While the catalog has no rows for the root (a fresh deployment, or one whose backups were purged), nothing tells the two directories apart: each instance creates its own identity file and store id and both start healthy, and the first backups then split the rows between two ids, which the next check reports as an identity mismatch that only `backup purge` resolves. Point every instance at one root before the first backup ([Backup and Restore manual](14-backup-and-restore.md#configuration)). To change the root later without losing the backups, reconfigure every instance to the new root, copy the old root's tree there without its `extenddb-store.json` (the new root has its own once the instances restart) and run `extenddb backup sync --move-store ` ([Moving a store to a new root](14-backup-and-restore.md#moving-a-store-to-a-new-root)). Each instance on a host needs its own `spool_path`; the spool's `.lock` refuses a second instance. An existing spool directory must be owned by the server's user with mode `0700`, as the root must; a spool created `0755` stops startup naming it. On SQLite this limit is enforced: `extenddb serve` holds an exclusive lock on `.lock` and a second server on the same file refuses to start. On PostgreSQL, MongoDB, and Cassandra it is not enforced. On PostgreSQL, if you start a second instance anyway, size `pool_size × instance_count + 3 × instance_count ≤ PostgreSQL max_connections`. diff --git a/docs/manuals/14-backup-and-restore.md b/docs/manuals/14-backup-and-restore.md index 7f3424abf..45abecc66 100644 --- a/docs/manuals/14-backup-and-restore.md +++ b/docs/manuals/14-backup-and-restore.md @@ -236,7 +236,10 @@ root before the first backup, and never copy `extenddb-store.json` between roots the old root's `extenddb-store.json` naming the old catalog; `extenddb backup sync --adopt-store` makes the new catalog its owner, and until then the store is unhealthy, or, on file-backed SQLite with the default root, unconfigured, with `CreateBackup` writing in-catalog backups ([The catalog is lost and -re-created](#the-catalog-is-lost-and-re-created)). Or configure a new root. +re-created](#the-catalog-is-lost-and-re-created)). Or configure a new root. A root that has to change while the +catalog stays (a new volume, a new bucket) keeps its backups when its tree is copied under the new root and +`extenddb backup sync --move-store ` re-points the rows ([Moving a store to a new +root](#moving-a-store-to-a-new-root)). `/health` reports the store as the `backup_store` component, for example `{"components":{"backup_store":{"kind":"filesystem","status":"healthy"}},"status":"healthy"}` (on MongoDB @@ -460,9 +463,23 @@ store. When the newest of those rows is the catalog's newest `AVAILABLE` backup, check looks for that backup's manifest, and `/health` reports `"reason":"manifest probe failed"`): both commands refuse ("sync changes no row while the store is unhealthy"), as do `CreateBackup` and the restore and delete of every backup in that store, and the state survives a restart. Putting that backup's prefix back from -a copy clears it. This release has no command that clears it without such a copy: `--prune-missing` refuses -while the store is unhealthy. To stay out of this state, take each catalog backup right after the deletions -your scheduler makes, so that the newest backup a catalog backup names is not one deleted later. +a copy clears it. When the backup is gone for good, remove its row: + +```bash +extenddb backup sync --forget-backup --config extenddb.toml +``` + +with the ARN the refusal and the server's log line name (`extenddb backup list` lists it too). First check that +the root is the one that holds the backups: an unmounted volume, or a root that is not shared by every server, +shows the same refusal, and forgetting its rows one by one ends with an empty mount point that a server's next +check takes as a new root. The command runs on the unhealthy store, removes that row outright and deletes no +object, then runs the store's check once, writing nothing to the root, and prints it: "The configured store's +check passes", or the next `AVAILABLE` row whose manifest is gone, with the same command for it; repeat for that one (the option can be given more than +once, and an ARN whose row is already gone counts as done, so the same command line can be run again). Once the check +passes, `extenddb backup sync --prune-missing` removes the older rows whose manifest is gone, and each running +server finds the store healthy at its next reconciler cycle. To stay out of this state, take each catalog +backup right after the deletions your scheduler makes, so that the newest backup a catalog backup names is not +one deleted later. Tables backed up after the copy can be restored, under their own names or others, once `backup sync` lists their backups; their writes after their last backup are lost. @@ -535,7 +552,8 @@ In both shapes plain `backup sync` refuses (exit 1, "owned by another catalog"). On SQLite every `backup sync` needs the server stopped, since it opens the database as the server does. On PostgreSQL, MongoDB and Cassandra it runs beside servers and holds the catalog's maintenance claim while it runs, during which `CreateBackup` and `RestoreTableFromBackup` on every instance answer -`LimitExceededException`; `--adopt-store` and `--prune-missing` first wait, as `purge` does, until no backup +`LimitExceededException`; `--adopt-store`, `--prune-missing`, `--move-store`, `--forget-store` and +`--forget-backup` first wait, as `purge` does, until no backup or restore job is running and no backup failed within `stale_job_after_secs` (a job that failed may still be writing to its prefix), at most `2 * stale_job_after_secs`, and then refuse. Sync exits 0 when it completed and 1 when it refused or skipped a prefix for a store or catalog error (run it again). It exits 6 when it @@ -569,6 +587,119 @@ only after the old one has exited. A root that a durable catalog owns is never a Every `extenddb backup` command refuses an in-memory database (exit 1), since it would open a new, empty catalog that ends with the command. +## Moving a store to a new root + +Each backup's row names its store's root as the catalog stores it (the canonical directory of a filesystem +store; `s3:///` for S3, with `?endpoint=` when an endpoint is set), so a tree copied to +a new root comes with no rows: configured with the new root, `extenddb backup sync` skips every copied prefix, +naming the row of the old store that holds it, and lists the old root's rows under "Rows of other stores". +`--move-store` re-points those rows at the configured store, and the backups keep their ARNs: + +1. Reconfigure every server of the catalog to the new root, on every host, and restart it (on SQLite stop it: + the move needs the serve lock). A server left on the old root keeps writing its backups there after the + move, and nothing detects it beyond the next sync's report of other stores. +2. Copy the old root's tree under the new root, whole and with the same relative paths, as the server's user, + keeping directories `0700` and files `0600`, and leave the old root's `extenddb-store.json` out of the + copy: the servers restarted in step 1 gave the new root its own (`rsync -a --exclude=/extenddb-store.json + /srv/backups/old/ /srv/backups/new/`; for S3, `aws s3 sync --exclude extenddb-store.json` between the two + prefixes). Finish the copy before the move: the move checks the copy once, and the reconciler later deletes + what the copy brought of the moved `FAILED` and `DELETED` backups. If the old root's `extenddb-store.json` + was copied over the new root's, every running server's store turns unhealthy (an identity mismatch). With + no backup taken under the new root since the restart, the move still runs; restart every server after it. + With one taken (a server kept serving between the steps), the move refuses with the same mismatch: + `extenddb backup list` names those backups under the new root's first store id; remove each row with + `extenddb backup sync --forget-backup ` (its bytes stay), run the move, whose sync adopts them again, + and restart every server. A `DELETING` row among them (a `DeleteBackup` that had not finished) is refused, + naming its store id as one the identity check refuses: its deletion waits on a check that cannot pass while + the copied file and that row's id disagree. Remove it with the same command run with `[backup.store] kind = + "disabled"` (its bytes stay), configure the new root again, run the move, whose sync adopts that backup + too, restart every server, and `DeleteBackup` it again. +3. With the configuration naming the new root, run: + + ```bash + extenddb backup sync --move-store /srv/backups/old --config extenddb.toml + ``` + + Give the old root byte for byte, as `extenddb backup list` printed it before the change or as plain sync's + report of other stores prints it. A root that differs by a trailing `/`, or by an S3 root's `?endpoint=` + suffix, has no rows; the command says so and lists the roots the catalog holds rows for. + +The move needs the configured store healthy. It takes the maintenance claim, waits for the running jobs of the +configured store and of the old root, and checks every row of the old root against the copy: + +- an `AVAILABLE` row needs its prefix complete under the new root: the manifest and its sidecar, the row's ARN, + every data file at its recorded size, and the row's size and item count; +- a `CREATING` row whose job is dead is resolved `AVAILABLE` when the copy holds its complete backup. A copy + without the complete backup stops the move and the row is left `CREATING`: a copy that lacks the manifest + or a data file says nothing about the old root's prefix, and a `FAILED` row would have the reconciler delete + the copy. Run a server configured with the old root (on SQLite, plain `extenddb backup sync` there), whose + reconciler resolves the row against the old root, then copy that prefix again; or remove the row with + `--forget-backup`. A `CREATING` row whose job still runs stops the move; +- a `DELETING` row has its deletion finished under the new root, which deletes what the copy brought of it; +- a `DELETING`, `DELETED` or `FAILED` row whose copied prefix holds a complete backup stops the move: moving + that row would have the reconciler delete the backup. + +Every row that stops the move is named with its reason in one run, and no row moves while any does (exit 1): + +``` +--- Stopping the move (no row is moved while any is listed here): + arn:aws:dynamodb:us-east-1::table/orders/backup/ (AVAILABLE, account ): not a complete copy under the configured root: data file data/000001.json.gz is not in the store; copy the prefix again (or remove the row with --forget-backup) +``` + +Copy what is missing and run the command again, or remove the row with `extenddb backup sync --forget-backup +` and run the move again; the prefix is then reported as an incomplete backup by every sync until the +missing file is copied, and the next sync adopts it. With no row stopping it, the move rewrites the rows, +logs each under `extenddb::audit`, and syncs the new root: + +``` +=== extenddb backup sync --move-store complete: 3 row(s) moved, 0 resolved, 0 deletion(s) finished, 0 row(s) stopping the move, 0 gone during the move, 0 left under the old root === +``` + +`DescribeBackup`, `ListBackups` and restores of the moved backups then work against the new root. A row still +under the old root after the rewrite (a `CreateBackup` of a server left on the old root) is listed and the +command exits 1: its prefix was written after the copy, so reconfigure that server, copy its prefix under the +new root (or `--forget-backup` it), and run the move again. A move interrupted after its rewrite (a signal, a +lost connection) has moved its rows: run plain `extenddb backup sync`, since a second `--move-store` then +finds no row under the old root and says so. A row of the old root in the report of other stores the move's +own sync prints (a `CreateBackup` whose insert landed during that sync) is named and the command exits 1 ("a +row landed under the old root during the move"). Nothing under the old root is read or changed, and no object +is copied. Before you delete the old root, run plain `extenddb backup sync` and check that its report of other +stores lists no row of the old root: a server still configured with it, or a `CreateBackup` that read the +maintenance claim before the move took it and inserted its row after the move's report, can leave a row there +whose bytes exist only under the old root. On Cassandra that report reads each root's rows whatever status +their status entry is under, so a row of the old root is listed as a row; a line that counts only "status +entries with no row of that status behind them" names no row, and `extenddb backup sync --forget-store ` removes those entries. A restore that failed because it read a store other than the configured one is +run again once its backup's rows are moved. The move's memory is the read of the old root's rows, about 23 KiB +per row on PostgreSQL and 27 on SQLite, above about 50 MiB, for a table with five GSIs, measured at 1,452 +bytes of table description per row (the figure grows with the description; the admin guide's `backup sync` +paragraph has the figures). + +To drop the rows of a root without moving them (a root that is gone, or one retired with its backups), +`extenddb backup sync --forget-store ` removes every row of that root outright, leaving no tombstone, +and deletes no object; it refuses the configured root. It runs on a store in any state and with no store +configured. Beside servers it waits a minute after taking the maintenance claim before its first removal of a +`FAILED` or `DELETED` row, and prints the wait: a reconciler cycle that began before the claim may be deleting +such a row's prefix, and its deletion stops within that time, checking its deadline before each removal, so +what it can still do once the command removes the row is the one removal (or one S3 request) in flight at that +deadline. That holds while the servers' processes keep running: a server paused for longer than the wait (a +stopped process, a frozen container or VM) in the middle of a deletion takes up that step when it resumes. A +`DELETING` row of another store keeps its bytes: if that root comes back, `backup sync` adopts the backup +again. It waits for running jobs as `purge` does, and refuses if one is still running when the wait ends; a +`CREATING` or `FAILED` row whose job writes a heartbeat after the wait, and a row a restore is reading, are +left and named (exit 1): run it again once the job ends, and for the restore once it ends or, if it failed, +once its target is removed with `extenddb backup remove-target --account
`. On Cassandra it also +sweeps the root's status entries: it deletes those that no row of the root stands behind, writes again under +its row's status one left under an earlier status (a status change whose entry batch failed), and says how +many of each. If the root comes back and is configured again, `backup sync` adopts its backups again. +`--forget-backup ` does the same for one backup; for a `DELETING` row of the configured store it first +waits as for a `DELETED` row, then finishes the deletion its `DeleteBackup` asked for (the per-row line says +the command deleted its objects), which needs the store's check to pass, and refuses the row otherwise. While +the configured root has no `extenddb-store.json` and the catalog holds rows for it (an unmounted volume), or +on SQLite the default root is missing while the catalog holds rows for it (moved away, or a link to a volume +that is gone), `--forget-backup` refuses that root's rows and writes nothing to it: mount the volume, or point +`[backup.store]` at the root that holds the backups, then run it again. + ## Legacy backups A backup written in the in-catalog format (by an earlier release, by a server without a configured store or diff --git a/tests/test_backup_job.py b/tests/test_backup_job.py index 0e48aeeec..04200e62e 100644 --- a/tests/test_backup_job.py +++ b/tests/test_backup_job.py @@ -68,7 +68,8 @@ the run exits 1 as incomplete; once it can be read, the next sync adopts it; - (PostgreSQL) while `--prune-missing` holds the maintenance claim, `backup - purge` and `--adopt-store` fail at once, and `--adopt-store` on a root + purge`, `--adopt-store`, `--move-store`, `--forget-store` and + `--forget-backup` fail at once, and `--adopt-store` on a root the catalog owns waits for a running job as `--prune-missing` does; - (PostgreSQL) on host-local roots, `--prune-missing` on the host that lacks a backup's bytes removes its row only, the other host keeps the @@ -84,6 +85,43 @@ deletion fence and leaves the bytes, and the adopting catalog's store stays healthy. +`backup sync --move-store`, `--forget-store` and `--forget-backup`, on the +three deployments (the steps that edit a row at the catalog on SQLite and +PostgreSQL): + +- a tree copied to a new root: sync there names the old store's rows, + `--move-store` refuses the configured root and a root that differs by a + trailing `/`, and stops on a FAILED row whose copy is a complete backup + and on a copy missing a data file, naming each and moving none; + `--forget-backup` of the incomplete one lets the move finish, the sync + after it skips that prefix until the file is copied, and a server on the + new root restores a moved backup; +- the rows of another root are left alone by a server's reconciler (a copy + of that root's tree under the configured root included) and reported by + sync; `--forget-store` removes them outright and deletes + nothing, after which the root's return has its prefixes adopted; +- a store left unhealthy by its newest backups' missing manifests is cleared + by `--forget-backup`, which prints the store's check, after which + `--prune-missing` runs; +- `--forget-backup` leaves a live CREATING row (PostgreSQL), an ARN spelt + with another region and an in-catalog backup, and runs with no store + configured; +- (PostgreSQL) a CreateBackup during `--move-store`'s wait for a running + job is refused by the maintenance claim, and the move finishes once the + job ends; +- (PostgreSQL, MongoDB) the backup manual's move procedure in its order + beside a running server, the tree copied without its identity file and a + CreateBackup between the restart and the copy: the server stays healthy, + restores a moved backup and backs up again; with the identity file + copied, the manual's remedy (`--forget-backup` of the backup taken + meanwhile, the move, a restart) ends in the same state; +- `--forget-backup` refuses a string that is no ARN before the claim, goes + past an ARN whose row is gone, and on an empty root under the rows runs no + store check and writes nothing there; +- (PostgreSQL) a forget option started while a DELETING row and a dead + RUNNING restore exist proceeds once a server's held-claim cycle has + finished the one and failed the other. + The binary at `EXTENDDB_BINARY` decides which deployment runs: a SQLite build, a PostgreSQL build (with `EXTENDDB_TEST_PG_CONNECTION_STRING`) or a MongoDB build (with `EXTENDDB_TEST_MONGODB_URI`, a replica set of this suite's own: the @@ -1712,8 +1750,15 @@ def test_sync_rebuilds_dropped_rows_and_adopts_only_verified_prefixes(deployment others[label] = arn if d.backend == "sqlite": # The serve lock held is the catalog refusal `serve` gives for - # it: exit 6, before any store or row is read. - for args in (("sync",), ("sync", "--prune-missing")): + # it: exit 6, before any store or row is read. The options that + # move or remove rows of other stores take it too. + for args in ( + ("sync",), + ("sync", "--prune-missing"), + ("sync", "--move-store", os.path.join(d.cwd, "elsewhere")), + ("sync", "--forget-store", os.path.join(d.cwd, "elsewhere")), + ("sync", "--forget-backup", kept), + ): beside = d.run("backup", *args) assert beside.returncode == 6, beside.stdout + beside.stderr assert "another extenddb process" in beside.stdout + beside.stderr, beside.stdout + beside.stderr @@ -1849,8 +1894,8 @@ def test_prune_missing_and_the_unhealthy_store_refusals(deployment, tmp_path): sync reports it with the `--prune-missing` hint and leaves it AVAILABLE; `--prune-missing` takes the maintenance claim and removes the row outright (no tombstone, BackupNotFound), deleting no object: the rest of - its prefix stays. An unhealthy store refuses plain sync and - `--prune-missing`, exit 1, with no row changed and nothing written to the + its prefix stays. An unhealthy store refuses plain sync, `--prune-missing` + and `--move-store`, exit 1, with no row changed and nothing written to the root, for each shape an unmounted volume gives: an empty root under the rows, a root holding only the identity file (the newest AVAILABLE row's manifest absent), and a root whose identity file carries another store @@ -1911,7 +1956,11 @@ def test_prune_missing_and_the_unhealthy_store_refusals(deployment, tmp_path): if content is not None: _private_file(os.path.join(d.root, "extenddb-store.json"), content) tree = _tree(d.root) - for args in (("sync",), ("sync", "--prune-missing")): + for args in ( + ("sync",), + ("sync", "--prune-missing"), + ("sync", "--move-store", os.path.join(d.cwd, "elsewhere")), + ): res = d.run("backup", *args) out = res.stdout + res.stderr assert res.returncode == 1, (shape, args, out) @@ -1945,8 +1994,10 @@ def test_a_second_maintenance_command_fails_at_once(deployment, tmp_path): """`backup sync --prune-missing` takes the maintenance claim and waits for a CREATING row whose owner beat just now (the row edited at the catalog, its heartbeat the database's now); while it waits, `backup - purge --confirm` and `backup sync --adopt-store` each fail within - seconds, exit 1, naming the one-at-a-time rule, and change nothing. The + purge --confirm`, `backup sync --adopt-store`, and `--move-store`, + `--forget-store` (of a root a second backup's row was edited to name) + and `--forget-backup` each fail within seconds, exit 1, naming the + one-at-a-time rule, and change nothing. The first command, signalled, stops before any change and releases the claim. `--adopt-store` on the root this catalog owns then takes the claim and waits for the same live job ("Waiting for running jobs to @@ -1963,8 +2014,17 @@ def test_a_second_maintenance_command_fails_at_once(deployment, tmp_path): server, ddb, creds = _started(d, str(tmp_path / "serve.log")) try: name, busy = _small_backed_up_table(ddb, "claim") + moved = ddb.create_backup(TableName=name, BackupName="elsewhere")["BackupDetails"]["BackupArn"] + _wait_backup_available(ddb, moved, timeout=60) finally: server.stop() + elsewhere = os.path.join(d.cwd, "elsewhere") + n = _catalog( + d, + "UPDATE backup_index SET store_root = ? WHERE account_id = ? AND backup_id = ?", + (elsewhere, ACCOUNT, _backup_id(moved)), + ) + assert n == 1 _set_status(d, busy, "CREATING") out_path = str(tmp_path / "first.out") with open(out_path, "wb") as out: @@ -1979,7 +2039,13 @@ def test_a_second_maintenance_command_fails_at_once(deployment, tmp_path): assert time.monotonic() < deadline, open(out_path).read() time.sleep(0.2) assert busy in open(out_path).read() or name in open(out_path).read(), open(out_path).read() - for args in (("purge", "--confirm"), ("sync", "--adopt-store")): + for args in ( + ("purge", "--confirm"), + ("sync", "--adopt-store"), + ("sync", "--move-store", elsewhere), + ("sync", "--forget-store", elsewhere), + ("sync", "--forget-backup", moved), + ): started = time.monotonic() res = d.run("backup", *args) took = time.monotonic() - started @@ -1989,6 +2055,7 @@ def test_a_second_maintenance_command_fails_at_once(deployment, tmp_path): assert took < 15, f"{args} took {took:.1f} s to refuse" print(f"{' '.join(args)}: refused in {took:.1f} s") assert first.poll() is None, "the first command stopped while the others ran" + assert _rows_present(d, [moved]) == 1, "a refused command removed the other root's row" first.send_signal(15) assert first.wait(timeout=30) == 1 text = open(out_path).read() @@ -2526,3 +2593,973 @@ def test_an_adopter_whose_put_lands_during_the_sync_ends_it_with_exit_1(deployme ) in text, text[-4000:] assert f" {arn}" in _adopted_section(text), text[-4000:] assert "=== extenddb backup sync complete" not in text, text[-4000:] + + +# --- `backup sync --move-store`, `--forget-store`, `--forget-backup` --------------- +# +# These cells change rows only through the binary, so they run on the MongoDB +# deployment too; the steps that edit a row at the catalog (`_catalog`) run on +# SQLite and PostgreSQL only and say so. + +_STORE_SECTION = re.compile( + r"^\[backup\.store\]\n(?:kind = .*\n)(?:\[backup\.store\.filesystem\]\npath = .*\n)?", re.M +) + + +def _set_root(d: _Deployment, path: str) -> None: + """Configure the deployment's filesystem store at `path`, replacing the + `[backup.store]` section the fixture or an earlier call wrote (SQLite's + fixture writes none: its root is the defaulted one).""" + with open(d.config) as f: + text = f.read() + section = f'[backup.store]\nkind = "filesystem"\n[backup.store.filesystem]\npath = "{path}"\n' + found = list(_STORE_SECTION.finditer(text)) + assert len(found) <= 1, text + if found: + text = text[: found[0].start()] + section + text[found[0].end() :] + else: + text += section + with open(d.config, "w") as f: + f.write(text) + + +def _set_store_disabled(d: _Deployment) -> None: + """Configure `[backup.store] kind = "disabled"`: CreateBackup writes an + in-catalog backup, and no store is configured.""" + with open(d.config) as f: + text = f.read() + found = list(_STORE_SECTION.finditer(text)) + section = '[backup.store]\nkind = "disabled"\n' + if found: + text = text[: found[0].start()] + section + text[found[0].end() :] + else: + text += section + with open(d.config, "w") as f: + f.write(text) + + +def _copy_root(src: str, dst: str) -> None: + """Copy a store tree as `cp -rp` does (contents, modes and times; the + identity file included).""" + shutil.copytree(src, dst, copy_function=shutil.copy2) + os.chmod(dst, 0o700) + + +def _copy_tree_into(src: str, dst: str, identity: bool = False) -> None: + """Copy a store tree into an existing root as the backup manual's move + procedure says, `rsync -a --exclude=/extenddb-store.json src/ dst/`: + contents, modes and times, the identity file at the top left out unless + `identity`.""" + + def top_identity(directory: str, names: list[str]) -> list[str]: + if not identity and os.path.samefile(directory, src): + return [n for n in names if n == "extenddb-store.json"] + return [] + + shutil.copytree(src, dst, copy_function=shutil.copy2, ignore=top_identity, dirs_exist_ok=True) + + +def _tree_without_identity(root: str) -> list[str]: + """`_tree` without the identity file at the top.""" + return [entry for entry in _tree(root) if not entry.startswith("extenddb-store.json")] + + +def _section(stdout: str, header: str) -> str: + """One `--- header` section of a command's report: the header's line + and the lines under it, up to a blank line or the next `---` or `===` + line.""" + assert header in stdout, f"{header!r} not in\n{stdout}" + lines = stdout.split(header, 1)[1].split("\n") + kept = [lines[0]] + for line in lines[1:]: + if not line or line.startswith(("---", "===")): + break + kept.append(line) + return "\n".join(kept) + + +OTHER_STORES = "--- Rows of other stores (left as they are; sync reads and changes none of them):" + + +def test_move_store_moves_a_copied_tree_and_stops_on_an_incomplete_copy(deployment, tmp_path): + """Three backups under the root; the tree copied to a new root and the + configuration pointed at it. Plain sync there skips each prefix, naming + the row of the old store that holds it, and its report of other stores + counts the old root's three AVAILABLE rows and names `--move-store` and + `--forget-store`. `--move-store` refuses the configured root and a root + that differs from the old one by a trailing `/` (printing the stores the + catalog knows). (SQLite and PostgreSQL) A FAILED row whose copied prefix + holds a complete backup stops the move, naming `--forget-backup`. With + one data file of the second backup left out of the copy, the move names + that row and that file, exits 1 and moves none; `--forget-backup` of it + removes the row and prints the store's check; the move then moves the + other two (audit lines), syncs the configured root, which skips the + incomplete prefix naming the missing file, and its report of other + stores is empty. Once the file is copied, sync adopts that backup. A + server on the new root describes and restores a moved backup. Nothing + under the old root changes.""" + d = deployment + old = os.path.realpath(d.root) + server, ddb, creds = _started(d, str(tmp_path / "serve.log")) + try: + name, b1 = _small_backed_up_table(ddb, "move") + b2 = ddb.create_backup(TableName=name, BackupName="two")["BackupDetails"]["BackupArn"] + _wait_backup_available(ddb, b2, timeout=60) + b3 = ddb.create_backup(TableName=name, BackupName="three")["BackupDetails"]["BackupArn"] + _wait_backup_available(ddb, b3, timeout=60) + finally: + server.stop() + store_id = _store_id(old) + res = d.run("backup", "sync") + assert res.returncode == 0, res.stdout + res.stderr + assert _section(res.stdout, OTHER_STORES).splitlines()[1:] == [" (none)"], res.stdout + old_tree = _tree(old) + + new = os.path.join(d.cwd, "backups-new") + _copy_root(old, new) + new = os.path.realpath(new) + _set_root(d, new) + res = d.run("backup", "sync") + out = res.stdout + res.stderr + assert res.returncode == 0, out + print(f"branch: {d.backend}\n{res.stdout}") + for arn in (b1, b2, b3): + line = ( + f" {_key_prefix(name, arn)}: a row of another store holds it (AVAILABLE; filesystem {old}, " + f"store id {store_id}); it is left as it is" + ) + assert line in res.stdout, f"{line!r} not in\n{res.stdout}" + others = _section(res.stdout, OTHER_STORES).splitlines()[1:] + assert others[0] == f" filesystem {old} (store id {store_id}): 3 AVAILABLE", others + assert "--move-store " in others[1] and "--forget-store " in others[1], others + assert len(others) == 2, others + assert _rows(d) == {}, "the configured store has no row before the move" + + res = d.run("backup", "sync", "--move-store", new) + out = res.stdout + res.stderr + assert res.returncode == 1 and "that is the configured store's root, so nothing would move" in out, out + res = d.run("backup", "sync", "--move-store", old + "/") + out = res.stdout + res.stderr + assert res.returncode == 1 and "the catalog has no backup row under that root" in out, out + known = _section(res.stdout, "--- The stores the catalog holds backup rows for:").splitlines()[1:] + assert known == [f" filesystem {old} (store id {store_id}): 3 AVAILABLE"], res.stdout + assert "Took the maintenance claim" not in res.stdout, res.stdout + + if d.catalog: + # A tombstone whose copied prefix holds a complete backup (its owner + # long dead: the move's wait passes over it). + _age_row(d, b3, "FAILED") + tree = _tree(new) + res = d.run("backup", "sync", "--move-store", old) + out = res.stdout + res.stderr + assert res.returncode == 1, out + stopping = _section(res.stdout, "--- Stopping the move (no row is moved while any is listed here):") + assert ( + f" {b3} (FAILED, account {ACCOUNT}): FAILED whose prefix under the configured root holds a " + "verified manifest" + ) in stopping and "--forget-backup" in stopping, stopping + assert len(stopping.splitlines()) == 2, stopping + assert "--move-store moved no row: 1 row(s) stop the move" in out, out + assert _tree(new) == tree, "the refused move changed the copy" + _set_status(d, b3, "AVAILABLE") + + # One data file of b2 left out of the copy. + data = os.path.join(_prefix(d, name, b2).replace(old, new, 1), "data") + dropped = sorted(os.listdir(data))[0] + aside = os.path.join(d.cwd, dropped) + os.rename(os.path.join(data, dropped), aside) + res = d.run("backup", "sync", "--move-store", old) + out = res.stdout + res.stderr + assert res.returncode == 1, out + stopping = _section(res.stdout, "--- Stopping the move (no row is moved while any is listed here):") + assert stopping.splitlines()[1:] == [ + f" {b2} (AVAILABLE, account {ACCOUNT}): not a complete copy under the configured root: data file " + f"data/{dropped} is not in the store; copy the prefix again (or remove the row with --forget-backup)" + ], stopping + assert "--move-store moved no row: 1 row(s) stop the move" in out, out + assert "=== extenddb backup sync complete" not in res.stdout, "a stopped move ran the sync" + assert _rows(d) == {} + + res = d.run("backup", "sync", "--forget-backup", b2) + out = res.stdout + res.stderr + assert res.returncode == 0, out + assert f" {b2}: removed: the row (AVAILABLE) is gone, and no object was deleted" in res.stdout, out + assert f"The configured store's check passes (store id {store_id})" in res.stdout, out + assert "removed a backup row outright; no object was deleted" in out, "no audit line" + + res = d.run("backup", "sync", "--move-store", old) + out = res.stdout + res.stderr + assert res.returncode == 0, out + assert ( + "=== extenddb backup sync --move-store complete: 2 row(s) moved, 0 resolved, 0 deletion(s) finished, " + "0 row(s) stopping the move, 0 gone during the move, 0 left under the old root ===" + ) in res.stdout, out + moved = _section(res.stdout, "--- Moved (each row now names the configured store):").splitlines()[1:] + assert sorted(moved) == sorted([f" {b1}", f" {b3}"]), moved + assert out.count("moved a backup row to the configured store; no object was copied or deleted") == 2, out + skipped = _section(res.stdout, "--- Skipped (not adopted):") + assert f" {_key_prefix(name, b2)}: " in skipped and f"data file data/{dropped} is not in the store" in skipped, skipped + assert _section(res.stdout, OTHER_STORES).splitlines()[1:] == [" (none)"], res.stdout + assert _rows(d) == {b1: "AVAILABLE", b3: "AVAILABLE"} + + os.rename(aside, os.path.join(data, dropped)) + res = d.run("backup", "sync") + assert res.returncode == 0, res.stdout + res.stderr + assert _adopted_section(res.stdout).splitlines()[1:-1] == [f" {b2}"], res.stdout + + server, ddb, creds = _started(d, str(tmp_path / "serve-new.log")) + try: + details = ddb.describe_backup(BackupArn=b1)["BackupDescription"]["BackupDetails"] + assert details["BackupStatus"] == "AVAILABLE" and details["BackupName"] == "nightly", details + target = f"{name}-r" + ddb.restore_table_from_backup(TargetTableName=target, BackupArn=b1) + _wait_active(ddb, target) + assert _count(ddb, target) == HAND_BUILT_ITEMS + finally: + server.stop() + assert _tree(old) == old_tree, "a command changed the old root" + + +def _age_row(d: _Deployment, arn: str, status: str) -> None: + """Set the backup's row to `status` with its heartbeat and completion in + 2000: a FAILED row whose owner is long dead and past every staleness + window, a DELETED tombstone past its retention.""" + n = _catalog( + d, + "UPDATE backup_index SET status = ?, heartbeat_at = ?, completed_at = ? " + "WHERE account_id = ? AND backup_id = ?", + (status, "2000-01-01T00:00:00.000Z", "2000-01-01T00:00:00.000Z", ACCOUNT, _backup_id(arn)), + ) + assert n == 1, f"{n} backup_index rows match {arn}" + + +def test_another_roots_rows_are_left_alone_until_forget_store_drops_them(deployment, tmp_path): + """Four backups under root B; the configuration then names a new, empty + root A. (SQLite and PostgreSQL: the rows of B are edited at the catalog + to a FAILED row and a DELETED tombstone dated 2000, past every window, + and a DELETING row.) A server on A runs three reconciler cycles: B's + tree is byte-identical after them and B's rows keep their statuses. + Plain sync on A reports B's rows per status and names the DELETING row. + `--forget-store` refuses the configured root A; `--forget-store B` + removes every row of B outright (no tombstone: `backup list` on B is + empty), deletes nothing under B and logs each removal; a second run + finds no row there. Configured with B again, sync adopts the four + prefixes as AVAILABLE backups.""" + d = deployment + b_root = os.path.realpath(d.root) + server, ddb, creds = _started(d, str(tmp_path / "serve.log")) + try: + name, a1 = _small_backed_up_table(ddb, "forget") + arns = [a1] + for label in ("two", "three", "four"): + arn = ddb.create_backup(TableName=name, BackupName=label)["BackupDetails"]["BackupArn"] + _wait_backup_available(ddb, arn, timeout=60) + arns.append(arn) + finally: + server.stop() + store_id = _store_id(b_root) + _, a2, a3, a4 = arns + if d.catalog: + _age_row(d, a2, "FAILED") + _set_status(d, a3, "DELETING") + _age_row(d, a4, "DELETED") + expected = {a1: "AVAILABLE", a2: "FAILED", a3: "DELETING", a4: "DELETED"} + counts = "1 AVAILABLE, 1 DELETING, 1 DELETED, 1 FAILED" + else: + expected = {arn: "AVAILABLE" for arn in arns} + counts = "4 AVAILABLE" + print(f"branch: {d.backend}, the rows of B: {counts}") + assert _rows(d) == expected + b_tree = _tree(b_root) + + # A holds a copy of B's tree (without B's identity file), as before a + # move: a duty of A's server acting on B's rows would find the bytes. + a_root = os.path.join(d.cwd, "backups-a") + os.mkdir(a_root, 0o700) + a_root = os.path.realpath(a_root) + _copy_tree_into(b_root, a_root) + a_tree = _tree_without_identity(a_root) + _set_root(d, a_root) + server, ddb, creds = _started(d, str(tmp_path / "serve-a.log")) + try: + time.sleep(3.5) # three cycles at reconcile_interval_secs = 1 + assert server.proc.poll() is None + finally: + server.stop() + assert _tree(b_root) == b_tree, "a reconciler on another root changed B" + assert _tree_without_identity(a_root) == a_tree, "a reconciler deleted the copy of another root's backups" + _set_root(d, b_root) + assert _rows(d) == expected, "a reconciler on another root changed B's rows" + _set_root(d, a_root) + + res = d.run("backup", "sync") + assert res.returncode == 0, res.stdout + res.stderr + others = _section(res.stdout, OTHER_STORES).splitlines()[1:] + assert others[0] == f" filesystem {b_root} (store id {store_id}): {counts}", others + named = [line for line in others[1:] if line.startswith(" ")] + assert named == ([f" DELETING {a3} (account {ACCOUNT})"] if d.catalog else []), others + + res = d.run("backup", "sync", "--forget-store", a_root) + out = res.stdout + res.stderr + assert res.returncode == 1 and "that is the configured store's root, which --forget-store refuses" in out, out + res = d.run("backup", "sync", "--forget-store", b_root) + out = res.stdout + res.stderr + assert res.returncode == 0, out + assert "=== extenddb backup sync --forget-store complete: 4 row(s) removed, 0 left ===" in res.stdout, out + # Beside servers the FAILED and DELETED removals wait out a deletion a + # server's reconciler may have started before the claim; on SQLite the + # serve lock excludes every server. + waited = "before the first removal of a FAILED or DELETED row" in res.stdout + assert waited == (d.catalog != "" and d.backend != "sqlite"), res.stdout + listed = _section(res.stdout, f"--- --forget-store {b_root} (filesystem): rows removed outright; no object is deleted:") + for arn in arns: + assert f" {arn} (" in listed, listed + assert listed.count(": removed: the row (") == 4, listed + assert out.count("removed a backup row outright; no object was deleted") == 4, out + assert _tree(b_root) == b_tree, "--forget-store changed B" + res = d.run("backup", "sync", "--forget-store", b_root) + out = res.stdout + res.stderr + assert res.returncode == 1 and "the catalog has no backup row under that root" in out, out + + _set_root(d, b_root) + assert _rows(d) == {}, "a tombstone was left" + res = d.run("backup", "sync") + assert res.returncode == 0, res.stdout + res.stderr + assert sorted(_adopted_section(res.stdout).splitlines()[1:-1]) == sorted(f" {arn}" for arn in arns), res.stdout + assert _rows(d) == {arn: "AVAILABLE" for arn in arns} + + +def test_forget_backup_clears_a_store_whose_newest_manifests_are_gone(deployment, tmp_path): + """The state a catalog restored from a dump can be in: of its four + backups, the manifests of the two newest and of the oldest are gone from + the store. The store is unhealthy (the manifest probe of the newest + AVAILABLE row), so plain sync refuses, naming `--forget-backup`, and + `--prune-missing` refuses, with no row changed. `--forget-backup + ` runs on the unhealthy store: it removes that row and prints + the store's check, which now names the next one and the command for it; + `--forget-backup ` removes that one and the check passes (beside + servers, the running server's `/health` turns healthy at its next + cycle). `--prune-missing` then removes the oldest row, and the second + one, whose manifest is there, stays. No object is deleted.""" + d = deployment + root = os.path.realpath(d.root) + server, ddb, creds = _started(d, str(tmp_path / "serve.log")) + try: + name, c1 = _small_backed_up_table(ddb, "heal") + arns = [c1] + for label in ("second", "third", "fourth"): + arn = ddb.create_backup(TableName=name, BackupName=label)["BackupDetails"]["BackupArn"] + _wait_backup_available(ddb, arn, timeout=60) + arns.append(arn) + _, c2, c3, c4 = arns + if d.backend == "sqlite": + server.stop() + print(f"branch: {d.backend}, the server {'stopped' if d.backend == 'sqlite' else 'running'}") + store_id = _store_id(root) + for arn in (c1, c3, c4): + os.remove(os.path.join(_prefix(d, name, arn), "extenddb-manifest.json")) + tree = _tree(root) + before = _rows(d) + for args in (("sync",), ("sync", "--prune-missing")): + res = d.run("backup", *args) + out = res.stdout + res.stderr + assert res.returncode == 1 and "is unhealthy" in out and c4 in out, (args, out) + assert f"remove its row with extenddb backup sync --forget-backup {c4}," in out, (args, out) + assert _rows(d) == before + + res = d.run("backup", "sync", "--forget-backup", c4) + out = res.stdout + res.stderr + assert res.returncode == 0, out + assert f"Store: filesystem {root} (unhealthy: " in res.stdout, out + assert f" {c4}: removed: the row (AVAILABLE) is gone, and no object was deleted" in res.stdout, out + assert "The configured store's check fails: " in res.stdout, out + assert f"run extenddb backup sync --forget-backup {c3}" in res.stdout, out + + res = d.run("backup", "sync", "--forget-backup", c3) + out = res.stdout + res.stderr + assert res.returncode == 0, out + assert f"The configured store's check passes (store id {store_id})" in res.stdout, out + if d.backend != "sqlite": + health = _wait_health( + d.port, + lambda h: h.get("components", {}).get("backup_store", {}).get("status") == "healthy", + server=server, + ) + print(f"the running server's store after the removals: {health['components']['backup_store']}") + + res = d.run("backup", "sync", "--prune-missing") + out = res.stdout + res.stderr + assert res.returncode == 0, out + assert f" {c1}: removed: the row is gone, and no object was deleted" in res.stdout, out + assert _rows(d) == {c2: "AVAILABLE"} + assert _tree(root) == tree, "an object was deleted" + finally: + server.stop() + + +def test_forget_backup_leaves_a_live_creating_row_and_an_in_catalog_backup(deployment, tmp_path): + """`--forget-backup` changes nothing for an ARN it cannot act on: (on + PostgreSQL, beside servers) a CREATING row of another root whose owner + beat just now is left with exit 1, and removed once its heartbeat is an + hour old; an ARN spelt with another region than the row's is refused, + naming the row's ARN; with `[backup.store] kind = "disabled"` (no store + configured, which both forget options accept) an earlier release's + in-catalog backup is reported as one with exit 1, the ARN after it in + the same command is not reached and keeps its row, and a second command + removes that row. The in-catalog backup is still AVAILABLE afterwards.""" + d = deployment + server, ddb, creds = _started(d, str(tmp_path / "serve.log")) + try: + name, e1 = _small_backed_up_table(ddb, "refuse") + e2 = ddb.create_backup(TableName=name, BackupName="two")["BackupDetails"]["BackupArn"] + _wait_backup_available(ddb, e2, timeout=60) + finally: + server.stop() + print(f"branch: {d.backend}") + if d.backend == "postgres": + elsewhere = os.path.join(d.cwd, "elsewhere") + n = _catalog( + d, + "UPDATE backup_index SET store_root = ?, status = 'CREATING', completed_at = NULL, " + "heartbeat_at = {now} WHERE account_id = ? AND backup_id = ?", + (elsewhere, ACCOUNT, _backup_id(e2)), + ) + assert n == 1 + res = d.run("backup", "sync", "--forget-backup", e2) + out = res.stdout + res.stderr + assert res.returncode == 1, out + assert f" {e2}: left: the CREATING row has a live owner" in res.stdout, out + assert f"--forget-backup did not remove the row of {e2}" in out, out + assert _rows_present(d, [e2]) == 1 + n = _catalog( + d, + "UPDATE backup_index SET heartbeat_at = {now} - interval '1 hour' WHERE account_id = ? AND backup_id = ?", + (ACCOUNT, _backup_id(e2)), + ) + assert n == 1 + res = d.run("backup", "sync", "--forget-backup", e2) + out = res.stdout + res.stderr + assert res.returncode == 0, out + assert f" {e2}: removed: the row (CREATING) is gone, and no object was deleted" in res.stdout, out + assert _rows_present(d, [e2]) == 0 + + region = e1.split(":")[3] + respelt = e1.replace(f":{region}:", ":eu-west-3:", 1) + assert respelt != e1 + res = d.run("backup", "sync", "--forget-backup", respelt) + out = res.stdout + res.stderr + assert res.returncode == 1, out + assert f" {respelt}: left: no row carries this ARN as written; the row of this backup is {e1}" in res.stdout, out + assert e1 in _rows(d) + + _set_store_disabled(d) + server = _Server(d.config, str(tmp_path / "serve-disabled.log")) + try: + _wait_health(d.port, lambda h: True, server=server) + ddb = _client(d.port, _provision(d.port, d.cert, ADMIN_PASSWORD)) + legacy = ddb.create_backup(TableName=name, BackupName="in-catalog")["BackupDetails"]["BackupArn"] + _wait_backup_available(ddb, legacy, timeout=60) + finally: + server.stop() + res = d.run("backup", "sync", "--forget-backup", legacy, "--forget-backup", e1) + out = res.stdout + res.stderr + assert res.returncode == 1, out + assert "Store: none (" in res.stdout, out + assert f" {legacy}: an in-catalog backup; --forget-backup changes nothing; DeleteBackup removes it" in res.stdout, out + assert f" {e1}: not reached" in res.stdout, out + assert "=== extenddb backup sync --forget-backup stopped: 0 row(s) removed, 2 not ===" in res.stdout, out + if d.catalog: + assert _rows_present(d, [e1]) == 1 + res = d.run("backup", "sync", "--forget-backup", e1) + out = res.stdout + res.stderr + assert res.returncode == 0, out + assert f" {e1}: removed: the row (AVAILABLE) is gone, and no object was deleted" in res.stdout, out + if d.catalog: + assert _rows_present(d, [e1]) == 0 + + server = _Server(d.config, str(tmp_path / "serve-disabled2.log")) + try: + _wait_health(d.port, lambda h: True, server=server) + ddb = _client(d.port, _provision(d.port, d.cert, ADMIN_PASSWORD)) + details = ddb.describe_backup(BackupArn=legacy)["BackupDescription"]["BackupDetails"] + assert details["BackupStatus"] == "AVAILABLE", details + finally: + server.stop() + + +def test_a_create_backup_during_the_moves_wait_is_refused(deployment, tmp_path): + """Beside servers (PostgreSQL): a CREATING row of the old root whose + owner beat just now; the tree copied to a new root, a server running on + it. `--move-store ` takes the claim and waits for that job; + while it waits, a CreateBackup on the server is refused with + LimitExceededException (the maintenance claim) and writes no row. Once + the job's row is AVAILABLE the move goes on and moves both rows, and the + server's next CreateBackup and a DescribeBackup of a moved backup + work.""" + d = deployment + if d.backend != "postgres": + pytest.skip("on SQLite the serve lock keeps every server stopped during the move") + old = os.path.realpath(d.root) + server, ddb, creds = _started(d, str(tmp_path / "serve.log")) + try: + name, m1 = _small_backed_up_table(ddb, "race") + m2 = ddb.create_backup(TableName=name, BackupName="two")["BackupDetails"]["BackupArn"] + _wait_backup_available(ddb, m2, timeout=60) + finally: + server.stop() + _set_status(d, m2, "CREATING") + new = os.path.join(d.cwd, "backups-new") + _copy_root(old, new) + new = os.path.realpath(new) + _set_root(d, new) + server, ddb, creds = _started(d, str(tmp_path / "serve-new.log")) + out_path = str(tmp_path / "move.out") + def every_row() -> int: + return _catalog(d, "SELECT backup_id FROM backup_index WHERE account_id = ?", (ACCOUNT,)) + + try: + before = every_row() + assert before == 2, before + with open(out_path, "wb") as out: + moving = subprocess.Popen( + [EXTENDDB_BINARY, "backup", "--config", d.config, "sync", "--move-store", old], + cwd=d.cwd, stdout=out, stderr=subprocess.STDOUT, stdin=subprocess.DEVNULL, + ) + try: + deadline = time.monotonic() + 60 + while "Waiting for running jobs to finish" not in open(out_path).read(): + assert moving.poll() is None, open(out_path).read() + assert time.monotonic() < deadline, open(out_path).read() + time.sleep(0.2) + assert _backup_id(m2) in open(out_path).read(), open(out_path).read() + with pytest.raises(ClientError) as refused: + ddb.create_backup(TableName=name, BackupName="during") + error = refused.value.response["Error"] + print(f"CreateBackup during the wait: {error}") + assert error["Code"] == "LimitExceededException", error + assert "maintenance claim" in error["Message"], error + assert every_row() == before, "the refused CreateBackup wrote a row" + _set_status(d, m2, "AVAILABLE") + assert moving.wait(timeout=60) == 0, open(out_path).read() + finally: + if moving.poll() is None: + moving.kill() + moving.wait(timeout=10) + text = open(out_path).read() + assert "=== extenddb backup sync --move-store complete: 2 row(s) moved" in text, text + assert _rows(d) == {m1: "AVAILABLE", m2: "AVAILABLE"} + after = ddb.create_backup(TableName=name, BackupName="after")["BackupDetails"]["BackupArn"] + _wait_backup_available(ddb, after, timeout=60) + details = ddb.describe_backup(BackupArn=m1)["BackupDescription"]["BackupDetails"] + assert details["BackupStatus"] == "AVAILABLE", details + finally: + server.stop() + + +def test_the_manuals_move_procedure_beside_a_running_server(deployment, tmp_path): + """The backup manual's "Moving a store to a new root" in its order, + beside a running server (PostgreSQL, MongoDB): one backup under the old + root; the configuration pointed at a new, empty root and the server + restarted (step 1), which gives the new root its own identity file; a + CreateBackup there before the copy (the server keeps serving); the old + tree copied in without its `extenddb-store.json` (step 2); then + `--move-store` (step 3), which moves the old row and keeps the new one. + The running server's store stays healthy throughout, it restores the + moved backup, and its next CreateBackup is AVAILABLE.""" + d = deployment + if d.backend == "sqlite": + pytest.skip("on SQLite the serve lock keeps every server stopped during the move") + old = os.path.realpath(d.root) + server, ddb, creds = _started(d, str(tmp_path / "serve.log")) + try: + name, b1 = _small_backed_up_table(ddb, "proc") + finally: + server.stop() + old_tree = _tree(old) + new = os.path.join(d.cwd, "backups-new") + os.mkdir(new, 0o700) + new = os.path.realpath(new) + _set_root(d, new) + healthy = lambda h: h.get("components", {}).get("backup_store", {}).get("status") == "healthy" # noqa: E731 + server, ddb, creds = _started(d, str(tmp_path / "serve-new.log")) + try: + _wait_health(d.port, healthy, server=server) + new_id = _store_id(new) + assert new_id != _store_id(old) + b2 = ddb.create_backup(TableName=name, BackupName="between")["BackupDetails"]["BackupArn"] + _wait_backup_available(ddb, b2, timeout=60) + _copy_tree_into(old, new) + assert _store_id(new) == new_id, "the copy brought the old identity file" + res = d.run("backup", "sync", "--move-store", old) + out = res.stdout + res.stderr + assert res.returncode == 0, out + assert ( + "=== extenddb backup sync --move-store complete: 1 row(s) moved, 0 resolved, 0 deletion(s) " + "finished, 0 row(s) stopping the move, 0 gone during the move, 0 left under the old root ===" + ) in res.stdout, out + time.sleep(2) # two cycles' checks of the running server + _wait_health(d.port, healthy, server=server) + target = f"{name}-r" + ddb.restore_table_from_backup(TargetTableName=target, BackupArn=b1) + _wait_active(ddb, target) + assert _count(ddb, target) == HAND_BUILT_ITEMS + b3 = ddb.create_backup(TableName=name, BackupName="after")["BackupDetails"]["BackupArn"] + _wait_backup_available(ddb, b3, timeout=60) + finally: + server.stop() + assert _rows(d) == {b1: "AVAILABLE", b2: "AVAILABLE", b3: "AVAILABLE"} + res = d.run("backup", "list") + assert res.stdout.count(f"store_id={new_id}") == 3, res.stdout + assert _tree(old) == old_tree, "the move changed the old root" + + +def test_a_copied_identity_file_is_undone_by_the_manuals_remedy(deployment, tmp_path): + """The manual's way out when `extenddb-store.json` was copied with the + tree over the new root's own (PostgreSQL, MongoDB): one backup under the + old root, the server restarted on the new root and a CreateBackup there, + then the whole tree copied, identity file included. The running server's + store turns unhealthy (an identity mismatch) and `--move-store` refuses + (the copied file's id against the row the new root's backup carries). + `extenddb backup list` names that backup under the new root's first id; + `--forget-backup` of it removes the row and deletes nothing; the move + then moves the old row and its sync adopts the forgotten backup again + under the copied file's id. After a restart every server's store is + healthy, both backups restore or describe, and a new CreateBackup is + AVAILABLE.""" + d = deployment + if d.backend == "sqlite": + pytest.skip("on SQLite the serve lock keeps every server stopped during the move") + old = os.path.realpath(d.root) + server, ddb, creds = _started(d, str(tmp_path / "serve.log")) + try: + name, b1 = _small_backed_up_table(ddb, "copied") + finally: + server.stop() + old_id = _store_id(old) + new = os.path.join(d.cwd, "backups-new") + os.mkdir(new, 0o700) + new = os.path.realpath(new) + _set_root(d, new) + status = lambda h: h.get("components", {}).get("backup_store", {}).get("status") # noqa: E731 + server, ddb, creds = _started(d, str(tmp_path / "serve-new.log")) + try: + _wait_health(d.port, lambda h: status(h) == "healthy", server=server) + first_id = _store_id(new) + b2 = ddb.create_backup(TableName=name, BackupName="between")["BackupDetails"]["BackupArn"] + _wait_backup_available(ddb, b2, timeout=60) + _copy_tree_into(old, new, identity=True) + assert _store_id(new) == old_id + health = _wait_health(d.port, lambda h: status(h) == "unhealthy", server=server) + print(f"the running server's store after the copy: {health['components']['backup_store']}") + res = d.run("backup", "sync", "--move-store", old) + out = res.stdout + res.stderr + assert res.returncode == 1 and "is unhealthy" in out and first_id in out, out + res = d.run("backup", "list") + assert f"{b2} store_root={new} store_id={first_id}" in res.stdout, res.stdout + tree = _tree(new) + res = d.run("backup", "sync", "--forget-backup", b2) + out = res.stdout + res.stderr + assert res.returncode == 0, out + assert f" {b2}: removed: the row (AVAILABLE) is gone, and no object was deleted" in res.stdout, out + assert f"The configured store's check passes (store id {old_id})" in res.stdout, out + assert _tree(new) == tree, "--forget-backup changed the root" + res = d.run("backup", "sync", "--move-store", old) + out = res.stdout + res.stderr + assert res.returncode == 0, out + assert "=== extenddb backup sync --move-store complete: 1 row(s) moved" in res.stdout, out + assert _adopted_section(res.stdout).splitlines()[1:-1] == [f" {b2}"], res.stdout + finally: + server.stop() + server, ddb, creds = _started(d, str(tmp_path / "serve-restarted.log")) + try: + _wait_health(d.port, lambda h: status(h) == "healthy", server=server) + target = f"{name}-r" + ddb.restore_table_from_backup(TargetTableName=target, BackupArn=b1) + _wait_active(ddb, target) + assert _count(ddb, target) == HAND_BUILT_ITEMS + details = ddb.describe_backup(BackupArn=b2)["BackupDescription"]["BackupDetails"] + assert details["BackupStatus"] == "AVAILABLE", details + b3 = ddb.create_backup(TableName=name, BackupName="after")["BackupDetails"]["BackupArn"] + _wait_backup_available(ddb, b3, timeout=60) + finally: + server.stop() + res = d.run("backup", "list") + assert res.stdout.count(f"store_id={old_id}") == 3, res.stdout + + +def test_forget_backup_checks_its_arns_first_goes_past_removed_ones_and_writes_no_store(deployment, tmp_path): + """`--forget-backup` refuses a string that is no ARN before it takes the + claim, with nothing changed; given again after a run that removed its + first ARN, it says that ARN carries no row and removes the next, exit 0. + With the root replaced by an empty directory (a volume not mounted under + the rows), it refuses the root's last row, exit 1, naming the unmounted + volume, and writes nothing to the root; run again, the same, where a + removal would have left a root with no rows that a server's check gives + a new identity file.""" + d = deployment + root = os.path.realpath(d.root) + server, ddb, creds = _started(d, str(tmp_path / "serve.log")) + try: + name, a = _small_backed_up_table(ddb, "again") + arns = [a] + for label in ("bbb", "ccc"): + arn = ddb.create_backup(TableName=name, BackupName=label)["BackupDetails"]["BackupArn"] + _wait_backup_available(ddb, arn, timeout=60) + arns.append(arn) + finally: + server.stop() + _, b, c = arns + + for given in ("", "not-an-arn"): + res = d.run("backup", "sync", "--forget-backup", given, "--forget-backup", a) + out = res.stdout + res.stderr + assert res.returncode == 1, out + assert f" {given!r}: not a backup ARN" in res.stdout.replace('"', "'"), out + assert "nothing was changed" in out, out + assert "Took the maintenance claim" not in res.stdout, out + assert set(_rows(d)) == set(arns) + + res = d.run("backup", "sync", "--forget-backup", a) + assert res.returncode == 0, res.stdout + res.stderr + res = d.run("backup", "sync", "--forget-backup", a, "--forget-backup", b) + out = res.stdout + res.stderr + assert res.returncode == 0, out + assert f" {a}: no row carries this ARN; nothing to remove" in res.stdout, out + assert f" {b}: removed: the row (AVAILABLE) is gone, and no object was deleted" in res.stdout, out + assert "=== extenddb backup sync --forget-backup complete: 1 row(s) removed, 0 not ===" in res.stdout, out + assert set(_rows(d)) == {c} + + aside = os.path.join(d.cwd, "aside") + os.rename(root, aside) + os.mkdir(root, 0o700) + res = d.run("backup", "sync") + out = res.stdout + res.stderr + assert res.returncode == 1 and "has no extenddb-store.json but the catalog has rows for it" in out, out + for run in ("first", "again"): + res = d.run("backup", "sync", "--forget-backup", c) + out = res.stdout + res.stderr + assert res.returncode == 1, f"{run}: {out}" + assert ( + f" {c}: left: the configured root has no extenddb-store.json while the catalog holds rows for " + "it (a volume that is not mounted?); mount it, or point [backup.store] at the root that holds " + "these backups, then run again" + ) in res.stdout, f"{run}: {out}" + assert ( + f"The configured store's check was not run: the root {root} had no extenddb-store.json when the " + "command opened it" + ) in res.stdout, f"{run}: {out}" + assert os.listdir(root) == [], f"{run}: the forget wrote to the empty root" + assert set(_rows(d)) == {c}, run + + +@pytest.mark.parametrize("shape", ["renamed away", "dangling link"]) +def test_forget_refuses_the_rows_of_a_missing_default_root(deployment, tmp_path, shape): + """SQLite's defaulted root missing while the catalog holds rows for it: + renamed away, or a symbolic link to a volume directory that is gone + (the rows carry the link's target). `--forget-backup` refuses the root's + row as an unmounted root's, exit 1, and says the root is missing with + rows rather than that no server has run with the database file; + `--forget-store` of the root refuses it as the configured root. Nothing + is created at the root's path, and the row is kept.""" + d = deployment + if d.backend != "sqlite": + pytest.skip("the defaulted root is SQLite's") + root = d.root + if shape == "dangling link": + volume = os.path.join(d.cwd, "volume") + os.mkdir(volume, 0o700) + label = os.path.join(volume, "backups") + os.mkdir(label, 0o700) + os.symlink(label, root) + else: + label = root + server, ddb, creds = _started(d, str(tmp_path / "serve.log")) + try: + _, arn = _small_backed_up_table(ddb, "gone") + finally: + server.stop() + if shape == "dangling link": + os.rename(volume, os.path.join(d.cwd, "volume-gone")) + assert os.path.islink(root) and not os.path.exists(root) + else: + os.rename(root, os.path.join(d.cwd, "aside")) + res = d.run("backup", "sync", "--forget-backup", arn) + out = res.stdout + res.stderr + assert res.returncode == 1, out + assert f"the default root {label} is missing while the catalog holds rows for it" in res.stdout, out + assert "no server has run with this database file" not in out, out + assert ( + f" {arn}: left: the configured root has no extenddb-store.json while the catalog holds rows for it" + ) in res.stdout, out + res = d.run("backup", "sync", "--forget-store", label) + out = res.stdout + res.stderr + assert res.returncode == 1 and "that is the configured store's root" in out, out + assert not os.path.exists(root), "the forget created the default root" + assert _rows_present(d, [arn]) == 1, "the row is kept" + + +def test_the_forget_options_write_nothing_to_a_fresh_root(deployment, tmp_path): + """A configured root with no identity file and no rows (a fresh root): + `--forget-backup` of an ARN no row carries, and `--forget-store` of a + root with no rows, open the store without writing to it; the root is + still empty afterwards, where the writing check of plain sync would + have created its identity file.""" + d = deployment + fresh = os.path.join(d.cwd, "fresh-root") + os.mkdir(fresh, 0o700) + fresh = os.path.realpath(fresh) + _set_root(d, fresh) + absent = f"arn:aws:dynamodb:us-east-1:{ACCOUNT}:table/x/backup/01700000000000-deadbeef" + res = d.run("backup", "sync", "--forget-backup", absent) + out = res.stdout + res.stderr + assert res.returncode == 0, out + assert f" {absent}: no row carries this ARN; nothing to remove" in res.stdout, out + assert ( + f"The configured store's check was not run: the root {fresh} had no extenddb-store.json when the " + "command opened it" + ) in res.stdout, out + assert os.listdir(fresh) == [], "the forget wrote to the fresh root" + res = d.run("backup", "sync", "--forget-store", os.path.join(d.cwd, "nowhere")) + out = res.stdout + res.stderr + assert res.returncode == 1 and "the catalog has no backup row under that root" in out, out + assert os.listdir(fresh) == [], "the forget wrote to the fresh root" + + +def _store_status(health: dict) -> "str | None": + """The backup store component's status in a /health answer.""" + return health.get("components", {}).get("backup_store", {}).get("status") + + +def test_forget_backup_beside_a_server_keeps_the_rows_of_an_unmounted_root(deployment, tmp_path): + """Beside a running server: the root replaced by an empty directory (a + volume not mounted under the rows), the server reports the identity file + missing. `--forget-backup` of the root's last row refuses it, exit 1, + and writes nothing; the server's checks then find the rows still there, + give the root no identity file and keep the store unhealthy, so no + CreateBackup lands on the empty mount point.""" + d = deployment + if d.backend == "sqlite": + pytest.skip("beside a server") + root = os.path.realpath(d.root) + server, ddb, creds = _started(d, str(tmp_path / "serve.log")) + try: + name, a = _small_backed_up_table(ddb, "unm") + os.rename(root, os.path.join(d.cwd, "aside")) + os.mkdir(root, 0o700) + _wait_health(d.port, lambda h: _store_status(h) == "unhealthy", server=server) + res = d.run("backup", "sync", "--forget-backup", a) + out = res.stdout + res.stderr + assert res.returncode == 1, out + assert f" {a}: left: the configured root has no extenddb-store.json" in res.stdout, out + assert os.listdir(root) == [], out + time.sleep(4) + assert os.listdir(root) == [], "a server's check wrote to the empty mount point" + assert _store_status(_health(d.port)) == "unhealthy" + assert set(_rows(d)) == {a} + finally: + server.stop() + + +def test_forget_backup_finishes_a_deleting_row_of_the_configured_store(deployment, tmp_path): + """A DELETING row of the configured store (its DeleteBackup's state + before the prefix is gone), no server running: `--forget-backup` deletes + the prefix first, says so, and removes the row; the next plain sync + adopts nothing. On SQLite the command's own pass over what the stopped + server left may finish the deletion first, and the line says so too.""" + d = deployment + if not d.catalog: + pytest.skip("edits the catalog") + server, ddb, creds = _started(d, str(tmp_path / "serve.log")) + try: + name, a = _small_backed_up_table(ddb, "deling") + finally: + server.stop() + _set_status(d, a, "DELETING") + prefix = _prefix(d, name, a) + assert os.listdir(prefix) + res = d.run("backup", "sync", "--forget-backup", a) + out = res.stdout + res.stderr + assert res.returncode == 0, out + assert ( + f" {a}: removed: the row is gone, and this command deleted its objects first (the row was " + "DELETING in the configured store: the deletion its DeleteBackup asked for)" + ) in res.stdout, out + assert "no object was deleted" not in res.stdout, out + assert not os.path.isdir(prefix) or os.listdir(prefix) == [], os.listdir(prefix) + assert a not in _rows(d) + res = d.run("backup", "sync") + out = res.stdout + res.stderr + assert res.returncode == 0, out + assert "=== extenddb backup sync complete: 0 backup(s) adopted" in res.stdout, out + assert a not in _rows(d) + + +def test_a_maintenance_command_waits_out_a_deleting_row_and_a_dead_restore(deployment, tmp_path): + """Beside a server (PostgreSQL): a DELETING row and a RUNNING restore + record whose owner is long dead (its target CREATING, both edited at the + catalog with the server stopped). `--forget-backup` of the restored + backup takes the claim and waits for the restore; a server started + while it waits runs its held-claim cycle, which finishes the DELETING + row's deletion and fails the dead restore and removes its target, and + the command then removes the backup's row (no longer in use), exit 0.""" + d = deployment + if d.backend != "postgres": + pytest.skip("beside servers, the restore record edited at the PostgreSQL catalog") + root = os.path.realpath(d.root) + server, ddb, creds = _started(d, str(tmp_path / "serve.log")) + try: + name, b1 = _small_backed_up_table(ddb, "dead") + b2 = ddb.create_backup(TableName=name, BackupName="two")["BackupDetails"]["BackupArn"] + _wait_backup_available(ddb, b2, timeout=60) + target = f"{name}-r" + ddb.restore_table_from_backup(TargetTableName=target, BackupArn=b1) + _wait_active(ddb, target) + finally: + server.stop() + n = _catalog( + d, + "UPDATE tables SET table_status = 'CREATING' WHERE account_id = ? AND table_name = ?", + (ACCOUNT, target), + ) + assert n == 1 + n = _catalog( + d, + "UPDATE table_restores SET status = 'RUNNING', file_index = NULL, items_applied = NULL, " + "failure_reason = NULL, completed_at = NULL, heartbeat_at = '1970-01-01T00:00:00Z' " + "WHERE table_id = (SELECT table_id FROM tables WHERE account_id = ? AND table_name = ?)", + (ACCOUNT, target), + ) + assert n == 1 + _set_status(d, b2, "DELETING") + out_path = str(tmp_path / "forget.out") + with open(out_path, "wb") as out: + forgetting = subprocess.Popen( + [EXTENDDB_BINARY, "backup", "--config", d.config, "sync", "--forget-backup", b1], + cwd=d.cwd, stdout=out, stderr=subprocess.STDOUT, stdin=subprocess.DEVNULL, + ) + server = None + try: + deadline = time.monotonic() + 60 + while "Waiting for running jobs to finish" not in open(out_path).read(): + assert forgetting.poll() is None, open(out_path).read() + assert time.monotonic() < deadline, open(out_path).read() + time.sleep(0.2) + assert target in open(out_path).read(), open(out_path).read() + server, ddb, creds = _started(d, str(tmp_path / "serve-held.log")) + assert forgetting.wait(timeout=120) == 0, open(out_path).read() + text = open(out_path).read() + assert f" {b1}: removed: the row (AVAILABLE) is gone, and no object was deleted" in text, text + with pytest.raises(ClientError) as gone: + ddb.describe_table(TableName=target) + assert gone.value.response["Error"]["Code"] == "ResourceNotFoundException", gone.value.response + log = server.log_text() + assert "a maintenance claim is held" in log, log[-4000:] + finally: + if forgetting.poll() is None: + forgetting.kill() + forgetting.wait(timeout=10) + if server: + server.stop() + assert _rows(d) == {b2: "DELETED"} + assert not os.path.exists(_prefix(d, name, b2)), "the DELETING row's deletion was not finished" + assert os.path.isdir(root) diff --git a/tests/test_backup_store_config.py b/tests/test_backup_store_config.py index cddfc0c11..0910d87e3 100644 --- a/tests/test_backup_store_config.py +++ b/tests/test_backup_store_config.py @@ -1534,6 +1534,16 @@ def test_backup_list_and_purge_on_an_empty_store_and_the_serve_lock(self, sqlite # it: exit 6, the code `migrate` uses for a locked catalog. assert res.returncode == 6, res.stdout + res.stderr assert "another extenddb process" in (res.stdout + res.stderr), res.stdout + res.stderr + # So is every option of `backup sync` that moves or removes rows, + # with or without a store to read. + for args in ( + ("sync", "--move-store", "/srv/elsewhere"), + ("sync", "--forget-store", "/srv/elsewhere"), + ("sync", "--forget-backup", "arn:aws:dynamodb:us-east-1:123456789012:table/t/backup/01700000000000-00000000"), + ): + res = d.run("backup", *args) + assert res.returncode == 6, (args, res.stdout + res.stderr) + assert "another extenddb process" in (res.stdout + res.stderr), (args, res.stdout + res.stderr) assert server.proc.poll() is None, "the server kept running" finally: server.stop()