From 892b3732b7ce7efd780d298a8b2e392d4c416896 Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Fri, 30 Oct 2026 19:34:24 +0000 Subject: [PATCH 01/45] feat(migrations): run Cassandra and MongoDB through the migration runner Cassandra and MongoDB now use the numbered catalog migrations and the runner that SQLite and PostgreSQL use. Each keyspace or database keeps a ledger, the status is read from the ledgers, serve refuses a catalog that does not match the binary, and every change runs under the migration lock. Cassandra: migration 1 is a CQL file for the catalog keyspace and one for each account keyspace, with every statement qualified by its keyspace. Migration 2 is the shared policy rewrite; 3 and 4 do not apply. Cassandra has no transactions: every statement of a body can be sent again without harm, and a body's ledger row is written at QUORUM after it. Migration 1's row and the catalog_version row go in one logged batch; for a later id the version row is written first, so a run stopped between the two leaves that id pending. The lock is a settings row written by lightweight transactions at SERIAL, with a 60 s TTL renewed every 15 s while the run makes progress; a holder whose renewals fail stops writing 25 s before the row can expire. Each request under the lock is bounded by 10 s and each schema statement by 20 s. After every schema statement the runner waits until every live node reports the same schema version, and a node that does not answer is not counted as agreeing (#637). init creates the catalog keyspace and its settings table, takes the lock, and runs everything else under it, the default account's migrations included. Under an authorizer other than AllowAllAuthorizer it grants the application role its permissions on the catalog keyspace and the default account's keyspace. CreateAccount runs the account keyspace's migrations through the runner. MongoDB: the bodies are code for the catalog and data databases, and a body's ledger row and the catalog_version row commit in one transaction. The lock and the server registrations are documents written with majority write concern. Commands connect with 10 s connect and server selection timeouts, and each request under the lock is bounded by 10 s. Both backends read the status under the lock as its holder (status_under_lock), so migration 2's scan over every principal keeps the lock. Migration 2's error names its first failed lookup whatever its kind, and the status names a migration pending in several stores once, with the number of stores. With every backend on the runner, the bootstrapper's migration methods, the old branches of init, migrate and verify, serve's resource-tag marker check and OperationsEngine::catalog_version are removed. Fixes #637. --- Cargo.lock | 1 + crates/app/src/cmd_init.rs | 107 +- crates/app/src/cmd_migrate.rs | 517 +- crates/app/src/cmd_migrate_preview_tests.rs | 237 - crates/app/src/cmd_migrate_runner.rs | 53 +- crates/app/src/cmd_migrate_runner_tests.rs | 107 + crates/app/src/cmd_verify.rs | 57 +- crates/app/src/lib.rs | 5 +- crates/core/src/settings_keys.rs | 17 - crates/server/src/serve.rs | 23 - crates/storage-cassandra/Cargo.toml | 8 + .../0001_baseline.cql} | 34 +- .../account/0001_baseline.expected.json | 400 ++ .../migrations/catalog/0001_baseline.cql | 358 ++ .../catalog/0001_baseline.expected.json | 1007 ++++ .../catalog/V001__initial_schema.cql | 213 - .../catalog/V002__backup_restore.cql | 65 - .../V003__account_scoped_idempotency.cql | 13 - .../catalog/V004__ttl_backfill_cursor.cql | 12 - ...p_definitions_and_restore_registration.cql | 32 - .../migrations/data/V002__backup_items.cql | 12 - crates/storage-cassandra/src/backup_engine.rs | 41 +- crates/storage-cassandra/src/bootstrapper.rs | 336 +- .../src/bootstrapper/guards.rs | 195 + crates/storage-cassandra/src/catalog_store.rs | 150 +- crates/storage-cassandra/src/config.rs | 161 +- crates/storage-cassandra/src/create_table.rs | 13 +- crates/storage-cassandra/src/engine.rs | 9 - crates/storage-cassandra/src/lib.rs | 163 +- .../src/management_store/mod.rs | 4 + crates/storage-cassandra/src/migrations.rs | 584 --- .../src/migrations/agreement.rs | 274 + .../src/migrations/backend.rs | 533 ++ .../src/migrations/backend/constructors.rs | 133 + .../src/migrations/backend/ledger.rs | 117 + .../src/migrations/commands.rs | 207 + .../src/migrations/connect.rs | 507 ++ .../storage-cassandra/src/migrations/cql.rs | 343 ++ .../src/migrations/guarded.rs | 153 + .../storage-cassandra/src/migrations/locks.rs | 566 +++ .../storage-cassandra/src/migrations/mod.rs | 599 +++ .../src/migrations/postcheck.rs | 262 + .../src/migrations/registrations.rs | 312 ++ .../storage-cassandra/src/migrations/tests.rs | 140 + crates/storage-cassandra/src/operations.rs | 30 +- crates/storage-cassandra/tests/common/mod.rs | 55 +- .../tests/direct/backup_engine.rs | 274 +- .../direct/concurrent_account_creation.rs | 2 +- .../tests/direct/keyspace_migrations.rs | 94 +- .../0002_resource_tag_condition_keys.txt | 14 + crates/storage-cassandra/tests/migrations.rs | 521 ++ .../tests/migrations/lock.rs | 299 ++ .../tests/migrations_stopped.rs | 516 ++ .../tests/migrations_support/mod.rs | 154 + crates/storage-mongodb/Cargo.toml | 16 +- .../catalog/0001_baseline.expected.json | 423 ++ .../data/0001_baseline.expected.json | 108 + crates/storage-mongodb/src/bootstrapper.rs | 632 +-- crates/storage-mongodb/src/catalog_store.rs | 11 + crates/storage-mongodb/src/lib.rs | 210 +- .../storage-mongodb/src/migrations/backend.rs | 497 ++ .../src/migrations/commands.rs | 176 + .../storage-mongodb/src/migrations/guarded.rs | 255 + .../storage-mongodb/src/migrations/locks.rs | 485 ++ .../src/migrations/m0001_baseline.rs | 235 + crates/storage-mongodb/src/migrations/mod.rs | 290 ++ .../src/migrations/postcheck.rs | 347 ++ .../src/migrations/registrations.rs | 277 + .../storage-mongodb/src/migrations/tests.rs | 80 + crates/storage-mongodb/src/operations.rs | 6 - .../tests/fixtures/golden/0001_baseline.txt | 557 ++ .../0002_resource_tag_condition_keys.txt | 83 + .../tests/fixtures/released/0.1.12.json | 2683 ++++++++++ .../tests/fixtures/released/0.1.13.json | 2683 ++++++++++ .../tests/fixtures/released/0.1.14.json | 2687 ++++++++++ .../tests/fixtures/released/README.md | 30 + .../released/accepted-differences.json | 8 + .../tests/fixtures/released/dotted-ttl.json | 4486 +++++++++++++++++ .../tests/fixtures/released/dump.js | 37 + .../tests/fixtures/released/load.js | 28 + .../tests/fixtures/released/make-fixture.sh | 91 + .../tests/fixtures/released/seed.py | 103 + .../storage-mongodb/tests/migrations_fresh.rs | 313 ++ .../tests/migrations_guards.rs | 294 ++ .../tests/migrations_holders.rs | 354 ++ .../storage-mongodb/tests/migrations_locks.rs | 274 + .../tests/migrations_released.rs | 313 ++ .../tests/migrations_support/fixture.rs | 206 + .../tests/migrations_support/mod.rs | 186 + crates/storage-postgres/src/bootstrapper.rs | 36 +- crates/storage-postgres/src/lib.rs | 2 +- crates/storage-postgres/src/operations.rs | 8 - crates/storage-sqlite/src/bootstrapper.rs | 36 - crates/storage-sqlite/src/lib.rs | 2 +- crates/storage-sqlite/src/operations.rs | 8 - crates/storage/src/backend.rs | 6 +- crates/storage/src/bootstrapper.rs | 127 +- crates/storage/src/migrations/advice.rs | 88 +- crates/storage/src/migrations/catalog.rs | 44 +- .../m0002_resource_tag_condition_keys.rs | 44 +- crates/storage/src/operations.rs | 8 - 101 files changed, 27683 insertions(+), 3229 deletions(-) delete mode 100644 crates/app/src/cmd_migrate_preview_tests.rs rename crates/storage-cassandra/migrations/{data/V001__initial_data_schema.cql => account/0001_baseline.cql} (74%) create mode 100644 crates/storage-cassandra/migrations/account/0001_baseline.expected.json create mode 100644 crates/storage-cassandra/migrations/catalog/0001_baseline.cql create mode 100644 crates/storage-cassandra/migrations/catalog/0001_baseline.expected.json delete mode 100644 crates/storage-cassandra/migrations/catalog/V001__initial_schema.cql delete mode 100644 crates/storage-cassandra/migrations/catalog/V002__backup_restore.cql delete mode 100644 crates/storage-cassandra/migrations/catalog/V003__account_scoped_idempotency.cql delete mode 100644 crates/storage-cassandra/migrations/catalog/V004__ttl_backfill_cursor.cql delete mode 100644 crates/storage-cassandra/migrations/catalog/V005__backup_definitions_and_restore_registration.cql delete mode 100644 crates/storage-cassandra/migrations/data/V002__backup_items.cql create mode 100644 crates/storage-cassandra/src/bootstrapper/guards.rs delete mode 100644 crates/storage-cassandra/src/migrations.rs create mode 100644 crates/storage-cassandra/src/migrations/agreement.rs create mode 100644 crates/storage-cassandra/src/migrations/backend.rs create mode 100644 crates/storage-cassandra/src/migrations/backend/constructors.rs create mode 100644 crates/storage-cassandra/src/migrations/backend/ledger.rs create mode 100644 crates/storage-cassandra/src/migrations/commands.rs create mode 100644 crates/storage-cassandra/src/migrations/connect.rs create mode 100644 crates/storage-cassandra/src/migrations/cql.rs create mode 100644 crates/storage-cassandra/src/migrations/guarded.rs create mode 100644 crates/storage-cassandra/src/migrations/locks.rs create mode 100644 crates/storage-cassandra/src/migrations/mod.rs create mode 100644 crates/storage-cassandra/src/migrations/postcheck.rs create mode 100644 crates/storage-cassandra/src/migrations/registrations.rs create mode 100644 crates/storage-cassandra/src/migrations/tests.rs create mode 100644 crates/storage-cassandra/tests/fixtures/golden/0002_resource_tag_condition_keys.txt create mode 100644 crates/storage-cassandra/tests/migrations.rs create mode 100644 crates/storage-cassandra/tests/migrations/lock.rs create mode 100644 crates/storage-cassandra/tests/migrations_stopped.rs create mode 100644 crates/storage-cassandra/tests/migrations_support/mod.rs create mode 100644 crates/storage-mongodb/migrations/catalog/0001_baseline.expected.json create mode 100644 crates/storage-mongodb/migrations/data/0001_baseline.expected.json create mode 100644 crates/storage-mongodb/src/migrations/backend.rs create mode 100644 crates/storage-mongodb/src/migrations/commands.rs create mode 100644 crates/storage-mongodb/src/migrations/guarded.rs create mode 100644 crates/storage-mongodb/src/migrations/locks.rs create mode 100644 crates/storage-mongodb/src/migrations/m0001_baseline.rs create mode 100644 crates/storage-mongodb/src/migrations/mod.rs create mode 100644 crates/storage-mongodb/src/migrations/postcheck.rs create mode 100644 crates/storage-mongodb/src/migrations/registrations.rs create mode 100644 crates/storage-mongodb/src/migrations/tests.rs create mode 100644 crates/storage-mongodb/tests/fixtures/golden/0001_baseline.txt create mode 100644 crates/storage-mongodb/tests/fixtures/golden/0002_resource_tag_condition_keys.txt create mode 100644 crates/storage-mongodb/tests/fixtures/released/0.1.12.json create mode 100644 crates/storage-mongodb/tests/fixtures/released/0.1.13.json create mode 100644 crates/storage-mongodb/tests/fixtures/released/0.1.14.json create mode 100644 crates/storage-mongodb/tests/fixtures/released/README.md create mode 100644 crates/storage-mongodb/tests/fixtures/released/accepted-differences.json create mode 100644 crates/storage-mongodb/tests/fixtures/released/dotted-ttl.json create mode 100644 crates/storage-mongodb/tests/fixtures/released/dump.js create mode 100644 crates/storage-mongodb/tests/fixtures/released/load.js create mode 100755 crates/storage-mongodb/tests/fixtures/released/make-fixture.sh create mode 100755 crates/storage-mongodb/tests/fixtures/released/seed.py create mode 100644 crates/storage-mongodb/tests/migrations_fresh.rs create mode 100644 crates/storage-mongodb/tests/migrations_guards.rs create mode 100644 crates/storage-mongodb/tests/migrations_holders.rs create mode 100644 crates/storage-mongodb/tests/migrations_locks.rs create mode 100644 crates/storage-mongodb/tests/migrations_released.rs create mode 100644 crates/storage-mongodb/tests/migrations_support/fixture.rs create mode 100644 crates/storage-mongodb/tests/migrations_support/mod.rs diff --git a/Cargo.lock b/Cargo.lock index ffe80fb18..ea03200fd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1904,6 +1904,7 @@ dependencies = [ "extenddb-auth", "extenddb-core", "extenddb-storage", + "extenddb-storage-cassandra", "futures", "rand 0.9.4", "serde", diff --git a/crates/app/src/cmd_init.rs b/crates/app/src/cmd_init.rs index 674ffe010..87378db2d 100755 --- a/crates/app/src/cmd_init.rs +++ b/crates/app/src/cmd_init.rs @@ -252,32 +252,9 @@ pub async fn run(args: InitArgs, release: &str) -> anyhow::Result { .await .map_err(crate::cmd_migrate::op_err)?; - let admin_result = if let Some(migrations) = extenddb_storage::migrations::installed() { - init_on_runner(bootstrapper.as_ref(), migrations, release).await? - } else { - // Take the same lock `migrate` uses around the schema work below, so an - // init cannot race a migrate running on another replica and fail on a - // duplicate pg_type entry from concurrent `CREATE TABLE IF NOT EXISTS`. - // Two concurrent inits cannot get this far: the second aborts earlier, at - // `create_catalog_db`, because the database already exists. The catalog - // database does exist by this point, so the lock connection can be opened. - // - // The lock is held through the bootstrap rows as well, not just the - // schema: a server that starts after the schema exists but before the - // encryption key is written fails with MissingEncryptionKey. - bootstrapper - .acquire_migration_lock() - .await - .map_err(crate::cmd_migrate::op_err)?; - let bootstrap_result = run_init_bootstrap(bootstrapper.as_ref()).await; - if let Err(e) = bootstrapper.release_migration_lock().await { - tracing::warn!( - "Failed to release migration lock: {}", - crate::cmd_migrate::op_err(e) - ); - } - bootstrap_result? - }; + let migrations = extenddb_storage::migrations::installed() + .ok_or_else(|| anyhow::anyhow!("no storage backend is installed"))?; + let admin_result = init_on_runner(bootstrapper.as_ref(), migrations, release).await?; if admin_result.already_existed { // Already printed by the bootstrap store. @@ -351,33 +328,26 @@ async fn init_on_runner( .lock(&target, INIT_LOCK_TIMEOUT) .await .map_err(|e| anyhow::anyhow!("{e}"))?; - println!("--- Creating the catalog through its migrations..."); - let report = run_locked(migrations, &target, &lock, release, RunKind::Create) - .await - .map_err(|e| anyhow::anyhow!("{e}"))?; - for note in &report.notes { - println!(" {note}"); + let result = async { + println!("--- Creating the catalog through its migrations..."); + let report = run_locked(migrations, &target, &lock, release, RunKind::Create) + .await + .map_err(|e| anyhow::anyhow!("{e}"))?; + for note in &report.notes { + println!(" {note}"); + } + crate::util::print_applied(&report, " ", "Applied migration"); + bootstrapper.hold(&lock); + let admin = write_bootstrap_rows(bootstrapper).await?; + bootstrapper + .record_data_connection() + .await + .map_err(crate::cmd_migrate::op_err)?; + Ok(admin) } - crate::util::print_applied(&report, " ", "Applied migration"); - let admin = write_bootstrap_rows(bootstrapper).await?; - bootstrapper - .record_data_connection() - .await - .map_err(crate::cmd_migrate::op_err)?; - drop(lock); - Ok(admin) -} - -/// Apply the catalog and data schema while the migration lock is held. Split out -/// of `run` so that the lock is released on every path, including errors. -/// Everything `init` does under the migration lock: the schema, then the -/// rows a server needs before it can start (encryption key, default account, -/// admin user). -async fn run_init_bootstrap( - bootstrapper: &dyn extenddb_storage::bootstrapper::Bootstrapper, -) -> anyhow::Result { - run_init_migrations(bootstrapper).await?; - write_bootstrap_rows(bootstrapper).await + .await; + migrations.release(lock).await; + result } /// The rows a server needs before it can start: encryption key, default @@ -404,39 +374,6 @@ async fn write_bootstrap_rows( .map_err(crate::cmd_migrate::op_err) } -async fn run_init_migrations( - bootstrapper: &dyn extenddb_storage::bootstrapper::Bootstrapper, -) -> anyhow::Result<()> { - // Check if catalog is already initialized. - let initialized = bootstrapper - .is_catalog_initialized() - .await - .map_err(crate::cmd_migrate::op_err)?; - - if initialized { - println!("--- Catalog already initialized. Use 'extenddb migrate' for pending migrations."); - } else { - bootstrapper - .run_catalog_migrations() - .await - .map_err(crate::cmd_migrate::op_err)?; - } - - // Record data database connection in catalog. - bootstrapper - .record_data_connection() - .await - .map_err(crate::cmd_migrate::op_err)?; - - // Initialize data database schema. - bootstrapper - .run_data_migrations() - .await - .map_err(crate::cmd_migrate::op_err)?; - - Ok(()) -} - #[cfg(test)] mod tests { use super::InitArgs; diff --git a/crates/app/src/cmd_migrate.rs b/crates/app/src/cmd_migrate.rs index 264fd6b1d..47162acdd 100755 --- a/crates/app/src/cmd_migrate.rs +++ b/crates/app/src/cmd_migrate.rs @@ -3,26 +3,16 @@ //! `extenddb migrate` — apply catalog schema migrations (REQ-CAT-014). //! -//! A backend on the migration runner (PostgreSQL, SQLite) takes -//! `cmd_migrate_runner`'s path, whose exit statuses are 0, 1, 3, 4, 5 and 6 -//! (see that module). This module is the bootstrapper's path, for Cassandra -//! and MongoDB: it reads the current catalog version, runs pending -//! migrations, and reports the result. Exit status: 0 when nothing is -//! pending or the migrations were applied; [`EXIT_PENDING`] (3) when work is -//! pending and `--yes` was not given (the preview); 1 on any error. A -//! scripted upgrade can run the preview under `set -e` by checking for 3 -//! explicitly. +//! The command's arguments and the error helpers the other commands share. +//! Every backend migrates through `cmd_migrate_runner`, whose exit statuses +//! are 0, 1, 3 (work pending and `--yes` not given), 4, 5 and 6 (see that +//! module). use clap::Args; -use extenddb_config as config; use extenddb_storage::error::StorageError; use extenddb_storage::management_store::OpError; -/// Exit status of a preview (`migrate` without `--yes`) that found work to do. -/// Distinct from 1 so a script can tell "pending" from "failed". -pub const EXIT_PENDING: u8 = extenddb_storage::migrations::exit::PENDING; - /// The config file `migrate` and `verify` read when `--config` is not given. pub(crate) const DEFAULT_CONFIG: &str = "extenddb.toml"; @@ -54,15 +44,15 @@ pub struct MigrateArgs { #[arg(long)] pub(crate) data_db: Option, - /// Cassandra admin user (the role `init` used); without it the - /// connection is attempted as `$USER` + /// Cassandra role to connect as instead of the role the config file + /// names (for example the role `init` used) #[arg(long)] - cassandra_user: Option, + pub(crate) cassandra_user: Option, - /// Cassandra admin password. + /// Password of the `--cassandra-user` role. /// Prefer `EXTENDDB_CASSANDRA_PASSWORD` to keep the value out of process arguments. #[arg(long)] - cassandra_pass: Option, + pub(crate) cassandra_pass: Option, /// Apply the pending migrations. Without it, migrate only reports what is /// pending and exits 3 if there is any (0 if everything is up to date) @@ -90,13 +80,6 @@ pub struct MigrateArgs { pub(crate) lock_timeout: Option, } -impl MigrateArgs { - /// The config file named by `--config`, else the default. - pub(crate) fn config_path(&self) -> &str { - self.config.as_deref().unwrap_or(DEFAULT_CONFIG) - } -} - /// The message of a bootstrapper error, without the variant name that `{e:?}` /// adds (which printed `Internal("Internal(\"Cannot connect ...\")")`). pub(crate) fn op_err(e: OpError) -> anyhow::Error { @@ -119,482 +102,10 @@ pub(crate) fn storage_err(e: StorageError) -> anyhow::Error { } } -/// Run `extenddb migrate`; returns the process exit status (see the module -/// docs). A backend on the migration runner takes `cmd_migrate_runner`'s -/// path; this one is the bootstrapper's. +/// Run `extenddb migrate`; returns the process exit status (see +/// `cmd_migrate_runner`). pub async fn run(args: MigrateArgs, release: &str) -> anyhow::Result { - if let Some(migrations) = extenddb_storage::migrations::installed() { - return crate::cmd_migrate_runner::run(&args, migrations, release).await; - } - if args.list || args.reconverge || args.forget.is_some() || args.lock_timeout.is_some() { - anyhow::bail!( - "--list, --reconverge, --forget and --lock-timeout need the migration ledger, and \ - the {} backend still migrates through its bootstrapper", - extenddb_storage::backend_name().unwrap_or("installed") - ); - } - let config_path = args.config_path(); - if !std::path::Path::new(config_path).exists() { - anyhow::bail!( - "Config file '{config_path}' not found. Run 'extenddb init' to set up a deployment, \ - or use --config to specify a different location.", - ); - } - // Load the config for validation and to reach the catalog for the policy - // rewrite: migrate drives the schema steps through the bootstrapper, but a - // malformed config should fail here rather than midway through a migration. - let app_config = config::load(config_path)?; - - println!("=== extenddb migrate ==="); - println!("Config: {config_path}"); - println!(); - - // Collect CLI args for backend-specific parsing. The environment-sourced - // secret is appended only to this in-process copy, not OS-visible argv. - let cli_args = backend_args( - std::env::args().collect(), - std::env::var_os("EXTENDDB_PG_PASSWORD"), - std::env::var_os("EXTENDDB_CASSANDRA_PASSWORD"), - )?; - - // Create bootstrapper via registry - let bootstrap = extenddb_storage::bootstrapper::create_bootstrapper(config_path, &cli_args) - .await - .map_err(storage_err)?; - - // Serialize concurrent migrators, such as several replicas starting at - // once, so they don't race each other applying schema changes. - bootstrap.acquire_migration_lock().await.map_err(op_err)?; - let result = match CatalogResourceTagStep::open(app_config.storage.connection_config()).await { - Ok(rtag) => apply_migrations(bootstrap.as_ref(), &args, &rtag).await, - Err(e) => Err(e), - }; - if let Err(e) = bootstrap.release_migration_lock().await { - tracing::warn!("Failed to release migration lock: {}", op_err(e)); - } - result -} - -/// The argument vector the backend bootstrapper parses: `argv`, plus -/// `--pg-pass` from `EXTENDDB_PG_PASSWORD` and `--cassandra-pass` from -/// `EXTENDDB_CASSANDRA_PASSWORD` when the flag itself was not given. -pub(crate) fn backend_args( - argv: Vec, - pg_pass_env: Option, - cassandra_pass_env: Option, -) -> anyhow::Result> { - let mut args = argv; - crate::util::append_secret_arg(&mut args, "--pg-pass", pg_pass_env, "EXTENDDB_PG_PASSWORD")?; - crate::util::append_secret_arg( - &mut args, - "--cassandra-pass", - cassandra_pass_env, - "EXTENDDB_CASSANDRA_PASSWORD", - )?; - Ok(args) -} - -/// Run the version checks and apply pending migrations while the migration lock -/// is held. Split out of `run` so that the lock is always released afterwards. -async fn apply_migrations( - bootstrap: &dyn extenddb_storage::bootstrapper::Bootstrapper, - args: &MigrateArgs, - rtag: &dyn ResourceTagStep, -) -> anyhow::Result { - // Show current catalog version. - println!("--- Checking current catalog version..."); - let current = bootstrap.read_catalog_version().await.map_err(op_err)?; - let current_display = current.as_deref().unwrap_or("none"); - println!(" Current version: {current_display}"); - - let expected = bootstrap.expected_catalog_version(); - refuse_newer_catalog(current.as_deref(), &expected)?; - let catalog_pending = current.as_deref() != Some(expected.as_str()); - - // Data migrations are tracked in the data database's own ledger, separate - // from and independent of the catalog version, so they must be checked (and - // applied) on their own — a release that only changes the data schema does - // not bump the catalog version. - println!("--- Checking data migrations..."); - let data_pending = bootstrap.pending_data_migrations().await.map_err(op_err)?; - if data_pending.is_empty() { - println!(" Data migrations up to date."); - } else { - println!(" Pending: {}", data_pending.join(", ")); - } - - // Detect table metadata damaged by the pre-fix UpdateTable (#259), which - // deleted a table's own key attribute definitions when a GSI was added. - // - // Detection is read-only and runs even when the catalog version is already - // current: the damage is in a row's contents, not in the schema, so an - // up-to-date deployment is exactly the case that needs checking. Nothing is - // written until --yes has been given, below. - println!("--- Checking table key metadata..."); - let detected = bootstrap - .repair_lost_sort_key_definitions(false) - .await - .map_err(op_err)?; - let repair_pending = !detected.repaired.is_empty(); - if !repair_pending && detected.needs_attention.is_empty() { - println!(" Table key metadata intact."); - } else { - for line in &detected.repaired { - println!(" Repairable: {line}"); - } - for line in &detected.needs_attention { - println!(" NEEDS ATTENTION: {line}"); - } - } - - // Detect stored policies that still name a dynamodb:ResourceTag condition - // key. Detection is read-only and runs on every migrate (like the repair - // above), so an interrupted run is finished by the next. The catalog is - // reached through a catalog-only management store, not the bootstrapper's - // schema connection. - println!("--- Checking resource-tag condition keys..."); - let (rtag_detect, marker_present) = rtag.detect().await?; - if rtag_detect.rewritten.is_empty() && rtag_detect.needs_attention.is_empty() { - println!(" No dynamodb:ResourceTag condition keys found."); - } else { - for r in &rtag_detect.rewritten { - println!( - " Rewrite: account {} {} policy {} ({} key(s))", - r.account_id, r.principal, r.policy_name, r.rewrites - ); - } - for n in &rtag_detect.needs_attention { - println!( - " NEEDS ATTENTION: account {} {} policy {} (aws:ResourceTag and \ - dynamodb:ResourceTag forms collide; left unchanged)", - n.account_id, n.principal, n.policy_name - ); - } - } - if rtag_detect.errors > 0 { - println!( - " WARNING: {} policy lookup(s) failed during detection.", - rtag_detect.errors - ); - } - let rtag_pending = !rtag_detect.rewritten.is_empty() || rtag_detect.errors > 0; - // A catalog with no marker must have one written, even when there is - // nothing to rewrite, so that serve will start. - let marker_pending = !marker_present; - - if !catalog_pending - && data_pending.is_empty() - && !repair_pending - && !rtag_pending - && !marker_pending - { - println!(); - if detected.needs_attention.is_empty() && rtag_detect.needs_attention.is_empty() { - println!( - "Everything is up to date (catalog version {expected}). No migrations needed." - ); - } else { - // Nothing is automatically applicable, but saying "everything is up - // to date" straight after a NEEDS ATTENTION line would be - // contradictory: those items require a human. - println!( - "No migrations needed (catalog version {expected}), but {} item(s) above \ - need manual attention.", - detected.needs_attention.len() + rtag_detect.needs_attention.len() - ); - } - return Ok(0); - } - - if !args.yes { - let mut what = Vec::new(); - if catalog_pending { - what.push(format!("catalog {current_display} -> {expected}")); - } - if !data_pending.is_empty() { - what.push(format!("data migrations [{}]", data_pending.join(", "))); - } - if repair_pending { - what.push(format!( - "table key metadata repairs [{}]", - detected.repaired.join("; ") - )); - } - if !rtag_detect.rewritten.is_empty() { - what.push(format!( - "resource-tag condition-key rewrites [{} policy(ies)]", - rtag_detect.rewritten.len() - )); - } - if marker_pending { - what.push("record the resource-tag migration marker".to_owned()); - } - if rtag_detect.errors > 0 { - what.push(format!( - "{} policy lookup(s) that failed and must be retried", - rtag_detect.errors - )); - } - if !rtag_detect.needs_attention.is_empty() { - what.push(format!( - "{} policy(ies) listed as NEEDS ATTENTION, which must be fixed by hand before \ - the migration can complete", - rtag_detect.needs_attention.len() - )); - } - // A preview, not a failure: report and exit EXIT_PENDING so a script - // under `set -e` can tell it from an error (exit 1). - println!(); - println!( - "Pending: {}. Nothing was changed; run again with --yes to apply \ - (exit status {EXIT_PENDING}: work pending).", - what.join("; ") - ); - return Ok(EXIT_PENDING); - } - - if catalog_pending { - bootstrap.run_catalog_migrations().await.map_err(op_err)?; - } - - // Always run data migrations: the data-database ledger is idempotent - // (already-applied migrations are skipped) and independent of the catalog - // version, so an upgrade that only touched the data schema is still applied. - bootstrap.run_data_migrations().await.map_err(op_err)?; - - // Apply the table key metadata repair after the schema migrations, so it never - // reads or writes a pre-migration catalog shape. - if repair_pending { - let applied = bootstrap - .repair_lost_sort_key_definitions(true) - .await - .map_err(op_err)?; - for line in &applied.repaired { - println!(" Restored sort key definition: {line}"); - } - } - - // Rewrite stored policies from dynamodb:ResourceTag to aws:ResourceTag, - // after the schema migrations. Every storage error is counted; on a - // non-zero count migrate reports it, does NOT record the completion - // marker, and exits non-zero, so the next run finishes the rewrite. No - // policy body is logged. - let rtag_applied = rtag.rewrite().await; - for r in &rtag_applied.rewritten { - println!( - " Rewrote resource-tag keys: account {} {} policy {} ({} key(s))", - r.account_id, r.principal, r.policy_name, r.rewrites - ); - } - for n in &rtag_applied.needs_attention { - println!( - " NEEDS ATTENTION: account {} {} policy {} left unchanged (collision)", - n.account_id, n.principal, n.policy_name - ); - } - if rtag_applied.errors > 0 { - anyhow::bail!( - "resource-tag migration incomplete: {} policy lookup(s) failed. \ - Re-run `extenddb migrate --yes` to finish; the migration marker was not recorded.", - rtag_applied.errors - ); - } - // A policy left unchanged because both resource-tag forms name the same - // tag key in one operator block is not migrated. Recording the marker - // would let serve evaluate it with the dynamodb: half absent, which can - // turn a Deny off. Withhold the marker so serve keeps refusing until the - // policy is fixed by hand and migrate is run again. - if !rtag_applied.needs_attention.is_empty() { - anyhow::bail!( - "resource-tag migration incomplete: {} policy(ies) above name both \ - aws:ResourceTag/ and dynamodb:ResourceTag/ for the same key in one \ - condition operator and were left unchanged. Remove one of the two forms from \ - each, then re-run `extenddb migrate --yes`; the migration marker was not recorded.", - rtag_applied.needs_attention.len() - ); - } - // Record the completion marker only after a clean rewrite. serve reads this - // row and refuses to start without it. - rtag.record_marker().await?; - - // Read new catalog version. - let new = bootstrap.read_catalog_version().await.map_err(op_err)?; - let new_display = new.as_deref().unwrap_or("none"); - - println!(); - println!("=== extenddb migrate complete ==="); - println!("Catalog version: {current_display} -> {new_display}"); - - Ok(0) -} - -/// Boxed future returned by [`ResourceTagStep`]. -type StepFuture<'a, T> = std::pin::Pin + Send + 'a>>; - -/// The resource-tag condition-key migration (stored policies rewritten from -/// `dynamodb:ResourceTag` to `aws:ResourceTag`, then a completion marker), -/// behind a trait so the rest of [`apply_migrations`] can be exercised without -/// a catalog. -trait ResourceTagStep: Sync { - /// The read-only detection report, and whether the completion marker is - /// recorded. - fn detect( - &self, - ) -> StepFuture<'_, anyhow::Result<(extenddb_storage::policy_migration::MigrationReport, bool)>>; - /// Rewrite the stored policies. - fn rewrite(&self) -> StepFuture<'_, extenddb_storage::policy_migration::MigrationReport>; - /// Record the completion marker `serve` requires. - fn record_marker(&self) -> StepFuture<'_, anyhow::Result<()>>; -} - -/// [`ResourceTagStep`] over the deployment's catalog. -struct CatalogResourceTagStep { - mgmt: Box, - settings: Box, -} - -impl CatalogResourceTagStep { - /// Open catalog-only management and settings stores (not the - /// bootstrapper's schema connection). - async fn open(connection: &str) -> anyhow::Result { - let mgmt = extenddb_storage::management_store_factory::create_management_store(connection) - .await - .map_err(|e| anyhow::anyhow!("{e}"))?; - let settings = extenddb_storage::settings_store::create_settings_store(connection) - .await - .map_err(|e| anyhow::anyhow!("{e}"))?; - Ok(Self { mgmt, settings }) - } -} - -impl ResourceTagStep for CatalogResourceTagStep { - fn detect( - &self, - ) -> StepFuture<'_, anyhow::Result<(extenddb_storage::policy_migration::MigrationReport, bool)>> - { - Box::pin(async move { - let report = extenddb_storage::policy_migration::migrate_resource_tag_condition_keys( - self.mgmt.as_ref(), - false, - ) - .await; - let marker_present = self - .settings - .get_setting(extenddb_core::settings_keys::RESOURCE_TAG_MIGRATION) - .await - .map_err(|e| anyhow::anyhow!("{e:?}"))? - .is_some(); - Ok((report, marker_present)) - }) - } - - fn rewrite(&self) -> StepFuture<'_, extenddb_storage::policy_migration::MigrationReport> { - Box::pin( - extenddb_storage::policy_migration::migrate_resource_tag_condition_keys( - self.mgmt.as_ref(), - true, - ), - ) - } - - fn record_marker(&self) -> StepFuture<'_, anyhow::Result<()>> { - Box::pin(async move { - self.settings - .set_setting( - extenddb_core::settings_keys::RESOURCE_TAG_MIGRATION, - extenddb_core::settings_keys::RESOURCE_TAG_MIGRATION_DONE, - ) - .await - .map_err(|e| anyhow::anyhow!("{e:?}")) - }) - } -} - -/// A catalog written by a newer binary is not a pending upgrade. The runners -/// write the compiled version after every walk, so letting an older binary -/// "migrate" it would stamp the older version onto the newer schema and let -/// that binary start against tables it does not understand. A stored version -/// this binary cannot parse is refused for the same reason: it is the one -/// case the guard cannot rule out being newer. The rule is shared with the -/// runners (`extenddb_storage::util::check_stored_catalog_version`). -fn refuse_newer_catalog(current: Option<&str>, expected: &str) -> anyhow::Result<()> { - use extenddb_core::version::CatalogVersion; - let expected_v = expected.parse::().map_err(|e| { - anyhow::anyhow!("this binary's expected catalog version {expected:?} is invalid: {e}") - })?; - extenddb_storage::util::check_stored_catalog_version(current, expected_v).map_err(|e| { - anyhow::anyhow!( - "{e} extenddb migrate does not downgrade a catalog or overwrite one it cannot read." - ) - }) -} - -#[cfg(test)] -#[path = "cmd_migrate_preview_tests.rs"] -mod preview_tests; - -#[cfg(test)] -mod backend_args_tests { - use super::backend_args; - - fn argv(extra: &[&str]) -> Vec { - ["extenddb", "migrate"] - .iter() - .chain(extra) - .map(|s| (*s).to_owned()) - .collect() - } - - #[test] - fn passwords_come_from_the_environment_when_the_flag_is_absent() { - let args = - backend_args(argv(&[]), Some("pgsecret".into()), Some("cass".into())).expect("args"); - assert_eq!( - args, - argv(&["--pg-pass", "pgsecret", "--cassandra-pass", "cass"]) - ); - } - - #[test] - fn a_flag_on_the_command_line_wins_over_the_environment() { - let given = argv(&["--cassandra-pass=fromflag", "--pg-pass", "pgflag"]); - let args = backend_args(given.clone(), Some("pgenv".into()), Some("cassenv".into())) - .expect("args"); - assert_eq!(args, given); - } - - #[test] - fn no_environment_adds_nothing() { - assert_eq!( - backend_args(argv(&[]), None, None).expect("args"), - argv(&[]) - ); - } -} - -#[cfg(test)] -mod newer_catalog_tests { - use super::refuse_newer_catalog; - - #[test] - fn older_or_equal_or_absent_is_allowed() { - assert!(refuse_newer_catalog(None, "0.0.4").is_ok()); - assert!(refuse_newer_catalog(Some("0.0.3"), "0.0.4").is_ok()); - assert!(refuse_newer_catalog(Some("0.0.4"), "0.0.4").is_ok()); - } - - #[test] - fn an_unparseable_stored_version_is_refused() { - for stored in ["garbage", "", "0.0", "0.0.4-rc1"] { - let err = refuse_newer_catalog(Some(stored), "0.0.4").expect_err(stored); - assert!(err.to_string().contains("refusing to overwrite"), "{err}"); - } - } - - #[test] - fn newer_is_refused() { - let err = refuse_newer_catalog(Some("0.0.5"), "0.0.4").expect_err("newer catalog"); - assert!(err.to_string().contains("does not downgrade"), "{err}"); - assert!(refuse_newer_catalog(Some("0.1.0"), "0.0.4").is_err()); - } + let migrations = extenddb_storage::migrations::installed() + .ok_or_else(|| anyhow::anyhow!("no storage backend is installed"))?; + crate::cmd_migrate_runner::run(&args, migrations, release).await } diff --git a/crates/app/src/cmd_migrate_preview_tests.rs b/crates/app/src/cmd_migrate_preview_tests.rs deleted file mode 100644 index 592ee11ef..000000000 --- a/crates/app/src/cmd_migrate_preview_tests.rs +++ /dev/null @@ -1,237 +0,0 @@ -// Copyright 2026 ExtendDB contributors -// SPDX-License-Identifier: Apache-2.0 - -//! The preview's exit status and that it changes nothing (it used to -//! exit 1 with "Error: --yes is required", which stops a -//! scripted upgrade under `set -e`). -use std::sync::Mutex; - -use super::{ResourceTagStep, StepFuture}; - -use async_trait::async_trait; -use extenddb_storage::bootstrapper::{AdminBootstrapResult, Bootstrapper, KeyDefinitionRepair}; -use extenddb_storage::management_store::{OpError, OpResult}; - -use super::{EXIT_PENDING, MigrateArgs, apply_migrations, op_err, storage_err}; - -struct Fake { - version: Mutex>, - data_pending: Mutex>, - catalog_runs: Mutex, - data_runs: Mutex, -} - -impl Fake { - fn new(version: Option<&str>, data_pending: &[&str]) -> Self { - Self { - version: Mutex::new(version.map(str::to_owned)), - data_pending: Mutex::new(data_pending.iter().map(|s| (*s).to_owned()).collect()), - catalog_runs: Mutex::new(0), - data_runs: Mutex::new(0), - } - } - fn runs(&self) -> (u32, u32) { - ( - *self.catalog_runs.lock().unwrap(), - *self.data_runs.lock().unwrap(), - ) - } -} - -fn unused() -> OpResult { - Err(OpError::Internal("not used by migrate".to_owned())) -} - -#[async_trait] -impl Bootstrapper for Fake { - async fn ensure_app_user(&self) -> OpResult<()> { - unused() - } - async fn grant_app_role_to_admin(&self) -> OpResult<()> { - unused() - } - async fn create_catalog_db(&self) -> OpResult<()> { - unused() - } - async fn create_data_db(&self) -> OpResult<()> { - unused() - } - async fn run_catalog_migrations(&self) -> OpResult<()> { - *self.catalog_runs.lock().unwrap() += 1; - *self.version.lock().unwrap() = Some(self.expected_catalog_version()); - Ok(()) - } - async fn run_data_migrations(&self) -> OpResult<()> { - *self.data_runs.lock().unwrap() += 1; - self.data_pending.lock().unwrap().clear(); - Ok(()) - } - async fn repair_lost_sort_key_definitions( - &self, - _apply: bool, - ) -> OpResult { - Ok(KeyDefinitionRepair::default()) - } - async fn pending_data_migrations(&self) -> OpResult> { - Ok(self.data_pending.lock().unwrap().clone()) - } - async fn record_data_connection(&self) -> OpResult<()> { - unused() - } - async fn bootstrap_encryption_key(&self) -> OpResult<()> { - unused() - } - async fn bootstrap_default_account(&self) -> OpResult<()> { - unused() - } - async fn bootstrap_admin_user( - &self, - _env_user: Option<&str>, - _env_password: Option<&str>, - ) -> OpResult { - unused() - } - async fn is_catalog_initialized(&self) -> OpResult { - Ok(true) - } - async fn list_table_names(&self) -> OpResult> { - unused() - } - async fn get_data_db_name(&self) -> OpResult> { - unused() - } - async fn drop_databases(&self, _data_db: &str) -> OpResult<()> { - unused() - } - async fn read_catalog_version(&self) -> OpResult> { - Ok(self.version.lock().unwrap().clone()) - } - fn expected_catalog_version(&self) -> String { - "0.0.4".to_owned() - } - fn catalog_database_name(&self) -> String { - "fake".to_owned() - } - fn endpoint_info(&self) -> String { - "fake".to_owned() - } - fn catalog_connection_url(&self) -> String { - "fake://".to_owned() - } - fn generate_backend_config_section(&self) -> String { - String::new() - } -} - -/// A catalog whose policies need no resource-tag rewrite and that already -/// records the completion marker. -struct NoResourceTags; - -impl ResourceTagStep for NoResourceTags { - fn detect( - &self, - ) -> StepFuture<'_, anyhow::Result<(extenddb_storage::policy_migration::MigrationReport, bool)>> - { - Box::pin(async { Ok((Default::default(), true)) }) - } - fn rewrite(&self) -> StepFuture<'_, extenddb_storage::policy_migration::MigrationReport> { - Box::pin(async { Default::default() }) - } - fn record_marker(&self) -> StepFuture<'_, anyhow::Result<()>> { - Box::pin(async { Ok(()) }) - } -} - -fn args(yes: bool) -> MigrateArgs { - MigrateArgs { - config: None, - pg_user: None, - pg_pass: None, - data_db: None, - cassandra_user: None, - cassandra_pass: None, - yes, - list: false, - reconverge: false, - forget: None, - lock_timeout: None, - } -} - -#[tokio::test] -async fn a_preview_with_pending_work_exits_3_and_changes_nothing() { - for (version, data) in [ - (Some("0.0.3"), &[][..]), - (Some("0.0.4"), &["rebuild backup_items (2 rows)"][..]), - (None, &["004_x.sql"][..]), - ] { - let fake = Fake::new(version, data); - let code = apply_migrations(&fake, &args(false), &NoResourceTags) - .await - .expect("preview"); - assert_eq!(code, EXIT_PENDING, "{version:?} {data:?}"); - assert_eq!(fake.runs(), (0, 0), "the preview applied something"); - assert_eq!( - fake.read_catalog_version().await.unwrap().as_deref(), - version - ); - } -} - -#[tokio::test] -async fn up_to_date_exits_0_with_or_without_yes() { - for yes in [false, true] { - let fake = Fake::new(Some("0.0.4"), &[]); - assert_eq!( - apply_migrations(&fake, &args(yes), &NoResourceTags) - .await - .unwrap(), - 0 - ); - assert_eq!(fake.runs(), (0, 0)); - } -} - -#[tokio::test] -async fn yes_applies_and_exits_0_and_a_second_preview_is_up_to_date() { - let fake = Fake::new(Some("0.0.3"), &["rebuild backup_items (2 rows)"]); - assert_eq!( - apply_migrations(&fake, &args(true), &NoResourceTags) - .await - .unwrap(), - 0 - ); - assert_eq!(fake.runs(), (1, 1)); - assert_eq!( - apply_migrations(&fake, &args(false), &NoResourceTags) - .await - .unwrap(), - 0 - ); -} - -#[tokio::test] -async fn a_newer_catalog_is_an_error_not_a_preview() { - let fake = Fake::new(Some("0.0.5"), &[]); - let err = apply_migrations(&fake, &args(false), &NoResourceTags) - .await - .expect_err("newer"); - assert!(err.to_string().contains("does not downgrade"), "{err}"); - assert_eq!(fake.runs(), (0, 0)); -} - -#[test] -fn errors_print_their_message_without_the_variant_name() { - let shown = format!( - "{:?}", - op_err(OpError::Internal("Cannot connect to PostgreSQL".to_owned())) - ); - assert_eq!(shown, "Cannot connect to PostgreSQL"); - let shown = format!( - "{:?}", - storage_err(extenddb_storage::error::StorageError::Internal( - "Cannot connect to PostgreSQL".to_owned() - )) - ); - assert_eq!(shown, "Cannot connect to PostgreSQL"); -} diff --git a/crates/app/src/cmd_migrate_runner.rs b/crates/app/src/cmd_migrate_runner.rs index b44894ff1..7c3271b6b 100644 --- a/crates/app/src/cmd_migrate_runner.rs +++ b/crates/app/src/cmd_migrate_runner.rs @@ -49,10 +49,11 @@ pub(crate) async fn run( let target = Target { location, data_store: args.data_db.clone(), - admin: admin_role( - args.pg_user.as_deref(), - args.pg_pass.as_deref(), + admin: admin_role_for( + extenddb_storage::backend_name(), + args, std::env::var_os("EXTENDDB_PG_PASSWORD"), + std::env::var_os("EXTENDDB_CASSANDRA_PASSWORD"), )?, data_connection: None, }; @@ -100,12 +101,41 @@ pub(crate) async fn run( })) } -/// The admin role `--pg-user` names, with `--pg-pass` or else the -/// `EXTENDDB_PG_PASSWORD` variable; `None` without `--pg-user`. +/// The admin role of the backend this binary serves: on Cassandra the role +/// the runner connects as instead of the config file's (`--cassandra-user`, +/// `--cassandra-pass` or `EXTENDDB_CASSANDRA_PASSWORD`); on every other +/// backend the PostgreSQL role of the pgvector attempt (`--pg-user`, +/// `--pg-pass` or `EXTENDDB_PG_PASSWORD`), which only PostgreSQL reads. The +/// other backend's flags are ignored, as they were before Cassandra moved +/// onto the runner. +fn admin_role_for( + backend: Option<&str>, + args: &MigrateArgs, + pg_password_env: Option, + cassandra_password_env: Option, +) -> anyhow::Result> { + if backend == Some("cassandra") { + admin_role( + args.cassandra_user.as_deref(), + args.cassandra_pass.as_deref(), + ("EXTENDDB_CASSANDRA_PASSWORD", cassandra_password_env), + ) + } else { + admin_role( + args.pg_user.as_deref(), + args.pg_pass.as_deref(), + ("EXTENDDB_PG_PASSWORD", pg_password_env), + ) + } +} + +/// The admin role a user flag names (`--pg-user` or `--cassandra-user`), +/// with its password flag or else the password variable `env_name` names; +/// `None` without the user flag. fn admin_role( user: Option<&str>, password: Option<&str>, - password_env: Option, + (env_name, password_env): (&str, Option), ) -> anyhow::Result> { let Some(user) = user else { return Ok(None); @@ -114,7 +144,7 @@ fn admin_role( (Some(password), _) => Some(password.to_owned()), (None, Some(env)) => Some( env.into_string() - .map_err(|_| anyhow::anyhow!("EXTENDDB_PG_PASSWORD must contain valid UTF-8"))?, + .map_err(|_| anyhow::anyhow!("{env_name} must contain valid UTF-8"))?, ), (None, None) => None, }; @@ -243,8 +273,9 @@ async fn apply( lock_timeout.as_secs() ); let lock = migrations.lock(target, lock_timeout).await?; - let report = run_locked(migrations, target, &lock, release, kind).await?; - drop(lock); + let report = run_locked(migrations, target, &lock, release, kind).await; + migrations.release(lock).await; + let report = report?; print_report(&report, kind); if report.applied.is_empty() && kind == RunKind::Migrate { // A catalog ahead by additive migrations has nothing to apply and @@ -289,7 +320,9 @@ async fn forget( lock_timeout: Duration, ) -> Result { let lock = migrations.lock(target, lock_timeout).await?; - forget_locked(migrations, target, &lock, id).await?; + let forgotten = forget_locked(migrations, target, &lock, id).await; + migrations.release(lock).await; + forgotten?; println!( "Forgot migration {id}: its ledger rows are deleted, and the next \ `extenddb migrate --yes` runs it again." diff --git a/crates/app/src/cmd_migrate_runner_tests.rs b/crates/app/src/cmd_migrate_runner_tests.rs index 2dca1de74..e8eb6d107 100644 --- a/crates/app/src/cmd_migrate_runner_tests.rs +++ b/crates/app/src/cmd_migrate_runner_tests.rs @@ -375,3 +375,110 @@ async fn the_preview_refuses_a_lock_timeout() { ); assert!(fake.calls().is_empty()); } + +fn role(user: &str, password: Option<&str>) -> Option { + Some(extenddb_storage::migrations::AdminRole { + user: user.to_owned(), + password: password.map(str::to_owned), + }) +} + +/// The admin role is the backend's own: a Cassandra binary reads +/// `--cassandra-user` with `--cassandra-pass` or +/// `EXTENDDB_CASSANDRA_PASSWORD` and ignores `--pg-user`; every other +/// binary reads `--pg-user` with `--pg-pass` or `EXTENDDB_PG_PASSWORD` and +/// ignores `--cassandra-user`. +#[test] +fn the_admin_role_is_chosen_by_the_backend() { + let both = parse(&[ + "--yes", + "--pg-user", + "pg_admin", + "--pg-pass", + "pg_secret", + "--cassandra-user", + "cass_admin", + "--cassandra-pass", + "cass_secret", + ]) + .expect("parse"); + let pick = |backend: Option<&str>, args| { + super::admin_role_for(backend, args, None, None).expect("admin role") + }; + assert_eq!( + pick(Some("cassandra"), &both), + role("cass_admin", Some("cass_secret")) + ); + assert_eq!( + pick(Some("postgres"), &both), + role("pg_admin", Some("pg_secret")) + ); + assert_eq!( + pick(Some("sqlite"), &both), + role("pg_admin", Some("pg_secret")) + ); + assert_eq!( + pick(Some("mongodb"), &both), + role("pg_admin", Some("pg_secret")) + ); + assert_eq!(pick(None, &both), role("pg_admin", Some("pg_secret"))); + + let cassandra_only = parse(&["--yes", "--cassandra-user", "cass_admin"]).expect("parse"); + assert_eq!(pick(Some("postgres"), &cassandra_only), None); + assert_eq!( + pick(Some("cassandra"), &cassandra_only), + role("cass_admin", None) + ); + + let pg_only = parse(&["--yes", "--pg-user", "pg_admin"]).expect("parse"); + assert_eq!(pick(Some("cassandra"), &pg_only), None); + assert_eq!(pick(Some("postgres"), &pg_only), role("pg_admin", None)); +} + +/// Each backend's password variable fills in for its password flag, and the +/// flag wins over the variable; the other backend's variable is ignored. +#[test] +fn the_password_variable_is_the_backends_own() { + let env = |value: &str| Some(std::ffi::OsString::from(value)); + let cassandra = parse(&["--yes", "--cassandra-user", "cass_admin"]).expect("parse"); + assert_eq!( + super::admin_role_for( + Some("cassandra"), + &cassandra, + env("pg_env"), + env("cass_env") + ) + .expect("admin role"), + role("cass_admin", Some("cass_env")) + ); + assert_eq!( + super::admin_role_for(Some("cassandra"), &cassandra, env("pg_env"), None) + .expect("admin role"), + role("cass_admin", None) + ); + let with_flag = parse(&[ + "--yes", + "--cassandra-user", + "cass_admin", + "--cassandra-pass", + "flag", + ]) + .expect("parse"); + assert_eq!( + super::admin_role_for(Some("cassandra"), &with_flag, None, env("cass_env")) + .expect("admin role"), + role("cass_admin", Some("flag")) + ); + + let postgres = parse(&["--yes", "--pg-user", "pg_admin"]).expect("parse"); + assert_eq!( + super::admin_role_for(Some("postgres"), &postgres, env("pg_env"), env("cass_env")) + .expect("admin role"), + role("pg_admin", Some("pg_env")) + ); + assert_eq!( + super::admin_role_for(Some("postgres"), &postgres, None, env("cass_env")) + .expect("admin role"), + role("pg_admin", None) + ); +} diff --git a/crates/app/src/cmd_verify.rs b/crates/app/src/cmd_verify.rs index 1b045e613..fe4c0196e 100755 --- a/crates/app/src/cmd_verify.rs +++ b/crates/app/src/cmd_verify.rs @@ -31,8 +31,6 @@ pub struct VerifyArgs { /// Run `extenddb verify`; returns the process exit status. pub async fn run(args: VerifyArgs, release: &str) -> anyhow::Result { let (app_config, config_file) = crate::util::command_config(args.config.as_deref())?; - let expected_version = - extenddb_storage::operations::catalog_version().unwrap_or_else(|_| "unknown".to_string()); // Parse connection string to get database name for display let parts = extenddb_storage::operations::parse_connection_string( @@ -54,59 +52,27 @@ pub async fn run(args: VerifyArgs, release: &str) -> anyhow::Result { // Read first, before anything opens the catalog: a store that does not // exist is reported, not created. - let status_code = match extenddb_storage::migrations::installed() { - Some(migrations) => { - let code = - check_migrations(migrations, config_file.as_deref(), &app_config, release).await; - if code == extenddb_storage::migrations::exit::ERROR { - println!(); - println!("=== UNHEALTHY: see the catalog check above ==="); - return Ok(code); - } - Some(code) - } - None => None, - }; + let migrations = extenddb_storage::migrations::installed() + .ok_or_else(|| anyhow::anyhow!("no storage backend is installed"))?; + let status_code = + check_migrations(migrations, config_file.as_deref(), &app_config, release).await; + if status_code == extenddb_storage::migrations::exit::ERROR { + println!(); + println!("=== UNHEALTHY: see the catalog check above ==="); + return Ok(status_code); + } - // Create settings and diagnostics store println!("--- Checking catalog connection..."); - let store = match extenddb_storage::settings_store::create_settings_store( + match extenddb_storage::settings_store::create_settings_store( app_config.storage.connection_config(), ) .await { - Ok(store) => { - println!(" OK: Connected to catalog."); - store - } + Ok(_) => println!(" OK: Connected to catalog."), Err(e) => { println!(" FAIL: Cannot connect to catalog database: {e}"); anyhow::bail!("Cannot proceed without catalog connection"); } - }; - - // Check 2: Catalog version (D-10: strict parsing), on a backend that - // still records one; the runner's status replaced it above. - if status_code.is_none() { - println!("--- Checking catalog version..."); - match store.get_setting("catalog_version").await { - Ok(Some(v)) => { - if v == expected_version { - println!(" OK: Catalog version {v}"); - } else { - println!(" WARN: Catalog version {v} (binary expects {expected_version})"); - errors += 1; - } - } - Ok(None) => { - println!(" FAIL: No catalog version found. Run 'extenddb init'."); - errors += 1; - } - Err(e) => { - println!(" FAIL: Failed to read catalog version: {e:?}"); - errors += 1; - } - } } // Check 3: Data database connection. @@ -137,7 +103,6 @@ pub async fn run(args: VerifyArgs, release: &str) -> anyhow::Result { println!(" Indexes: {index_count}"); println!(); - let status_code = status_code.unwrap_or(0); if errors == 0 && status_code == 0 { println!("=== HEALTHY: All checks passed ==="); return Ok(0); diff --git a/crates/app/src/lib.rs b/crates/app/src/lib.rs index 153470867..157f29d9b 100644 --- a/crates/app/src/lib.rs +++ b/crates/app/src/lib.rs @@ -124,9 +124,8 @@ pub fn run(build: BuildInfo) -> anyhow::Result<()> { Ok(()) } Command::Migrate(args) => { - // 0 up to date or applied; cmd_migrate::EXIT_PENDING for a preview - // that found work; on the migration ledger, the status's code; - // errors exit 1 through main. + // 0 up to date or applied, else the catalog status's code (3 for + // a preview that found work); errors exit 1 through main. let code = run_interactive(cmd_migrate::run(args, build.version))?; if code != 0 { std::process::exit(i32::from(code)); diff --git a/crates/core/src/settings_keys.rs b/crates/core/src/settings_keys.rs index ceca1cb56..27ab58054 100644 --- a/crates/core/src/settings_keys.rs +++ b/crates/core/src/settings_keys.rs @@ -100,23 +100,6 @@ pub const VECTOR_INDEX_MIN_CREATING_MS: &str = "vector_index_min_creating_ms"; /// Unset, nothing waits and no branch is taken. pub const VECTOR_ALLOCATION_PHASE_DELAY_MS: &str = "vector_allocation_phase_delay_ms"; -/// Marker recording that the resource-tag condition-key migration has run on -/// this catalog, or that the catalog was created after the migration existed -/// and so has nothing to rewrite. -/// -/// `extenddb migrate --yes` writes it after rewriting every stored policy from -/// `dynamodb:ResourceTag/*` to `aws:ResourceTag/*` with no storage errors; -/// `init` and every fresh-catalog bootstrap write it on a new catalog. `serve` -/// reads this one row at startup and refuses to start when it is absent, naming -/// `extenddb migrate --yes`, so a pre-migration catalog is never served under -/// the new evaluator (which leaves `dynamodb:ResourceTag/*` absent). The value -/// is informational; presence of the key is the signal. -pub const RESOURCE_TAG_MIGRATION: &str = "resource_tag_migration"; - -/// The value written for [`RESOURCE_TAG_MIGRATION`] when the migration is -/// complete. -pub const RESOURCE_TAG_MIGRATION_DONE: &str = "applied"; - /// Resolve a caller-supplied settings key to its canonical name. /// /// Accepting the old name keeps `extenddb settings set gsi_propagation_delay_ms 0` diff --git a/crates/server/src/serve.rs b/crates/server/src/serve.rs index ee52ba9f2..c26aa3872 100644 --- a/crates/server/src/serve.rs +++ b/crates/server/src/serve.rs @@ -361,29 +361,6 @@ async fn serve_inner(params: ServeParams, port: u16) -> anyhow::Result<()> { let cred_store = components.credential_store; let runtime_hooks = components.runtime_hooks; - // Resource-tag migration guard. ExtendDB evaluates resource tags only as - // aws:ResourceTag/* and leaves dynamodb:ResourceTag/* absent, so a catalog - // whose stored policies still name the dynamodb: form must be rewritten by - // `extenddb migrate --yes` before this binary serves it. `migrate`, `init` - // and every fresh-catalog bootstrap record a settings marker; refuse to - // start when it is missing. One settings-row read, no policy scan. A - // backend on the migration runner read the catalog's status in its - // factory instead, and migration 1 deletes the marker. - if extenddb_storage::migrations::installed().is_none() - && catalog_store - .get_setting(extenddb_core::settings_keys::RESOURCE_TAG_MIGRATION) - .await - .map_err(|e| anyhow::anyhow!("reading the resource-tag migration marker: {e:?}"))? - .is_none() - { - anyhow::bail!( - "this catalog has not had the resource-tag condition-key migration applied. \ - Run `extenddb migrate --yes` before starting the server. It rewrites stored \ - policies from dynamodb:ResourceTag to aws:ResourceTag, which this build \ - evaluates, and records a marker the server checks at startup." - ); - } - // Dev mode: seed the credentials the SDK may sign with and verify against // them (see `dev_credentials` for the vending contract). Seeded here so it // works for both file-backed and bootstrap-on-serve (in-memory) diff --git a/crates/storage-cassandra/Cargo.toml b/crates/storage-cassandra/Cargo.toml index 92e3fab95..234d2bf9a 100644 --- a/crates/storage-cassandra/Cargo.toml +++ b/crates/storage-cassandra/Cargo.toml @@ -29,3 +29,11 @@ crc32fast = { workspace = true } zeroize = { workspace = true } cdrs-tokio = { workspace = true } chrono = { workspace = true } + +[features] +# Entry points the integration tests build catalogs with. +test-util = ["extenddb-storage/test-util"] + +[dev-dependencies] +extenddb-storage = { workspace = true, features = ["test-util"] } +extenddb-storage-cassandra = { path = ".", features = ["test-util"] } diff --git a/crates/storage-cassandra/migrations/data/V001__initial_data_schema.cql b/crates/storage-cassandra/migrations/account/0001_baseline.cql similarity index 74% rename from crates/storage-cassandra/migrations/data/V001__initial_data_schema.cql rename to crates/storage-cassandra/migrations/account/0001_baseline.cql index 3b93cf952..9e69dab9f 100644 --- a/crates/storage-cassandra/migrations/data/V001__initial_data_schema.cql +++ b/crates/storage-cassandra/migrations/account/0001_baseline.cql @@ -1,8 +1,21 @@ -- Copyright 2026 ExtendDB contributors -- SPDX-License-Identifier: Apache-2.0 --- Data schema for user-facing DynamoDB resources +-- +-- Migration 1, baseline, Cassandra account keyspace. +-- +-- Creates the v1.0 schema in an empty account keyspace and brings an account +-- keyspace created by any released version to the same schema. It is the +-- statements of the legacy data files V001 and V002 in order, with every +-- CREATE using IF NOT EXISTS. Tables are named without a keyspace: the runner +-- writes the account keyspace in front of each one, and waits for every live +-- node to agree on the schema after each CREATE. CreateAccount runs this file +-- through the same runner on a new account's keyspace. +-- +-- The runner splits this file on the semicolon character, so no comment holds +-- one. --- Migration tracking table (must be first) +-- The ledger of releases before 1.0, kept so their migrate finds every file +-- it knows recorded and runs none of them. CREATE TABLE IF NOT EXISTS schema_history ( version int PRIMARY KEY, description text, @@ -131,3 +144,20 @@ CREATE TABLE IF NOT EXISTS ttl_repair_inflight ( created_at timestamp, PRIMARY KEY ((worker_partition), repair_id) ); + +-- Legacy V002: logical backup payloads. Each backup is split into bounded +-- partitions so a large DynamoDB table does not become one unbounded +-- Cassandra partition. + +CREATE TABLE IF NOT EXISTS backup_items ( + backup_arn text, + bucket int, + item_index bigint, + item_data text, + PRIMARY KEY ((backup_arn, bucket), item_index) +) WITH CLUSTERING ORDER BY (item_index ASC); + +-- Every legacy data file, recorded in the legacy ledger. The insert names no +-- applied_at, so a row a release wrote keeps its time. +INSERT INTO schema_history (version, description) VALUES (1, 'initial_data_schema'); +INSERT INTO schema_history (version, description) VALUES (2, 'backup_items'); diff --git a/crates/storage-cassandra/migrations/account/0001_baseline.expected.json b/crates/storage-cassandra/migrations/account/0001_baseline.expected.json new file mode 100644 index 000000000..0079d5d90 --- /dev/null +++ b/crates/storage-cassandra/migrations/account/0001_baseline.expected.json @@ -0,0 +1,400 @@ +{ + "columns": { + "backup_items.backup_arn": { + "kind": "partition_key", + "position": 0, + "clustering_order": "none", + "type": "text" + }, + "backup_items.bucket": { + "kind": "partition_key", + "position": 1, + "clustering_order": "none", + "type": "int" + }, + "backup_items.item_data": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "backup_items.item_index": { + "kind": "clustering", + "position": 0, + "clustering_order": "asc", + "type": "bigint" + }, + "gsi_pending.id": { + "kind": "clustering", + "position": 1, + "clustering_order": "asc", + "type": "timeuuid" + }, + "gsi_pending.index_context": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "gsi_pending.last_ready_at": { + "kind": "static", + "position": -1, + "clustering_order": "none", + "type": "timestamp" + }, + "gsi_pending.new_item": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "gsi_pending.old_item": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "gsi_pending.ready_at": { + "kind": "clustering", + "position": 0, + "clustering_order": "asc", + "type": "timestamp" + }, + "gsi_pending.table_id": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "gsi_pending.worker_partition": { + "kind": "partition_key", + "position": 0, + "clustering_order": "none", + "type": "int" + }, + "index_propagation_holds.index_id": { + "kind": "clustering", + "position": 0, + "clustering_order": "asc", + "type": "text" + }, + "index_propagation_holds.table_id": { + "kind": "partition_key", + "position": 0, + "clustering_order": "none", + "type": "text" + }, + "schema_history.applied_at": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "timestamp" + }, + "schema_history.description": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "schema_history.version": { + "kind": "partition_key", + "position": 0, + "clustering_order": "none", + "type": "int" + }, + "stream_records.created_at": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "timestamp" + }, + "stream_records.event_name": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "stream_records.record_data": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "stream_records.sequence_number": { + "kind": "clustering", + "position": 0, + "clustering_order": "asc", + "type": "text" + }, + "stream_records.shard_id": { + "kind": "partition_key", + "position": 0, + "clustering_order": "none", + "type": "text" + }, + "stream_records.table_id": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "stream_shards.created_at": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "timestamp" + }, + "stream_shards.ending_sequence_number": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "stream_shards.parent_shard_id": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "stream_shards.shard_id": { + "kind": "partition_key", + "position": 0, + "clustering_order": "none", + "type": "text" + }, + "stream_shards.starting_sequence_number": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "stream_shards.table_id": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "transaction_ledger.client_token": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "transaction_ledger.items_blob": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "transaction_ledger.request_fingerprint": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "transaction_ledger.started_at": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "bigint" + }, + "transaction_ledger.state": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "transaction_ledger.txn_id": { + "kind": "partition_key", + "position": 0, + "clustering_order": "none", + "type": "uuid" + }, + "ttl_expiration_buckets.bucket": { + "kind": "clustering", + "position": 1, + "clustering_order": "asc", + "type": "bigint" + }, + "ttl_expiration_buckets.generation": { + "kind": "clustering", + "position": 0, + "clustering_order": "asc", + "type": "uuid" + }, + "ttl_expiration_buckets.shard": { + "kind": "clustering", + "position": 2, + "clustering_order": "asc", + "type": "int" + }, + "ttl_expiration_buckets.table_id": { + "kind": "partition_key", + "position": 0, + "clustering_order": "none", + "type": "text" + }, + "ttl_expirations.bucket": { + "kind": "partition_key", + "position": 2, + "clustering_order": "none", + "type": "bigint" + }, + "ttl_expirations.expires_at": { + "kind": "clustering", + "position": 0, + "clustering_order": "asc", + "type": "bigint" + }, + "ttl_expirations.generation": { + "kind": "partition_key", + "position": 1, + "clustering_order": "none", + "type": "uuid" + }, + "ttl_expirations.key_data": { + "kind": "clustering", + "position": 2, + "clustering_order": "asc", + "type": "text" + }, + "ttl_expirations.key_hash": { + "kind": "clustering", + "position": 1, + "clustering_order": "asc", + "type": "text" + }, + "ttl_expirations.shard": { + "kind": "partition_key", + "position": 3, + "clustering_order": "none", + "type": "int" + }, + "ttl_expirations.state": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "ttl_expirations.table_id": { + "kind": "partition_key", + "position": 0, + "clustering_order": "none", + "type": "text" + }, + "ttl_expirations.work_data": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "ttl_expirations.work_id": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "uuid" + }, + "ttl_reconcile_pending.account_id": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "ttl_reconcile_pending.id": { + "kind": "clustering", + "position": 0, + "clustering_order": "asc", + "type": "timeuuid" + }, + "ttl_reconcile_pending.key_data": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "ttl_reconcile_pending.table_id": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "ttl_reconcile_pending.table_name": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "ttl_reconcile_pending.worker_partition": { + "kind": "partition_key", + "position": 0, + "clustering_order": "none", + "type": "int" + }, + "ttl_repair_inflight.account_id": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "ttl_repair_inflight.created_at": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "timestamp" + }, + "ttl_repair_inflight.generation": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "uuid" + }, + "ttl_repair_inflight.key_data": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "ttl_repair_inflight.repair_id": { + "kind": "clustering", + "position": 0, + "clustering_order": "asc", + "type": "uuid" + }, + "ttl_repair_inflight.table_id": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "ttl_repair_inflight.table_name": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "ttl_repair_inflight.worker_partition": { + "kind": "partition_key", + "position": 0, + "clustering_order": "none", + "type": "int" + }, + "ttl_repair_inflight_partitions.worker_partition": { + "kind": "partition_key", + "position": 0, + "clustering_order": "none", + "type": "int" + } + }, + "indexes": { + "idx_txn_started_at": { + "table": "transaction_ledger", + "kind": "COMPOSITES", + "target": "started_at" + }, + "stream_shards_table_id_idx": { + "table": "stream_shards", + "kind": "COMPOSITES", + "target": "table_id" + } + } +} diff --git a/crates/storage-cassandra/migrations/catalog/0001_baseline.cql b/crates/storage-cassandra/migrations/catalog/0001_baseline.cql new file mode 100644 index 000000000..0719fdc60 --- /dev/null +++ b/crates/storage-cassandra/migrations/catalog/0001_baseline.cql @@ -0,0 +1,358 @@ +-- Copyright 2026 ExtendDB contributors +-- SPDX-License-Identifier: Apache-2.0 +-- +-- Migration 1, baseline, Cassandra catalog keyspace. +-- +-- Creates the v1.0 catalog schema in an empty keyspace and brings a catalog +-- created by any released version to the same schema. It is the statements +-- of the legacy catalog files V001 to V005 in order, unchanged apart from +-- these rules, so every statement is safe to run on a catalog that already +-- has some or all of their objects: +-- +-- * every CREATE uses IF NOT EXISTS, and every column a later legacy file +-- added stays an ALTER TABLE ... ADD IF NOT EXISTS, because folding it into +-- the CREATE TABLE would leave every existing catalog without it +-- * no statement writes a settings row: every seeded setting has an equal +-- default in code, and the runner writes the version +-- * tables are named without a keyspace: the runner writes the catalog +-- keyspace in front of each one, and waits for every live node to agree on +-- the schema after each CREATE and ALTER +-- +-- The runner splits this file on the semicolon character, so no comment holds +-- one. + +-- The ledger of releases before 1.0, kept so their migrate finds every file +-- it knows recorded and runs none of them. +CREATE TABLE IF NOT EXISTS schema_history ( + version int PRIMARY KEY, + description text, + applied_at timestamp +); + +-- Settings table (key-value store) +CREATE TABLE IF NOT EXISTS settings ( + key text PRIMARY KEY, + value text +); + +-- Accounts table +CREATE TABLE IF NOT EXISTS accounts ( + account_id text PRIMARY KEY, + account_name text, + created_at timestamp +); + +-- Tables (per-account table definitions) +CREATE TABLE IF NOT EXISTS tables ( + account_id text, + table_name text, + table_id text, + key_schema text, + attribute_definitions text, + table_status text, + created_at timestamp, + billing_mode text, + provisioned_throughput text, + on_demand_throughput text, + table_class text, + sse_specification text, + stream_specification text, + table_size_bytes bigint, + item_count bigint, + table_arn text, + ttl_attribute text, + deletion_protection_enabled boolean, + status_transition_at timestamp, + stream_label text, + ttl_index_ready boolean, + -- Fences TTL enable, backfill, sweep, and cleanup cycles against each other. + ttl_generation uuid, + ttl_sweep_owner uuid, + ttl_cleanup_generation uuid, + PRIMARY KEY ((account_id), table_name) +); +CREATE INDEX IF NOT EXISTS ON tables (table_id); + +-- Indexes (GSI/LSI metadata) +CREATE TABLE IF NOT EXISTS indexes ( + table_id text, + index_name text, + index_id text, + index_type text, + key_schema text, + projection text, + index_status text, + provisioned_throughput text, + propagation_delay_ms int, + backfill_heartbeat timestamp, + PRIMARY KEY ((table_id), index_name) +); + +-- Tags (resource tagging) +CREATE TABLE IF NOT EXISTS tags ( + resource_arn text, + tag_key text, + tag_value text, + PRIMARY KEY ((resource_arn), tag_key) +); + +-- Metrics (metrics collection) +CREATE TABLE IF NOT EXISTS metrics ( + bucket timestamp, + metric text, + table_name text, + index_name text, + operation text, + sum double, + count bigint, + min double, + max double, + PRIMARY KEY ((bucket, metric, table_name, index_name, operation)) +); + +-- Login attempts (security auditing) +CREATE TABLE IF NOT EXISTS login_attempts ( + principal text, + attempted_at timestamp, + success boolean, + source_ip text, + PRIMARY KEY ((principal), attempted_at) +) WITH CLUSTERING ORDER BY (attempted_at DESC); + +-- Admin users table +CREATE TABLE IF NOT EXISTS admin_users ( + admin_name text PRIMARY KEY, + password_hash text, + created_at timestamp +); + +-- IAM users table +CREATE TABLE IF NOT EXISTS iam_users ( + account_id text, + user_name text, + user_arn text, + password_hash text, + created_at timestamp, + PRIMARY KEY ((account_id), user_name) +); + +-- IAM groups table +CREATE TABLE IF NOT EXISTS iam_groups ( + account_id text, + group_name text, + group_arn text, + created_at timestamp, + PRIMARY KEY ((account_id), group_name) +); + +-- IAM roles table +CREATE TABLE IF NOT EXISTS iam_roles ( + account_id text, + role_name text, + role_arn text, + trust_policy text, + permissions_boundary_arn text, + created_at timestamp, + PRIMARY KEY ((account_id), role_name) +); + +-- IAM policies table +CREATE TABLE IF NOT EXISTS iam_policies ( + account_id text, + principal_type text, + principal_name text, + policy_name text, + policy_document text, + created_at timestamp, + PRIMARY KEY ((account_id), principal_type, principal_name, policy_name) +); + +-- Access keys table +CREATE TABLE IF NOT EXISTS access_keys ( + access_key_id text PRIMARY KEY, + account_id text, + user_name text, + secret_key_encrypted blob, + is_active boolean, + created_at timestamp +); + +-- IAM sessions table +CREATE TABLE IF NOT EXISTS iam_sessions ( + session_token text PRIMARY KEY, + account_id text, + role_name text, + session_name text, + access_key_id text, + secret_key_encrypted blob, + session_tags text, + session_policy text, + expires_at timestamp, + created_at timestamp +); + +-- IAM user tags table +CREATE TABLE IF NOT EXISTS iam_user_tags ( + account_id text, + user_name text, + tag_key text, + tag_value text, + PRIMARY KEY ((account_id, user_name), tag_key) +); + +-- IAM role tags table +CREATE TABLE IF NOT EXISTS iam_role_tags ( + account_id text, + role_name text, + tag_key text, + tag_value text, + PRIMARY KEY ((account_id, role_name), tag_key) +); + +-- IAM group members table +CREATE TABLE IF NOT EXISTS iam_group_members ( + account_id text, + group_name text, + user_name text, + PRIMARY KEY ((account_id, group_name), user_name) +); + +-- IAM permissions boundaries table +CREATE TABLE IF NOT EXISTS iam_permissions_boundaries ( + account_id text, + principal_type text, + principal_name text, + policy_document text, + PRIMARY KEY (account_id, principal_type, principal_name) +); + +-- Legacy V002: logical backup metadata and continuous backup settings. + +-- Authoritative lookup for ARN-addressed operations. Include account_id in the +-- partition key so a caller cannot even address another account's backup row. +CREATE TABLE IF NOT EXISTS backups_by_arn ( + account_id text, + backup_arn text, + backup_name text, + table_id text, + table_name text, + table_arn text, + backup_status text, + backup_type text, + backup_size_bytes bigint, + item_count bigint, + key_schema text, + attribute_definitions text, + billing_mode text, + provisioned_throughput text, + stream_specification text, + table_created_at timestamp, + created_at timestamp, + PRIMARY KEY ((account_id, backup_arn)) +); + +-- Denormalized list access path for all backups owned by an account. table_id +-- records the exact source-table incarnation even if the name is later reused. +CREATE TABLE IF NOT EXISTS backups_by_account ( + account_id text, + created_at timestamp, + backup_arn text, + backup_name text, + table_id text, + table_name text, + table_arn text, + backup_size_bytes bigint, + PRIMARY KEY ((account_id), created_at, backup_arn) +) WITH CLUSTERING ORDER BY (created_at DESC, backup_arn ASC); + +-- ListBackups filters by table name, and backups intentionally outlive their +-- source table. Keep the name in the partition key so backups from an earlier +-- table incarnation remain discoverable after delete/recreate. table_id retains +-- the immutable source identity used by the authoritative metadata. +CREATE TABLE IF NOT EXISTS backups_by_table ( + account_id text, + table_name text, + created_at timestamp, + backup_arn text, + backup_name text, + table_id text, + table_arn text, + backup_size_bytes bigint, + PRIMARY KEY ((account_id, table_name), created_at, backup_arn) +) WITH CLUSTERING ORDER BY (created_at DESC, backup_arn ASC); + +CREATE TABLE IF NOT EXISTS continuous_backups ( + account_id text, + table_name text, + pitr_enabled boolean, + PRIMARY KEY ((account_id), table_name) +); + +-- Legacy V003: scope TransactWriteItems client tokens to their owning account. + +CREATE TABLE IF NOT EXISTS idempotency_tokens_by_account ( + account_id text, + "token" text, + fingerprint text, + created_at timestamp, + PRIMARY KEY ((account_id), "token") +); + +-- Legacy V004: durable resume point for the TTL enable backfill. +-- +-- JSON blob {"generation": , "last_key": } written after each +-- scanned page. A backfill that dies mid-scan (host crash, deploy) resumes +-- from here instead of rescanning the table from the top. A cursor whose +-- generation does not match the table's current ttl_generation is ignored. +ALTER TABLE tables ADD IF NOT EXISTS ttl_backfill_cursor text; + +-- Legacy V005: backup table definitions and restore registration. +-- +-- backups_by_arn.definition holds the BackupTableDefinition JSON a backup +-- records at CreateBackup (GSIs, LSIs, billing, throughput, table class, SSE, +-- on-demand throughput, vector index names, TTL, stream specification). A row +-- written before this migration has no value here and restores as a legacy +-- backup (keys and billing only). +-- +-- backups_by_arn.active_restores maps a restore target's table_id to its table +-- name while that restore copies from the backup. It is written and cleared +-- with lightweight transactions on the backup's own partition, the same +-- partition DeleteBackup moves to DELETING with a lightweight transaction, so +-- the two are serialized by Paxos on one partition. +-- +-- tables.restore_* describe a restore target: the source backup and table, the +-- RestoreDateTime to report, and the owner and heartbeat of the process +-- copying into it. The heartbeat is refreshed while the copy runs, and the +-- abandoned-restore sweep removes a target whose heartbeat has gone stale. +ALTER TABLE backups_by_arn ADD IF NOT EXISTS definition text; +ALTER TABLE backups_by_arn ADD IF NOT EXISTS active_restores map; +ALTER TABLE tables ADD IF NOT EXISTS restore_source_backup_arn text; +ALTER TABLE tables ADD IF NOT EXISTS restore_source_table_arn text; +ALTER TABLE tables ADD IF NOT EXISTS restore_date_time timestamp; +ALTER TABLE tables ADD IF NOT EXISTS restore_owner uuid; +ALTER TABLE tables ADD IF NOT EXISTS restore_heartbeat timestamp; + +-- One row per running server, written at QUORUM when it starts and again +-- every 15 s with a TTL of 45 s. migrate --yes waits while a row exists. +-- started_at is the server's start time in RFC 3339, which every renewal +-- writes again unchanged. +CREATE TABLE IF NOT EXISTS extenddb_servers ( + server_id text PRIMARY KEY, + host text, + pid int, + release text, + started_at text +); + +-- Releases before 1.0 refuse a catalog without this row and insert it again +-- from their migrate. Its presence beside a ledger tells v1.0 that one of +-- them wrote to the catalog. +DELETE FROM settings WHERE key = 'resource_tag_migration'; + +-- Every legacy catalog file, recorded in the legacy ledger. The insert names +-- no applied_at, so a row a release wrote keeps its time. +INSERT INTO schema_history (version, description) VALUES (1, 'initial_schema'); +INSERT INTO schema_history (version, description) VALUES (2, 'backup_restore'); +INSERT INTO schema_history (version, description) VALUES (3, 'account_scoped_idempotency'); +INSERT INTO schema_history (version, description) VALUES (4, 'ttl_backfill_cursor'); +INSERT INTO schema_history (version, description) VALUES (5, 'backup_definitions_and_restore_registration'); diff --git a/crates/storage-cassandra/migrations/catalog/0001_baseline.expected.json b/crates/storage-cassandra/migrations/catalog/0001_baseline.expected.json new file mode 100644 index 000000000..d0e32e5aa --- /dev/null +++ b/crates/storage-cassandra/migrations/catalog/0001_baseline.expected.json @@ -0,0 +1,1007 @@ +{ + "columns": { + "access_keys.access_key_id": { + "kind": "partition_key", + "position": 0, + "clustering_order": "none", + "type": "text" + }, + "access_keys.account_id": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "access_keys.created_at": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "timestamp" + }, + "access_keys.is_active": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "boolean" + }, + "access_keys.secret_key_encrypted": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "blob" + }, + "access_keys.user_name": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "accounts.account_id": { + "kind": "partition_key", + "position": 0, + "clustering_order": "none", + "type": "text" + }, + "accounts.account_name": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "accounts.created_at": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "timestamp" + }, + "admin_users.admin_name": { + "kind": "partition_key", + "position": 0, + "clustering_order": "none", + "type": "text" + }, + "admin_users.created_at": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "timestamp" + }, + "admin_users.password_hash": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "backups_by_account.account_id": { + "kind": "partition_key", + "position": 0, + "clustering_order": "none", + "type": "text" + }, + "backups_by_account.backup_arn": { + "kind": "clustering", + "position": 1, + "clustering_order": "asc", + "type": "text" + }, + "backups_by_account.backup_name": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "backups_by_account.backup_size_bytes": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "bigint" + }, + "backups_by_account.created_at": { + "kind": "clustering", + "position": 0, + "clustering_order": "desc", + "type": "timestamp" + }, + "backups_by_account.table_arn": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "backups_by_account.table_id": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "backups_by_account.table_name": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "backups_by_arn.account_id": { + "kind": "partition_key", + "position": 0, + "clustering_order": "none", + "type": "text" + }, + "backups_by_arn.active_restores": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "map" + }, + "backups_by_arn.attribute_definitions": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "backups_by_arn.backup_arn": { + "kind": "partition_key", + "position": 1, + "clustering_order": "none", + "type": "text" + }, + "backups_by_arn.backup_name": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "backups_by_arn.backup_size_bytes": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "bigint" + }, + "backups_by_arn.backup_status": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "backups_by_arn.backup_type": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "backups_by_arn.billing_mode": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "backups_by_arn.created_at": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "timestamp" + }, + "backups_by_arn.definition": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "backups_by_arn.item_count": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "bigint" + }, + "backups_by_arn.key_schema": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "backups_by_arn.provisioned_throughput": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "backups_by_arn.stream_specification": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "backups_by_arn.table_arn": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "backups_by_arn.table_created_at": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "timestamp" + }, + "backups_by_arn.table_id": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "backups_by_arn.table_name": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "backups_by_table.account_id": { + "kind": "partition_key", + "position": 0, + "clustering_order": "none", + "type": "text" + }, + "backups_by_table.backup_arn": { + "kind": "clustering", + "position": 1, + "clustering_order": "asc", + "type": "text" + }, + "backups_by_table.backup_name": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "backups_by_table.backup_size_bytes": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "bigint" + }, + "backups_by_table.created_at": { + "kind": "clustering", + "position": 0, + "clustering_order": "desc", + "type": "timestamp" + }, + "backups_by_table.table_arn": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "backups_by_table.table_id": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "backups_by_table.table_name": { + "kind": "partition_key", + "position": 1, + "clustering_order": "none", + "type": "text" + }, + "continuous_backups.account_id": { + "kind": "partition_key", + "position": 0, + "clustering_order": "none", + "type": "text" + }, + "continuous_backups.pitr_enabled": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "boolean" + }, + "continuous_backups.table_name": { + "kind": "clustering", + "position": 0, + "clustering_order": "asc", + "type": "text" + }, + "extenddb_servers.host": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "extenddb_servers.pid": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "int" + }, + "extenddb_servers.release": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "extenddb_servers.server_id": { + "kind": "partition_key", + "position": 0, + "clustering_order": "none", + "type": "text" + }, + "extenddb_servers.started_at": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "iam_group_members.account_id": { + "kind": "partition_key", + "position": 0, + "clustering_order": "none", + "type": "text" + }, + "iam_group_members.group_name": { + "kind": "partition_key", + "position": 1, + "clustering_order": "none", + "type": "text" + }, + "iam_group_members.user_name": { + "kind": "clustering", + "position": 0, + "clustering_order": "asc", + "type": "text" + }, + "iam_groups.account_id": { + "kind": "partition_key", + "position": 0, + "clustering_order": "none", + "type": "text" + }, + "iam_groups.created_at": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "timestamp" + }, + "iam_groups.group_arn": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "iam_groups.group_name": { + "kind": "clustering", + "position": 0, + "clustering_order": "asc", + "type": "text" + }, + "iam_permissions_boundaries.account_id": { + "kind": "partition_key", + "position": 0, + "clustering_order": "none", + "type": "text" + }, + "iam_permissions_boundaries.policy_document": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "iam_permissions_boundaries.principal_name": { + "kind": "clustering", + "position": 1, + "clustering_order": "asc", + "type": "text" + }, + "iam_permissions_boundaries.principal_type": { + "kind": "clustering", + "position": 0, + "clustering_order": "asc", + "type": "text" + }, + "iam_policies.account_id": { + "kind": "partition_key", + "position": 0, + "clustering_order": "none", + "type": "text" + }, + "iam_policies.created_at": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "timestamp" + }, + "iam_policies.policy_document": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "iam_policies.policy_name": { + "kind": "clustering", + "position": 2, + "clustering_order": "asc", + "type": "text" + }, + "iam_policies.principal_name": { + "kind": "clustering", + "position": 1, + "clustering_order": "asc", + "type": "text" + }, + "iam_policies.principal_type": { + "kind": "clustering", + "position": 0, + "clustering_order": "asc", + "type": "text" + }, + "iam_role_tags.account_id": { + "kind": "partition_key", + "position": 0, + "clustering_order": "none", + "type": "text" + }, + "iam_role_tags.role_name": { + "kind": "partition_key", + "position": 1, + "clustering_order": "none", + "type": "text" + }, + "iam_role_tags.tag_key": { + "kind": "clustering", + "position": 0, + "clustering_order": "asc", + "type": "text" + }, + "iam_role_tags.tag_value": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "iam_roles.account_id": { + "kind": "partition_key", + "position": 0, + "clustering_order": "none", + "type": "text" + }, + "iam_roles.created_at": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "timestamp" + }, + "iam_roles.permissions_boundary_arn": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "iam_roles.role_arn": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "iam_roles.role_name": { + "kind": "clustering", + "position": 0, + "clustering_order": "asc", + "type": "text" + }, + "iam_roles.trust_policy": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "iam_sessions.access_key_id": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "iam_sessions.account_id": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "iam_sessions.created_at": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "timestamp" + }, + "iam_sessions.expires_at": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "timestamp" + }, + "iam_sessions.role_name": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "iam_sessions.secret_key_encrypted": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "blob" + }, + "iam_sessions.session_name": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "iam_sessions.session_policy": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "iam_sessions.session_tags": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "iam_sessions.session_token": { + "kind": "partition_key", + "position": 0, + "clustering_order": "none", + "type": "text" + }, + "iam_user_tags.account_id": { + "kind": "partition_key", + "position": 0, + "clustering_order": "none", + "type": "text" + }, + "iam_user_tags.tag_key": { + "kind": "clustering", + "position": 0, + "clustering_order": "asc", + "type": "text" + }, + "iam_user_tags.tag_value": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "iam_user_tags.user_name": { + "kind": "partition_key", + "position": 1, + "clustering_order": "none", + "type": "text" + }, + "iam_users.account_id": { + "kind": "partition_key", + "position": 0, + "clustering_order": "none", + "type": "text" + }, + "iam_users.created_at": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "timestamp" + }, + "iam_users.password_hash": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "iam_users.user_arn": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "iam_users.user_name": { + "kind": "clustering", + "position": 0, + "clustering_order": "asc", + "type": "text" + }, + "idempotency_tokens_by_account.account_id": { + "kind": "partition_key", + "position": 0, + "clustering_order": "none", + "type": "text" + }, + "idempotency_tokens_by_account.created_at": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "timestamp" + }, + "idempotency_tokens_by_account.fingerprint": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "idempotency_tokens_by_account.token": { + "kind": "clustering", + "position": 0, + "clustering_order": "asc", + "type": "text" + }, + "indexes.backfill_heartbeat": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "timestamp" + }, + "indexes.index_id": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "indexes.index_name": { + "kind": "clustering", + "position": 0, + "clustering_order": "asc", + "type": "text" + }, + "indexes.index_status": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "indexes.index_type": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "indexes.key_schema": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "indexes.projection": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "indexes.propagation_delay_ms": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "int" + }, + "indexes.provisioned_throughput": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "indexes.table_id": { + "kind": "partition_key", + "position": 0, + "clustering_order": "none", + "type": "text" + }, + "login_attempts.attempted_at": { + "kind": "clustering", + "position": 0, + "clustering_order": "desc", + "type": "timestamp" + }, + "login_attempts.principal": { + "kind": "partition_key", + "position": 0, + "clustering_order": "none", + "type": "text" + }, + "login_attempts.source_ip": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "login_attempts.success": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "boolean" + }, + "metrics.bucket": { + "kind": "partition_key", + "position": 0, + "clustering_order": "none", + "type": "timestamp" + }, + "metrics.count": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "bigint" + }, + "metrics.index_name": { + "kind": "partition_key", + "position": 3, + "clustering_order": "none", + "type": "text" + }, + "metrics.max": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "double" + }, + "metrics.metric": { + "kind": "partition_key", + "position": 1, + "clustering_order": "none", + "type": "text" + }, + "metrics.min": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "double" + }, + "metrics.operation": { + "kind": "partition_key", + "position": 4, + "clustering_order": "none", + "type": "text" + }, + "metrics.sum": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "double" + }, + "metrics.table_name": { + "kind": "partition_key", + "position": 2, + "clustering_order": "none", + "type": "text" + }, + "schema_history.applied_at": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "timestamp" + }, + "schema_history.description": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "schema_history.version": { + "kind": "partition_key", + "position": 0, + "clustering_order": "none", + "type": "int" + }, + "settings.key": { + "kind": "partition_key", + "position": 0, + "clustering_order": "none", + "type": "text" + }, + "settings.value": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "tables.account_id": { + "kind": "partition_key", + "position": 0, + "clustering_order": "none", + "type": "text" + }, + "tables.attribute_definitions": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "tables.billing_mode": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "tables.created_at": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "timestamp" + }, + "tables.deletion_protection_enabled": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "boolean" + }, + "tables.item_count": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "bigint" + }, + "tables.key_schema": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "tables.on_demand_throughput": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "tables.provisioned_throughput": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "tables.restore_date_time": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "timestamp" + }, + "tables.restore_heartbeat": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "timestamp" + }, + "tables.restore_owner": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "uuid" + }, + "tables.restore_source_backup_arn": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "tables.restore_source_table_arn": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "tables.sse_specification": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "tables.status_transition_at": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "timestamp" + }, + "tables.stream_label": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "tables.stream_specification": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "tables.table_arn": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "tables.table_class": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "tables.table_id": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "tables.table_name": { + "kind": "clustering", + "position": 0, + "clustering_order": "asc", + "type": "text" + }, + "tables.table_size_bytes": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "bigint" + }, + "tables.table_status": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "tables.ttl_attribute": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "tables.ttl_backfill_cursor": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "tables.ttl_cleanup_generation": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "uuid" + }, + "tables.ttl_generation": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "uuid" + }, + "tables.ttl_index_ready": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "boolean" + }, + "tables.ttl_sweep_owner": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "uuid" + }, + "tags.resource_arn": { + "kind": "partition_key", + "position": 0, + "clustering_order": "none", + "type": "text" + }, + "tags.tag_key": { + "kind": "clustering", + "position": 0, + "clustering_order": "asc", + "type": "text" + }, + "tags.tag_value": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + } + }, + "indexes": { + "tables_table_id_idx": { + "table": "tables", + "kind": "COMPOSITES", + "target": "table_id" + } + } +} diff --git a/crates/storage-cassandra/migrations/catalog/V001__initial_schema.cql b/crates/storage-cassandra/migrations/catalog/V001__initial_schema.cql deleted file mode 100644 index 001d9fd14..000000000 --- a/crates/storage-cassandra/migrations/catalog/V001__initial_schema.cql +++ /dev/null @@ -1,213 +0,0 @@ --- Copyright 2026 ExtendDB contributors --- SPDX-License-Identifier: Apache-2.0 --- Consolidated catalog schema for extenddb (catalog version 0.0.2). --- This is the complete schema applied on fresh installs. - --- Migration tracking table (must be first) -CREATE TABLE IF NOT EXISTS schema_history ( - version int PRIMARY KEY, - description text, - applied_at timestamp -); - --- Settings table (key-value store) -CREATE TABLE IF NOT EXISTS settings ( - key text PRIMARY KEY, - value text -); - --- Accounts table -CREATE TABLE IF NOT EXISTS accounts ( - account_id text PRIMARY KEY, - account_name text, - created_at timestamp -); - --- Tables (per-account table definitions) -CREATE TABLE IF NOT EXISTS tables ( - account_id text, - table_name text, - table_id text, - key_schema text, - attribute_definitions text, - table_status text, - created_at timestamp, - billing_mode text, - provisioned_throughput text, - on_demand_throughput text, - table_class text, - sse_specification text, - stream_specification text, - table_size_bytes bigint, - item_count bigint, - table_arn text, - ttl_attribute text, - deletion_protection_enabled boolean, - status_transition_at timestamp, - stream_label text, - ttl_index_ready boolean, - -- Fences TTL enable, backfill, sweep, and cleanup cycles against each other. - ttl_generation uuid, - ttl_sweep_owner uuid, - ttl_cleanup_generation uuid, - PRIMARY KEY ((account_id), table_name) -); -CREATE INDEX IF NOT EXISTS ON tables (table_id); - --- Indexes (GSI/LSI metadata) -CREATE TABLE IF NOT EXISTS indexes ( - table_id text, - index_name text, - index_id text, - index_type text, - key_schema text, - projection text, - index_status text, - provisioned_throughput text, - propagation_delay_ms int, - backfill_heartbeat timestamp, - PRIMARY KEY ((table_id), index_name) -); - --- Tags (resource tagging) -CREATE TABLE IF NOT EXISTS tags ( - resource_arn text, - tag_key text, - tag_value text, - PRIMARY KEY ((resource_arn), tag_key) -); - --- Metrics (metrics collection) -CREATE TABLE IF NOT EXISTS metrics ( - bucket timestamp, - metric text, - table_name text, - index_name text, - operation text, - sum double, - count bigint, - min double, - max double, - PRIMARY KEY ((bucket, metric, table_name, index_name, operation)) -); - --- Login attempts (security auditing) -CREATE TABLE IF NOT EXISTS login_attempts ( - principal text, - attempted_at timestamp, - success boolean, - source_ip text, - PRIMARY KEY ((principal), attempted_at) -) WITH CLUSTERING ORDER BY (attempted_at DESC); - --- Admin users table -CREATE TABLE IF NOT EXISTS admin_users ( - admin_name text PRIMARY KEY, - password_hash text, - created_at timestamp -); - --- IAM users table -CREATE TABLE IF NOT EXISTS iam_users ( - account_id text, - user_name text, - user_arn text, - password_hash text, - created_at timestamp, - PRIMARY KEY ((account_id), user_name) -); - --- IAM groups table -CREATE TABLE IF NOT EXISTS iam_groups ( - account_id text, - group_name text, - group_arn text, - created_at timestamp, - PRIMARY KEY ((account_id), group_name) -); - --- IAM roles table -CREATE TABLE IF NOT EXISTS iam_roles ( - account_id text, - role_name text, - role_arn text, - trust_policy text, - permissions_boundary_arn text, - created_at timestamp, - PRIMARY KEY ((account_id), role_name) -); - --- IAM policies table -CREATE TABLE IF NOT EXISTS iam_policies ( - account_id text, - principal_type text, - principal_name text, - policy_name text, - policy_document text, - created_at timestamp, - PRIMARY KEY ((account_id), principal_type, principal_name, policy_name) -); - --- Access keys table -CREATE TABLE IF NOT EXISTS access_keys ( - access_key_id text PRIMARY KEY, - account_id text, - user_name text, - secret_key_encrypted blob, - is_active boolean, - created_at timestamp -); - --- IAM sessions table -CREATE TABLE IF NOT EXISTS iam_sessions ( - session_token text PRIMARY KEY, - account_id text, - role_name text, - session_name text, - access_key_id text, - secret_key_encrypted blob, - session_tags text, - session_policy text, - expires_at timestamp, - created_at timestamp -); - --- IAM user tags table -CREATE TABLE IF NOT EXISTS iam_user_tags ( - account_id text, - user_name text, - tag_key text, - tag_value text, - PRIMARY KEY ((account_id, user_name), tag_key) -); - --- IAM role tags table -CREATE TABLE IF NOT EXISTS iam_role_tags ( - account_id text, - role_name text, - tag_key text, - tag_value text, - PRIMARY KEY ((account_id, role_name), tag_key) -); - --- IAM group members table -CREATE TABLE IF NOT EXISTS iam_group_members ( - account_id text, - group_name text, - user_name text, - PRIMARY KEY ((account_id, group_name), user_name) -); - --- IAM permissions boundaries table -CREATE TABLE IF NOT EXISTS iam_permissions_boundaries ( - account_id text, - principal_type text, - principal_name text, - policy_document text, - PRIMARY KEY (account_id, principal_type, principal_name) -); - --- Seed settings -INSERT INTO settings (key, value) VALUES ('catalog_version', '0.0.1'); -INSERT INTO settings (key, value) VALUES ('control_plane_delay_seconds', '0.25'); -INSERT INTO settings (key, value) VALUES ('resource_tag_migration', 'applied'); diff --git a/crates/storage-cassandra/migrations/catalog/V002__backup_restore.cql b/crates/storage-cassandra/migrations/catalog/V002__backup_restore.cql deleted file mode 100644 index 29709b542..000000000 --- a/crates/storage-cassandra/migrations/catalog/V002__backup_restore.cql +++ /dev/null @@ -1,65 +0,0 @@ --- Copyright 2026 ExtendDB contributors --- SPDX-License-Identifier: Apache-2.0 --- Logical backup metadata and continuous backup settings. - --- Authoritative lookup for ARN-addressed operations. Include account_id in the --- partition key so a caller cannot even address another account's backup row. -CREATE TABLE IF NOT EXISTS backups_by_arn ( - account_id text, - backup_arn text, - backup_name text, - table_id text, - table_name text, - table_arn text, - backup_status text, - backup_type text, - backup_size_bytes bigint, - item_count bigint, - key_schema text, - attribute_definitions text, - billing_mode text, - provisioned_throughput text, - stream_specification text, - table_created_at timestamp, - created_at timestamp, - PRIMARY KEY ((account_id, backup_arn)) -); - --- Denormalized list access path for all backups owned by an account. table_id --- records the exact source-table incarnation even if the name is later reused. -CREATE TABLE IF NOT EXISTS backups_by_account ( - account_id text, - created_at timestamp, - backup_arn text, - backup_name text, - table_id text, - table_name text, - table_arn text, - backup_size_bytes bigint, - PRIMARY KEY ((account_id), created_at, backup_arn) -) WITH CLUSTERING ORDER BY (created_at DESC, backup_arn ASC); - --- ListBackups filters by table name, and backups intentionally outlive their --- source table. Keep the name in the partition key so backups from an earlier --- table incarnation remain discoverable after delete/recreate. table_id retains --- the immutable source identity used by the authoritative metadata. -CREATE TABLE IF NOT EXISTS backups_by_table ( - account_id text, - table_name text, - created_at timestamp, - backup_arn text, - backup_name text, - table_id text, - table_arn text, - backup_size_bytes bigint, - PRIMARY KEY ((account_id, table_name), created_at, backup_arn) -) WITH CLUSTERING ORDER BY (created_at DESC, backup_arn ASC); - -CREATE TABLE IF NOT EXISTS continuous_backups ( - account_id text, - table_name text, - pitr_enabled boolean, - PRIMARY KEY ((account_id), table_name) -); - -INSERT INTO settings (key, value) VALUES ('catalog_version', '0.0.2'); diff --git a/crates/storage-cassandra/migrations/catalog/V003__account_scoped_idempotency.cql b/crates/storage-cassandra/migrations/catalog/V003__account_scoped_idempotency.cql deleted file mode 100644 index 957134988..000000000 --- a/crates/storage-cassandra/migrations/catalog/V003__account_scoped_idempotency.cql +++ /dev/null @@ -1,13 +0,0 @@ --- Copyright 2026 ExtendDB contributors --- SPDX-License-Identifier: Apache-2.0 --- Scope TransactWriteItems client tokens to their owning account. - -CREATE TABLE IF NOT EXISTS idempotency_tokens_by_account ( - account_id text, - "token" text, - fingerprint text, - created_at timestamp, - PRIMARY KEY ((account_id), "token") -); - -INSERT INTO settings (key, value) VALUES ('catalog_version', '0.0.3'); diff --git a/crates/storage-cassandra/migrations/catalog/V004__ttl_backfill_cursor.cql b/crates/storage-cassandra/migrations/catalog/V004__ttl_backfill_cursor.cql deleted file mode 100644 index 83473e8d9..000000000 --- a/crates/storage-cassandra/migrations/catalog/V004__ttl_backfill_cursor.cql +++ /dev/null @@ -1,12 +0,0 @@ --- Copyright 2026 ExtendDB contributors --- SPDX-License-Identifier: Apache-2.0 --- Durable resume point for the TTL enable backfill. --- --- JSON blob {"generation": , "last_key": } written after each --- scanned page. A backfill that dies mid-scan (host crash, deploy) resumes --- from here instead of rescanning the table from the top. A cursor whose --- generation does not match the table's current ttl_generation is ignored. --- --- NOTE for future migrations: the migration runner splits files on the --- semicolon character, including inside comments. Do not use one in a comment. -ALTER TABLE tables ADD IF NOT EXISTS ttl_backfill_cursor text; diff --git a/crates/storage-cassandra/migrations/catalog/V005__backup_definitions_and_restore_registration.cql b/crates/storage-cassandra/migrations/catalog/V005__backup_definitions_and_restore_registration.cql deleted file mode 100644 index 8edfce760..000000000 --- a/crates/storage-cassandra/migrations/catalog/V005__backup_definitions_and_restore_registration.cql +++ /dev/null @@ -1,32 +0,0 @@ --- Copyright 2026 ExtendDB contributors --- SPDX-License-Identifier: Apache-2.0 --- Backup table definitions and restore registration (catalog version 0.0.4). --- --- backups_by_arn.definition holds the BackupTableDefinition JSON a backup --- records at CreateBackup (GSIs, LSIs, billing, throughput, table class, SSE, --- on-demand throughput, vector index names, TTL, stream specification). A row --- written before this migration has no value here and restores as a legacy --- backup (keys and billing only). --- --- backups_by_arn.active_restores maps a restore target's table_id to its table --- name while that restore copies from the backup. It is written and cleared --- with lightweight transactions on the backup's own partition, the same --- partition DeleteBackup moves to DELETING with a lightweight transaction, so --- the two are serialized by Paxos on one partition. --- --- tables.restore_* describe a restore target: the source backup and table, the --- RestoreDateTime to report, and the owner and heartbeat of the process --- copying into it. The heartbeat is refreshed while the copy runs, and the --- abandoned-restore sweep removes a target whose heartbeat has gone stale. --- --- NOTE: the migration runner splits files on the semicolon character, --- including inside comments. Do not use one in a comment. -ALTER TABLE backups_by_arn ADD IF NOT EXISTS definition text; -ALTER TABLE backups_by_arn ADD IF NOT EXISTS active_restores map; -ALTER TABLE tables ADD IF NOT EXISTS restore_source_backup_arn text; -ALTER TABLE tables ADD IF NOT EXISTS restore_source_table_arn text; -ALTER TABLE tables ADD IF NOT EXISTS restore_date_time timestamp; -ALTER TABLE tables ADD IF NOT EXISTS restore_owner uuid; -ALTER TABLE tables ADD IF NOT EXISTS restore_heartbeat timestamp; - -INSERT INTO settings (key, value) VALUES ('catalog_version', '0.0.4'); diff --git a/crates/storage-cassandra/migrations/data/V002__backup_items.cql b/crates/storage-cassandra/migrations/data/V002__backup_items.cql deleted file mode 100644 index 8b63b6d78..000000000 --- a/crates/storage-cassandra/migrations/data/V002__backup_items.cql +++ /dev/null @@ -1,12 +0,0 @@ --- Copyright 2026 ExtendDB contributors --- SPDX-License-Identifier: Apache-2.0 --- Logical backup payloads. Each backup is split into bounded partitions so a --- large DynamoDB table does not become one unbounded Cassandra partition. - -CREATE TABLE IF NOT EXISTS backup_items ( - backup_arn text, - bucket int, - item_index bigint, - item_data text, - PRIMARY KEY ((backup_arn, bucket), item_index) -) WITH CLUSTERING ORDER BY (item_index ASC); diff --git a/crates/storage-cassandra/src/backup_engine.rs b/crates/storage-cassandra/src/backup_engine.rs index 7c070934e..8c911cf09 100755 --- a/crates/storage-cassandra/src/backup_engine.rs +++ b/crates/storage-cassandra/src/backup_engine.rs @@ -140,15 +140,6 @@ fn registration_exhausted( } } -/// See [`CassandraEngine::require_migrated_catalog`]. -fn unmigrated_catalog_refusal(pending: &[&str]) -> StorageError { - StorageError::Unsupported(format!( - "Backups are unavailable until the ExtendDB catalog is migrated: it does not record \ - catalog migration {}. Run `extenddb migrate --yes` with the admin credentials.", - pending.join(", ") - )) -} - #[derive(Debug)] struct StoredBackup { backup_arn: String, @@ -181,7 +172,8 @@ impl StoredBackup { .as_rust::>() .map_err(|e| StorageError::Internal(format!("Parse active_restores: {e}")))? .unwrap_or_default(), - // Absent before V005 has been applied, and null when empty. + // Null when empty. Every catalog this release serves has the + // column (catalog migration 1 creates it). Ok(None) | Err(_) => HashMap::new(), }; Ok(Self { @@ -399,29 +391,6 @@ struct HeartbeatTarget { } impl CassandraEngine { - /// Refuse a backup operation on a catalog that has not run every catalog - /// migration this binary carries. Catalog migration V005 adds the columns - /// CreateBackup, DeleteBackup and RestoreTableFromBackup write; without - /// it they failed with an internal error (`Undefined column name ...`), - /// which SDKs retry. `Unsupported` reaches the client as - /// `ValidationException` with this message. Table, item, ListBackups and - /// DescribeBackup operations do not call this: they work on such a - /// catalog. - pub(crate) async fn require_migrated_catalog(&self) -> Result<(), StorageError> { - use std::sync::atomic::Ordering; - if self.catalog_migrated.load(Ordering::Acquire) { - return Ok(()); - } - let pending = - crate::migrations::pending_catalog_migrations(&self.session, &self.catalog_keyspace()) - .await?; - if pending.is_empty() { - self.catalog_migrated.store(true, Ordering::Release); - return Ok(()); - } - Err(unmigrated_catalog_refusal(&pending)) - } - async fn load_backup( &self, account_id: &str, @@ -1426,7 +1395,6 @@ impl BackupEngine for CassandraEngine { let backup_name = backup_name.to_owned(); Box::pin(async move { Self::validate_account_id(&account_id)?; - self.require_migrated_catalog().await?; // fetch_table_key_info enforces the DynamoDB requirement that only // ACTIVE tables can be backed up. @@ -1739,7 +1707,6 @@ impl BackupEngine for CassandraEngine { /// the registration check and the DELETING claim. const CLAIM_ATTEMPTS: usize = 3; - self.require_migrated_catalog().await?; let mut backup = self.load_backup(&account_id, &backup_arn).await?; if backup.backup_status != "AVAILABLE" && backup.backup_status != "DELETING" { return Err(backup_not_found(&backup_arn)); @@ -1888,10 +1855,6 @@ impl BackupEngine for CassandraEngine { tokio::spawn(async move { // Inside the task, like the registration, so the first poll // still spawns before any await. - if let Err(error) = engine.require_migrated_catalog().await { - let _ = registered_tx.send(Err(error)); - return; - } match engine.register_restore_target(&account_id, request).await { Ok((description, job, heartbeat)) => { // The receiver is gone if the request was dropped; diff --git a/crates/storage-cassandra/src/bootstrapper.rs b/crates/storage-cassandra/src/bootstrapper.rs index 8d4353ae5..b4e98dfd8 100644 --- a/crates/storage-cassandra/src/bootstrapper.rs +++ b/crates/storage-cassandra/src/bootstrapper.rs @@ -15,11 +15,13 @@ use extenddb_storage::bootstrapper::{ }; use extenddb_storage::management_store::{OpError, OpResult}; +use extenddb_storage::migrations::{AdminRole, HeldLock, Target}; + use crate::config::CassandraStorageConfig; use crate::engine::CassandraEngine; -use crate::migrations; -use crate::CATALOG_VERSION; +mod guards; +use guards::{HeldLease, check_name}; // Helper structs for parsing Cassandra rows #[derive(Debug, Clone, TryFromRow)] @@ -32,16 +34,17 @@ struct KeyspaceNameRow { keyspace_name: String, } -#[derive(Debug, Clone, TryFromRow)] -struct SettingRow { - value: String, -} - /// Cassandra bootstrapper for init/destroy/migrate operations. pub struct CassandraBootstrapper { engine: CassandraEngine, config: BootstrapConfig, keyspace_prefix: String, + /// The connection string of the admin session, without credentials; + /// [`Bootstrapper::init_target`] adds the admin role. + engine_connection_string: String, + /// The lease of the migration lock `init` holds while the bootstrap rows + /// are written, set by [`Bootstrapper::hold`]. + held: HeldLease, } impl CassandraBootstrapper { @@ -58,9 +61,43 @@ impl CassandraBootstrapper { engine, config: bootstrap_config, keyspace_prefix: cassandra_config.keyspace_prefix.clone(), + engine_connection_string: format!( + "{}/{}", + cassandra_config.contact_points.join(","), + cassandra_config.keyspace_prefix + ), + held: HeldLease::default(), }) } + /// Grant the application role what `migrate`, `serve` and CreateAccount + /// need under `CassandraAuthorizer` on `keyspace`, which `init` creates as + /// the admin role, so the application role holds no permission on it + /// yet: see [`crate::migrations::grants`]. A role that creates a keyspace + /// holds every permission on it, so the keyspaces CreateAccount creates + /// need no grant. Under `AllowAllAuthorizer` nothing is granted. + async fn grant_app_role(&self, keyspace: &str, account: bool) -> OpResult<()> { + let session = self.engine.session_arc(); + let role = &self.config.app_user; + // Checked against the held lease before each request, so a holder + // whose lease has lapsed sends no further GRANT. + let what = format!("Grant on {keyspace}"); + let granted = crate::migrations::grant_role( + &session, + keyspace, + role, + account, + || self.held.check_run(&what), + || self.held.progress(), + ) + .await + .map_err(|e| OpError::Internal(e.to_string()))?; + if let Some(authorizer) = granted { + println!(" Granted '{role}' its permissions on {keyspace} ({authorizer})."); + } + Ok(()) + } + /// Create a bootstrapper from config file and CLI args. Parses /// Cassandra-specific arguments and merges with config. pub async fn from_config( @@ -194,6 +231,11 @@ impl CassandraBootstrapper { replication_factor.map_or(rf_from_config, |rf| rf.parse::().unwrap_or(1)); let resolved_app_user = extenddb_user.unwrap_or(user); let resolved_app_password = extenddb_pass.unwrap_or(password); + check_name("the application role (--extenddb-user)", &resolved_app_user)?; + check_name( + "the keyspace prefix (--keyspace-prefix)", + &resolved_keyspace_prefix, + )?; // Extract host and port from first contact point let (host, port) = resolved_contact_points @@ -346,26 +388,42 @@ impl Bootstrapper for CassandraBootstrapper { Ok(()) } + /// Create the catalog keyspace and its `settings` table with + /// `IF NOT EXISTS`, wait for schema agreement, and grant the application + /// role what it needs. The migration lock is a row of `settings`, so + /// `init` creates the table before it takes the lock; the catalog is + /// created under the lock, which refuses one that is already + /// initialized. Two `init`s that start together can both send these two + /// statements before either holds the lock; that window is accepted. A + /// keyspace of the catalog's name that holds tables and is not a catalog + /// is refused before anything is written to it. async fn create_catalog_db(&self) -> OpResult<()> { let keyspace = self.engine.catalog_keyspace(); println!("--- Creating catalog keyspace '{keyspace}'..."); - - if self - .engine - .keyspace_exists(&keyspace) + let session = self.engine.session_arc(); + let status = crate::migrations::catalog_status(session.clone(), &self.keyspace_prefix) .await - .map_err(|e| OpError::Internal(e.to_string()))? - { - return Err(OpError::AlreadyExists(format!( - "Catalog keyspace '{keyspace}' already exists. Run 'destroy' first, then re-run 'init'." - ))); + .map_err(|e| OpError::Internal(e.to_string()))?; + if status == extenddb_storage::migrations::CatalogStatus::Uninitialized { + let foreign = crate::migrations::foreign_contents(&session, &keyspace) + .await + .map_err(|e| OpError::Internal(e.to_string()))?; + if !foreign.is_empty() { + return Err(OpError::AlreadyExists(format!( + "Keyspace '{keyspace}' exists and is not an ExtendDB catalog: it holds {}. \ + Choose another --keyspace-prefix.", + foreign.join(", ") + ))); + } } - self.engine .create_keyspace(&keyspace) .await .map_err(|e| OpError::Internal(format!("Create catalog keyspace: {e}")))?; - + crate::migrations::create_lock_table(&session, &keyspace) + .await + .map_err(|e| OpError::Internal(e.to_string()))?; + self.grant_app_role(&keyspace, false).await?; println!(" Created."); Ok(()) } @@ -376,92 +434,48 @@ impl Bootstrapper for CassandraBootstrapper { Ok(()) } - async fn run_catalog_migrations(&self) -> OpResult<()> { - let keyspace = self.engine.catalog_keyspace(); - migrations::run_catalog_migrations(&self.engine.session_arc(), &keyspace).await - } - - async fn run_data_migrations(&self) -> OpResult<()> { - // Data schema is isolated per account, so upgrades must visit every - // existing account keyspace. New accounts receive the same migration - // list from ensure_account_keyspace/bootstrap_default_account. - let query = format!( - "SELECT account_id FROM {}.accounts", - self.engine.catalog_keyspace() - ); - let rows = self - .engine - .session() - .query(query) - .await - .map_err(|e| OpError::Internal(format!("List accounts for migration: {e}")))? - .response_body() - .map_err(|e| OpError::Internal(format!("Parse accounts for migration: {e}")))? - .into_rows() - .unwrap_or_default(); - - for row in rows { - let account_id: String = crate::cassandra_util::get_column::( - &row, - "account_id", - "run_data_migrations", - )?; - let keyspace = self.engine.account_keyspace(&account_id); - migrations::run_data_migrations(&self.engine.session_arc(), &keyspace).await?; - } - Ok(()) - } - - async fn pending_data_migrations(&self) -> OpResult> { - migrations::pending_data_migrations( - &self.engine.session_arc(), - &self.engine.catalog_keyspace(), - |account_id| self.engine.account_keyspace(account_id), - ) - .await - } - async fn record_data_connection(&self) -> OpResult<()> { // Not applicable for Cassandra (no separate data connection) Ok(()) } + /// Keep the lease of the lock `init` holds, checked before each + /// bootstrap row; a lock another backend took carries none. + fn hold(&self, lock: &HeldLock) { + if let Some(lease) = crate::migrations::lease_of(lock) { + self.held.hold(lease); + } + } + + /// Store a new encryption key unless the catalog has one, with + /// `INSERT ... IF NOT EXISTS`, so a key a concurrent `init` stored first + /// is never replaced: credentials a server encrypted with it stay + /// readable. Nothing else writes that row. async fn bootstrap_encryption_key(&self) -> OpResult<()> { let keyspace = self.engine.catalog_keyspace(); println!("--- Generating AES-256-GCM encryption key..."); - - // Check if key already exists - let check_cql = - format!("SELECT value FROM {keyspace}.settings WHERE key = 'encryption_key'"); - let exists = self - .engine - .session() - .query(check_cql) - .await - .ok() - .and_then(|frame| frame.response_body().ok()) - .and_then(cdrs_tokio::frame::message_response::ResponseBody::into_rows) - .is_some_and(|rows| !rows.is_empty()); - - if exists { - println!("--- Encryption key already exists, skipping."); - return Ok(()); - } - let key_b64 = generate_encryption_key(); - - // Store key - let insert_cql = format!("INSERT INTO {keyspace}.settings (key, value) VALUES (?, ?)"); - self.engine - .session() - .query_with_values( - insert_cql, - cdrs_tokio::query_values!("encryption_key", key_b64), + let insert_cql = format!( + "INSERT INTO {keyspace}.settings (key, value) VALUES ('encryption_key', ?) IF NOT \ + EXISTS" + ); + let applied = self + .held + .request( + "Store encryption key", + crate::cassandra_util::apply_lwt::( + &self.engine.session_arc(), + &insert_cql, + cdrs_tokio::query_values!(key_b64), + "bootstrap_encryption_key", + ), ) - .await - .map_err(|e| OpError::Internal(format!("Store encryption key: {e}")))?; - - println!(" Encryption key stored."); + .await?; + if applied { + println!(" Encryption key stored."); + } else { + println!("--- Encryption key already exists, skipping."); + } Ok(()) } @@ -476,16 +490,18 @@ impl Bootstrapper for CassandraBootstrapper { VALUES (?, ?, toTimestamp(now())) IF NOT EXISTS" ); - let applied = crate::cassandra_util::apply_lwt( - &self.engine.session_arc(), - &insert_cql, - cdrs_tokio::query_values!(account_id.as_str(), account_name), - "bootstrap_default_account", - ) - .await - .map_err(|e: extenddb_storage::error::StorageError| { - OpError::Internal(format!("Create account: {e}")) - })?; + let applied = self + .held + .request( + "Create account", + crate::cassandra_util::apply_lwt::( + &self.engine.session_arc(), + &insert_cql, + cdrs_tokio::query_values!(account_id.as_str(), account_name), + "bootstrap_default_account", + ), + ) + .await?; if !applied { println!("--- Default account already exists, skipping."); @@ -494,17 +510,49 @@ impl Bootstrapper for CassandraBootstrapper { println!("--- Default account created"); - // Create account-specific keyspace + // The keyspace, then every account migration through the runner, + // as CreateAccount does, all under the lease of the lock `init` + // holds: the CREATE KEYSPACE through the held lease with the bound of + // a schema statement under the lock, and the account run checking + // the lease before each write and recording progress on it. let account_keyspace = self.engine.account_keyspace(&account_id); println!("--- Creating account keyspace..."); - self.engine - .create_keyspace(&account_keyspace) - .await - .map_err(|e| OpError::Internal(format!("Create account keyspace: {e}")))?; - - // Run data migrations in account keyspace - println!("--- Running data migrations for default account..."); - migrations::run_data_migrations(&self.engine.session_arc(), &account_keyspace).await?; + self.held + .request_within( + "Create account keyspace", + crate::migrations::LOCKED_DDL_TIMEOUT, + self.engine.create_keyspace(&account_keyspace), + ) + .await?; + println!("--- Running the account migrations for the default account..."); + self.held.check("Migrate the default account's keyspace")?; + let session = self.engine.session_arc(); + let release = env!("CARGO_PKG_VERSION"); + let run = match self.held.handle() { + Some(lease) => { + crate::migrations::migrate_account_under( + session, + &self.keyspace_prefix, + &account_keyspace, + release, + lease, + ) + .await + } + None => { + crate::migrations::migrate_account( + session, + &self.keyspace_prefix, + &account_keyspace, + release, + ) + .await + } + }; + run.map_err(|e| OpError::Internal(format!("Migrate the default account's keyspace: {e}")))?; + // `init` created this keyspace as the admin role, so the application + // role holds no permission on it yet. + self.grant_app_role(&account_keyspace, true).await?; Ok(()) } @@ -536,16 +584,18 @@ impl Bootstrapper for CassandraBootstrapper { "INSERT INTO {keyspace}.admin_users (admin_name, password_hash) VALUES (?, ?) IF NOT EXISTS" ); - let applied = crate::cassandra_util::apply_lwt( - &self.engine.session_arc(), - &insert_cql, - cdrs_tokio::query_values!(username, password_hash), - "bootstrap_admin_user", - ) - .await - .map_err(|e: extenddb_storage::error::StorageError| { - OpError::Internal(format!("Create admin user: {e}")) - })?; + let applied = self + .held + .request( + "Create admin user", + crate::cassandra_util::apply_lwt::( + &self.engine.session_arc(), + &insert_cql, + cdrs_tokio::query_values!(username, password_hash), + "bootstrap_admin_user", + ), + ) + .await?; if !applied { println!("--- Admin user already exists, skipping."); @@ -567,11 +617,6 @@ impl Bootstrapper for CassandraBootstrapper { }) } - async fn is_catalog_initialized(&self) -> OpResult { - let keyspace = self.engine.catalog_keyspace(); - migrations::table_exists(&self.engine.session_arc(), &keyspace, "settings").await - } - async fn list_table_names(&self) -> OpResult> { let keyspace = self.engine.catalog_keyspace(); @@ -645,31 +690,18 @@ impl Bootstrapper for CassandraBootstrapper { Ok(()) } - async fn read_catalog_version(&self) -> OpResult> { - let keyspace = self.engine.catalog_keyspace(); - - if !migrations::table_exists(&self.engine.session_arc(), &keyspace, "settings").await? { - return Ok(None); + /// The deployment `init` creates, reached as the admin role `init` + /// connects with. + fn init_target(&self) -> Target { + Target { + location: self.engine_connection_string.clone(), + data_store: None, + admin: Some(AdminRole { + user: self.config.admin_user.clone(), + password: self.config.admin_password.clone(), + }), + data_connection: None, } - - let query = format!("SELECT value FROM {keyspace}.settings WHERE key = 'catalog_version'"); - let version = self - .engine - .session() - .query(query) - .await - .ok() - .and_then(|frame| frame.response_body().ok()) - .and_then(cdrs_tokio::frame::message_response::ResponseBody::into_rows) - .and_then(|mut rows| rows.pop()) - .and_then(|row| TryFromRowTrait::try_from_row(row).ok()) - .map(|r: SettingRow| r.value); - - Ok(version) - } - - fn expected_catalog_version(&self) -> String { - CATALOG_VERSION.to_string() } fn catalog_database_name(&self) -> String { diff --git a/crates/storage-cassandra/src/bootstrapper/guards.rs b/crates/storage-cassandra/src/bootstrapper/guards.rs new file mode 100644 index 000000000..8ba099692 --- /dev/null +++ b/crates/storage-cassandra/src/bootstrapper/guards.rs @@ -0,0 +1,195 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! The checks `init`'s bootstrap writes go through: the lease of the +//! migration lock `init` holds while it writes them, and the names it writes +//! into its CQL statements. + +use extenddb_storage::management_store::{OpError, OpResult}; +use extenddb_storage::migrations::MigrationError; + +use crate::migrations::{LeaseHandle, REQUEST_TIMEOUT}; + +/// The lease of the lock `init` holds, checked before each bootstrap write +/// as the runner checks it before each migration request: a process paused +/// past its deadline writes nothing after another process may have taken +/// the lock. Every write waits at most the request timeout for its answer +/// and records progress when it completes. Holds nothing until `init` hands +/// the lock over, and for every command other than `init`. +#[derive(Default)] +pub(super) struct HeldLease { + lease: std::sync::OnceLock, +} + +impl HeldLease { + pub(super) fn hold(&self, lease: LeaseHandle) { + let _ = self.lease.set(lease); + } + + /// Refuse a write once the held lease's deadline has passed. + pub(super) fn check(&self, what: &str) -> OpResult<()> { + match self.lease.get() { + Some(lease) => lease + .check() + .map_err(|e| OpError::Internal(format!("{what}: {e}"))), + None => Ok(()), + } + } + + /// [`Self::check`] as a runner error, for code that takes a check + /// closure. + pub(super) fn check_run(&self, what: &str) -> Result<(), MigrationError> { + match self.lease.get() { + Some(lease) => lease + .check() + .map_err(|e| MigrationError::Store(format!("{what}: {e}"))), + None => Ok(()), + } + } + + /// The held lease, for a run that sends its own requests under it. + pub(super) fn handle(&self) -> Option { + self.lease.get().cloned() + } + + /// Record progress on the held lease: a request sent under it completed. + pub(super) fn progress(&self) { + if let Some(lease) = self.lease.get() { + lease.progress(); + } + } + + /// Send `request` after the lease check, wait at most the request + /// timeout for its answer, and record progress when it completes. + pub(super) async fn request( + &self, + what: &str, + request: impl std::future::Future>, + ) -> OpResult { + self.request_within(what, REQUEST_TIMEOUT, request).await + } + + /// [`Self::request`] waiting at most `bound` for the answer: a schema + /// statement gets the bound of one sent under the lock. + pub(super) async fn request_within( + &self, + what: &str, + bound: std::time::Duration, + request: impl std::future::Future>, + ) -> OpResult { + self.check(what)?; + let answer = tokio::time::timeout(bound, request).await; + self.progress(); + match answer { + Ok(Ok(value)) => Ok(value), + Ok(Err(e)) => Err(OpError::Internal(format!("{what}: {e}"))), + Err(_) => Err(OpError::Internal(format!( + "{what}: no answer within {} s", + bound.as_secs() + ))), + } + } +} + +/// Refuse an application role or keyspace prefix holding anything other +/// than ASCII letters, digits and `_`. `init` writes the role between single +/// quotes and the keyspaces unquoted into its `CREATE ROLE`, `CREATE +/// KEYSPACE` and `GRANT` statements, so a quote or a space in either would +/// break a statement partway through `init`. +pub(super) fn check_name( + what: &str, + value: &str, +) -> Result<(), extenddb_storage::error::StorageError> { + if !value.is_empty() && value.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') { + return Ok(()); + } + Err(extenddb_storage::error::StorageError::Internal(format!( + "{what} {value:?} may hold only ASCII letters, digits and '_'" + ))) +} + +#[cfg(test)] +mod tests { + use super::HeldLease; + use crate::bootstrapper::CassandraBootstrapper; + + /// `init` writes its bootstrap rows under the migration lock: a process + /// paused past its lease's deadline sends none of them, and one that + /// holds no lock (every other command) sends them unchecked. + #[tokio::test] + async fn a_bootstrap_write_is_refused_once_the_held_lease_lapses() { + use std::sync::atomic::{AtomicBool, Ordering}; + + let sent = AtomicBool::new(false); + let write = || async { + sent.store(true, Ordering::SeqCst); + Ok::(1) + }; + + let unheld = HeldLease::default(); + assert_eq!(unheld.request("a row", write()).await.unwrap(), 1); + assert!(sent.swap(false, Ordering::SeqCst)); + assert!(unheld.check("a row").is_ok()); + + let held = HeldLease::default(); + held.hold(crate::migrations::LeaseHandle::lapsed("p2m_catalog")); + let refused = held.request("Create admin user", write()).await; + let message = format!("{refused:?}"); + assert!( + message.contains("Create admin user") + && message.contains("migration lock in p2m_catalog.settings was not renewed"), + "{message}" + ); + assert!(!sent.load(Ordering::SeqCst), "the write was sent"); + assert!(held.check("Create account").is_err()); + + let failed = unheld + .request("a row", async { + Err::("refused by the node".to_owned()) + }) + .await; + assert_eq!( + format!("{failed:?}"), + format!( + "{:?}", + Err::(super::OpError::Internal( + "a row: refused by the node".to_owned() + )) + ) + ); + } + + /// The application role and the keyspace prefix go into `init`'s CQL + /// statements, so a value holding anything but ASCII letters, digits + /// and `_` is refused before `init` connects. The contact point is a + /// closed port: without the check the call fails on the connection + /// instead, with another message. + #[tokio::test] + async fn a_role_or_prefix_with_another_character_is_refused_before_connecting() { + for (flag, value) in [ + ("--extenddb-user", "app'role"), + ("--extenddb-user", "app role"), + ("--extenddb-user", ""), + ("--keyspace-prefix", "pfx'x"), + ("--keyspace-prefix", "pfx-x"), + ] { + let args: Vec = ["--cassandra-contact-points", "127.0.0.1:1", flag, value] + .iter() + .map(|s| (*s).to_owned()) + .collect(); + let message = + match CassandraBootstrapper::from_config("/nonexistent/x.toml", &args).await { + Err(e) => e.to_string(), + Ok(_) => panic!("{flag} {value:?} was accepted"), + }; + assert!( + message.contains(flag) + && message.contains(&format!("{value:?}")) + && message.contains("ASCII letters, digits and '_'"), + "{flag} {value:?}: {message}" + ); + } + for value in ["extenddb", "app_role_2", "A1", "_x"] { + assert!(super::check_name("a name", value).is_ok(), "{value}"); + } + } +} diff --git a/crates/storage-cassandra/src/catalog_store.rs b/crates/storage-cassandra/src/catalog_store.rs index e0adc3f6f..1b739647a 100644 --- a/crates/storage-cassandra/src/catalog_store.rs +++ b/crates/storage-cassandra/src/catalog_store.rs @@ -24,6 +24,12 @@ type CassandraSession = Session< RoundRobinLoadBalancingStrategy, >; +/// The replication map of a keyspace with `NetworkTopologyStrategy` and +/// `replication_factor` replicas in `datacenter`. +pub(crate) fn network_topology(datacenter: &str, replication_factor: u32) -> String { + format!("{{'class': 'NetworkTopologyStrategy', '{datacenter}': {replication_factor}}}") +} + /// Cassandra-backed catalog store for settings, metrics, and rate limiting. /// /// Holds a session to the catalog keyspace. Created once at startup @@ -31,14 +37,22 @@ type CassandraSession = Session< pub struct CassandraCatalogStore { session: Arc, keyspace_prefix: String, - datacenter: String, - replication_factor: u32, + /// The replication CreateAccount gives a new account keyspace, as a CQL + /// map literal; `None` for a store opened from a connection string, + /// which knows no replication and refuses to create a keyspace. + replication: Option, + /// The release recorded in the ledger rows of a new account keyspace. + release: String, /// Cached encryption key (immutable after bootstrap). Avoids /// per-request DB query on access key and assume-role operations. encryption_key: Option>, /// TTL in seconds applied to every metrics row on insert. /// Cassandra expires rows natively; `prune_metrics` is a no-op. metrics_ttl_seconds: i32, + /// The crash-and-rerun tests' request budget, shared with the migration + /// backend, which fails a policy write once it is spent. + #[cfg(any(test, feature = "test-util"))] + stop_after: Option>, } impl CassandraCatalogStore { @@ -53,13 +67,60 @@ impl CassandraCatalogStore { Self { session, keyspace_prefix, - datacenter, - replication_factor, + replication: Some(network_topology(&datacenter, replication_factor)), + release: env!("CARGO_PKG_VERSION").to_owned(), encryption_key: None, metrics_ttl_seconds: 86400, + #[cfg(any(test, feature = "test-util"))] + stop_after: None, } } + /// A store for the catalog's rows alone, opened from a connection string + /// (`settings`, `verify`, the diagnostics, a migration's code body). It + /// knows no replication, so CreateAccount on it fails instead of + /// creating a keyspace with a guessed one. + #[must_use] + pub fn catalog_only(session: Arc, keyspace_prefix: String) -> Self { + Self { + session, + keyspace_prefix, + replication: None, + release: env!("CARGO_PKG_VERSION").to_owned(), + encryption_key: None, + metrics_ttl_seconds: 86400, + #[cfg(any(test, feature = "test-util"))] + stop_after: None, + } + } + + /// Share the crash-and-rerun tests' request budget: each policy write + /// takes one request, and fails once the budget is spent. + #[cfg(any(test, feature = "test-util"))] + pub(crate) fn stop_after(&mut self, left: Arc) { + self.stop_after = Some(left); + } + + /// Fail a policy write once the crash-and-rerun tests' budget is spent, + /// as a migrator killed before the write stops. + pub(crate) fn test_stop(&self) -> OpResult<()> { + #[cfg(any(test, feature = "test-util"))] + if crate::migrations::stopped(self.stop_after.as_deref()) { + return Err(OpError::Internal( + "stopped by the test hook before a policy write".to_owned(), + )); + } + Ok(()) + } + + /// Record `release` in the ledger rows of the account keyspaces this + /// store creates: the server's own release. + #[must_use] + pub fn recording_release(mut self, release: &str) -> Self { + release.clone_into(&mut self.release); + self + } + /// Create a new catalog store with a pre-loaded encryption key. #[must_use] pub fn with_encryption_key( @@ -72,10 +133,12 @@ impl CassandraCatalogStore { Self { session, keyspace_prefix, - datacenter, - replication_factor, + replication: Some(network_topology(&datacenter, replication_factor)), + release: env!("CARGO_PKG_VERSION").to_owned(), encryption_key: Some(Arc::from(encryption_key.as_str())), metrics_ttl_seconds: 86400, + #[cfg(any(test, feature = "test-util"))] + stop_after: None, } } @@ -101,60 +164,35 @@ impl CassandraCatalogStore { format!("{}_account_{}", self.keyspace_prefix, account_id) } - /// Ensure an account keyspace exists (idempotent). - /// Creates the keyspace with `NetworkTopologyStrategy` if it doesn't exist. + /// Ensure an account keyspace exists with every account migration + /// (idempotent): create it when missing, then run the account + /// migrations on it through the migration runner, which waits for + /// schema agreement after each statement and records each one in the + /// keyspace's ledger. A retried CreateAccount whose earlier call stopped + /// after the `accounts` row repairs the keyspace the same way. pub(crate) async fn ensure_account_keyspace(&self, account_id: &str) -> OpResult<()> { let keyspace_name = self.account_keyspace(account_id); - - // Check if keyspace already exists to avoid re-running migrations on every call. - let exists_result = self - .session - .query_with_values( - "SELECT keyspace_name FROM system_schema.keyspaces WHERE keyspace_name = ?", - cdrs_tokio::query_values!(keyspace_name.as_str()), - ) - .await - .ok() - .and_then(|r| r.response_body().ok()) - .is_some_and(|b| !b.into_rows().unwrap_or_default().is_empty()); - - if exists_result { - // Keyspace exists but may have been created by a concurrent caller that - // hasn't finished running migrations yet. Always run migrations — they are - // idempotent (CREATE TABLE IF NOT EXISTS) so re-running is safe. - return crate::migrations::run_data_migrations(&self.session, &keyspace_name) - .await - .map_err(|e| { - tracing::error!( - "Failed to run data migrations for {}: {:?}", - keyspace_name, - e - ); - OpError::Internal("Failed to initialize account storage".to_owned()) - }); - } - + let failed = |e: &dyn std::fmt::Display| { + tracing::error!("Failed to initialize account keyspace {keyspace_name}: {e}"); + OpError::Internal("Failed to initialize account storage".to_owned()) + }; + let Some(replication) = &self.replication else { + return Err(failed( + &"this store was opened without the deployment's replication", + )); + }; let cql = format!( - "CREATE KEYSPACE IF NOT EXISTS {} WITH replication = {{'class': 'NetworkTopologyStrategy', '{}': {}}}", - keyspace_name, self.datacenter, self.replication_factor + "CREATE KEYSPACE IF NOT EXISTS {keyspace_name} WITH replication = {replication}" ); - - self.session.query(cql).await.map_err(|e| { - tracing::error!("Failed to create account keyspace {}: {}", keyspace_name, e); - OpError::Internal("Failed to initialize account storage".to_owned()) - })?; - - crate::migrations::run_data_migrations(&self.session, &keyspace_name) - .await - .map_err(|e| { - tracing::error!( - "Failed to run data migrations for {}: {:?}", - keyspace_name, - e - ); - OpError::Internal("Failed to initialize account storage".to_owned()) - })?; - + self.session.query(cql).await.map_err(|e| failed(&e))?; + crate::migrations::migrate_account( + self.session.clone(), + &self.keyspace_prefix, + &keyspace_name, + &self.release, + ) + .await + .map_err(|e| failed(&e))?; Ok(()) } diff --git a/crates/storage-cassandra/src/config.rs b/crates/storage-cassandra/src/config.rs index d631fd7bd..b1a2f461a 100644 --- a/crates/storage-cassandra/src/config.rs +++ b/crates/storage-cassandra/src/config.rs @@ -60,11 +60,83 @@ fn default_max_connections() -> u32 { 10 } +/// The parts of a Cassandra connection string. +#[derive(Clone, PartialEq, Eq)] +pub struct ConnectionParts { + pub contact_points: Vec, + pub keyspace_prefix: String, + pub username: Option, + pub password: Option, +} + +impl std::fmt::Debug for ConnectionParts { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ConnectionParts") + .field("contact_points", &self.contact_points) + .field("keyspace_prefix", &self.keyspace_prefix) + .field("username", &self.username) + .field("password", &self.password.as_ref().map(|_| "")) + .finish() + } +} + +/// Every byte outside ASCII letters, digits and `-._~` as `%XX`, so a role or +/// password holding `@`, `:`, `/` or `,` cannot be read as another part. +fn percent_encode(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + for byte in s.bytes() { + if byte.is_ascii_alphanumeric() || b"-._~".contains(&byte) { + out.push(char::from(byte)); + } else { + out.push_str(&format!("%{byte:02X}")); + } + } + out +} + +/// The inverse of [`percent_encode`]; a `%` not followed by two hex digits +/// stays as it is. +fn percent_decode(s: &str) -> String { + let bytes = s.as_bytes(); + let mut out = Vec::with_capacity(bytes.len()); + let mut i = 0; + while i < bytes.len() { + let hex = bytes + .get(i + 1..i + 3) + .and_then(|h| std::str::from_utf8(h).ok()) + .and_then(|h| u8::from_str_radix(h, 16).ok()); + match (bytes[i], hex) { + (b'%', Some(byte)) => { + out.push(byte); + i += 3; + } + (byte, _) => { + out.push(byte); + i += 1; + } + } + } + String::from_utf8_lossy(&out).into_owned() +} + impl CassandraStorageConfig { - /// Build the connection string in JDBC-style format: `host1,host2/keyspace_prefix` + /// Build the connection string: `[user[:password]@]host1,host2/keyspace_prefix`. + /// + /// The role and password are the config's, percent-encoded, so a command + /// that receives only the connection string (`verify`, `settings`, + /// `migrate`) connects with the role the server uses. + /// [`Self::redact_connection_string`] hides the password wherever the + /// string is shown. fn build_connection_string(&self) -> String { let hosts = self.contact_points.join(","); - format!("{}/{}", hosts, self.keyspace_prefix) + let userinfo = match (&self.username, &self.password) { + (Some(user), Some(password)) => { + format!("{}:{}@", percent_encode(user), percent_encode(password)) + } + (Some(user), None) => format!("{}@", percent_encode(user)), + (None, _) => String::new(), + }; + format!("{userinfo}{hosts}/{}", self.keyspace_prefix) } /// Ensure the cached connection string is populated. @@ -82,17 +154,52 @@ impl CassandraStorageConfig { } /// Parse a JDBC-style connection string into contact points and keyspace prefix. - /// Format: "`host1:port1,host2:port2/keyspace_prefix`" + /// Format: "`[user[:password]@]host1:port1,host2:port2/keyspace_prefix`" /// Returns (`contact_points`, `keyspace_prefix`) #[must_use] pub fn parse_connection_string(conn_str: &str) -> (Vec, String) { - if let Some((hosts, keyspace)) = conn_str.split_once('/') { - let contact_points = hosts.split(',').map(|s| s.trim().to_string()).collect(); - (contact_points, keyspace.trim().to_string()) - } else { - // No keyspace specified, use default - let contact_points = conn_str.split(',').map(|s| s.trim().to_string()).collect(); - (contact_points, default_keyspace_prefix()) + let parts = Self::connection_parts(conn_str); + (parts.contact_points, parts.keyspace_prefix) + } + + /// Every part of a connection string [`Self::build_connection_string`] + /// wrote, the role and password decoded. + #[must_use] + pub fn connection_parts(conn_str: &str) -> ConnectionParts { + let (userinfo, rest) = match conn_str.rsplit_once('@') { + Some((userinfo, rest)) => (Some(userinfo), rest), + None => (None, conn_str), + }; + let (username, password) = match userinfo { + Some(info) => match info.split_once(':') { + Some((user, password)) => { + (Some(percent_decode(user)), Some(percent_decode(password))) + } + None => (Some(percent_decode(info)), None), + }, + None => (None, None), + }; + let (hosts, keyspace_prefix) = match rest.split_once('/') { + Some((hosts, keyspace)) => (hosts, keyspace.trim().to_string()), + None => (rest, default_keyspace_prefix()), + }; + ConnectionParts { + contact_points: hosts.split(',').map(|s| s.trim().to_string()).collect(), + keyspace_prefix, + username, + password, + } + } + + /// The connection string with its password replaced by `***`. + #[must_use] + pub fn redact_connection_string(conn_str: &str) -> String { + match conn_str.rsplit_once('@') { + Some((userinfo, rest)) => match userinfo.split_once(':') { + Some((user, _)) => format!("{user}:***@{rest}"), + None => conn_str.to_owned(), + }, + None => conn_str.to_owned(), } } @@ -183,6 +290,40 @@ mod tests { assert_eq!(config.connection_config(), "/extenddb"); } + #[test] + fn the_connection_string_carries_the_role_and_hides_the_password_when_shown() { + let mut config = CassandraStorageConfig::new(vec!["h1:9042".to_string()]); + config.username = Some("app:role".to_string()); + config.password = Some("p@ss/w,rd%".to_string()); + config.rebuild_cached_connection_string(); + let conn = config.connection_config(); + assert_eq!(conn, "app%3Arole:p%40ss%2Fw%2Crd%25@h1:9042/extenddb"); + let parts = CassandraStorageConfig::connection_parts(conn); + assert_eq!(parts.username.as_deref(), Some("app:role")); + assert_eq!(parts.password.as_deref(), Some("p@ss/w,rd%")); + assert_eq!(parts.contact_points, ["h1:9042"]); + assert_eq!(parts.keyspace_prefix, "extenddb"); + assert_eq!( + CassandraStorageConfig::parse_connection_string(conn), + (vec!["h1:9042".to_string()], "extenddb".to_string()) + ); + let shown = CassandraStorageConfig::redact_connection_string(conn); + assert_eq!(shown, "app%3Arole:***@h1:9042/extenddb"); + assert!(!format!("{parts:?}").contains("p@ss")); + // Without a password nothing is hidden, and without a role nothing + // is added. + config.password = None; + config.rebuild_cached_connection_string(); + assert_eq!(config.connection_config(), "app%3Arole@h1:9042/extenddb"); + config.username = None; + config.rebuild_cached_connection_string(); + assert_eq!(config.connection_config(), "h1:9042/extenddb"); + assert_eq!( + CassandraStorageConfig::redact_connection_string("h1:9042/extenddb"), + "h1:9042/extenddb" + ); + } + #[test] fn test_toml_deserialization() { let toml_str = r#" diff --git a/crates/storage-cassandra/src/create_table.rs b/crates/storage-cassandra/src/create_table.rs index 9bf36d3ad..024c74ce9 100644 --- a/crates/storage-cassandra/src/create_table.rs +++ b/crates/storage-cassandra/src/create_table.rs @@ -219,11 +219,10 @@ impl CassandraEngine { ))); } - // Insert table metadata with LWT (IF NOT EXISTS). Only a restore - // target names the restore columns (added by catalog migration V005), - // so an ordinary CreateTable also works on a catalog that has not run - // V005; CQL rejects a statement naming an undefined column even when - // no value is bound to it. + // Insert table metadata with LWT (IF NOT EXISTS). Every catalog this + // release serves has the restore columns (catalog migration 1 creates + // them). Only a restore target has values for them, so only its insert + // names them, and an ordinary table's row leaves them null. let mut columns = String::from( "account_id, table_name, table_id, table_arn, key_schema, \ attribute_definitions, billing_mode, provisioned_throughput, stream_specification, \ @@ -311,8 +310,8 @@ impl CassandraEngine { // A restore into a name that is a CREATING restore target is // refused as such, naming that target's source backup (the // service answers TableInUseException); any other taken name - // is TableAlreadyExists. The restore column is absent on a - // catalog without V005, which reads as "not a restore target". + // is TableAlreadyExists. An ordinary table's row has a null + // restore column, which reads as "not a restore target". if restore.is_some() { let status: Option = row.get_by_name("table_status").ok().flatten(); let source: Option = diff --git a/crates/storage-cassandra/src/engine.rs b/crates/storage-cassandra/src/engine.rs index d897602e7..a70db87fb 100644 --- a/crates/storage-cassandra/src/engine.rs +++ b/crates/storage-cassandra/src/engine.rs @@ -101,14 +101,6 @@ pub struct CassandraEngine { /// runs (`tables.restore_owner`). Random per process start, so a restore /// left behind by a crashed process is never mistaken for a live one. pub(crate) restore_owner: uuid::Uuid, - - /// Set once `schema_history` was seen to record every catalog migration - /// this binary carries; until then each CreateBackup, DeleteBackup and - /// RestoreTableFromBackup re-reads it (see `require_migrated_catalog`), so - /// a `migrate --yes` run while the server is up takes effect without a - /// restart. Never cleared: migrations are not undone under a running - /// server. - pub(crate) catalog_migrated: Arc, } /// Cap on concurrently in-flight detached TTL claim releases. @@ -151,7 +143,6 @@ impl CassandraEngine { std::collections::HashMap::new(), )), restore_owner: uuid::Uuid::new_v4(), - catalog_migrated: Arc::new(std::sync::atomic::AtomicBool::new(false)), }) } diff --git a/crates/storage-cassandra/src/lib.rs b/crates/storage-cassandra/src/lib.rs index e21bf181e..f7335d72e 100644 --- a/crates/storage-cassandra/src/lib.rs +++ b/crates/storage-cassandra/src/lib.rs @@ -7,8 +7,6 @@ //! as the underlying database. It provides a DynamoDB-compatible API backed //! by Cassandra's distributed architecture. -pub const CATALOG_VERSION: &str = "0.0.4"; - /// Default control-plane delay (seconds) when the `control_plane_delay_seconds` /// setting is absent or unparseable: CreateTable, the control-plane worker /// and the console's default all read this one value. @@ -49,10 +47,6 @@ pub use config::CassandraStorageConfig; pub use engine::{CassandraEngine, CassandraSession}; pub use metadata_engine::TtlBackfillCursor; -use cdrs_tokio::authenticators::StaticPasswordAuthenticatorProvider; -use cdrs_tokio::cluster::NodeTcpConfigBuilder; -use cdrs_tokio::cluster::session::{SessionBuilder, TcpSessionBuilder}; -use cdrs_tokio::load_balancing::RoundRobinLoadBalancingStrategy; use cdrs_tokio::types::IntoRustByName; use extenddb_storage::hooks::{ServerRuntimeHooks, WorkerContext}; use extenddb_storage::server_components::{BackendError, ServerComponents}; @@ -67,6 +61,8 @@ struct CassandraRuntimeHooks { engine: Arc, control_plane_notify: Arc, gsi_worker_guard: std::sync::OnceLock, + /// The registration this server holds while it serves. + registration: Arc, } #[async_trait::async_trait] @@ -172,6 +168,11 @@ impl ServerRuntimeHooks for CassandraRuntimeHooks { .await; }); + let registration = tokio::spawn(hold_registration( + self.registration.clone(), + ctx.shutdown.clone(), + )); + vec![ control_plane, transaction_recovery, @@ -181,6 +182,7 @@ impl ServerRuntimeHooks for CassandraRuntimeHooks { ttl_repair, ttl_audit, restore_sweep, + registration, ] } @@ -189,6 +191,41 @@ impl ServerRuntimeHooks for CassandraRuntimeHooks { } } +/// Hold the serving registration for the server's life: release it when +/// the server shuts down, so a migration need not wait for its TTL, and end +/// the process with exit status 1 once it could not be renewed for 30 s, +/// before its row expires and a migration could start under this server. +/// The exit runs the `atexit` handler that removes the PID file. +async fn hold_registration( + registration: Arc, + shutdown: extenddb_storage::hooks::CancellationToken, +) { + let mut lost = registration.lost(); + loop { + tokio::select! { + () = shutdown.cancelled() => { + registration.release().await; + return; + } + changed = lost.changed() => { + if changed.is_err() { + return; + } + let reason = lost.borrow().clone(); + if let Some(reason) = reason { + let message = format!( + "extenddb fatal: {reason}; the server stops, because a migration can \ + start on the catalog once the serving registration is gone" + ); + tracing::error!("{message}"); + eprintln!("{message}"); + std::process::exit(1); + } + } + } + } +} + // ============================================================================ // Backend Registration // ============================================================================ @@ -258,57 +295,37 @@ pub fn backend() -> extenddb_storage::Backend { }) }, server_components: server_components_factory, - // Migrates through its bootstrapper until it moves onto the runner. - catalog_migrations: None, + catalog_migrations: &migrations::CassandraCatalogMigrations, } } /// Build a `CassandraCatalogStore` from a bare connection string. /// -/// Used by the `settings_store` and `diagnostics_store` factories, which -/// receive only a connection string (no full config object). +/// Used by the `settings_store`, `management_store` and `diagnostics_store` +/// factories, which receive only a connection string (no full config +/// object): `extenddb settings`, `verify` and the diagnostics. The string +/// carries the role the config file names, so they connect as the server +/// does, with its 10 s bound on the connect. async fn make_catalog_store_from_connection_string( connection_string: &str, ) -> Result { - let (contact_points, keyspace_prefix) = - CassandraStorageConfig::parse_connection_string(connection_string); - - if contact_points.is_empty() { - return Err("No contact points provided".to_string()); - } - - let mut node_builder = NodeTcpConfigBuilder::new(); - for contact_point in &contact_points { - node_builder = node_builder.with_contact_point(contact_point.clone().into()); - } - let auth_provider = Arc::new(StaticPasswordAuthenticatorProvider::new( - "cassandra", - "cassandra", - )); - node_builder = node_builder.with_authenticator_provider(auth_provider); - - let cluster_config = node_builder - .build() - .await - .map_err(|e| format!("Failed to build cluster config: {e}"))?; - - let session = TcpSessionBuilder::new(RoundRobinLoadBalancingStrategy::new(), cluster_config) - .build() - .await - .map_err(|e| format!("Failed to create session: {e}"))?; - - Ok(CassandraCatalogStore::new( - Arc::new(session), - keyspace_prefix, - "datacenter1".to_string(), - 1, + let deployment = migrations::Deployment::from_target( + &extenddb_storage::migrations::Target::at(connection_string), + ) + .map_err(|e| e.to_string())?; + let session = deployment.connect().await.map_err(|e| e.to_string())?; + Ok(CassandraCatalogStore::catalog_only( + session, + deployment.keyspace_prefix, )) } fn server_components_factory( config: &dyn extenddb_storage::config::StorageConfig, region: &str, - _options: extenddb_storage::server_components::ServerComponentsOptions, + // `bootstrap_if_uninitialized` is not honored: an uninitialized catalog + // fails with the explicit-`init` guidance, as on PostgreSQL. + options: extenddb_storage::server_components::ServerComponentsOptions, ) -> std::pin::Pin< Box> + Send>, > { @@ -323,12 +340,42 @@ fn server_components_factory( BackendError::InitializationFailed("Expected CassandraStorageConfig".to_string()) })?; - let engine = CassandraEngine::new(cassandra_config, ®ion) - .await - .map_err(|e| BackendError::ConnectionFailed { - backend: "cassandra".to_string(), - details: e.to_string(), - })?; + // Register and read the catalog's status before the engine starts: + // creating it runs the control-plane recovery, which writes to the + // catalog. The status session uses the engine's role and settings. + let deployment = migrations::Deployment::from_config(cassandra_config); + let status_session = + deployment + .connect() + .await + .map_err(|e| BackendError::ConnectionFailed { + backend: "cassandra".to_string(), + details: e.to_string(), + })?; + let host = cassandra_config + .instance_id + .clone() + .unwrap_or_else(|| "unknown".to_owned()); + let registration = migrations::register_server( + status_session, + &cassandra_config.keyspace_prefix, + &host, + options.release, + ) + .await + .map_err(BackendError::Catalog)?; + let registration = Arc::new(registration); + + let engine = match CassandraEngine::new(cassandra_config, ®ion).await { + Ok(engine) => engine, + Err(e) => { + registration.release().await; + return Err(BackendError::ConnectionFailed { + backend: "cassandra".to_string(), + details: e.to_string(), + }); + } + }; let control_plane_notify = engine.control_plane_notify.clone(); let engine = Arc::new(engine); @@ -366,13 +413,16 @@ fn server_components_factory( BackendError::InitializationFailed(format!("Failed to parse encryption key: {e}")) })?; - let catalog_store = Arc::new(CassandraCatalogStore::with_encryption_key( - engine.session.clone(), - cassandra_config.keyspace_prefix.clone(), - cassandra_config.datacenter.clone(), - cassandra_config.replication_factor, - enc_key.clone(), - )); + let catalog_store = Arc::new( + CassandraCatalogStore::with_encryption_key( + engine.session.clone(), + cassandra_config.keyspace_prefix.clone(), + cassandra_config.datacenter.clone(), + cassandra_config.replication_factor, + enc_key.clone(), + ) + .recording_release(options.release), + ); let credential_store: Arc = Arc::new(credential_store::CassandraCredentialStore::new( @@ -385,6 +435,7 @@ fn server_components_factory( engine: engine.clone(), control_plane_notify, gsi_worker_guard: std::sync::OnceLock::new(), + registration, }); Ok(ServerComponents { diff --git a/crates/storage-cassandra/src/management_store/mod.rs b/crates/storage-cassandra/src/management_store/mod.rs index a38bfb5e8..28bf7bae1 100755 --- a/crates/storage-cassandra/src/management_store/mod.rs +++ b/crates/storage-cassandra/src/management_store/mod.rs @@ -356,6 +356,7 @@ impl extenddb_storage::management_store::ManagementStore for CassandraCatalogSto let policy_name = policy_name.to_string(); let document = document.clone(); Box::pin(async move { + self.test_stop()?; self.put_policy_impl( &account_id, &principal_type, @@ -411,6 +412,7 @@ impl extenddb_storage::management_store::ManagementStore for CassandraCatalogSto let user_name = user_name.to_string(); let document = document.clone(); Box::pin(async move { + self.test_stop()?; self.set_boundary_impl(&account_id, "user", &user_name, &document) .await }) @@ -452,6 +454,7 @@ impl extenddb_storage::management_store::ManagementStore for CassandraCatalogSto let role_name = role_name.to_string(); let document = document.clone(); Box::pin(async move { + self.test_stop()?; self.set_boundary_impl(&account_id, "role", &role_name, &document) .await }) @@ -592,6 +595,7 @@ impl extenddb_storage::management_store::ManagementStore for CassandraCatalogSto let session_token = session_token.to_string(); let document = document.clone(); Box::pin(async move { + self.test_stop()?; self.update_session_policy_impl(&session_token, &document) .await }) diff --git a/crates/storage-cassandra/src/migrations.rs b/crates/storage-cassandra/src/migrations.rs deleted file mode 100644 index 8eabe63b4..000000000 --- a/crates/storage-cassandra/src/migrations.rs +++ /dev/null @@ -1,584 +0,0 @@ -// Copyright 2026 ExtendDB Contributors -// SPDX-License-Identifier: Apache-2.0 - -//! Cassandra schema migrations. -//! -//! ## Migration Versioning -//! -//! This module uses Flyway-style versioning for migration files: -//! -//! - Format: `V###__description.cql` (e.g., `V001__initial_schema.cql`) -//! - Version: Integer extracted from `V###__` prefix (e.g., 1 from V001) -//! - Description: Text after double underscore (e.g., `initial_schema`) -//! -//! Flyway is a popular database migration tool that uses this naming convention. -//! See: -//! -//! ## Migration Tracking -//! -//! Applied migrations are tracked in the `schema_history` table: -//! - `version` (int): Extracted from filename (V001 → 1) -//! - `description` (text): Extracted from filename (`V001__initial_schema.cql` → `initial_schema`) -//! - `applied_at` (timestamp): When the migration was applied -//! -//! Migrations are applied in order by version number. Already-applied migrations -//! are skipped based on the version number in the tracking table. -//! -//! ## Adding New Migrations -//! -//! 1. Create a new file: `migrations/catalog/V###__description.cql` -//! 2. Use the next sequential version number (e.g., V003 after V002) -//! 3. Add the migration to the `CATALOG_MIGRATIONS` array below -//! 4. Migrations are embedded at compile time via `include_str!()` -//! 5. Name tables without a keyspace. The runner writes the keyspace it -//! migrates into each `CREATE TABLE`, `CREATE INDEX`, `ALTER TABLE` and -//! `INSERT INTO` statement, and rejects any other statement form. - -use cdrs_tokio::cluster::TcpConnectionManager; -use cdrs_tokio::cluster::session::Session; -use cdrs_tokio::load_balancing::RoundRobinLoadBalancingStrategy; -use cdrs_tokio::transport::TransportTcp; -use extenddb_storage::management_store::{OpError, OpResult}; -use std::sync::Arc; -use tokio::sync::Mutex; - -/// Serializes migration runs across the whole process (issue #408). -/// -/// Migrations set the keyspace with `USE ` on the shared cdrs-tokio -/// `Session` and then run UNQUALIFIED DDL (`CREATE TABLE IF NOT EXISTS `). -/// cdrs-tokio keeps one current keyspace per `Session`, and the whole process -/// shares one `Session`, so two concurrent migration runs (e.g. two accounts -/// created at once) interleave their `USE` and one account's DDL lands in the -/// other's keyspace — while its own `schema_history` still records the -/// migration as applied, so it never retries. Holding this lock for the whole -/// `USE`-then-DDL sequence makes each run atomic with respect to the shared -/// keyspace state. Both catalog and data migrations take the SAME lock, because -/// they race on the same `Session`. -/// -/// The lock is held only during a migration run (rare: account/catalog -/// creation), never on the request hot path. -static MIGRATION_LOCK: Mutex<()> = Mutex::const_new(()); - -type CassandraSession = Session< - TransportTcp, - TcpConnectionManager, - RoundRobinLoadBalancingStrategy, ->; - -/// Embedded catalog migration files, applied in order. -pub(crate) const CATALOG_MIGRATIONS: &[(&str, &str)] = &[ - ( - "V001__initial_schema.cql", - include_str!("../migrations/catalog/V001__initial_schema.cql"), - ), - ( - "V002__backup_restore.cql", - include_str!("../migrations/catalog/V002__backup_restore.cql"), - ), - ( - "V003__account_scoped_idempotency.cql", - include_str!("../migrations/catalog/V003__account_scoped_idempotency.cql"), - ), - ( - "V004__ttl_backfill_cursor.cql", - include_str!("../migrations/catalog/V004__ttl_backfill_cursor.cql"), - ), - ( - "V005__backup_definitions_and_restore_registration.cql", - include_str!("../migrations/catalog/V005__backup_definitions_and_restore_registration.cql"), - ), -]; - -/// Embedded data migration files, applied in order. -pub(crate) const DATA_MIGRATIONS: &[(&str, &str)] = &[ - ( - "V001__initial_data_schema.cql", - include_str!("../migrations/data/V001__initial_data_schema.cql"), - ), - ( - "V002__backup_items.cql", - include_str!("../migrations/data/V002__backup_items.cql"), - ), -]; - -/// Run catalog migrations, skipping already-applied ones. -pub async fn run_catalog_migrations( - session: &Arc, - keyspace: &str, -) -> OpResult<()> { - // Hold the process-wide migration lock for the whole USE-then-DDL sequence - // so a concurrent run cannot move the Session's current keyspace (issue - // #408). Held until this function returns. - let _guard = MIGRATION_LOCK.lock().await; - - println!("--- Running catalog migrations..."); - - for (filename, sql) in CATALOG_MIGRATIONS { - if is_migration_applied(session, keyspace, filename).await? { - println!(" {filename} — already applied, skipping."); - continue; - } - println!(" Applying {filename}..."); - execute_migration(session, keyspace, filename, sql).await?; - record_migration(session, keyspace, filename).await?; - } - println!(" Migrations applied."); - Ok(()) -} - -/// Run data database migrations. -pub async fn run_data_migrations(session: &Arc, keyspace: &str) -> OpResult<()> { - // Hold the process-wide migration lock for the whole USE-then-DDL sequence - // so a concurrent run cannot move the Session's current keyspace out from - // under this account's unqualified DDL (issue #408). Held until return. - let _guard = MIGRATION_LOCK.lock().await; - - println!("--- Running data migrations..."); - - for (filename, sql) in DATA_MIGRATIONS { - if is_migration_applied(session, keyspace, filename).await? { - println!(" {filename} — already applied, skipping."); - continue; - } - println!(" Applying {filename}..."); - execute_migration(session, keyspace, filename, sql).await?; - record_migration(session, keyspace, filename).await?; - } - println!(" Data migrations applied."); - Ok(()) -} - -/// Check if a table exists in the given keyspace. -pub(crate) async fn table_exists( - session: &Arc, - keyspace: &str, - table_name: &str, -) -> OpResult { - let query = - "SELECT table_name FROM system_schema.tables WHERE keyspace_name = ? AND table_name = ?"; - let exists = session - .query_with_values(query, cdrs_tokio::query_values!(keyspace, table_name)) - .await - .ok() - .and_then(|frame| frame.response_body().ok()) - .and_then(cdrs_tokio::frame::message_response::ResponseBody::into_rows) - .is_some_and(|rows| !rows.is_empty()); - Ok(exists) -} - -/// Return the union of data migrations pending in any account keyspace. -/// -/// A migration is pending if at least one existing account has not recorded it. -/// This keeps a partially completed multi-account upgrade retryable. -pub(crate) async fn pending_data_migrations( - session: &Arc, - catalog_keyspace: &str, - account_keyspace_fn: impl Fn(&str) -> String, -) -> OpResult> { - let query = format!("SELECT account_id FROM {catalog_keyspace}.accounts"); - let rows = crate::cassandra_util::query_rows::( - &Arc::clone(session), - &query, - cdrs_tokio::query_values!(), - "pending_data_migrations", - ) - .await?; - - let mut pending = std::collections::BTreeSet::new(); - for row in rows { - let account_id: String = - crate::cassandra_util::get_column(&row, "account_id", "pending_data_migrations")?; - let keyspace = account_keyspace_fn(&account_id); - for (filename, _sql) in DATA_MIGRATIONS { - if !is_migration_applied(session, &keyspace, filename).await? { - pending.insert((*filename).to_owned()); - } - } - } - Ok(pending.into_iter().collect()) -} - -/// Check if a migration has already been applied. -async fn is_migration_applied( - session: &Arc, - keyspace: &str, - filename: &str, -) -> OpResult { - let version = migration_version(filename)?; - - let check_cql = format!("SELECT version FROM {keyspace}.schema_history WHERE version = ?"); - - let applied = session - .query_with_values(check_cql, cdrs_tokio::query_values!(version)) - .await - .ok() - .and_then(|frame| frame.response_body().ok()) - .and_then(cdrs_tokio::frame::message_response::ResponseBody::into_rows) - .is_some_and(|rows| !rows.is_empty()); - - Ok(applied) -} - -/// Execute a migration by splitting on semicolons and running each statement -/// with the table it names written as `keyspace.table`. -/// -/// The runners never select a keyspace with `USE` (#632). cdrs-tokio runs a -/// `USE` on one pooled connection and copies it to the others from a -/// background task, so a statement relying on it can run in the keyspace the -/// connection it lands on had selected before. -async fn execute_migration( - session: &Arc, - keyspace: &str, - filename: &str, - sql: &str, -) -> OpResult<()> { - for statement in migration_statements(keyspace, filename, sql)? { - session - .query(statement) - .await - .map_err(|e| OpError::Internal(format!("Migration failed: {e}")))?; - } - Ok(()) -} - -/// The statements of migration file `filename`, split on semicolons, each -/// qualified with `keyspace` by [`qualify_statement`]. If one cannot be -/// qualified, the file fails before any of its statements runs. -#[doc(hidden)] -pub fn migration_statements(keyspace: &str, filename: &str, sql: &str) -> OpResult> { - sql.split(';') - .map(str::trim) - .filter(|statement| !statement.is_empty()) - .map(|statement| qualify_statement(keyspace, filename, statement)) - .collect() -} - -/// `statement` with the table it names written as `keyspace.table`. -/// -/// Recognises `CREATE TABLE [IF NOT EXISTS]`, -/// `CREATE INDEX [IF NOT EXISTS] [name] ON`, `ALTER TABLE [IF EXISTS]` and -/// `INSERT INTO`, each naming an unquoted table without a keyspace. Any other -/// statement is an error naming `filename`, so no statement runs in whatever -/// keyspace the session has selected. -fn qualify_statement(keyspace: &str, filename: &str, statement: &str) -> OpResult { - let body = skip_comments(statement); - if body.is_empty() { - return Err(OpError::Internal(format!( - "Migration file {filename} has a fragment between semicolons that holds only a \ - comment (a semicolon inside a comment, or a comment after the last statement): {}", - statement.lines().next().unwrap_or_default() - ))); - } - let at = table_name_offset(body).ok_or_else(|| { - OpError::Internal(format!( - "Migration file {filename} has a statement that names no table the runner can \ - qualify with a keyspace: {}", - body.lines().next().unwrap_or_default() - )) - })?; - let at = statement.len() - body.len() + at; - Ok(format!( - "{}{keyspace}.{}", - &statement[..at], - &statement[at..] - )) -} - -/// `statement` from its first character outside whitespace and `--` or `//` -/// line comments. -fn skip_comments(mut statement: &str) -> &str { - loop { - statement = statement.trim_start(); - if !statement.starts_with("--") && !statement.starts_with("//") { - return statement; - } - statement = statement.split_once('\n').map_or("", |(_, rest)| rest); - } -} - -/// The byte offset in `body` of the table name, for the statement forms -/// [`qualify_statement`] recognises. -fn table_name_offset(body: &str) -> Option { - fn is(word: Option<(usize, &str)>, keyword: &str) -> bool { - word.is_some_and(|(_, word)| word.eq_ignore_ascii_case(keyword)) - } - let mut words = words(body).peekable(); - let (first, second) = (words.next(), words.next()); - if is(first, "CREATE") && (is(second, "TABLE") || is(second, "INDEX")) { - if words - .next_if(|&(_, w)| w.eq_ignore_ascii_case("IF")) - .is_some() - && !(is(words.next(), "NOT") && is(words.next(), "EXISTS")) - { - return None; - } - // CREATE INDEX [name] ON table: the index name is optional. - if is(second, "INDEX") && !is(words.next(), "ON") && !is(words.next(), "ON") { - return None; - } - } else if is(first, "ALTER") && is(second, "TABLE") { - if words - .next_if(|&(_, w)| w.eq_ignore_ascii_case("IF")) - .is_some() - && !is(words.next(), "EXISTS") - { - return None; - } - } else if !(is(first, "INSERT") && is(second, "INTO")) { - return None; - } - let (at, table) = words.next()?; - let mut chars = table.chars(); - let unquoted = chars.next().is_some_and(|c| c.is_ascii_alphabetic()) - && chars.all(|c| c.is_ascii_alphanumeric() || c == '_'); - unquoted.then_some(at) -} - -/// The words of `body` with their byte offsets, a word being a run of -/// characters ended by whitespace or an opening parenthesis. -fn words(body: &str) -> impl Iterator { - let mut at = 0; - std::iter::from_fn(move || { - let rest = &body[at..]; - let start = at + (rest.len() - rest.trim_start().len()); - let word = &body[start..]; - let end = word - .find(|c: char| c.is_whitespace() || c == '(') - .unwrap_or(word.len()); - at = start + end; - (end > 0).then(|| (start, &word[..end])) - }) -} - -/// Record a migration in the `schema_history` table. -async fn record_migration( - session: &Arc, - keyspace: &str, - filename: &str, -) -> OpResult<()> { - // Extract version and description from filename - let version = migration_version(filename)?; - - let description = filename - .split("__") - .nth(1) - .and_then(|s| s.strip_suffix(".cql")) - .unwrap_or("unknown"); - - let record_cql = format!( - "INSERT INTO {keyspace}.schema_history (version, description, applied_at) VALUES (?, ?, toTimestamp(now()))" - ); - - session - .query_with_values(record_cql, cdrs_tokio::query_values!(version, description)) - .await - .map_err(|e| OpError::Internal(format!("Record migration: {e}")))?; - - Ok(()) -} - -/// The version a migration file records in `schema_history`: the integer of -/// its `V###__` prefix (`V001__initial_schema.cql` -> 1). -fn migration_version(filename: &str) -> OpResult { - filename - .strip_prefix("V") - .and_then(|s| s.split("__").next()) - .and_then(|s| s.parse().ok()) - .ok_or_else(|| OpError::Internal(format!("Invalid migration filename: {filename}"))) -} - -/// The catalog migration files this binary carries that the catalog's -/// `schema_history` does not record, in order. Unlike the runner's per-file -/// check, a failed read is an error, not "pending". -pub(crate) async fn pending_catalog_migrations( - session: &Arc, - catalog_keyspace: &str, -) -> Result, extenddb_storage::error::StorageError> { - use extenddb_storage::error::StorageError; - let rows = crate::cassandra_util::query_rows::( - session, - &format!("SELECT version FROM {catalog_keyspace}.schema_history"), - cdrs_tokio::query_values!(), - "pending_catalog_migrations", - ) - .await?; - let mut recorded = std::collections::BTreeSet::new(); - for row in &rows { - let version: i32 = - crate::cassandra_util::get_column(row, "version", "pending_catalog_migrations")?; - recorded.insert(version); - } - let mut pending = Vec::new(); - for (filename, _sql) in CATALOG_MIGRATIONS { - let version = migration_version(filename).map_err(|e| match e { - OpError::Validation(m) - | OpError::AlreadyExists(m) - | OpError::NotFound(m) - | OpError::HasDependents(m) - | OpError::Internal(m) - | OpError::Database { message: m, .. } => StorageError::Internal(m), - })?; - if !recorded.contains(&version) { - pending.push(*filename); - } - } - Ok(pending) -} - -#[cfg(test)] -mod tests { - #[test] - fn every_migration_file_has_a_distinct_version() { - let mut seen = std::collections::BTreeSet::new(); - for (filename, _) in super::CATALOG_MIGRATIONS { - let v = super::migration_version(filename).expect(filename); - assert!(seen.insert(v), "{filename}: version {v} repeated"); - } - assert_eq!( - super::migration_version("V005__backup_definitions_and_restore_registration.cql") - .expect("V005"), - 5 - ); - assert!(super::migration_version("005_x.cql").is_err()); - } - - #[test] - fn qualifies_each_statement_form_with_the_keyspace() { - for (statement, qualified) in [ - ( - "CREATE TABLE IF NOT EXISTS settings (\n key text PRIMARY KEY\n)", - "CREATE TABLE IF NOT EXISTS ks_a.settings (\n key text PRIMARY KEY\n)", - ), - ( - "create table accounts(account_id text PRIMARY KEY)", - "create table ks_a.accounts(account_id text PRIMARY KEY)", - ), - ( - "CREATE INDEX IF NOT EXISTS ON tables (table_id)", - "CREATE INDEX IF NOT EXISTS ON ks_a.tables (table_id)", - ), - ( - "CREATE INDEX ON tables (table_id)", - "CREATE INDEX ON ks_a.tables (table_id)", - ), - ( - "CREATE INDEX idx ON tables (table_id)", - "CREATE INDEX idx ON ks_a.tables (table_id)", - ), - ( - "CREATE INDEX IF NOT EXISTS idx_txn_started_at ON transaction_ledger (started_at)", - "CREATE INDEX IF NOT EXISTS idx_txn_started_at ON ks_a.transaction_ledger (started_at)", - ), - ( - "ALTER TABLE tables ADD IF NOT EXISTS ttl_backfill_cursor text", - "ALTER TABLE ks_a.tables ADD IF NOT EXISTS ttl_backfill_cursor text", - ), - ( - "ALTER TABLE IF EXISTS tables ADD IF NOT EXISTS c text", - "ALTER TABLE IF EXISTS ks_a.tables ADD IF NOT EXISTS c text", - ), - ( - "alter table if exists tables with comment = 'x'", - "alter table if exists ks_a.tables with comment = 'x'", - ), - ( - "-- Seed settings\n// two comments\nINSERT INTO settings (key) VALUES ('a')", - "-- Seed settings\n// two comments\nINSERT INTO ks_a.settings (key) VALUES ('a')", - ), - ] { - let got = - super::qualify_statement("ks_a", "V999__test.cql", statement).expect(statement); - assert_eq!(got, qualified); - } - } - - #[test] - fn rejects_statements_it_cannot_qualify() { - for statement in [ - "UPDATE settings SET value = 'x' WHERE key = 'y'", - "DROP TABLE settings", - "CREATE TABLE other.settings (key text PRIMARY KEY)", - "CREATE TABLE \"Settings\" (key text PRIMARY KEY)", - "CREATE TABLE IF EXISTS settings (key text PRIMARY KEY)", - "CREATE INDEX IF EXISTS idx ON tables (table_id)", - "ALTER TABLE IF NOT EXISTS tables ADD c text", - "CREATE INDEX idx settings (table_id)", - "-- a comment and no statement", - ] { - assert!( - super::qualify_statement("ks_a", "V999__test.cql", statement).is_err(), - "{statement}" - ); - } - } - - /// A semicolon inside a comment, or a comment after a file's last - /// statement, leaves a fragment that holds only a comment. - #[test] - fn a_fragment_of_only_a_comment_fails_naming_the_file() { - let sql = "CREATE TABLE t (k int PRIMARY KEY);\n-- a note after the last statement\n"; - let Err(super::OpError::Internal(message)) = - super::migration_statements("ks_a", "V999__note.cql", sql) - else { - panic!("a fragment that holds only a comment was accepted"); - }; - assert!( - message.contains("V999__note.cql") - && message.contains("holds only a comment") - && message.ends_with("-- a note after the last statement"), - "{message}" - ); - } - - /// Every statement of every embedded migration file, as the runner sends - /// it, is the file's statement with the keyspace written in once, in front - /// of its table: after the statement's leading comments, as a word of its - /// own after `ON` in a `CREATE INDEX`, after `INTO` in an `INSERT INTO`, - /// and after `TABLE` or `EXISTS` otherwise, and not in front of the `IF` - /// that opens an `IF EXISTS` or `IF NOT EXISTS` clause. - #[test] - fn every_migration_statement_names_the_keyspace() { - let files = super::CATALOG_MIGRATIONS - .iter() - .chain(super::DATA_MIGRATIONS); - for (filename, sql) in files { - let sent = super::migration_statements("ks_a", filename, sql).expect(filename); - let written: Vec<&str> = sql - .split(';') - .map(str::trim) - .filter(|s| !s.is_empty()) - .collect(); - assert_eq!(sent.len(), written.len(), "{filename}"); - for (sent, written) in sent.iter().zip(written) { - assert_eq!(sent.matches("ks_a.").count(), 1, "{filename}: {sent}"); - assert_eq!(sent.replacen("ks_a.", "", 1), written, "{filename}"); - let (before, after) = super::skip_comments(sent) - .split_once("ks_a.") - .unwrap_or_else(|| panic!("{filename}: keyspace in a leading comment: {sent}")); - let keyword = if before.ends_with(char::is_whitespace) { - before.split_whitespace().next_back().unwrap_or_default() - } else { - "" - }; - let table = after - .split(|c: char| c.is_whitespace() || c == '(') - .next() - .unwrap_or_default(); - let mut words = super::skip_comments(written) - .split_whitespace() - .map(str::to_ascii_uppercase); - let expected: &[&str] = match (words.next().as_deref(), words.next().as_deref()) { - (Some("CREATE"), Some("INDEX")) => &["ON"], - (Some("INSERT"), Some("INTO")) => &["INTO"], - _ => &["TABLE", "EXISTS"], - }; - assert!( - expected.iter().any(|k| keyword.eq_ignore_ascii_case(k)) - && !table.is_empty() - && !table.eq_ignore_ascii_case("IF"), - "{filename}: keyspace not in front of the table: {sent}" - ); - } - } - } -} diff --git a/crates/storage-cassandra/src/migrations/agreement.rs b/crates/storage-cassandra/src/migrations/agreement.rs new file mode 100644 index 000000000..1bb052199 --- /dev/null +++ b/crates/storage-cassandra/src/migrations/agreement.rs @@ -0,0 +1,274 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Waiting until every live node agrees on the schema (#637). +//! +//! A statement that follows a `CREATE TABLE` can reach a node that does not +//! have the table yet. The runner waits after every DDL statement, before +//! every ledger row and before it releases the lock, until every live node +//! reports the same `schema_version`. Each node is asked for its own version +//! in `system.local`, on a connection to that node: reading `system.local` +//! from one node and `system.peers` from another never sees the second node's +//! own version, and a node that is down keeps its last version in its peers' +//! `system.peers`, so the wait in #457 could not agree while a node was down. +//! A node the driver reports down is not live and is left out; it takes the +//! schema from its peers when it comes back. A node the driver does not +//! report down, and that does not answer the version read within the +//! request timeout, is still in the topology the next statement is routed +//! over, so its schema is unknown and the wait goes on; when the agreement +//! timeout passes the error names it. cdrs-tokio 9.0.2 has no agreement wait +//! of its own. + +use std::time::{Duration, Instant}; + +use cdrs_tokio::cluster::topology::NodeState; +use cdrs_tokio::consistency::Consistency; +use cdrs_tokio::frame::{Envelope, Flags, Version}; +use cdrs_tokio::transport::CdrsTransport; +use extenddb_storage::migrations::MigrationError; + +use super::connect::{REQUEST_TIMEOUT, column}; +use crate::cassandra_util::CassandraSession; + +/// How long the runner waits for agreement before it fails the step. +pub const AGREEMENT_TIMEOUT: Duration = Duration::from_secs(30); + +/// How often the versions are read again while they differ. +const POLL: Duration = Duration::from_millis(200); + +/// One node the driver does not report down: its schema version, or no +/// answer within the request timeout. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct NodeVersion { + pub(crate) node: String, + /// `None` when the node did not answer within the request timeout. It is + /// still up in the driver's topology, so the next statement may reach it, + /// and its schema is unknown. + pub(crate) version: Option, +} + +/// Wait until every live node reports the same schema version, for up to +/// `timeout`. +/// +/// # Errors +/// +/// The versions still differ, a live node did not answer, or no node +/// answered, when `timeout` passes; the error lists each node's version or +/// that it did not answer. +pub(crate) async fn wait_for_agreement( + session: &CassandraSession, + timeout: Duration, +) -> Result<(), MigrationError> { + wait_for_agreement_with(session, timeout, || {}).await +} + +/// [`wait_for_agreement`] calling `answered` after each round in which a +/// node answered: a run under the migration lock records progress there, so +/// a wait that lasts longer than the lease's stall while the nodes answer +/// does not stop the renewal. +/// +/// # Errors +/// +/// As [`wait_for_agreement`]. +pub(crate) async fn wait_for_agreement_with( + session: &CassandraSession, + timeout: Duration, + answered: impl Fn(), +) -> Result<(), MigrationError> { + wait_over(|| live_versions(session), timeout, answered).await +} + +/// The wait itself, over `round`, which reads the live nodes' versions once. +async fn wait_over( + mut round: Round, + timeout: Duration, + answered: impl Fn(), +) -> Result<(), MigrationError> +where + Round: FnMut() -> Read, + Read: std::future::Future>, +{ + let started = Instant::now(); + loop { + let versions = round().await; + if any_answered(&versions) { + answered(); + } + if agree(&versions) { + return Ok(()); + } + if started.elapsed() >= timeout { + let listed: Vec = versions + .iter() + .map(|v| match &v.version { + Some(version) => format!("{} at {version}", v.node), + None => format!( + "{} did not answer within {} s", + v.node, + REQUEST_TIMEOUT.as_secs() + ), + }) + .collect(); + return Err(MigrationError::Store(format!( + "the live Cassandra nodes did not agree on the schema within {} s: {}", + // A round reads every live node, each within the request + // timeout, so the wait can end past `timeout`: say how long + // it waited. + started.elapsed().as_secs().max(timeout.as_secs()), + if listed.is_empty() { + "no node answered".to_owned() + } else { + listed.join(", ") + } + ))); + } + tokio::time::sleep(POLL).await; + } +} + +/// Whether a node answered in a round of the wait: what a run under the +/// lock records as progress. +pub(crate) fn any_answered(versions: &[NodeVersion]) -> bool { + versions.iter().any(|v| v.version.is_some()) +} + +/// Whether the nodes agree: at least one node answered, every node +/// answered, and every answer is the same version. A node that did not +/// answer is unknown, not absent: it may serve the next statement. +pub(crate) fn agree(versions: &[NodeVersion]) -> bool { + let Some(first) = versions.first().and_then(|first| first.version.as_ref()) else { + return false; + }; + versions.iter().all(|v| v.version.as_ref() == Some(first)) +} + +/// The schema version of every node the driver does not report down, read +/// from that node's own `system.local` on a connection to it; `None` for a +/// node that cannot be reached, or does not answer, within the request +/// timeout. +async fn live_versions(session: &CassandraSession) -> Vec { + let metadata = session.cluster_metadata(); + let mut versions = Vec::new(); + for node in metadata.nodes().values() { + if node.state() == NodeState::Down { + continue; + } + let address = node.broadcast_rpc_address().to_string(); + let read = async { + let connection = node.persistent_connection().await.ok()?; + let envelope = Envelope::new_req_query( + "SELECT schema_version FROM system.local".to_owned(), + Consistency::One, + None, + false, + None, + None, + None, + None, + None, + None, + Flags::empty(), + Version::V4, + ); + let answer = connection.write_envelope(&envelope, false).await.ok()?; + let row = answer + .response_body() + .ok()? + .into_rows()? + .into_iter() + .next()?; + let version: uuid::Uuid = column(&row, "schema_version", "read schema_version").ok()?; + Some(version.to_string()) + }; + let version = tokio::time::timeout(REQUEST_TIMEOUT, read) + .await + .ok() + .flatten(); + versions.push(NodeVersion { + node: address, + version, + }); + } + versions.sort_by(|a, b| a.node.cmp(&b.node)); + versions +} + +#[cfg(test)] +mod tests { + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::time::Duration; + + use super::{NodeVersion, agree, any_answered, wait_over}; + + fn at(node: &str, version: &str) -> NodeVersion { + NodeVersion { + node: node.to_owned(), + version: Some(version.to_owned()), + } + } + + fn silent(node: &str) -> NodeVersion { + NodeVersion { + node: node.to_owned(), + version: None, + } + } + + #[test] + fn the_nodes_agree_only_when_one_answered_and_all_answers_match() { + assert!(!agree(&[])); + assert!(agree(&[at("a", "v1")])); + assert!(agree(&[at("a", "v1"), at("b", "v1")])); + assert!(!agree(&[at("a", "v1"), at("b", "v2")])); + } + + /// A node the driver does not report down, and that did not answer the + /// version read, may serve the next statement, so the nodes do not agree + /// until it answers (finding 3). + #[test] + fn a_live_node_that_did_not_answer_is_not_counted_as_agreeing() { + assert!(!agree(&[silent("a")])); + assert!(!agree(&[at("a", "v1"), silent("b")])); + assert!(!agree(&[silent("a"), at("b", "v1")])); + assert!(!agree(&[at("a", "v1"), silent("b"), at("c", "v1")])); + assert!(agree(&[at("a", "v1"), at("b", "v1"), at("c", "v1")])); + } + + /// A round in which a node answered is progress of a run under the + /// lock, even when the nodes do not agree yet; a round in which none + /// answered is not. + #[test] + fn a_round_with_an_answer_is_progress_and_one_without_is_not() { + assert!(any_answered(&[at("a", "v1"), at("b", "v2")])); + assert!(any_answered(&[silent("a"), at("b", "v1")])); + assert!(!any_answered(&[silent("a"), silent("b")])); + assert!(!any_answered(&[])); + } + + /// The wait reports each round in which a node answered, including one + /// in which the nodes still differ, and not a round in which none did. + #[tokio::test] + async fn the_wait_reports_each_round_in_which_a_node_answered() { + let rounds = [ + vec![silent("a"), silent("b")], + vec![at("a", "v1"), at("b", "v2")], + vec![at("a", "v2"), at("b", "v2")], + ]; + let read = AtomicUsize::new(0); + let answered = AtomicUsize::new(0); + let waited = wait_over( + || { + let versions = rounds[read.fetch_add(1, Ordering::SeqCst)].clone(); + async move { versions } + }, + Duration::from_secs(30), + || { + answered.fetch_add(1, Ordering::SeqCst); + }, + ) + .await; + assert!(waited.is_ok(), "{waited:?}"); + assert_eq!(read.load(Ordering::SeqCst), 3); + assert_eq!(answered.load(Ordering::SeqCst), 2); + } +} diff --git a/crates/storage-cassandra/src/migrations/backend.rs b/crates/storage-cassandra/src/migrations/backend.rs new file mode 100644 index 000000000..d30aee1aa --- /dev/null +++ b/crates/storage-cassandra/src/migrations/backend.rs @@ -0,0 +1,533 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The Cassandra [`MigrationBackend`]. +//! +//! Stores: the catalog keyspace, and one account keyspace per row of the +//! catalog's `accounts` table. Every store has a ledger, +//! `extenddb_migrations`, which the runner creates with the first row it +//! writes. Cassandra has no transactions: every statement of a body is +//! idempotent, and a body's ledger row is written at `QUORUM` after the body +//! completes, so a crash runs the body again. Only the holder of the +//! migration lock writes ledger rows, except CreateAccount, which records the +//! rows of the one keyspace it creates while its server is registered, so no +//! migration can run. + +mod constructors; +mod ledger; + +use std::sync::Arc; + +use async_trait::async_trait; +use cdrs_tokio::consistency::Consistency; +use extenddb_storage::management_store::ManagementStore; +use extenddb_storage::migrations::{ + CatalogTraces, Done, HasManagementStore, LedgerColumns, LedgerRow, Migration, MigrationBackend, + MigrationError, MigrationId, Released, Step, StoreKind, StoreRef, record_rows, +}; + +use super::agreement::{AGREEMENT_TIMEOUT, wait_for_agreement_with}; +use super::connect::{ + LOCKED_DDL_TIMEOUT, UNLOCKED_DDL_TIMEOUT, column, has_columns, nullable, replication_of, rows, + send, send_within, table_exists, +}; +use super::cql::statements; +use super::guarded::LeasedManagementStore; +use super::locks::{LeaseHandle, MigrationLock}; +use super::{ACCOUNT_EXPECTED, CATALOG_EXPECTED, RELEASED, migration, postcheck}; +use crate::cassandra_util::CassandraSession; +use crate::catalog_store::CassandraCatalogStore; + +/// The handle code bodies receive: the session, the keyspace names and the +/// catalog's management store (migration 2 rewrites policies through it), +/// every call of which is bounded by the request timeout and, under the +/// lock, checked against the lease. +pub struct CassandraStores { + pub(crate) session: Arc, + pub(crate) keyspace_prefix: String, + management: LeasedManagementStore, +} + +impl CassandraStores { + fn new( + session: Arc, + keyspace_prefix: &str, + lease: Option, + ) -> Self { + Self { + management: LeasedManagementStore::new( + CassandraCatalogStore::catalog_only(session.clone(), keyspace_prefix.to_owned()), + lease, + ), + session, + keyspace_prefix: keyspace_prefix.to_owned(), + } + } + + pub(crate) fn catalog_keyspace(&self) -> String { + format!("{}_catalog", self.keyspace_prefix) + } +} + +impl HasManagementStore for CassandraStores { + fn management_store(&self) -> &dyn ManagementStore { + &self.management + } +} + +/// The account keyspaces a run works on. +#[derive(Debug, Clone, PartialEq, Eq)] +enum Accounts { + /// The keyspace of every row of the catalog's `accounts` table. + All, + /// One keyspace: CreateAccount's. + One(String), + /// None: the catalog keyspace alone, for tests whose shared catalogs + /// hold `accounts` rows of keyspaces they dropped. + #[cfg(any(test, feature = "test-util"))] + None, +} + +/// The Cassandra backend for the runner, for one deployment. +pub struct CassandraMigrations<'l> { + stores: CassandraStores, + accounts: Accounts, + lock: Option<&'l MigrationLock>, + /// The lease of a lock another part of the command holds, for a run + /// that does not own the lock: the account run `init` sends for the + /// default account while it holds the migration lock. Checked and + /// renewed like [`Self::lock`]'s. + lease: Option, + /// migrator killed at that point stops: `None` outside the tests. + stop_after: Option>, + /// The application role, when this run connects as another role: an + /// account keyspace the run creates belongs to the role it connects as, + /// so the application role is granted its permissions on it. + app_role: Option, +} + +/// The bound on a schema statement sent by a run that holds the lock +/// (`locked`) or by CreateAccount's run, which holds no lease: under the +/// lock [`LOCKED_DDL_TIMEOUT`], which is shorter than the lease's margin, so +/// a statement sent before the holder's deadline is done before its row can +/// expire, and shorter than its stall; without one the longer bound a loaded +/// node needs. +pub(super) const fn ddl_bound(locked: bool) -> std::time::Duration { + if locked { + LOCKED_DDL_TIMEOUT + } else { + UNLOCKED_DDL_TIMEOUT + } +} + +/// Whether the test hook's request budget is spent; takes one request. +pub(crate) fn stopped(left: Option<&std::sync::atomic::AtomicUsize>) -> bool { + use std::sync::atomic::Ordering; + let Some(left) = left else { + return false; + }; + // A compare-exchange loop: `fetch_update` is deprecated on current + // stable and its replacement does not exist on 1.88. + let mut n = left.load(Ordering::SeqCst); + loop { + let Some(next) = n.checked_sub(1) else { + return true; + }; + match left.compare_exchange(n, next, Ordering::SeqCst, Ordering::SeqCst) { + Ok(_) => return false, + Err(actual) => n = actual, + } + } +} + +impl CassandraMigrations<'_> { + fn session(&self) -> &CassandraSession { + &self.stores.session + } + + /// Stop before a write when the lock's deadline has passed, or when the + /// test hook's budget is spent. + fn check(&self) -> Result<(), MigrationError> { + if stopped(self.stop_after.as_deref()) { + return Err(MigrationError::Store( + "stopped by the test hook before a write, as a killed migrator stops".to_owned(), + )); + } + match (self.lock, &self.lease) { + (Some(lock), _) => lock.check(), + (None, Some(lease)) => lease.check(), + (None, None) => Ok(()), + } + } + + /// The bound on a schema statement: [`LOCKED_DDL_TIMEOUT`] under the + /// lock, which keeps it shorter than the lease's margin, and + /// [`UNLOCKED_DDL_TIMEOUT`] for CreateAccount's runs, which hold no + /// lease. + pub(super) fn ddl_timeout(&self) -> std::time::Duration { + ddl_bound(self.lock.is_some() || self.lease.is_some()) + } + + /// Record progress of a locked run: a request of the runner's completed. + fn progress(&self) { + match (self.lock, &self.lease) { + (Some(lock), _) => lock.progress(), + (None, Some(lease)) => lease.progress(), + (None, None) => {} + } + } + + /// Wait for schema agreement, as progress of a locked run. + async fn agree(&self) -> Result<(), MigrationError> { + wait_for_agreement_with(self.session(), AGREEMENT_TIMEOUT, || self.progress()).await?; + self.progress(); + Ok(()) + } + + /// Run `work` under the lock's lease when there is one. A run under a + /// lease alone (`init`'s default account, `for_account_under`) is not + /// raced against the lease's end here: it runs account-store bodies + /// only, and every one of those is a file today. A code body there + /// would send nothing after the deadline through the management store + /// it is handed, which checks the lease before each call; a request it + /// sent on `CassandraStores::session` itself would not be checked. + async fn guarded( + &self, + work: impl std::future::Future>, + ) -> Result { + match self.lock { + Some(lock) => lock.guard(work).await, + None => work.await, + } + } + + /// Execute a file body's statements in `keyspace`, checking the lock + /// before each one, recording progress when each completes, and waiting + /// for schema agreement after each DDL statement. + async fn run_file(&self, keyspace: &str, name: &str, text: &str) -> Result<(), MigrationError> { + for statement in statements(keyspace, name, text)? { + self.check()?; + send_within( + self.session(), + &statement.text, + cdrs_tokio::query_values!(), + Consistency::Quorum, + None, + &format!("migration file {name} in {keyspace}"), + self.ddl_timeout(), + ) + .await?; + self.progress(); + if statement.ddl { + self.agree().await?; + } + } + Ok(()) + } + + /// Create a missing account keyspace with the catalog keyspace's + /// replication, read from `system_schema`: an `accounts` row whose + /// keyspace a stopped CreateAccount never created (Cassandra item 5). + async fn ensure_account_keyspace(&self, keyspace: &str) -> Result<(), MigrationError> { + if replication_of(self.session(), keyspace).await?.is_some() { + return Ok(()); + } + let catalog = self.stores.catalog_keyspace(); + let replication = replication_of(self.session(), &catalog) + .await? + .ok_or_else(|| { + MigrationError::Store(format!("the catalog keyspace {catalog} does not exist")) + })?; + self.check()?; + send_within( + self.session(), + &format!("CREATE KEYSPACE IF NOT EXISTS {keyspace} WITH replication = {replication}"), + cdrs_tokio::query_values!(), + Consistency::Quorum, + None, + &format!("create the account keyspace {keyspace}"), + self.ddl_timeout(), + ) + .await?; + self.progress(); + self.agree().await?; + // The keyspace belongs to the role this run connects as; the server + // serves the account as the application role. The lock is checked + // before each GRANT, as before every other write. + if let Some(role) = &self.app_role { + super::grant_role( + self.session(), + keyspace, + role, + true, + || self.check(), + || self.progress(), + ) + .await?; + } + Ok(()) + } + + /// Whether this run writes the deployment's `catalog_version` row. + /// CreateAccount adds a keyspace with every account migration, which + /// leaves the deployment's version as it was. + fn writes_catalog_version(&self) -> bool { + !matches!(self.accounts, Accounts::One(_)) + } +} + +/// The writes that record a body in one keyspace. +struct KeyspaceLedger<'a> { + backend: &'a CassandraMigrations<'a>, + keyspace: &'a str, + /// The `catalog_version` to write with this keyspace's row. Migration + /// 1's row and the version go in one logged batch: a stop between two + /// writes would leave that row beside a 0.1.x version row, which the + /// next run reads as a catalog an older release wrote to. A later row + /// goes on its own, after the version, since its notes can be long and a + /// logged batch is refused past the node's `batch_size_fail_threshold` + /// (50 KB by default). A stop between them leaves the version ahead of + /// the ledger and the id pending, which the next run applies; the other + /// order left the last id's version behind with nothing pending. + version: Option, + /// Whether the row's batch wrote `version`. + wrote_version: std::sync::atomic::AtomicBool, +} + +#[async_trait] +impl MigrationBackend for CassandraMigrations<'_> { + type Stores = CassandraStores; + + fn backend_name(&self) -> &'static str { + "cassandra" + } + + fn migration(&self, id: MigrationId) -> Migration { + migration(id) + } + + fn released(&self) -> &'static [Released] { + RELEASED + } + + fn stores(&self) -> &CassandraStores { + &self.stores + } + + async fn store_names(&self, kind: StoreKind) -> Result, MigrationError> { + match kind { + StoreKind::Catalog => Ok(vec![self.stores.catalog_keyspace()]), + StoreKind::Data => Ok(Vec::new()), + StoreKind::Account => match &self.accounts { + Accounts::One(keyspace) => Ok(vec![keyspace.clone()]), + #[cfg(any(test, feature = "test-util"))] + Accounts::None => Ok(Vec::new()), + Accounts::All => { + let catalog = self.stores.catalog_keyspace(); + self.agree().await?; + // Another program's keyspace of the same name may hold an + // `accounts` table of its own. + if !has_columns(self.session(), &catalog, "accounts", &["account_id"]).await? { + return Ok(Vec::new()); + } + let found = rows( + self.session(), + &format!("SELECT account_id FROM {catalog}.accounts"), + cdrs_tokio::query_values!(), + Consistency::Quorum, + "list the accounts", + ) + .await?; + found + .iter() + .map(|row| { + let id: String = column(row, "account_id", "list the accounts")?; + Ok(format!("{}_account_{id}", self.stores.keyspace_prefix)) + }) + .collect() + } + }, + } + } + + async fn read_ledger(&self, store: &StoreRef) -> Result, MigrationError> { + // A ledger that does not exist reads as no rows only once every live + // node agrees on the schema; any other failure is an error. + self.agree().await?; + if !table_exists(self.session(), &store.name, "extenddb_migrations").await? { + return Ok(Vec::new()); + } + let context = format!("read the ledger of {}", store.name); + let found = rows( + self.session(), + &format!( + "SELECT id, name, kind, checksum, outcome, notes, toUnixTimestamp(applied_at) AS \ + applied_ms, applied_by FROM {}.extenddb_migrations", + store.name + ), + cdrs_tokio::query_values!(), + Consistency::Quorum, + &context, + ) + .await?; + found + .iter() + .map(|row| { + let text = |name: &str| -> Result { + Ok(nullable::(row, name, &context)?.unwrap_or_default()) + }; + let applied_ms: Option = nullable(row, "applied_ms", &context)?; + let applied_at = applied_ms + .and_then(chrono::DateTime::from_timestamp_millis) + .map(|t| t.to_rfc3339_opts(chrono::SecondsFormat::Millis, true)) + .unwrap_or_default(); + let columns: LedgerColumns = ( + column(row, "id", &context)?, + text("name")?, + text("kind")?, + text("checksum")?, + text("outcome")?, + text("notes")?, + applied_at, + text("applied_by")?, + ); + LedgerRow::from_columns(Some(&store.name), columns) + }) + .collect() + } + + async fn read_traces(&self) -> Result { + let catalog = self.stores.catalog_keyspace(); + // A `settings` table without the catalog's columns is another + // program's, and holds no trace of a release. + if !has_columns(self.session(), &catalog, "settings", &["key", "value"]).await? { + return Ok(CatalogTraces::default()); + } + let found = rows( + self.session(), + &format!( + "SELECT key, value FROM {catalog}.settings WHERE key IN ('catalog_version', \ + 'resource_tag_migration')" + ), + cdrs_tokio::query_values!(), + Consistency::Quorum, + "read the catalog's version rows", + ) + .await?; + let pairs = found + .iter() + .map(|row| { + Ok(( + column::(row, "key", "read settings")?, + nullable::(row, "value", "read settings")?.unwrap_or_default(), + )) + }) + .collect::, MigrationError>>()?; + Ok(CatalogTraces::from_settings(&pairs)) + } + + /// A catalog keyspace with no ledger row and no `catalog_version` row is + /// not a catalog yet: `init` created its `settings` table before taking + /// the lock, or an `init` stopped part way through migration 1. No + /// release wrote data to it, so a code body's `detect` has nothing to + /// find, and the tables it reads may not exist until migration 1 runs. + async fn catalog_is_empty(&self) -> Result { + let catalog = StoreRef { + kind: StoreKind::Catalog, + name: self.stores.catalog_keyspace(), + }; + Ok(self.read_ledger(&catalog).await?.is_empty() + && self.read_traces().await?.catalog_version.is_none()) + } + + async fn before_first_write( + &self, + _from: Option, + _to: u32, + ) -> Result, MigrationError> { + Ok(Vec::new()) + } + + async fn apply( + &self, + store: &StoreRef, + id: MigrationId, + step: Step<'_, CassandraStores>, + row: Option, + catalog_version: Option, + ) -> Result { + if store.kind == StoreKind::Account { + self.ensure_account_keyspace(&store.name).await?; + } + let done = match step { + Step::File { name, text } => { + self.run_file(&store.name, name, text).await?; + Done::default() + } + Step::Code(body) => { + self.check()?; + self.guarded(body.run(&self.stores)).await? + } + Step::Record => Done::default(), + }; + // The first time migration 1 is applied to a store, the store must + // match the v1.0 schema; a later --reconverge leaves its row as it + // is and is not checked, since later migrations change the schema. + if id == MigrationId::Baseline && row.is_some() { + self.agree().await?; + let expected = match store.kind { + StoreKind::Catalog => CATALOG_EXPECTED, + StoreKind::Account | StoreKind::Data => ACCOUNT_EXPECTED, + }; + postcheck::check(self.session(), expected, &store.name).await?; + } + self.agree().await?; + let ledger = KeyspaceLedger { + backend: self, + keyspace: &store.name, + version: catalog_version.filter(|_| self.writes_catalog_version()), + wrote_version: std::sync::atomic::AtomicBool::new(false), + }; + record_rows(&ledger, row, &done.notes, catalog_version).await?; + Ok(done) + } + + async fn write_catalog_version(&self, version: u32) -> Result<(), MigrationError> { + if !self.writes_catalog_version() { + return Ok(()); + } + self.check()?; + send( + self.session(), + &format!( + "INSERT INTO {}.settings (key, value) VALUES ('catalog_version', ?)", + self.stores.catalog_keyspace() + ), + cdrs_tokio::query_values!(version.to_string()), + Consistency::Quorum, + None, + "write the catalog version", + ) + .await + .map(drop) + } + + async fn delete_ledger_row(&self, store: &StoreRef, id: u32) -> Result<(), MigrationError> { + self.check()?; + let id = i32::try_from(id) + .map_err(|_| MigrationError::Store(format!("migration id {id} is too large")))?; + send( + self.session(), + &format!( + "DELETE FROM {}.extenddb_migrations WHERE id = ?", + store.name + ), + cdrs_tokio::query_values!(id), + Consistency::Quorum, + None, + &format!("forget migration {id} in {}", store.name), + ) + .await + .map(drop) + } +} diff --git a/crates/storage-cassandra/src/migrations/backend/constructors.rs b/crates/storage-cassandra/src/migrations/backend/constructors.rs new file mode 100644 index 000000000..65f175a1b --- /dev/null +++ b/crates/storage-cassandra/src/migrations/backend/constructors.rs @@ -0,0 +1,133 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The constructors of [`CassandraMigrations`], one for each kind of run, +//! and the settings a run is built with. + +use std::sync::Arc; + +use super::{Accounts, CassandraMigrations, CassandraStores}; +use crate::cassandra_util::CassandraSession; +use crate::migrations::locks::{LeaseHandle, MigrationLock}; + +impl<'l> CassandraMigrations<'l> { + /// The deployment's every store, read only: for the status. + #[must_use] + pub fn for_status(session: Arc, keyspace_prefix: &str) -> Self { + Self { + stores: CassandraStores::new(session, keyspace_prefix, None), + accounts: Accounts::All, + lock: None, + lease: None, + stop_after: None, + app_role: None, + } + } + + /// The deployment's every store, for a run under `lock`. + #[must_use] + pub fn locked(lock: &'l MigrationLock, keyspace_prefix: &str) -> Self { + Self { + stores: CassandraStores::new( + lock.session().clone(), + keyspace_prefix, + Some(lock.handle()), + ), + accounts: Accounts::All, + lock: Some(lock), + lease: None, + stop_after: None, + app_role: None, + } + } + + /// The catalog keyspace alone, for a run under `lock`: for tests. + #[cfg(any(test, feature = "test-util"))] + #[must_use] + pub fn locked_catalog_only(lock: &'l MigrationLock, keyspace_prefix: &str) -> Self { + Self { + stores: CassandraStores::new( + lock.session().clone(), + keyspace_prefix, + Some(lock.handle()), + ), + accounts: Accounts::None, + lock: Some(lock), + lease: None, + stop_after: None, + app_role: None, + } + } + + /// One account keyspace, for CreateAccount: the runner records the + /// account migrations in that keyspace only. + #[must_use] + pub fn for_account( + session: Arc, + keyspace_prefix: &str, + keyspace: &str, + ) -> Self { + Self { + stores: CassandraStores::new(session, keyspace_prefix, None), + accounts: Accounts::One(keyspace.to_owned()), + lock: None, + lease: None, + stop_after: None, + app_role: None, + } + } + + /// One account keyspace, for `init`'s default account, under the lease + /// of the migration lock `init` holds: the lease is checked before each + /// write, each completed request records progress on it, and each schema + /// statement gets the bound of one sent under the lock. + #[must_use] + pub(crate) fn for_account_under( + session: Arc, + keyspace_prefix: &str, + keyspace: &str, + lease: LeaseHandle, + ) -> Self { + Self { + stores: CassandraStores::new(session, keyspace_prefix, Some(lease.clone())), + accounts: Accounts::One(keyspace.to_owned()), + lock: None, + lease: Some(lease), + stop_after: None, + app_role: None, + } + } + + /// Grant `role`, the application role, its permissions on an account + /// keyspace this run creates: for a run that connects as another role + /// (`migrate --cassandra-user`). + #[must_use] + pub fn granting(mut self, role: Option) -> Self { + self.app_role = role; + self + } + + /// Stop this run before its `requests`-th write, as a migrator killed + /// there stops: for the crash-and-rerun tests. Every write the backend + /// sends counts (a file's statements, the ledger table and its rows, the + /// version row, an account keyspace, the start of a code body) and so does + /// each policy write of migration 2's body; reads do not. + #[cfg(any(test, feature = "test-util"))] + #[must_use] + pub fn stop_after(mut self, requests: usize) -> Self { + let left = Arc::new(std::sync::atomic::AtomicUsize::new(requests)); + self.stores.management.stop_after(Arc::clone(&left)); + self.stop_after = Some(left); + self + } + + /// The writes [`Self::stop_after`]'s budget still allows: a run given + /// `usize::MAX` that finishes has sent `usize::MAX` minus this many. + #[cfg(any(test, feature = "test-util"))] + #[must_use] + pub fn requests_left(&self) -> Option { + self.stop_after + .as_deref() + .map(|left| left.load(std::sync::atomic::Ordering::SeqCst)) + } +} diff --git a/crates/storage-cassandra/src/migrations/backend/ledger.rs b/crates/storage-cassandra/src/migrations/backend/ledger.rs new file mode 100644 index 000000000..5e807856d --- /dev/null +++ b/crates/storage-cassandra/src/migrations/backend/ledger.rs @@ -0,0 +1,117 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The writes that record a body in one keyspace's ledger. + +use async_trait::async_trait; +use cdrs_tokio::consistency::Consistency; +use extenddb_storage::migrations::{ + LedgerRow, LedgerWrites, MigrationBackend, MigrationError, MigrationId, +}; + +use super::KeyspaceLedger; +use crate::migrations::connect::{send, send_within}; + +#[async_trait] +impl LedgerWrites for KeyspaceLedger<'_> { + type Error = MigrationError; + + async fn create_ledger(&self) -> Result<(), MigrationError> { + self.backend.check()?; + send_within( + self.backend.session(), + &format!( + "CREATE TABLE IF NOT EXISTS {}.extenddb_migrations (id int PRIMARY KEY, name \ + text, kind text, checksum text, outcome text, notes text, applied_at timestamp, \ + applied_by text)", + self.keyspace + ), + cdrs_tokio::query_values!(), + Consistency::Quorum, + None, + &format!("create the ledger of {}", self.keyspace), + self.backend.ddl_timeout(), + ) + .await?; + self.backend.agree().await + } + + async fn insert_row(&self, row: &LedgerRow) -> Result<(), MigrationError> { + let id = i32::try_from(row.id) + .map_err(|_| MigrationError::Store(format!("migration id {} is too large", row.id)))?; + let insert = format!( + "INSERT INTO {}.extenddb_migrations (id, name, kind, checksum, outcome, notes, \ + applied_at, applied_by) VALUES (?, ?, ?, ?, ?, ?, toTimestamp(now()), ?)", + self.keyspace + ); + // A later id's version goes first, on its own (see + // `KeyspaceLedger::version`); `write_version` then skips it. + if let Some(version) = self + .version + .filter(|_| row.id != MigrationId::Baseline.number()) + { + self.backend.write_catalog_version(version).await?; + self.wrote_version + .store(true, std::sync::atomic::Ordering::SeqCst); + } + self.backend.check()?; + let version = self + .version + .filter(|_| row.id == MigrationId::Baseline.number()); + let Some(version) = version else { + return send( + self.backend.session(), + &insert, + cdrs_tokio::query_values!( + id, + row.name.as_str(), + row.kind.as_str(), + row.checksum.as_str(), + row.outcome.as_str(), + row.notes.as_str(), + row.applied_by.as_str() + ), + Consistency::Quorum, + None, + &format!("record migration {} in {}", row.id, self.keyspace), + ) + .await + .map(drop); + }; + send( + self.backend.session(), + &format!( + "BEGIN BATCH {insert}; INSERT INTO {}.settings (key, value) VALUES \ + ('catalog_version', ?); APPLY BATCH", + self.backend.stores.catalog_keyspace() + ), + cdrs_tokio::query_values!( + id, + row.name.as_str(), + row.kind.as_str(), + row.checksum.as_str(), + row.outcome.as_str(), + row.notes.as_str(), + row.applied_by.as_str(), + version.to_string() + ), + Consistency::Quorum, + None, + &format!( + "record migration {} in {} with the catalog version", + row.id, self.keyspace + ), + ) + .await?; + self.wrote_version + .store(true, std::sync::atomic::Ordering::SeqCst); + Ok(()) + } + + async fn write_version(&self, version: u32) -> Result<(), MigrationError> { + if self.wrote_version.load(std::sync::atomic::Ordering::SeqCst) { + return Ok(()); + } + self.backend.write_catalog_version(version).await + } +} diff --git a/crates/storage-cassandra/src/migrations/commands.rs b/crates/storage-cassandra/src/migrations/commands.rs new file mode 100644 index 000000000..9f315abe9 --- /dev/null +++ b/crates/storage-cassandra/src/migrations/commands.rs @@ -0,0 +1,207 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The Cassandra backend's [`CatalogMigrations`], which `init`, `migrate` +//! and `verify` call through the installed backend. +//! +//! A command's [`Target`] gives the config file's connection string, which +//! carries the role the config names, and the admin role the command was +//! given (`--cassandra-user`), which replaces it. Every connect is bounded +//! by 10 s. + +use std::time::Duration; + +use async_trait::async_trait; +use extenddb_storage::migrations::{ + CatalogMigrations, CatalogStatus, HeldLock, LedgerRow, Listed, MigrationError, MigrationId, + RunKind, RunReport, Runner, Target, +}; + +use super::backend::CassandraMigrations; +use super::connect::{Deployment, has_columns}; +use super::locks::{LeaseHandle, LeaseTimes, MigrationLock}; +use super::registrations::wait_for_no_servers; +use super::{catalog_status, foreign_contents, migration, status_holding}; +use crate::cassandra_util::CassandraSession; + +/// The Cassandra backend's catalog migrations for the commands. +pub struct CassandraCatalogMigrations; + +/// What the migration lock carries for [`CatalogMigrations::run`] and +/// [`CatalogMigrations::forget`]. +struct Held { + lock: MigrationLock, + keyspace_prefix: String, + /// The application role, when the command connects as another role. + app_role: Option, +} + +fn held(lock: &HeldLock) -> Result<&Held, MigrationError> { + lock.get::().ok_or_else(|| { + MigrationError::Refused("the migration lock was not taken by the Cassandra backend".into()) + }) +} + +/// The lease of a lock this backend took, which `init`'s bootstrapper checks +/// before each row it writes; `None` for another backend's lock. +pub(crate) fn lease_of(lock: &HeldLock) -> Option { + held(lock).ok().map(|held| held.lock.handle()) +} + +/// Refuse an uninitialized catalog keyspace that holds another program's +/// tables or `settings` rows, naming them: the status's advice for +/// `Uninitialized` is to run `extenddb init`, which refuses such a keyspace, +/// so the preview, `verify` and `migrate --yes` say what `init` would say. +/// +/// # Errors +/// +/// [`MigrationError::Refused`] naming the contents; a read failed. +async fn refuse_foreign( + session: &CassandraSession, + deployment: &Deployment, + status: &CatalogStatus, +) -> Result<(), MigrationError> { + if *status != CatalogStatus::Uninitialized { + return Ok(()); + } + let catalog = deployment.catalog_keyspace(); + let foreign = foreign_contents(session, &catalog).await?; + if foreign.is_empty() { + return Ok(()); + } + Err(MigrationError::Refused(format!( + "the keyspace {catalog} is not an ExtendDB catalog: it holds {}. Nothing was written to \ + it; check keyspace_prefix in the config file", + foreign.join(", ") + ))) +} + +#[async_trait] +impl CatalogMigrations for CassandraCatalogMigrations { + fn listed(&self, id: MigrationId) -> Listed { + Listed::of(&migration(id)) + } + + fn backup_advice(&self, target: &Target) -> String { + match Deployment::from_target(target) { + Ok(d) => format!( + "back up the Cassandra catalog keyspace {} and every account keyspace {}, with \ + a snapshot on every node taken while every server is stopped", + d.catalog_keyspace(), + d.account_keyspace("") + ), + Err(_) => "back up the Cassandra catalog keyspace and every account keyspace, with \ + a snapshot on every node taken while every server is stopped" + .to_owned(), + } + } + + async fn status(&self, target: &Target) -> Result { + let deployment = Deployment::from_target(target)?; + let session = deployment.connect().await?; + let status = catalog_status(session.clone(), &deployment.keyspace_prefix).await?; + refuse_foreign(&session, &deployment, &status).await?; + Ok(status) + } + + /// Read as the holder, on the lock's session, so the requests of a long + /// status read are progress that keeps the lock ([`status_holding`]). + /// Another program's keyspace is refused as [`Self::status`] refuses it. + async fn status_under_lock( + &self, + lock: &HeldLock, + target: &Target, + ) -> Result { + let held = held(lock)?; + let deployment = Deployment::from_target(target)?; + let status = status_holding(&held.lock, &held.keyspace_prefix).await?; + refuse_foreign(held.lock.session(), &deployment, &status).await?; + Ok(status) + } + + async fn catalog_ledger(&self, target: &Target) -> Result, MigrationError> { + let deployment = Deployment::from_target(target)?; + let session = deployment.connect().await?; + let migrations = CassandraMigrations::for_status(session, &deployment.keyspace_prefix); + Runner::new(&migrations, "").catalog_ledger().await + } + + /// Take the lock, then wait for every registered server to go away, + /// both within `timeout`: a migration needs every replica of the catalog + /// keyspace, and no server may run while the catalog changes. An + /// uninitialized keyspace that holds another program's tables or + /// `settings` rows is refused before anything is written to it, and + /// named, since `init` refuses it too. A keyspace without the catalog's + /// `settings` table is not a catalog; `init` creates it before it takes + /// the lock. + async fn lock(&self, target: &Target, timeout: Duration) -> Result { + let deployment = Deployment::from_target(target)?; + let session = deployment.connect().await?; + let catalog = deployment.catalog_keyspace(); + let status = catalog_status(session.clone(), &deployment.keyspace_prefix).await?; + refuse_foreign(&session, &deployment, &status).await?; + if !has_columns(&session, &catalog, "settings", &["key", "value"]).await? { + return Err(MigrationError::Status(CatalogStatus::Uninitialized)); + } + let lock = MigrationLock::acquire(session, &catalog, LeaseTimes::DEFAULT, timeout).await?; + if let Err(e) = wait_for_no_servers(&lock, timeout).await { + lock.release().await; + return Err(e); + } + Ok(HeldLock::new( + target.location.as_str(), + Held { + lock, + app_role: deployment.app_role_if_other(), + keyspace_prefix: deployment.keyspace_prefix, + }, + )) + } + + async fn run( + &self, + lock: &HeldLock, + release: &str, + kind: RunKind, + ) -> Result { + let held = held(lock)?; + let migrations = CassandraMigrations::locked(&held.lock, &held.keyspace_prefix) + .granting(held.app_role.clone()); + let runner = Runner::new(&migrations, release); + match kind { + RunKind::Create | RunKind::Migrate => runner.migrate().await, + RunKind::Reconverge => runner.reconverge().await, + } + } + + async fn forget(&self, lock: &HeldLock, id: u32) -> Result<(), MigrationError> { + let held = held(lock)?; + let migrations = CassandraMigrations::locked(&held.lock, &held.keyspace_prefix); + Runner::new(&migrations, "").forget(id).await + } + + async fn release(&self, lock: HeldLock) { + if let Ok(held) = held(&lock) { + held.lock.release().await; + } + } +} + +#[cfg(test)] +mod tests { + use extenddb_storage::migrations::{CatalogMigrations, Target}; + + use super::CassandraCatalogMigrations; + + #[test] + fn the_backup_advice_names_the_keyspaces_and_no_password() { + let advice = + CassandraCatalogMigrations.backup_advice(&Target::at("app:pw@127.0.0.1:9042/acme")); + assert_eq!( + advice, + "back up the Cassandra catalog keyspace acme_catalog and every account keyspace \ + acme_account_, with a snapshot on every node taken while every server \ + is stopped" + ); + } +} diff --git a/crates/storage-cassandra/src/migrations/connect.rs b/crates/storage-cassandra/src/migrations/connect.rs new file mode 100644 index 000000000..924052bce --- /dev/null +++ b/crates/storage-cassandra/src/migrations/connect.rs @@ -0,0 +1,507 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! How the runner reaches a deployment: one session built the way the server +//! builds its own, with the credentials the command was given, and every +//! request bounded by [`REQUEST_TIMEOUT`], except a schema statement, which +//! gets [`LOCKED_DDL_TIMEOUT`] under the lock and [`UNLOCKED_DDL_TIMEOUT`] +//! outside it. + +use std::sync::Arc; +use std::time::Duration; + +use cdrs_tokio::consistency::Consistency; +use cdrs_tokio::frame::Envelope; +use cdrs_tokio::query::QueryValues; +use cdrs_tokio::statement::StatementParamsBuilder; +use cdrs_tokio::types::rows::Row; +use extenddb_storage::migrations::{MigrationError, Target}; + +use crate::cassandra_util::CassandraSession; +use crate::config::CassandraStorageConfig; +use crate::engine::CassandraEngine; + +/// The bound on every request the runner, the lock and the registration +/// send, apart from a schema statement. It is shorter than the lock's margin +/// (25 s), so a request sent before the holder's deadline has finished +/// before the lock can expire. +pub(crate) const REQUEST_TIMEOUT: Duration = Duration::from_secs(10); + +/// The bound on a schema statement sent under the migration lock. A schema +/// statement waits behind every other schema change on its coordinator: +/// measured through cqlsh on one node while ten migration tests ran on it in +/// parallel, 90 table creations, alterations and drops took a median of +/// 1.9 s, 5 of them 10 s or more (10.6, 11.3, 14.8, 24.4 and 76.8 s), where +/// the same statements took about 1 s on the idle node. The bound covers +/// three of those five and is the largest round value inside the lease: it +/// is under the margin (25 s), so a statement sent before the holder's +/// deadline has finished before its row can expire, and under the stall +/// (30 s), so a statement that takes the whole bound does not stop the +/// renewal, since the runner records progress when it completes. The two +/// statements beyond it are beyond the lease too, and a run that meets one +/// fails on it and is run again. +pub(crate) const LOCKED_DDL_TIMEOUT: Duration = Duration::from_secs(20); + +/// The bound on a schema statement sent outside the migration lock: +/// CreateAccount's account migrations, which hold no lease and run while +/// the server serves, and the `settings` table that `init` and `migrate` +/// create before the lock exists. A CREATE TABLE waits behind every other +/// schema change on its coordinator, and in the direct integration suite and +/// the pytest suite, run in parallel on one node, such a statement took +/// longer than [`REQUEST_TIMEOUT`]. +pub(crate) const UNLOCKED_DDL_TIMEOUT: Duration = Duration::from_secs(60); + +/// A deployment: where it is and the role a command connects as. +#[derive(Clone, PartialEq, Eq)] +pub struct Deployment { + pub contact_points: Vec, + pub keyspace_prefix: String, + pub username: Option, + pub password: Option, + /// The role the config names, which the server serves as: `username` + /// unless the command connects as an admin role (`--cassandra-user`). + pub app_role: Option, +} + +impl std::fmt::Debug for Deployment { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Deployment") + .field("contact_points", &self.contact_points) + .field("keyspace_prefix", &self.keyspace_prefix) + .field("username", &self.username) + .field("password", &self.password.as_ref().map(|_| "")) + .field("app_role", &self.app_role) + .finish() + } +} + +impl Deployment { + /// The deployment a server's config names, with the role it serves as. + #[must_use] + pub fn from_config(config: &CassandraStorageConfig) -> Self { + Self { + contact_points: config.contact_points.clone(), + keyspace_prefix: config.keyspace_prefix.clone(), + username: config.username.clone(), + password: config.password.clone(), + app_role: config.username.clone(), + } + } + + /// The deployment a command's target names. The location is the config + /// file's connection string, which carries the role the config names; + /// an admin role the command was given (`--cassandra-user`) replaces it. + /// + /// # Errors + /// + /// The location names no contact point. + pub fn from_target(target: &Target) -> Result { + let parts = CassandraStorageConfig::connection_parts(&target.location); + if parts.contact_points.iter().all(|c| c.trim().is_empty()) { + return Err(MigrationError::Refused(format!( + "the Cassandra connection string {:?} names no contact point", + CassandraStorageConfig::redact_connection_string(&target.location) + ))); + } + let (username, password) = match &target.admin { + Some(admin) => (Some(admin.user.clone()), admin.password.clone()), + None => (parts.username.clone(), parts.password), + }; + Ok(Self { + contact_points: parts.contact_points, + keyspace_prefix: parts.keyspace_prefix, + username, + password, + app_role: parts.username, + }) + } + + /// The application role when the command connects as another role, so + /// a keyspace it creates belongs to that other role and the application + /// role must be granted its permissions on it; `None` otherwise. + #[must_use] + pub fn app_role_if_other(&self) -> Option { + self.app_role + .clone() + .filter(|role| Some(role) != self.username.as_ref()) + } + + /// The catalog keyspace. + #[must_use] + pub fn catalog_keyspace(&self) -> String { + format!("{}_catalog", self.keyspace_prefix) + } + + /// The keyspace of account `account_id`. + #[must_use] + pub fn account_keyspace(&self, account_id: &str) -> String { + format!("{}_account_{account_id}", self.keyspace_prefix) + } + + /// A session, built as the engine builds its own: the same credential + /// path and a 10 s bound on the connect. + /// + /// # Errors + /// + /// The connect fails or does not finish within 10 s. + pub async fn connect(&self) -> Result, MigrationError> { + let mut config = CassandraStorageConfig::new(self.contact_points.clone()); + config.keyspace_prefix.clone_from(&self.keyspace_prefix); + config.username.clone_from(&self.username); + config.password.clone_from(&self.password); + CassandraEngine::create_session(&config) + .await + .map(Arc::new) + .map_err(|e| { + MigrationError::store( + format!("connect to Cassandra at {}", self.contact_points.join(",")), + e, + ) + }) + } +} + +/// Send `cql` at `consistency` (and `serial` for a lightweight transaction), +/// bounded by [`REQUEST_TIMEOUT`]. +/// +/// # Errors +/// +/// The request fails or times out; `context` names it. +pub(crate) async fn send( + session: &CassandraSession, + cql: &str, + values: QueryValues, + consistency: Consistency, + serial: Option, + context: &str, +) -> Result { + send_within( + session, + cql, + values, + consistency, + serial, + context, + REQUEST_TIMEOUT, + ) + .await +} + +/// [`send`] bounded by `timeout` instead of [`REQUEST_TIMEOUT`]. +/// +/// # Errors +/// +/// The request fails or times out; `context` names it. +pub(crate) async fn send_within( + session: &CassandraSession, + cql: &str, + values: QueryValues, + consistency: Consistency, + serial: Option, + context: &str, + timeout: Duration, +) -> Result { + let mut params = StatementParamsBuilder::new() + .with_consistency(consistency) + .with_values(values); + if let Some(serial) = serial { + params = params.with_serial_consistency(serial); + } + match tokio::time::timeout(timeout, session.query_with_params(cql, params.build())).await { + Ok(Ok(envelope)) => Ok(envelope), + Ok(Err(e)) => { + let code = match &e { + cdrs_tokio::error::Error::Server { body, .. } => { + Some(format!("{:#06x}", body.ty.to_error_code())) + } + _ => None, + }; + Err(MigrationError::coded(context, e, code)) + } + Err(_) => Err(MigrationError::Store(format!( + "{context}: no answer within {} s", + timeout.as_secs() + ))), + } +} + +/// The error codes of a request that may succeed when sent again: +/// Unavailable, Overloaded, IsBootstrapping, WriteTimeout and ReadTimeout. +const TRANSIENT_CODES: [&str; 5] = ["0x1000", "0x1001", "0x1002", "0x1100", "0x1200"]; + +/// Whether a request that failed with `error` may succeed when sent again: +/// no answer, a connection that failed, or a server error whose code is in +/// [`TRANSIENT_CODES`]. Decided by the code, never by the message. +pub(crate) fn retryable(error: &MigrationError) -> bool { + error + .sqlstate() + .is_none_or(|code| TRANSIENT_CODES.contains(&code)) +} + +/// The rows of a read sent with [`send`]. +/// +/// # Errors +/// +/// As [`send`], or the answer holds no rows. +pub(crate) async fn rows( + session: &CassandraSession, + cql: &str, + values: QueryValues, + consistency: Consistency, + context: &str, +) -> Result, MigrationError> { + let envelope = send(session, cql, values, consistency, None, context).await?; + envelope + .response_body() + .map_err(|e| MigrationError::store(context, e))? + .into_rows() + .ok_or_else(|| MigrationError::Store(format!("{context}: the answer holds no rows"))) +} + +/// A column of a row, or an error naming it. +/// +/// # Errors +/// +/// The column is missing or has another type. +pub(crate) fn column(row: &Row, name: &str, context: &str) -> Result +where + Row: cdrs_tokio::types::IntoRustByName, +{ + use cdrs_tokio::types::IntoRustByName; + row.get_r_by_name(name) + .map_err(|e| MigrationError::store(format!("{context}: column {name}"), e)) +} + +/// A column that may be null. +/// +/// # Errors +/// +/// The column is missing or has another type. +pub(crate) fn nullable(row: &Row, name: &str, context: &str) -> Result, MigrationError> +where + Row: cdrs_tokio::types::IntoRustByName, +{ + use cdrs_tokio::types::IntoRustByName; + row.get_by_name(name) + .map_err(|e| MigrationError::store(format!("{context}: column {name}"), e)) +} + +/// A `map` column, empty when null. +/// +/// # Errors +/// +/// The column is missing or has another type. +pub(crate) fn text_map( + row: &Row, + name: &str, + context: &str, +) -> Result, MigrationError> { + use cdrs_tokio::types::AsRust; + let Some(map) = nullable::(row, name, context)? else { + return Ok(std::collections::HashMap::new()); + }; + let map = map + .as_rust::>() + .map_err(|e| MigrationError::store(format!("{context}: column {name}"), e))?; + Ok(map.unwrap_or_default()) +} + +/// Whether `table` exists in `keyspace`, read from `system_schema`, which +/// every node holds the same once the schema agrees. +/// +/// # Errors +/// +/// The read fails. +pub(crate) async fn table_exists( + session: &CassandraSession, + keyspace: &str, + table: &str, +) -> Result { + let found = rows( + session, + "SELECT table_name FROM system_schema.tables WHERE keyspace_name = ? AND table_name = ?", + cdrs_tokio::query_values!(keyspace, table), + Consistency::One, + &format!("read whether {keyspace}.{table} exists"), + ) + .await?; + Ok(!found.is_empty()) +} + +/// Whether `table` exists in `keyspace` with every column of `columns`. +/// +/// # Errors +/// +/// The read fails. +pub(crate) async fn has_columns( + session: &CassandraSession, + keyspace: &str, + table: &str, + columns: &[&str], +) -> Result { + let found = rows( + session, + "SELECT column_name FROM system_schema.columns WHERE keyspace_name = ? AND table_name = ?", + cdrs_tokio::query_values!(keyspace, table), + Consistency::One, + &format!("read the columns of {keyspace}.{table}"), + ) + .await?; + let names = found + .iter() + .map(|row| column::(row, "column_name", "read columns")) + .collect::, _>>()?; + Ok(columns.iter().all(|c| names.iter().any(|n| n == c))) +} + +/// The tables of `keyspace`, empty when the keyspace does not exist. +/// +/// # Errors +/// +/// The read fails. +pub(crate) async fn tables_of( + session: &CassandraSession, + keyspace: &str, +) -> Result, MigrationError> { + let found = rows( + session, + "SELECT table_name FROM system_schema.tables WHERE keyspace_name = ?", + cdrs_tokio::query_values!(keyspace), + Consistency::One, + &format!("list the tables of {keyspace}"), + ) + .await?; + let mut names = found + .iter() + .map(|row| column::(row, "table_name", "list tables")) + .collect::, _>>()?; + names.sort_unstable(); + Ok(names) +} + +/// The replication of `keyspace` as `system_schema` records it, written as +/// the map literal a `CREATE KEYSPACE` takes; `None` when the keyspace does +/// not exist. +/// +/// # Errors +/// +/// The read fails. +pub(crate) async fn replication_of( + session: &CassandraSession, + keyspace: &str, +) -> Result, MigrationError> { + let found = rows( + session, + "SELECT replication FROM system_schema.keyspaces WHERE keyspace_name = ?", + cdrs_tokio::query_values!(keyspace), + Consistency::One, + &format!("read the replication of {keyspace}"), + ) + .await?; + let Some(row) = found.first() else { + return Ok(None); + }; + let map = text_map(row, "replication", "read the replication")?; + Ok(Some(replication_literal(&map))) +} + +/// A replication map as a CQL map literal, in a stable order. +pub(crate) fn replication_literal(map: &std::collections::HashMap) -> String { + let mut entries: Vec<(&String, &String)> = map.iter().collect(); + entries.sort(); + let quote = |s: &str| format!("'{}'", s.replace('\'', "''")); + let body: Vec = entries + .into_iter() + .map(|(k, v)| format!("{}: {}", quote(k), quote(v))) + .collect(); + format!("{{{}}}", body.join(", ")) +} + +#[cfg(test)] +mod tests { + use extenddb_storage::migrations::{AdminRole, MigrationError, Target}; + + use super::{Deployment, TRANSIENT_CODES, replication_literal, retryable}; + + /// The errors a request may be sent again after, decided by the code + /// (finding 4): no answer or a failed connection, which carry no code, + /// and the five transient server codes; a server error with any other + /// code is final. + #[test] + fn a_request_is_sent_again_after_a_transient_code_and_never_after_another() { + let coded = + |code: &str| MigrationError::coded("send", "server error", Some(code.to_owned())); + assert!(retryable(&MigrationError::Store( + "send: no answer within 10 s".to_owned() + ))); + assert!(retryable(&MigrationError::coded( + "send", + "connection reset", + None + ))); + assert_eq!( + TRANSIENT_CODES, + ["0x1000", "0x1001", "0x1002", "0x1100", "0x1200"], + "Unavailable, Overloaded, IsBootstrapping, WriteTimeout, ReadTimeout" + ); + for code in TRANSIENT_CODES { + assert!(retryable(&coded(code)), "{code}"); + } + for code in [ + "0x0000", "0x000a", "0x1101", "0x1201", "0x1300", "0x1400", "0x1500", "0x2000", + "0x2100", "0x2200", "0x2300", "0x2400", "0x2500", + ] { + assert!(!retryable(&coded(code)), "{code}"); + } + } + + #[test] + fn a_command_connects_as_the_config_role_unless_given_an_admin() { + let target = Target::at("app:s%40cret@10.0.0.1:9042,10.0.0.2:9042/acme"); + let deployment = Deployment::from_target(&target).expect("deployment"); + assert_eq!( + deployment.contact_points, + ["10.0.0.1:9042", "10.0.0.2:9042"] + ); + assert_eq!(deployment.catalog_keyspace(), "acme_catalog"); + assert_eq!(deployment.account_keyspace("123"), "acme_account_123"); + assert_eq!(deployment.username.as_deref(), Some("app")); + assert_eq!(deployment.password.as_deref(), Some("s@cret")); + let shown = format!("{deployment:?}"); + assert!(!shown.contains("s@cret"), "{shown}"); + + let target = Target { + admin: Some(AdminRole { + user: "dba".to_owned(), + password: Some("pw".to_owned()), + }), + ..target + }; + let deployment = Deployment::from_target(&target).expect("deployment"); + assert_eq!(deployment.username.as_deref(), Some("dba")); + assert_eq!(deployment.password.as_deref(), Some("pw")); + + let none = Deployment::from_target(&Target::at("/acme")).expect_err("no contact point"); + assert!(none.to_string().contains("no contact point"), "{none}"); + } + + #[test] + fn a_replication_map_is_written_back_as_it_was_read() { + let map = [ + ( + "class", + "org.apache.cassandra.locator.NetworkTopologyStrategy", + ), + ("dc1", "3"), + ("dc2", "1"), + ] + .into_iter() + .map(|(k, v)| (k.to_owned(), v.to_owned())) + .collect(); + assert_eq!( + replication_literal(&map), + "{'class': 'org.apache.cassandra.locator.NetworkTopologyStrategy', 'dc1': '3', \ + 'dc2': '1'}" + ); + } +} diff --git a/crates/storage-cassandra/src/migrations/cql.rs b/crates/storage-cassandra/src/migrations/cql.rs new file mode 100644 index 000000000..4f1c1864d --- /dev/null +++ b/crates/storage-cassandra/src/migrations/cql.rs @@ -0,0 +1,343 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! A CQL body's statements, each with the keyspace written in front of the +//! table it names (#632). +//! +//! The runner never selects a keyspace with `USE`. cdrs-tokio runs a `USE` on +//! one pooled connection and copies it to the others from a background task, +//! so a statement that relied on it could run in the keyspace the connection +//! it landed on had selected before. A body names its tables without a +//! keyspace, and [`statements`] writes the keyspace of the store the runner +//! migrates in front of each one. + +use extenddb_storage::migrations::{MigrationError, split_comments}; + +/// One statement of a body, qualified with the store's keyspace. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct Statement { + /// The statement, without comments, with `keyspace.table` in it. + pub(crate) text: String, + /// A `CREATE` or `ALTER`: the runner waits for every live node to agree + /// on the schema after it. + pub(crate) ddl: bool, +} + +/// The statements of body `file` with comments removed, split on the +/// semicolons outside quotes, each with its table written as +/// `keyspace.table`. A statement whose table cannot be qualified fails the +/// whole file before any statement runs. +/// +/// # Errors +/// +/// A statement is not one of the forms [`qualify`] recognises. +pub(crate) fn statements( + keyspace: &str, + file: &str, + text: &str, +) -> Result, MigrationError> { + let (code, _) = split_comments(text, true); + split(&code) + .into_iter() + .map(|statement| qualify(keyspace, file, statement)) + .collect() +} + +/// `code` split on the semicolons outside single and double quotes, each +/// piece trimmed and the empty ones dropped. +fn split(code: &str) -> Vec<&str> { + let mut pieces = Vec::new(); + let mut quote: Option = None; + let mut start = 0; + for (i, c) in code.char_indices() { + match (quote, c) { + (None, '\'' | '"') => quote = Some(c), + (Some(q), c) if c == q => quote = None, + (None, ';') => { + pieces.push(&code[start..i]); + start = i + 1; + } + _ => {} + } + } + pieces.push(&code[start..]); + pieces + .into_iter() + .map(str::trim) + .filter(|p| !p.is_empty()) + .collect() +} + +/// `statement` with its table written as `keyspace.table`. Recognises +/// `CREATE TABLE [IF NOT EXISTS] t`, `CREATE INDEX [IF NOT EXISTS] [name] +/// ON t`, `ALTER TABLE [IF EXISTS] t`, `INSERT INTO t` and `DELETE FROM t`, +/// each naming an unquoted table without a keyspace. +fn qualify(keyspace: &str, file: &str, statement: &str) -> Result { + let first_line = statement.lines().next().unwrap_or_default(); + let refuse = || { + MigrationError::Store(format!( + "migration file {file} has a statement that names no table the runner can qualify \ + with a keyspace: {first_line}" + )) + }; + let (at, ddl) = table_offset(statement).ok_or_else(refuse)?; + let table: String = statement[at..] + .chars() + .take_while(|c| c.is_ascii_alphanumeric() || *c == '_') + .collect(); + let next = statement[at + table.len()..].chars().next(); + let starts_well = table + .chars() + .next() + .is_some_and(|c| c.is_ascii_alphabetic() || c == '_'); + if !starts_well || matches!(next, Some('.' | '"')) { + return Err(refuse()); + } + Ok(Statement { + text: format!("{}{keyspace}.{}", &statement[..at], &statement[at..]), + ddl, + }) +} + +/// The byte offset of the table name in `statement`, and whether the +/// statement changes the schema. +fn table_offset(statement: &str) -> Option<(usize, bool)> { + let words: Vec<(usize, &str)> = words(statement).collect(); + let is = |i: usize, keyword: &str| { + words + .get(i) + .is_some_and(|(_, w)| w.eq_ignore_ascii_case(keyword)) + }; + let at = |i: usize| words.get(i).map(|(offset, _)| *offset); + if is(0, "CREATE") && is(1, "TABLE") { + let i = if is(2, "IF") { 5 } else { 2 }; + if i == 5 && !(is(3, "NOT") && is(4, "EXISTS")) { + return None; + } + return at(i).map(|o| (o, true)); + } + if is(0, "CREATE") && is(1, "INDEX") { + let i = if is(2, "IF") { + if !(is(3, "NOT") && is(4, "EXISTS")) { + return None; + } + 5 + } else { + 2 + }; + // The index name is optional: `ON` is at i or i + 1. + let on = if is(i, "ON") { i } else { i + 1 }; + if !is(on, "ON") { + return None; + } + return at(on + 1).map(|o| (o, true)); + } + if is(0, "ALTER") && is(1, "TABLE") { + // `IF EXISTS` comes before the table. `IF NOT EXISTS` is not a form + // of `ALTER TABLE`, and `IF` cannot name a table unquoted. + let i = if is(2, "IF") { + if !is(3, "EXISTS") { + return None; + } + 4 + } else { + 2 + }; + return at(i).map(|o| (o, true)); + } + if (is(0, "INSERT") && is(1, "INTO")) || (is(0, "DELETE") && is(1, "FROM")) { + return at(2).map(|o| (o, false)); + } + None +} + +/// The words of `statement` with their byte offsets. A word ends at +/// whitespace, `(` or `,`. +fn words(statement: &str) -> impl Iterator { + let mut rest = statement; + let mut offset = 0; + std::iter::from_fn(move || { + let skip = rest.len() + - rest + .trim_start_matches(|c: char| c.is_whitespace() || c == '(' || c == ',') + .len(); + offset += skip; + rest = &rest[skip..]; + if rest.is_empty() { + return None; + } + let len = rest + .find(|c: char| c.is_whitespace() || c == '(' || c == ',') + .unwrap_or(rest.len()); + let word = (offset, &rest[..len]); + offset += len; + rest = &rest[len..]; + Some(word) + }) +} + +#[cfg(test)] +mod tests { + use super::{Statement, statements}; + + fn one(sql: &str) -> Result { + let mut all = statements("ks", "f.cql", sql).map_err(|e| e.to_string())?; + assert_eq!(all.len(), 1, "{sql}"); + Ok(all.remove(0)) + } + + #[test] + fn every_form_is_qualified_once_in_front_of_its_table() { + let cases = [ + ( + "CREATE TABLE IF NOT EXISTS settings (key text PRIMARY KEY)", + "CREATE TABLE IF NOT EXISTS ks.settings (key text PRIMARY KEY)", + true, + ), + ( + "create table t(a int primary key)", + "create table ks.t(a int primary key)", + true, + ), + ( + "CREATE INDEX IF NOT EXISTS idx ON tables (status)", + "CREATE INDEX IF NOT EXISTS idx ON ks.tables (status)", + true, + ), + ("CREATE INDEX ON t (c)", "CREATE INDEX ON ks.t (c)", true), + ( + "CREATE INDEX idx ON tables (table_id)", + "CREATE INDEX idx ON ks.tables (table_id)", + true, + ), + ( + "ALTER TABLE tables ADD IF NOT EXISTS c text", + "ALTER TABLE ks.tables ADD IF NOT EXISTS c text", + true, + ), + ( + "ALTER TABLE IF EXISTS tables ADD IF NOT EXISTS c text", + "ALTER TABLE IF EXISTS ks.tables ADD IF NOT EXISTS c text", + true, + ), + ( + "alter table if exists tables with comment = 'x'", + "alter table if exists ks.tables with comment = 'x'", + true, + ), + ( + "INSERT INTO schema_history (version) VALUES (1)", + "INSERT INTO ks.schema_history (version) VALUES (1)", + false, + ), + ( + "DELETE FROM settings WHERE key = 'x'", + "DELETE FROM ks.settings WHERE key = 'x'", + false, + ), + ]; + for (input, want, ddl) in cases { + assert_eq!( + one(input), + Ok(Statement { + text: want.to_owned(), + ddl + }), + "{input}" + ); + } + } + + #[test] + fn a_statement_the_runner_cannot_qualify_fails_the_file() { + for sql in [ + "UPDATE settings SET value = 'x' WHERE key = 'y'", + "CREATE TABLE other.t (a int PRIMARY KEY)", + "CREATE TABLE \"T\" (a int PRIMARY KEY)", + "CREATE TABLE IF EXISTS t (a int PRIMARY KEY)", + "CREATE TABLE 9t (a int PRIMARY KEY)", + "CREATE INDEX IF NOT EXISTS i t (c)", + "CREATE INDEX IF EXISTS idx ON tables (table_id)", + "ALTER TABLE IF NOT EXISTS tables ADD c text", + "ALTER TABLE IF tables ADD c text", + "USE ks", + "DROP TABLE t", + ] { + let error = one(sql).expect_err(sql); + assert!( + error.contains("f.cql") && error.contains("qualify"), + "{error}" + ); + } + // One bad statement refuses the whole file, before anything runs. + assert!(statements("ks", "f.cql", "CREATE TABLE a (x int PRIMARY KEY); USE ks").is_err()); + } + + #[test] + fn comments_and_quoted_semicolons_do_not_split_a_statement() { + let all = statements( + "ks", + "f.cql", + "-- a comment\nCREATE TABLE a (x text PRIMARY KEY); // another\n\ + INSERT INTO a (x) VALUES ('p;q');\n/* block */", + ) + .expect("statements"); + let texts: Vec<&str> = all.iter().map(|s| s.text.as_str()).collect(); + assert_eq!( + texts, + [ + "CREATE TABLE ks.a (x text PRIMARY KEY)", + "INSERT INTO ks.a (x) VALUES ('p;q')" + ] + ); + } + + /// Every statement of both baselines, as the runner sends it, is the + /// statement as written with the keyspace written in once, as a word of + /// its own in front of the table: after `ON` in a `CREATE INDEX`, after + /// `INTO` in an `INSERT INTO`, after `FROM` in a `DELETE FROM`, and after + /// `TABLE` or `EXISTS` otherwise, never in front of the `IF` that opens + /// an `IF EXISTS` or `IF NOT EXISTS` clause. + #[test] + fn every_statement_of_both_baselines_is_qualified() { + for (file, text) in [ + ("catalog", super::super::CATALOG_BASELINE), + ("account", super::super::ACCOUNT_BASELINE), + ] { + let all = statements("ks", file, text).expect(file); + let (code, _) = super::split_comments(text, true); + let written = super::split(&code); + assert!(!all.is_empty(), "{file}"); + assert_eq!(all.len(), written.len(), "{file}"); + for (statement, written) in all.iter().zip(written) { + let sent = statement.text.as_str(); + assert_eq!(sent.matches("ks.").count(), 1, "{sent}"); + assert_eq!(sent.replacen("ks.", "", 1), written, "{file}"); + assert!(!sent.contains("--"), "{sent}"); + let (before, after) = sent.split_once("ks.").unwrap_or_default(); + let keyword = if before.ends_with(char::is_whitespace) { + before.split_whitespace().next_back().unwrap_or_default() + } else { + "" + }; + let table = after + .split(|c: char| c.is_whitespace() || c == '(') + .next() + .unwrap_or_default(); + let mut words = written.split_whitespace().map(str::to_ascii_uppercase); + let expected: &[&str] = match (words.next().as_deref(), words.next().as_deref()) { + (Some("CREATE"), Some("INDEX")) => &["ON"], + (Some("INSERT"), Some("INTO")) => &["INTO"], + (Some("DELETE"), Some("FROM")) => &["FROM"], + _ => &["TABLE", "EXISTS"], + }; + assert!( + expected.iter().any(|k| keyword.eq_ignore_ascii_case(k)) + && !table.is_empty() + && !table.eq_ignore_ascii_case("IF"), + "{file}: keyspace not in front of the table: {sent}" + ); + } + } + } +} diff --git a/crates/storage-cassandra/src/migrations/guarded.rs b/crates/storage-cassandra/src/migrations/guarded.rs new file mode 100644 index 000000000..b1ac6f851 --- /dev/null +++ b/crates/storage-cassandra/src/migrations/guarded.rs @@ -0,0 +1,153 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The management store migration 2's shared body runs through under the +//! migration lock. +//! +//! The runner bounds every request it sends itself by +//! [`REQUEST_TIMEOUT`](super::connect::REQUEST_TIMEOUT) and checks the lock's +//! deadline before each one. A code body's requests go through +//! [`CassandraCatalogStore`], whose session carries a connection timeout and +//! no request timeout, so a request that never gets an answer was cut only +//! by the lease: no progress for the stall, then the deadline, 35 to 65 s +//! after the last answer. This store wraps every delegated call: the lease +//! is checked first, so a holder paused past its deadline sends nothing +//! more; the call waits at most the request timeout for its answer; and a +//! call that completes is progress, as a request of the runner's is, so the +//! renewal keeps the lock through a scan that lasts longer than the stall. + +use std::time::Duration; + +use extenddb_storage::management_store::{ + AccessKeyCreated, AccountDetail, GroupDetail, GroupListEntry, ManagementStore, OpError, + OpResult, RoleDetail, RoleListEntry, SessionPolicyEntry, UserDetail, UserListEntry, +}; +use futures::future::BoxFuture; + +use super::connect::REQUEST_TIMEOUT; +use super::locks::LeaseHandle; +use crate::catalog_store::CassandraCatalogStore; + +/// [`CassandraCatalogStore`] with the lease checked before every call, the +/// request timeout on every call, and progress recorded when a call +/// completes. +pub(crate) struct LeasedManagementStore { + inner: CassandraCatalogStore, + lease: Option, + /// The bound on one call: the request timeout, or a shorter one in tests. + bound: Duration, +} + +impl LeasedManagementStore { + pub(crate) fn new(inner: CassandraCatalogStore, lease: Option) -> Self { + Self { + inner, + lease, + bound: REQUEST_TIMEOUT, + } + } + + /// Stop the policy writes of migration 2's body when the test hook's + /// request budget is spent. + #[cfg(any(test, feature = "test-util"))] + pub(crate) fn stop_after(&mut self, left: std::sync::Arc) { + self.inner.stop_after(left); + } + + fn lease_ok(&self) -> OpResult<()> { + match &self.lease { + Some(lease) => lease.check().map_err(|e| OpError::Internal(e.to_string())), + None => Ok(()), + } + } + + /// Record that a call completed, as the runner does after each of its + /// requests, so the renewal task keeps the lock while a scan runs. + fn progressed(&self) { + if let Some(lease) = &self.lease { + lease.progress(); + } + } + + /// A call that had no answer within the bound. + fn no_answer(&self, what: &str) -> OpError { + OpError::Internal(format!( + "{what}: no answer within {} s", + self.bound.as_secs() + )) + } +} + +/// Delegate each method to the inner store after checking the lease, wait +/// at most the bound for its answer, and record progress when the call +/// completes. +macro_rules! checked { + ($( fn $name:ident(&self $(, $arg:ident: $ty:ty)* $(,)?) -> $ret:ty; )+) => { + $( + fn $name(&self $(, $arg: $ty)*) -> $ret { + if let Err(e) = self.lease_ok() { + return Box::pin(async move { Err(e) }); + } + let call = self.inner.$name($($arg),*); + Box::pin(async move { + let answer = tokio::time::timeout(self.bound, call).await; + self.progressed(); + answer.unwrap_or_else(|_| Err(self.no_answer(stringify!($name)))) + }) + } + )+ + }; +} + +impl ManagementStore for LeasedManagementStore { + checked! { + fn list_all_accounts(&self) -> BoxFuture<'_, OpResult>>; + fn create_account(&self, account_id: &str, account_name: &str) -> BoxFuture<'_, OpResult<()>>; + fn delete_account(&self, account_id: &str) -> BoxFuture<'_, OpResult<()>>; + fn default_account_id(&self) -> BoxFuture<'_, OpResult>>; + fn list_all_accounts_full(&self) -> BoxFuture<'_, OpResult>>; + fn list_accounts_for(&self, account_id: &str) -> BoxFuture<'_, OpResult>>; + fn get_account_detail(&self, account_id: &str) -> BoxFuture<'_, OpResult>>; + fn dashboard_counts(&self) -> BoxFuture<'_, OpResult<(i64, i64)>>; + fn create_user(&self, account_id: &str, user_name: &str, password_hash: Option<&str>) -> BoxFuture<'_, OpResult<()>>; + fn delete_user(&self, account_id: &str, user_name: &str) -> BoxFuture<'_, OpResult<()>>; + fn list_users(&self, account_id: &str) -> BoxFuture<'_, OpResult>>; + fn get_user_detail(&self, account_id: &str, user_name: &str) -> BoxFuture<'_, OpResult>>; + fn verify_iam_user_password(&self, account_id: &str, user_name: &str, password: &str) -> BoxFuture<'_, OpResult>; + fn change_user_password(&self, account_id: &str, user_name: &str, password_hash: &str) -> BoxFuture<'_, OpResult<()>>; + fn tag_user(&self, account_id: &str, user_name: &str, tags: &[(String, String)]) -> BoxFuture<'_, OpResult<()>>; + fn untag_user(&self, account_id: &str, user_name: &str, tag_keys: &[String]) -> BoxFuture<'_, OpResult<()>>; + fn list_user_tags(&self, account_id: &str, user_name: &str) -> BoxFuture<'_, OpResult>>; + fn create_group(&self, account_id: &str, group_name: &str) -> BoxFuture<'_, OpResult<()>>; + fn delete_group(&self, account_id: &str, group_name: &str) -> BoxFuture<'_, OpResult<()>>; + fn list_groups(&self, account_id: &str) -> BoxFuture<'_, OpResult>>; + fn get_group_detail(&self, account_id: &str, group_name: &str) -> BoxFuture<'_, OpResult>>; + fn add_group_member(&self, account_id: &str, group_name: &str, user_name: &str) -> BoxFuture<'_, OpResult<()>>; + fn remove_group_member(&self, account_id: &str, group_name: &str, user_name: &str) -> BoxFuture<'_, OpResult<()>>; + fn create_role(&self, account_id: &str, role_name: &str, trust_policy: &serde_json::Value) -> BoxFuture<'_, OpResult<()>>; + fn delete_role(&self, account_id: &str, role_name: &str) -> BoxFuture<'_, OpResult<()>>; + fn list_roles(&self, account_id: &str) -> BoxFuture<'_, OpResult>>; + fn get_role_detail(&self, account_id: &str, role_name: &str) -> BoxFuture<'_, OpResult>>; + fn get_role_trust_policy(&self, account_id: &str, role_name: &str) -> BoxFuture<'_, OpResult>>; + fn tag_role(&self, account_id: &str, role_name: &str, tags: &[(String, String)]) -> BoxFuture<'_, OpResult<()>>; + fn untag_role(&self, account_id: &str, role_name: &str, tag_keys: &[String]) -> BoxFuture<'_, OpResult<()>>; + fn list_role_tags(&self, account_id: &str, role_name: &str) -> BoxFuture<'_, OpResult>>; + fn put_policy(&self, account_id: &str, principal_type: &str, principal_name: &str, policy_name: &str, document: &serde_json::Value) -> BoxFuture<'_, OpResult<()>>; + fn delete_policy(&self, account_id: &str, principal_type: &str, principal_name: &str, policy_name: &str) -> BoxFuture<'_, OpResult<()>>; + fn list_policies(&self, account_id: &str, principal_type: &str, principal_name: &str) -> BoxFuture<'_, OpResult>>; + fn set_user_boundary(&self, account_id: &str, user_name: &str, document: &serde_json::Value) -> BoxFuture<'_, OpResult<()>>; + fn get_user_boundary(&self, account_id: &str, user_name: &str) -> BoxFuture<'_, OpResult>>; + fn delete_user_boundary(&self, account_id: &str, user_name: &str) -> BoxFuture<'_, OpResult<()>>; + fn set_role_boundary(&self, account_id: &str, role_name: &str, document: &serde_json::Value) -> BoxFuture<'_, OpResult<()>>; + fn get_role_boundary(&self, account_id: &str, role_name: &str) -> BoxFuture<'_, OpResult>>; + fn delete_role_boundary(&self, account_id: &str, role_name: &str) -> BoxFuture<'_, OpResult<()>>; + fn create_access_key(&self, account_id: &str, user_name: &str) -> BoxFuture<'_, OpResult>; + fn delete_access_key(&self, account_id: &str, user_name: &str, key_id: &str) -> BoxFuture<'_, OpResult<()>>; + fn list_access_keys(&self, account_id: &str, user_name: &str) -> BoxFuture<'_, OpResult>>; + fn import_access_key(&self, account_id: &str, user_name: &str, access_key_id: &str, secret_access_key: &str) -> BoxFuture<'_, OpResult<()>>; + fn store_session(&self, session_token: &str, access_key_id: &str, secret_key_encrypted: &[u8], account_id: &str, role_name: &str, session_name: &str, session_tags: &Option, session_policy: &Option, expires_at: time::OffsetDateTime) -> BoxFuture<'_, OpResult<()>>; + fn list_session_policies(&self) -> BoxFuture<'_, OpResult>>; + fn update_session_policy(&self, session_token: &str, document: &serde_json::Value) -> BoxFuture<'_, OpResult<()>>; + fn fetch_caller_tags(&self, account_id: &str, resource: &str) -> BoxFuture<'_, OpResult>>; + } +} diff --git a/crates/storage-cassandra/src/migrations/locks.rs b/crates/storage-cassandra/src/migrations/locks.rs new file mode 100644 index 000000000..9fa56c69e --- /dev/null +++ b/crates/storage-cassandra/src/migrations/locks.rs @@ -0,0 +1,566 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The migration lock. +//! +//! The lock is the `migration_lock` row of the catalog's `settings` table, +//! written only by lightweight transactions at `SERIAL`: taken with +//! `INSERT ... IF NOT EXISTS USING TTL 60`, renewed every 15 s with an +//! `UPDATE` that sets the whole row (its one value column) to the holder's +//! token on the condition that it still holds it, and released with +//! `DELETE ... IF value = `. The token is drawn when the lock is +//! created and never changes, so a renewal whose result is unknown after a +//! timeout is retried with the same condition. `SERIAL` runs Paxos across +//! every datacenter: with two datacenters and the link between them cut, +//! `LOCAL_SERIAL` let one holder in each take the lock. +//! +//! The holder keeps a deadline on a monotonic clock, the last successful +//! renewal plus the TTL minus a 25 s margin, and the runner checks it before +//! every statement and every ledger write ([`MigrationLock::check`]). Each +//! request is bounded by 10 s, shorter than the margin, so nothing the holder +//! sent before its deadline still runs when the row expires. The renewal +//! stops when the runner has made no progress for 30 s, so a hung holder +//! loses the lock instead of keeping it. + +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use cdrs_tokio::consistency::Consistency; +use cdrs_tokio::types::rows::Row; +use extenddb_storage::migrations::MigrationError; +use tokio::sync::watch; +use tokio::task::JoinHandle; +use tokio::time::Instant; + +use super::agreement::{AGREEMENT_TIMEOUT, wait_for_agreement, wait_for_agreement_with}; +use super::connect::{column, nullable, retryable, rows, send, table_exists}; +use crate::cassandra_util::CassandraSession; + +/// The `settings` key of the lock row. +pub(crate) const LOCK_KEY: &str = "migration_lock"; + +/// The times of the lock's lease. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct LeaseTimes { + /// The row's TTL. + pub ttl: Duration, + /// How often the holder renews. + pub renew_every: Duration, + /// Subtracted from the TTL to give the holder's deadline. + pub margin: Duration, + /// How long the runner may make no progress before the renewal stops. + pub stall: Duration, +} + +impl LeaseTimes { + /// 60 s TTL, renewed every 15 s, a 25 s margin and a 30 s stall. + pub const DEFAULT: Self = Self { + ttl: Duration::from_secs(60), + renew_every: Duration::from_secs(15), + margin: Duration::from_secs(25), + stall: Duration::from_secs(30), + }; + + /// The holder's deadline after a renewal sent at `sent`. + #[must_use] + pub fn deadline_after(&self, sent: Instant) -> Instant { + sent + self.ttl.saturating_sub(self.margin) + } + + fn ttl_secs(&self) -> i32 { + i32::try_from(self.ttl.as_secs().max(1)).unwrap_or(i32::MAX) + } +} + +/// What the holder and its renewal task share. +struct Lease { + deadline: Mutex, + progress: Mutex, + lost: watch::Sender, +} + +impl Lease { + fn deadline(&self) -> Instant { + *self + .deadline + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } + + fn set_deadline(&self, deadline: Instant) { + *self + .deadline + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = deadline; + } + + fn last_progress(&self) -> Instant { + *self + .progress + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } +} + +fn lost_error(keyspace: &str) -> MigrationError { + MigrationError::Store(format!( + "the migration lock in {keyspace}.settings was not renewed in time, so another migration \ + may hold it; this run stopped before its next write. Run 'extenddb migrate --yes' again" + )) +} + +/// A lock's lease as seen by code that runs under the lock without holding +/// it: the deadline check before a request, and the progress a completed +/// request records. +#[derive(Clone)] +pub(crate) struct LeaseHandle { + lease: Arc, + keyspace: String, +} + +impl LeaseHandle { + /// Stop before a request when the holder's deadline has passed. + /// + /// # Errors + /// + /// The deadline has passed: the lease may have lapsed, and another + /// migrator may hold the lock. + pub(crate) fn check(&self) -> Result<(), MigrationError> { + if Instant::now() >= self.lease.deadline() { + return Err(lost_error(&self.keyspace)); + } + Ok(()) + } + + /// Record progress, which keeps the renewal going. + pub(crate) fn progress(&self) { + *self + .lease + .progress + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Instant::now(); + } + + /// A handle on a lease whose deadline has already passed: for the tests + /// of code that must refuse a request after it. + #[cfg(any(test, feature = "test-util"))] + pub(crate) fn lapsed(keyspace: &str) -> Self { + let (lost, _) = watch::channel(true); + Self { + lease: Arc::new(Lease { + deadline: Mutex::new(Instant::now()), + progress: Mutex::new(Instant::now()), + lost, + }), + keyspace: keyspace.to_owned(), + } + } +} + +/// The catalog's migration lock, held until [`MigrationLock::release`] or +/// until it is dropped, after which its row expires within one TTL. +pub struct MigrationLock { + session: Arc, + keyspace: String, + token: String, + lease: Arc, + renewal: JoinHandle<()>, +} + +impl std::fmt::Debug for MigrationLock { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("MigrationLock") + .field("keyspace", &self.keyspace) + .field("token", &self.token) + .finish_non_exhaustive() + } +} + +/// Whether a lightweight transaction applied, and the lock value it found. +fn lwt_outcome(rows: &[Row], context: &str) -> Result<(bool, Option), MigrationError> { + let row = rows + .first() + .ok_or_else(|| MigrationError::Store(format!("{context}: no [applied] row")))?; + let applied: bool = column(row, "[applied]", context)?; + // A condition on a row that does not exist answers with `[applied]` + // alone, without the row's columns. + let value: Option = if applied { + None + } else { + nullable::(row, "value", context).unwrap_or(None) + }; + Ok((applied, value)) +} + +async fn lwt( + session: &CassandraSession, + cql: &str, + values: cdrs_tokio::query::QueryValues, + context: &str, +) -> Result<(bool, Option), MigrationError> { + let envelope = send( + session, + cql, + values, + Consistency::Quorum, + Some(Consistency::Serial), + context, + ) + .await?; + let rows = envelope + .response_body() + .map_err(|e| MigrationError::store(context, e))? + .into_rows() + .ok_or_else(|| MigrationError::Store(format!("{context}: the answer holds no rows")))?; + lwt_outcome(&rows, context) +} + +impl MigrationLock { + /// Take the lock in `keyspace`'s `settings` table, retrying every second + /// for up to `timeout` while another process holds it, then wait for + /// schema agreement: a holder that took an expired lock may follow one + /// whose statements have not reached every node. + /// + /// # Errors + /// + /// [`MigrationError::Locked`] (exit 6) when the lock is still held after + /// `timeout`; a store error when Cassandra cannot reach a majority of the + /// catalog's replicas at `SERIAL` for that long. + pub async fn acquire( + session: Arc, + keyspace: &str, + times: LeaseTimes, + timeout: Duration, + ) -> Result { + let token = uuid::Uuid::new_v4().to_string(); + let started = Instant::now(); + let cql = format!( + "INSERT INTO {keyspace}.settings (key, value) VALUES ('{LOCK_KEY}', ?) IF NOT EXISTS \ + USING TTL {}", + times.ttl_secs() + ); + loop { + let sent = Instant::now(); + let outcome = lwt( + &session, + &cql, + cdrs_tokio::query_values!(token.as_str()), + "take the migration lock", + ) + .await; + match outcome { + // A retry after an unknown result finds this lock's own token. + Ok((true, _)) => return Self::start(session, keyspace, token, times, sent).await, + Ok((false, Some(value))) if value == token => { + return Self::start(session, keyspace, token, times, sent).await; + } + Ok((false, holder)) => { + if started.elapsed() >= timeout { + return Err(MigrationError::Locked(format!( + "another migration holds the catalog's migration lock (the \ + {LOCK_KEY} row of {keyspace}.settings, holder {}) after {} s; \ + retry when it finishes, or after the row's {} s TTL if its \ + process died", + holder.unwrap_or_else(|| "unknown".to_owned()), + timeout.as_secs(), + times.ttl.as_secs() + ))); + } + } + Err(e) if !retryable(&e) => return Err(e), + Err(e) => { + if started.elapsed() >= timeout { + return Err(MigrationError::Locked(format!( + "the catalog's migration lock could not be taken within {} s: {e}; \ + a migration needs a majority of every datacenter's replicas of \ + {keyspace}", + timeout.as_secs() + ))); + } + } + } + tokio::time::sleep(Duration::from_secs(1).min(times.renew_every)).await; + } + } + + async fn start( + session: Arc, + keyspace: &str, + token: String, + times: LeaseTimes, + sent: Instant, + ) -> Result { + let (lost, _) = watch::channel(false); + let lease = Arc::new(Lease { + deadline: Mutex::new(times.deadline_after(sent)), + progress: Mutex::new(Instant::now()), + lost, + }); + let renewal = tokio::spawn(renew( + session.clone(), + keyspace.to_owned(), + token.clone(), + times, + lease.clone(), + )); + let lock = Self { + session, + keyspace: keyspace.to_owned(), + token, + lease, + renewal, + }; + wait_for_agreement_with(&lock.session, AGREEMENT_TIMEOUT, || lock.progress()).await?; + lock.progress(); + Ok(lock) + } + + /// The keyspace whose `settings` table holds the lock. + #[must_use] + pub fn keyspace(&self) -> &str { + &self.keyspace + } + + /// The session the lock was taken on. + #[must_use] + pub fn session(&self) -> &Arc { + &self.session + } + + /// Stop before writing when the holder's deadline has passed. + /// + /// # Errors + /// + /// The deadline has passed: the lease may have lapsed, and another + /// migrator may hold the lock. + pub fn check(&self) -> Result<(), MigrationError> { + self.handle().check() + } + + fn lost_error(&self) -> MigrationError { + lost_error(&self.keyspace) + } + + /// A handle on this lock's lease for code that runs under the lock + /// without holding it: the store a code body's requests go through. + #[must_use] + pub(crate) fn handle(&self) -> LeaseHandle { + LeaseHandle { + lease: Arc::clone(&self.lease), + keyspace: self.keyspace.clone(), + } + } + + /// Record that the runner made progress, which keeps the renewal going. + pub fn progress(&self) { + self.handle().progress(); + } + + /// Resolves once the holder's deadline has passed. + pub async fn expired(&self) { + let mut lost = self.lease.lost.subscribe(); + // `wait_for` returns at once when the value already holds. + let _ = lost.wait_for(|lost| *lost).await; + } + + /// Run `work` while the lease holds: `work` is dropped, and the run + /// stops with an error, when the holder's deadline passes first. Each + /// time `work` is woken counts as progress, so a body that is waiting + /// on Cassandra keeps the lock and one that hangs loses it. + /// + /// # Errors + /// + /// The deadline passed, or `work` failed. + pub async fn guard( + &self, + work: impl std::future::Future>, + ) -> Result { + self.check()?; + let progressing = Progressing { + inner: std::pin::pin!(work), + lock: self, + }; + tokio::select! { + result = progressing => result, + () = self.expired() => Err(self.lost_error()), + } + } + + /// Release the lock: wait for schema agreement, so the next holder + /// starts on a schema every node has, then delete the row if this lock + /// still holds it. A second call does nothing. + pub async fn release(&self) { + self.renewal.abort(); + if self.check().is_err() { + return; + } + // Later calls and checks see the lease as ended. + self.lease.set_deadline(Instant::now()); + if let Err(e) = wait_for_agreement(&self.session, AGREEMENT_TIMEOUT).await { + tracing::warn!("schema agreement before releasing the migration lock: {e}"); + } + let cql = format!( + "DELETE FROM {}.settings WHERE key = '{LOCK_KEY}' IF value = ?", + self.keyspace + ); + if let Err(e) = lwt( + &self.session, + &cql, + cdrs_tokio::query_values!(self.token.as_str()), + "release the migration lock", + ) + .await + { + tracing::warn!("{e}; the lock expires within its TTL"); + } + } +} + +impl Drop for MigrationLock { + fn drop(&mut self) { + self.renewal.abort(); + } +} + +impl MigrationLock { + /// Stop renewing and leave the row, as a holder whose process was + /// killed does: the lease lapses within one TTL. For the lease tests. + #[cfg(any(test, feature = "test-util"))] + pub fn abandon(self) { + self.renewal.abort(); + } +} + +/// A future that marks progress on the lock each time it is polled. +struct Progressing<'a, F> { + inner: std::pin::Pin<&'a mut F>, + lock: &'a MigrationLock, +} + +impl std::future::Future for Progressing<'_, F> { + type Output = F::Output; + + fn poll( + mut self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll { + self.lock.progress(); + self.inner.as_mut().poll(cx) + } +} + +/// The renewal task: renew every `renew_every` while the runner makes +/// progress, and mark the lease lost once its deadline passes. +async fn renew( + session: Arc, + keyspace: String, + token: String, + times: LeaseTimes, + lease: Arc, +) { + let cql = format!( + "UPDATE {keyspace}.settings USING TTL {} SET value = ? WHERE key = '{LOCK_KEY}' IF value \ + = ?", + times.ttl_secs() + ); + let mut next = Instant::now() + times.renew_every; + loop { + tokio::time::sleep_until(next.min(lease.deadline())).await; + let now = Instant::now(); + if now >= lease.deadline() { + let _ = lease.lost.send(true); + return; + } + if now < next { + continue; + } + if now.duration_since(lease.last_progress()) >= times.stall { + tracing::warn!( + "the migration has made no progress for {} s; the migration lock is no longer \ + renewed", + times.stall.as_secs() + ); + next = lease.deadline(); + continue; + } + let sent = Instant::now(); + let outcome = lwt( + &session, + &cql, + cdrs_tokio::query_values!(token.as_str(), token.as_str()), + "renew the migration lock", + ) + .await; + match outcome { + Ok((true, _)) => { + lease.set_deadline(times.deadline_after(sent)); + next = sent + times.renew_every; + } + Ok((false, holder)) => { + tracing::error!( + "the migration lock was lost (it is held by {}); stopping", + holder.unwrap_or_else(|| "nobody".to_owned()) + ); + lease.set_deadline(Instant::now()); + let _ = lease.lost.send(true); + return; + } + // Unknown result: retry soon with the same condition, which the + // token makes safe. + Err(e) => { + tracing::warn!("{e}; retrying"); + next = Instant::now() + times.renew_every / 3; + } + } + } +} + +/// Whether a migration holds the lock, read at `SERIAL`; `None` when the +/// catalog has no `settings` table. +/// +/// # Errors +/// +/// The read fails. +pub(crate) async fn lock_holder( + session: &CassandraSession, + keyspace: &str, +) -> Result, MigrationError> { + if !table_exists(session, keyspace, "settings").await? { + return Ok(None); + } + let found = rows( + session, + &format!("SELECT value FROM {keyspace}.settings WHERE key = '{LOCK_KEY}'"), + cdrs_tokio::query_values!(), + Consistency::Serial, + "read the migration lock", + ) + .await?; + found + .first() + .map(|row| nullable::(row, "value", "read the migration lock")) + .transpose() + .map(Option::flatten) +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use tokio::time::Instant; + + use super::LeaseTimes; + + #[test] + fn the_deadline_is_the_ttl_minus_the_margin_after_the_renewal_was_sent() { + let sent = Instant::now(); + let times = LeaseTimes::DEFAULT; + assert_eq!(times.deadline_after(sent), sent + Duration::from_secs(35)); + // Each request's bound is shorter than the margin, so a statement + // sent before the deadline is done before the row can expire. + assert!(super::super::connect::REQUEST_TIMEOUT < times.margin); + // The holder renews several times before its deadline. + assert!(times.renew_every * 2 < times.ttl.saturating_sub(times.margin)); + assert_eq!(times.ttl_secs(), 60); + } +} diff --git a/crates/storage-cassandra/src/migrations/mod.rs b/crates/storage-cassandra/src/migrations/mod.rs new file mode 100644 index 000000000..d886ee364 --- /dev/null +++ b/crates/storage-cassandra/src/migrations/mod.rs @@ -0,0 +1,599 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The Cassandra backend's catalog migrations. +//! +//! Stores: the catalog keyspace ([`StoreKind::Catalog`]) and one keyspace +//! per account ([`StoreKind::Account`]). [`migration`] is the table, an +//! exhaustive `match` from every [`MigrationId`] to this backend's bodies. +//! [`RELEASED`] pins the released bodies; the release PR that raises +//! `RELEASED_THROUGH` pastes the lines the checksum test prints. +//! +//! Entry points: [`catalog_status`] reads with no lock and no write; +//! [`MigrationLock`] is the lock a run holds and [`ServingRegistration`] +//! what a server holds while it serves; [`migrate_account`] is how +//! CreateAccount and `init`'s default account run the account migrations on +//! a new keyspace; [`CassandraCatalogMigrations`] is how the commands reach +//! them. +//! +//! Every statement names its keyspace (#632), and the runner waits for +//! every live node to agree on the schema after each DDL statement (#637). + +mod agreement; +mod backend; +mod commands; +mod connect; +mod cql; +mod guarded; +mod locks; +pub mod postcheck; +mod registrations; + +use std::sync::Arc; + +use extenddb_storage::file_body; +use extenddb_storage::migrations::m0002_resource_tag_condition_keys::{ + self, ResourceTagConditionKeys, +}; +use extenddb_storage::migrations::{ + CatalogStatus, Kind, Migration, MigrationError, MigrationId, Released, RunReport, Runner, + StoreBody, StoreKind, +}; + +pub use agreement::AGREEMENT_TIMEOUT; +#[cfg(any(test, feature = "test-util"))] +pub(crate) use backend::stopped; +pub use backend::{CassandraMigrations, CassandraStores}; +pub use commands::CassandraCatalogMigrations; +pub(crate) use commands::lease_of; +pub use connect::Deployment; +pub(crate) use connect::{LOCKED_DDL_TIMEOUT, REQUEST_TIMEOUT}; +pub(crate) use locks::LeaseHandle; +pub use locks::{LeaseTimes, MigrationLock}; +pub use registrations::ServingRegistration; + +use crate::cassandra_util::CassandraSession; + +/// Migration 1's catalog body, for the tests that read it. +#[cfg(test)] +pub(crate) const CATALOG_BASELINE: &str = file_body!("migrations/catalog", "0001_baseline.cql"); + +/// Migration 1's account body, for the tests that read it. +#[cfg(test)] +pub(crate) const ACCOUNT_BASELINE: &str = file_body!("migrations/account", "0001_baseline.cql"); + +/// The definitions migration 1 creates in an empty catalog keyspace, which +/// the post-check compares the keyspace with the first time migration 1 is +/// applied to it. +pub(crate) const CATALOG_EXPECTED: &str = + include_str!("../../migrations/catalog/0001_baseline.expected.json"); + +/// The same for an account keyspace. +pub(crate) const ACCOUNT_EXPECTED: &str = + include_str!("../../migrations/account/0001_baseline.expected.json"); + +/// Checksums of the released bodies (none until v1.0). +pub(crate) const RELEASED: &[Released] = &[]; + +/// This backend's entry for every id. Public so `migrate --list` can print +/// each migration's kind without connecting. +#[must_use] +pub fn migration(id: MigrationId) -> Migration { + match id { + MigrationId::Baseline => Migration::Applies { + kind: Kind::Breaking, + convergent: true, + bodies: vec![ + file_body!( + StoreKind::Catalog, + "migrations/catalog", + "0001_baseline.cql" + ), + file_body!( + StoreKind::Account, + "migrations/account", + "0001_baseline.cql" + ), + ], + }, + MigrationId::ResourceTagConditionKeys => Migration::Applies { + kind: Kind::Breaking, + convergent: true, + bodies: vec![StoreBody::code( + StoreKind::Catalog, + &ResourceTagConditionKeys, + m0002_resource_tag_condition_keys::SOURCE, + )], + }, + MigrationId::SortKeyDefinitions => Migration::NotApplicable( + "the Cassandra backend first shipped in v0.1.14, after the #259 fix in v0.1.5, so no \ + Cassandra catalog holds a sort key definition the pre-fix UpdateTable deleted", + ), + MigrationId::IndexTableBaseKey => Migration::NotApplicable( + "Cassandra index tables are keyed by their own primary key, so there is no base key \ + index to add", + ), + } +} + +/// The catalog status. Read-only: no lock is taken and nothing is written, +/// so it runs beside a server. A catalog keyspace that does not exist, or +/// that holds no ledger and no `catalog_version` row, is `Uninitialized`. +/// +/// # Errors +/// +/// A read fails, including the ledger of an account keyspace, which the +/// error names. +pub async fn catalog_status( + session: Arc, + keyspace_prefix: &str, +) -> Result { + let migrations = CassandraMigrations::for_status(session, keyspace_prefix); + // A read records nothing, so no release is named. + Runner::new(&migrations, "").status().await +} + +/// The catalog status, read as the holder of `lock`, which `migrate --yes` +/// and `--forget` read before they act: each request a code body's `detect` +/// sends goes through the leased management store, which checks the lease +/// before it and records progress when it completes, and each ledger read's +/// schema-agreement wait records progress. A status that runs migration 2's +/// `detect` over every principal of a large catalog then keeps the lock +/// past the stall, where a read through [`catalog_status`] records nothing +/// and the renewal stops after 30 s. +/// +/// # Errors +/// +/// As [`catalog_status`], or the lock's deadline passed during the read. +pub async fn status_holding( + lock: &MigrationLock, + keyspace_prefix: &str, +) -> Result { + let migrations = CassandraMigrations::locked(lock, keyspace_prefix); + Runner::new(&migrations, "").status().await +} + +/// Serializes the account migrations a process runs (#453): two +/// CreateAccount calls for the same account run the same DDL one after the +/// other, never at once. +static ACCOUNT_MIGRATIONS: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + +/// Run every account migration this binary carries on `keyspace`, which +/// must exist, through the runner: each body's statements with schema +/// agreement after every DDL statement, and its ledger row after it. +/// CreateAccount and `init`'s default account call it once the keyspace +/// exists, before the account is usable. The catalog's rows are read and +/// none is written. +/// +/// # Errors +/// +/// The catalog's status forbids a run, or a body fails. +pub async fn migrate_account( + session: Arc, + keyspace_prefix: &str, + keyspace: &str, + release: &str, +) -> Result { + let _serialized = ACCOUNT_MIGRATIONS.lock().await; + let migrations = CassandraMigrations::for_account(session, keyspace_prefix, keyspace); + Runner::new(&migrations, release).migrate().await +} + +/// [`migrate_account`] under the lease of the migration lock the caller +/// holds: `init`'s default account. Every write is checked against the +/// lease, every completed request records progress on it, and each schema +/// statement gets the bound of one sent under the lock. +/// +/// # Errors +/// +/// The lease has lapsed, the catalog's status forbids a run, or a body +/// fails. +pub(crate) async fn migrate_account_under( + session: Arc, + keyspace_prefix: &str, + keyspace: &str, + release: &str, + lease: LeaseHandle, +) -> Result { + let _serialized = ACCOUNT_MIGRATIONS.lock().await; + let migrations = + CassandraMigrations::for_account_under(session, keyspace_prefix, keyspace, lease); + Runner::new(&migrations, release).migrate().await +} + +/// [`migrate_account_under`] with a lease whose deadline has already +/// passed: for the test that an account run under a lapsed lease writes +/// nothing. +/// +/// # Errors +/// +/// As [`migrate_account_under`]; with a lapsed lease, the first write. +#[cfg(any(test, feature = "test-util"))] +pub async fn migrate_account_under_lapsed_lease( + session: Arc, + keyspace_prefix: &str, + keyspace: &str, +) -> Result { + let lease = LeaseHandle::lapsed(&format!("{keyspace_prefix}_catalog")); + migrate_account_under(session, keyspace_prefix, keyspace, "test", lease).await +} + +/// The bound a schema statement gets in a run under `lock`, in `init`'s +/// default-account run under `lock`'s lease, and in CreateAccount's run, +/// which holds neither: for the test that pins which run gets which bound. +#[cfg(any(test, feature = "test-util"))] +#[must_use] +pub fn schema_statement_bounds( + lock: &MigrationLock, + keyspace_prefix: &str, +) -> [std::time::Duration; 3] { + let account = format!("{keyspace_prefix}_account_000000000000"); + [ + CassandraMigrations::locked(lock, keyspace_prefix).ddl_timeout(), + CassandraMigrations::for_account_under( + lock.session().clone(), + keyspace_prefix, + &account, + lock.handle(), + ) + .ddl_timeout(), + CassandraMigrations::for_account(lock.session().clone(), keyspace_prefix, &account) + .ddl_timeout(), + ] +} + +/// Take the migration lock on the deployment `keyspace_prefix` names, apply +/// every pending migration to its catalog keyspace alone, and release the +/// lock. For tests that build a catalog without `init` and create their +/// account keyspaces with [`migrate_account`]; the shared test catalogs hold +/// `accounts` rows whose keyspaces the tests have dropped. +/// +/// # Errors +/// +/// The lock cannot be taken within `timeout`, or as [`Runner::migrate`]. +#[cfg(any(test, feature = "test-util"))] +pub async fn migrate_catalog_keyspace( + session: Arc, + keyspace_prefix: &str, + release: &str, + timeout: std::time::Duration, +) -> Result { + let catalog = format!("{keyspace_prefix}_catalog"); + create_lock_table(&session, &catalog).await?; + let lock = MigrationLock::acquire(session, &catalog, LeaseTimes::DEFAULT, timeout).await?; + let result = { + let migrations = CassandraMigrations::locked_catalog_only(&lock, keyspace_prefix); + Runner::new(&migrations, release).migrate().await + }; + lock.release().await; + result +} + +/// Wait until every live node reports the same schema version, as the +/// runner does after each DDL statement: for tests that create keyspaces +/// and tables themselves before a step that reads them. +/// +/// # Errors +/// +/// The nodes did not agree within [`AGREEMENT_TIMEOUT`]. +#[cfg(any(test, feature = "test-util"))] +pub async fn wait_for_schema_agreement(session: &CassandraSession) -> Result<(), MigrationError> { + agreement::wait_for_agreement(session, AGREEMENT_TIMEOUT).await +} + +/// Run migration 1's body for stores of `kind` in `keyspace` and record +/// nothing, as a release before 1.0 left a catalog with these tables and no +/// ledger: for the tests of adoption. +/// +/// # Errors +/// +/// A statement or the agreement wait fails. +#[cfg(any(test, feature = "test-util"))] +pub async fn run_baseline_unrecorded( + session: &CassandraSession, + keyspace: &str, + kind: StoreKind, +) -> Result<(), MigrationError> { + let baseline = migration(MigrationId::Baseline); + let Some(extenddb_storage::migrations::Body::File { name, text }) = baseline + .bodies() + .iter() + .find(|b| b.store == kind) + .map(|b| &b.body) + else { + return Err(MigrationError::Store(format!( + "migration 1 has no file body for the {} store", + kind.as_str() + ))); + }; + for statement in cql::statements(keyspace, name, text)? { + // No lock is held here, so a schema statement gets the bound of one + // outside the lock. + let bound = if statement.ddl { + connect::UNLOCKED_DDL_TIMEOUT + } else { + connect::REQUEST_TIMEOUT + }; + connect::send_within( + session, + &statement.text, + cdrs_tokio::query_values!(), + cdrs_tokio::consistency::Consistency::Quorum, + None, + name, + bound, + ) + .await?; + if statement.ddl { + agreement::wait_for_agreement(session, AGREEMENT_TIMEOUT).await?; + } + } + Ok(()) +} + +#[cfg(test)] +mod tests; + +/// Send one statement that takes no values, bounded by the request timeout: +/// `init`'s grants, which run outside the runner. +/// +/// # Errors +/// +/// The statement fails or gets no answer within the timeout. +pub(crate) async fn send_statement( + session: &CassandraSession, + statement: &str, + context: &str, +) -> Result<(), MigrationError> { + connect::send( + session, + statement, + cdrs_tokio::query_values!(), + cdrs_tokio::consistency::Consistency::Quorum, + None, + context, + ) + .await + .map(drop) +} + +/// The bound on [`create_lock_table`]'s CREATE TABLE. The statement runs +/// before the lock exists, because the lock is a row of that table, so it +/// holds no lease and gets the bound of a schema statement outside the lock. +const SETTINGS_TABLE_BOUND: std::time::Duration = connect::UNLOCKED_DDL_TIMEOUT; + +/// Create the catalog's `settings` table, where the migration lock lives, +/// with `IF NOT EXISTS`, waiting for schema agreement before the statement +/// (the keyspace was just created) and after it: `init` runs this before it +/// takes the lock, because the lock is a row of that table. The statement is +/// bounded by [`SETTINGS_TABLE_BOUND`]. +/// +/// # Errors +/// +/// The statement or the agreement wait fails. +pub async fn create_lock_table( + session: &CassandraSession, + keyspace: &str, +) -> Result<(), MigrationError> { + // The keyspace was created just before, and the CREATE TABLE can reach + // a node that does not have it yet (#637). + agreement::wait_for_agreement(session, AGREEMENT_TIMEOUT).await?; + connect::send_within( + session, + &format!( + "CREATE TABLE IF NOT EXISTS {keyspace}.settings (key text PRIMARY KEY, value text)" + ), + cdrs_tokio::query_values!(), + cdrs_tokio::consistency::Consistency::Quorum, + None, + "create the settings table", + SETTINGS_TABLE_BOUND, + ) + .await?; + agreement::wait_for_agreement(session, AGREEMENT_TIMEOUT).await +} + +/// What an uninitialized `keyspace` holds that makes it another program's: +/// the tables migration 1 does not create, a `settings` table without the +/// catalog's `key` and `value` columns, and `settings` rows other than the +/// migration lock's. Empty when the keyspace does not exist, and for +/// one an `init` created and stopped in before migration 1 recorded +/// anything, which holds only a catalog's tables (migration 1 converges +/// them) and at most the lock row. Read before anything is written to the +/// keyspace: `init` and the migration lock refuse it when this is not +/// empty, so another program's `settings` table never receives the lock +/// row. Call it only for a keyspace whose status is `Uninitialized`, since +/// a catalog's `settings` table holds many rows. +/// +/// # Errors +/// +/// `system_schema` or the `settings` table cannot be read. +pub async fn foreign_contents( + session: &CassandraSession, + keyspace: &str, +) -> Result, MigrationError> { + let expected: postcheck::Definitions = serde_json::from_str(CATALOG_EXPECTED) + .map_err(|e| MigrationError::Store(format!("read the expected definitions: {e}")))?; + let ours: std::collections::BTreeSet<&str> = expected + .columns + .keys() + .filter_map(|k| k.split_once('.').map(|(t, _)| t)) + .collect(); + let tables = connect::tables_of(session, keyspace).await?; + let mut foreign: Vec = tables + .iter() + .filter(|t| *t != "extenddb_migrations" && !ours.contains(t.as_str())) + .map(|t| format!("the table {t}")) + .collect(); + if !connect::has_columns(session, keyspace, "settings", &["key", "value"]).await? { + if tables.iter().any(|t| t == "settings") { + foreign + .push("the table settings without the catalog's key and value columns".to_owned()); + } + } else { + let found = connect::rows( + session, + &format!("SELECT key FROM {keyspace}.settings LIMIT 3"), + cdrs_tokio::query_values!(), + cdrs_tokio::consistency::Consistency::Quorum, + &format!("read the settings rows of {keyspace}"), + ) + .await?; + for row in &found { + let key: String = connect::column(row, "key", "read the settings rows")?; + if key != locks::LOCK_KEY { + foreign.push(format!("the settings row '{key}'")); + } + } + } + Ok(foreign) +} + +/// The authorizer the node runs, read from `system_views.settings` +/// (Cassandra 4.0 and later). +/// +/// # Errors +/// +/// The setting cannot be read. +pub async fn authorizer(session: &CassandraSession) -> Result { + let found = connect::rows( + session, + "SELECT value FROM system_views.settings WHERE name = 'authorizer'", + cdrs_tokio::query_values!(), + cdrs_tokio::consistency::Consistency::One, + "read the authorizer from system_views.settings, which Cassandra 4.0 and later have", + ) + .await?; + let value = found + .first() + .map(|row| connect::nullable::(row, "value", "read the authorizer")) + .transpose()? + .flatten(); + value.ok_or_else(|| { + MigrationError::Store("system_views.settings has no authorizer row".to_owned()) + }) +} + +/// The statements that grant `role` what `migrate`, `serve` and +/// CreateAccount need on `keyspace` under `CassandraAuthorizer`: SELECT, +/// MODIFY, CREATE and ALTER; on an account keyspace DROP as well, because +/// DeleteTable drops the account's item and index tables and DeleteAccount +/// drops the keyspace; on the catalog keyspace CREATE on ALL KEYSPACES, for +/// the keyspaces CreateAccount creates. +pub(crate) fn grants(keyspace: &str, role: &str, account: bool) -> Vec { + let mut permissions = vec!["SELECT", "MODIFY", "CREATE", "ALTER"]; + if account { + permissions.push("DROP"); + } + let mut statements: Vec = permissions + .iter() + .map(|p| format!("GRANT {p} ON KEYSPACE {keyspace} TO '{role}'")) + .collect(); + if !account { + statements.push(format!("GRANT CREATE ON ALL KEYSPACES TO '{role}'")); + } + statements +} + +/// Send [`grants`] for `role` on `keyspace`, each bounded by the request +/// timeout, unless the node runs `AllowAllAuthorizer`, which supports no +/// GRANT and allows everything. Returns the authorizer when it granted. +/// A keyspace belongs to the role that created it, so `init`, which creates +/// the catalog and the default account's keyspace as the admin role, and a +/// run as an admin role that creates a missing account keyspace grant the +/// application role its permissions with this. Under a lease, `check` runs +/// before the authorizer read and before each GRANT, and `progress` after +/// each request completes, so a holder whose lease has lapsed sends no +/// further GRANT. +/// +/// # Errors +/// +/// `check` fails, the authorizer cannot be read, or a GRANT fails. +pub(crate) async fn grant_role( + session: &CassandraSession, + keyspace: &str, + role: &str, + account: bool, + check: impl Fn() -> Result<(), MigrationError>, + progress: impl Fn(), +) -> Result, MigrationError> { + check()?; + let authorizer = authorizer(session).await?; + progress(); + if authorizer.ends_with("AllowAllAuthorizer") { + return Ok(None); + } + for statement in grants(keyspace, role, account) { + check()?; + send_statement(session, &statement, "grant the application role") + .await + .map_err(|e| { + MigrationError::Store(format!("{statement} failed under {authorizer}: {e}")) + })?; + progress(); + } + Ok(Some(authorizer)) +} + +/// Register a server and read the catalog's status, before anything opens +/// the engine, so a server that refuses has written nothing else. The +/// registration is written at `QUORUM` first, then the migration lock is +/// read at `SERIAL` and the ledgers at `QUORUM`: a migration that starts in +/// between waits for this server, and one that holds the lock makes it exit +/// 6. A catalog without the registration table has not had migration 1, and +/// its status says what to run. +/// +/// # Errors +/// +/// [`MigrationError::Locked`] (exit 6) while a migration holds the lock; +/// [`MigrationError::Status`] for a status a server refuses; or a read or +/// write failed. +pub async fn register_server( + session: Arc, + keyspace_prefix: &str, + host: &str, + release: &str, +) -> Result { + let catalog = format!("{keyspace_prefix}_catalog"); + agreement::wait_for_agreement(&session, AGREEMENT_TIMEOUT).await?; + if !connect::table_exists(&session, &catalog, "extenddb_servers").await? { + let status = catalog_status(session, keyspace_prefix).await?; + return Err(match status { + CatalogStatus::Current { .. } | CatalogStatus::AheadCompatible { .. } => { + MigrationError::Store(format!( + "{catalog}.extenddb_servers, which migration 1 creates, does not exist" + )) + } + status => MigrationError::Status(status), + }); + } + let registration = + ServingRegistration::register(session.clone(), &catalog, host, release).await?; + let checked = async { + if let Some(holder) = locks::lock_holder(&session, &catalog).await? { + return Err(MigrationError::Locked(format!( + "a migration holds the catalog's migration lock (holder {holder}); start the \ + server again when it finishes" + ))); + } + match catalog_status(session.clone(), keyspace_prefix).await? { + status @ CatalogStatus::AheadCompatible { .. } => { + tracing::warn!( + "{status}; serving it, because a binary may serve migrations it does not \ + know when every one of them is additive" + ); + Ok(()) + } + CatalogStatus::Current { .. } => Ok(()), + status => Err(MigrationError::Status(status)), + } + } + .await; + match checked { + Ok(()) => Ok(registration), + Err(e) => { + registration.release().await; + Err(e) + } + } +} diff --git a/crates/storage-cassandra/src/migrations/postcheck.rs b/crates/storage-cassandra/src/migrations/postcheck.rs new file mode 100644 index 000000000..f3dbfec99 --- /dev/null +++ b/crates/storage-cassandra/src/migrations/postcheck.rs @@ -0,0 +1,262 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The post-check of migration 1: a keyspace's definitions in +//! `system_schema` compared with the ones migration 1 creates in an empty +//! keyspace, embedded from `0001_baseline.expected.json` beside each file. +//! +//! A definition is a column's kind (partition key, clustering, regular, +//! static), position, clustering order and type, and a secondary index's +//! table, kind and target; column order in the table is ignored. Every +//! object the expected file names is checked, and objects it does not name +//! are ignored, because account keyspaces also hold the per-table tables +//! that CreateTable makes. The definitions are read from `system_schema`, +//! never from the data directory: two coordinators creating the same table +//! at once can leave an orphan table directory on disk while `system_schema` +//! holds one table id per name. A difference stops migration 1 and names the +//! object, before its ledger row is written, so the next run applies it +//! again. + +use std::collections::{BTreeMap, HashMap}; + +use cdrs_tokio::consistency::Consistency; +use extenddb_storage::migrations::MigrationError; +use serde::{Deserialize, Serialize}; + +use super::connect::{column, rows, text_map}; +use crate::cassandra_util::CassandraSession; + +/// One column's definition. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Column { + pub kind: String, + pub position: i32, + pub clustering_order: String, + #[serde(rename = "type")] + pub data_type: String, +} + +/// One secondary index's definition. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Index { + pub table: String, + pub kind: String, + pub target: String, +} + +/// A keyspace's definitions: columns keyed `table.column`, indexes by name. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct Definitions { + pub columns: BTreeMap, + pub indexes: BTreeMap, +} + +/// The runner's ledger, which no migration body creates. +const LEDGER: &str = "extenddb_migrations"; + +/// The definitions of `keyspace` in `system_schema`, the runner's ledger +/// left out. +/// +/// # Errors +/// +/// A read of `system_schema` fails. +pub async fn describe( + session: &CassandraSession, + keyspace: &str, +) -> Result { + let mut definitions = Definitions::default(); + let found = rows( + session, + "SELECT table_name, column_name, kind, position, clustering_order, type FROM \ + system_schema.columns WHERE keyspace_name = ?", + cdrs_tokio::query_values!(keyspace), + Consistency::One, + &format!("read the columns of {keyspace} from system_schema"), + ) + .await?; + for row in &found { + let context = "read system_schema.columns"; + let table: String = column(row, "table_name", context)?; + if table == LEDGER { + continue; + } + let name: String = column(row, "column_name", context)?; + definitions.columns.insert( + format!("{table}.{name}"), + Column { + kind: column(row, "kind", context)?, + position: column(row, "position", context)?, + clustering_order: column(row, "clustering_order", context)?, + data_type: column(row, "type", context)?, + }, + ); + } + let found = rows( + session, + "SELECT table_name, index_name, kind, options FROM system_schema.indexes WHERE \ + keyspace_name = ?", + cdrs_tokio::query_values!(keyspace), + Consistency::One, + &format!("read the indexes of {keyspace} from system_schema"), + ) + .await?; + for row in &found { + let context = "read system_schema.indexes"; + let options: HashMap = text_map(row, "options", context)?; + definitions.indexes.insert( + column(row, "index_name", context)?, + Index { + table: column(row, "table_name", context)?, + kind: column(row, "kind", context)?, + target: options.get("target").cloned().unwrap_or_default(), + }, + ); + } + Ok(definitions) +} + +/// The differences between `expected` and `actual`, one line per object: +/// a column or index of `expected` missing or defined otherwise, and a +/// column `actual` has in a table `expected` names that `expected` lacks. +#[must_use] +pub fn compare(expected: &Definitions, actual: &Definitions) -> Vec { + let mut differences = Vec::new(); + for (key, want) in &expected.columns { + match actual.columns.get(key) { + None => differences.push(format!("column {key} is missing")), + Some(found) if found == want => {} + Some(found) => { + differences.push(format!("column {key} is {found:?}, expected {want:?}")); + } + } + } + let named = |key: &str| { + let table = key.split_once('.').map_or(key, |(t, _)| t); + expected + .columns + .keys() + .any(|k| k.split_once('.').is_some_and(|(t, _)| t == table)) + }; + for key in actual.columns.keys() { + if named(key) && !expected.columns.contains_key(key) { + differences.push(format!("column {key} is not part of the v1.0 schema")); + } + } + for (name, want) in &expected.indexes { + match actual.indexes.get(name) { + None => differences.push(format!("index {name} is missing")), + Some(found) if found == want => {} + Some(found) => { + differences.push(format!("index {name} is {found:?}, expected {want:?}")); + } + } + } + differences +} + +/// Compare `keyspace` with `expected_json`; stop naming every difference. +/// +/// # Errors +/// +/// The expected file cannot be read, `system_schema` cannot be read, or the +/// keyspace differs. +pub(crate) async fn check( + session: &CassandraSession, + expected_json: &str, + keyspace: &str, +) -> Result<(), MigrationError> { + let expected: Definitions = serde_json::from_str(expected_json).map_err(|e| { + MigrationError::Store(format!( + "the expected definitions embedded in this binary cannot be read: {e}" + )) + })?; + let actual = describe(session, keyspace).await?; + let differences = compare(&expected, &actual); + if differences.is_empty() { + return Ok(()); + } + Err(MigrationError::Store(format!( + "migration 1 left {keyspace} different from the v1.0 schema, so it was not recorded: {}", + differences.join("; ") + ))) +} + +#[cfg(test)] +mod tests { + use super::{Column, Definitions, Index, compare}; + + fn col(kind: &str, data_type: &str) -> Column { + Column { + kind: kind.to_owned(), + position: -1, + clustering_order: "none".to_owned(), + data_type: data_type.to_owned(), + } + } + + fn expected() -> Definitions { + let mut d = Definitions::default(); + d.columns.insert("t.k".into(), col("partition_key", "text")); + d.columns.insert("t.v".into(), col("regular", "text")); + d.indexes.insert( + "i".into(), + Index { + table: "t".into(), + kind: "COMPOSITES".into(), + target: "v".into(), + }, + ); + d + } + + #[test] + fn the_check_names_every_difference_and_ignores_unnamed_tables() { + let want = expected(); + let mut same = want.clone(); + // A table the expected file does not name (a per-table table) is + // ignored. + same.columns + .insert("items_x.pk".into(), col("partition_key", "blob")); + assert!(compare(&want, &same).is_empty()); + + let mut other = want.clone(); + other.columns.remove("t.v"); + other + .columns + .insert("t.k".into(), col("partition_key", "int")); + other + .columns + .insert("t.extra".into(), col("regular", "text")); + other.indexes.clear(); + let differences = compare(&want, &other); + assert_eq!(differences.len(), 4, "{differences:?}"); + for needle in [ + "t.v is missing", + "t.k is", + "t.extra is not part", + "index i is missing", + ] { + assert!( + differences.iter().any(|d| d.contains(needle)), + "{needle}: {differences:?}" + ); + } + } + + #[test] + fn the_embedded_expected_files_can_be_read() { + for text in [ + super::super::CATALOG_EXPECTED, + super::super::ACCOUNT_EXPECTED, + ] { + let definitions: Definitions = serde_json::from_str(text).expect("expected json"); + assert!(!definitions.columns.is_empty()); + assert!( + !definitions + .columns + .keys() + .any(|k| k.starts_with("extenddb_migrations.")) + ); + } + } +} diff --git a/crates/storage-cassandra/src/migrations/registrations.rs b/crates/storage-cassandra/src/migrations/registrations.rs new file mode 100644 index 000000000..3d30a9879 --- /dev/null +++ b/crates/storage-cassandra/src/migrations/registrations.rs @@ -0,0 +1,312 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The serving registration. +//! +//! A server registers in `extenddb_servers`: one row, written at `QUORUM` +//! with a TTL of 45 s and again every 15 s at `LOCAL_QUORUM`. A server that +//! cannot renew for 30 s stops (before its row expires), so `migrate --yes`, +//! which waits while a row exists, cannot run beside it. + +use std::sync::Arc; +use std::time::Duration; + +use cdrs_tokio::consistency::Consistency; +use extenddb_storage::migrations::MigrationError; +use tokio::sync::watch; +use tokio::task::JoinHandle; +use tokio::time::Instant; + +use super::connect::{nullable, rows, send, table_exists}; +use super::locks::MigrationLock; +use crate::cassandra_util::CassandraSession; + +/// The times of a serving registration. +const REGISTRATION_TTL_SECS: i32 = 45; +const REGISTRATION_RENEW: Duration = Duration::from_secs(15); +const REGISTRATION_LOST_AFTER: Duration = Duration::from_secs(30); + +/// One running server, as `migrate --yes` names it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct Server { + pub(crate) host: String, + pub(crate) pid: i32, + pub(crate) release: String, + pub(crate) started_at: String, +} + +impl std::fmt::Display for Server { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "extenddb serve {} on {} (process {}, started {})", + self.release, self.host, self.pid, self.started_at + ) + } +} + +/// The servers registered in `keyspace`, read at `ALL`: every replica of +/// the catalog must answer. A catalog without the table has none. +/// +/// # Errors +/// +/// A replica did not answer. +pub(crate) async fn registered_servers( + session: &CassandraSession, + keyspace: &str, +) -> Result, MigrationError> { + if !table_exists(session, keyspace, "extenddb_servers").await? { + return Ok(Vec::new()); + } + let found = rows( + session, + &format!("SELECT host, pid, release, started_at FROM {keyspace}.extenddb_servers"), + cdrs_tokio::query_values!(), + Consistency::All, + "read the running servers at consistency ALL", + ) + .await?; + found + .iter() + .map(|row| { + Ok(Server { + host: nullable::(row, "host", "read a server")?.unwrap_or_default(), + pid: nullable::(row, "pid", "read a server")?.unwrap_or_default(), + release: nullable::(row, "release", "read a server")?.unwrap_or_default(), + started_at: nullable::(row, "started_at", "read a server")? + .unwrap_or_default(), + }) + }) + .collect() +} + +/// Wait, while holding `lock`, for every registered server to go away, for +/// up to `timeout`. +/// +/// # Errors +/// +/// [`MigrationError::Locked`] (exit 6) naming the servers, or the error of +/// the read at `ALL`, when `timeout` passes first. +pub(crate) async fn wait_for_no_servers( + lock: &MigrationLock, + timeout: Duration, +) -> Result<(), MigrationError> { + let started = Instant::now(); + loop { + lock.check()?; + let read = registered_servers(lock.session(), lock.keyspace()).await; + lock.progress(); + let outcome = match read { + Ok(servers) if servers.is_empty() => return Ok(()), + Ok(servers) => format!( + "{} server(s) serve this catalog: {}. Stop them, then run the migration again", + servers.len(), + servers + .iter() + .map(ToString::to_string) + .collect::>() + .join("; ") + ), + Err(e) => format!( + "the running servers could not be read at consistency ALL ({e}); a migration \ + needs every replica of {} reachable. Nodes the driver reports down: {}", + lock.keyspace(), + down_nodes(lock.session()) + ), + }; + if started.elapsed() >= timeout { + return Err(MigrationError::Locked(format!( + "after waiting {} s, {outcome}", + timeout.as_secs() + ))); + } + tokio::time::sleep(Duration::from_secs(1)).await; + } +} + +/// The nodes the driver reports down, or "none". +fn down_nodes(session: &CassandraSession) -> String { + let metadata = session.cluster_metadata(); + let mut down: Vec = metadata + .nodes() + .values() + .filter(|n| n.state() == cdrs_tokio::cluster::topology::NodeState::Down) + .map(|n| n.broadcast_rpc_address().to_string()) + .collect(); + down.sort(); + if down.is_empty() { + "none".to_owned() + } else { + down.join(", ") + } +} + +/// A server's registration, renewed until it is released or lost. +pub struct ServingRegistration { + session: Arc, + keyspace: String, + server_id: String, + lost: watch::Receiver>, + renewal: JoinHandle<()>, +} + +impl ServingRegistration { + /// Write this server's row at `QUORUM`, so starting needs a majority of + /// the catalog's replicas, as migrating does, and start renewing it. + /// + /// # Errors + /// + /// The write fails. + pub async fn register( + session: Arc, + keyspace: &str, + host: &str, + release: &str, + ) -> Result { + let server_id = uuid::Uuid::new_v4().to_string(); + let started_at = chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true); + let pid = i32::try_from(std::process::id()).unwrap_or(i32::MAX); + let cql = format!( + "INSERT INTO {keyspace}.extenddb_servers (server_id, host, pid, release, started_at) \ + VALUES (?, ?, ?, ?, ?) USING TTL {REGISTRATION_TTL_SECS}" + ); + let values = || { + cdrs_tokio::query_values!(server_id.as_str(), host, pid, release, started_at.as_str()) + }; + send( + &session, + &cql, + values(), + Consistency::Quorum, + None, + "register this server in extenddb_servers", + ) + .await?; + let (lost_tx, lost) = watch::channel(None); + let renewal = tokio::spawn(renew_registration(session.clone(), cql, values(), lost_tx)); + Ok(Self { + session, + keyspace: keyspace.to_owned(), + server_id, + lost, + renewal, + }) + } + + /// Changes to `Some(reason)` when the registration could not be renewed + /// for 30 s. + #[must_use] + pub fn lost(&self) -> watch::Receiver> { + self.lost.clone() + } + + /// Stop renewing and delete the row, so a migration need not wait for + /// its TTL. + pub async fn release(&self) { + self.renewal.abort(); + let cql = format!( + "DELETE FROM {}.extenddb_servers WHERE server_id = ?", + self.keyspace + ); + if let Err(e) = send( + &self.session, + &cql, + cdrs_tokio::query_values!(self.server_id.as_str()), + Consistency::Quorum, + None, + "remove this server from extenddb_servers", + ) + .await + { + tracing::warn!("{e}; the row expires within {REGISTRATION_TTL_SECS} s"); + } + } +} + +impl Drop for ServingRegistration { + fn drop(&mut self) { + self.renewal.abort(); + } +} + +async fn renew_registration( + session: Arc, + cql: String, + values: cdrs_tokio::query::QueryValues, + lost: watch::Sender>, +) { + let mut last_ok = Instant::now(); + let mut next = last_ok + REGISTRATION_RENEW; + loop { + tokio::time::sleep_until(next.min(last_ok + REGISTRATION_LOST_AFTER)).await; + if last_ok.elapsed() >= REGISTRATION_LOST_AFTER { + let _ = lost.send(Some(format!( + "the serving registration was not renewed for {} s", + REGISTRATION_LOST_AFTER.as_secs() + ))); + return; + } + if Instant::now() < next { + continue; + } + let sent = Instant::now(); + // LOCAL_QUORUM, so a running server stays up while a remote + // datacenter is unreachable. + match send( + &session, + &cql, + values.clone(), + Consistency::LocalQuorum, + None, + "renew the serving registration", + ) + .await + { + Ok(_) => { + last_ok = sent; + next = sent + REGISTRATION_RENEW; + } + Err(e) => { + tracing::warn!("{e}; retrying"); + next = Instant::now() + REGISTRATION_RENEW / 3; + } + } + } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + #[test] + fn a_server_stops_before_its_row_expires() { + assert!( + super::REGISTRATION_LOST_AFTER + < Duration::from_secs(super::REGISTRATION_TTL_SECS as u64) + ); + assert!(super::REGISTRATION_RENEW < super::REGISTRATION_LOST_AFTER); + } + + /// The times ADR-0023 (decision 5) and the module documentation give the + /// lock and the registration: a 60 s lock row renewed every 15 s, acted + /// on until 25 s before it expires and no longer renewed after 30 s + /// without progress; a 45 s registration renewed every 15 s, whose server + /// stops after 30 s without a renewal. + #[test] + fn the_lease_and_registration_times_are_adr_0023s() { + use crate::migrations::LeaseTimes; + + assert_eq!( + LeaseTimes::DEFAULT, + LeaseTimes { + ttl: Duration::from_secs(60), + renew_every: Duration::from_secs(15), + margin: Duration::from_secs(25), + stall: Duration::from_secs(30), + } + ); + assert_eq!(super::REGISTRATION_TTL_SECS, 45); + assert_eq!(super::REGISTRATION_RENEW, Duration::from_secs(15)); + assert_eq!(super::REGISTRATION_LOST_AFTER, Duration::from_secs(30)); + } +} diff --git a/crates/storage-cassandra/src/migrations/tests.rs b/crates/storage-cassandra/src/migrations/tests.rs new file mode 100644 index 000000000..e2943deb5 --- /dev/null +++ b/crates/storage-cassandra/src/migrations/tests.rs @@ -0,0 +1,140 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Tests of the Cassandra migration table that need no database. + +use extenddb_storage::migrations::{ + Body, Kind, Migration, MigrationId, RELEASED_THROUGH, StoreKind, check_released, + refuse_additive, +}; + +use super::{ACCOUNT_BASELINE, CATALOG_BASELINE, RELEASED, migration}; + +#[test] +fn the_released_bodies_match_their_checksums() { + let lines = check_released("cassandra", migration, RELEASED, RELEASED_THROUGH) + .unwrap_or_else(|e| panic!("{e}")); + println!("{lines}"); +} + +#[test] +fn no_migration_is_additive_before_the_old_binary_job() { + refuse_additive("cassandra", migration).unwrap_or_else(|e| panic!("{e}")); +} + +#[test] +fn every_migration_records_a_row_in_the_catalog_and_names_its_stores() { + for id in MigrationId::ALL { + let kinds = migration(id).store_kinds(); + assert_eq!(kinds.first(), Some(&StoreKind::Catalog), "{id}"); + assert!(!kinds.contains(&StoreKind::Data), "{id}"); + } + assert_eq!( + migration(MigrationId::Baseline).store_kinds(), + [StoreKind::Catalog, StoreKind::Account] + ); + assert_eq!( + migration(MigrationId::ResourceTagConditionKeys).store_kinds(), + [StoreKind::Catalog] + ); + for id in [ + MigrationId::SortKeyDefinitions, + MigrationId::IndexTableBaseKey, + ] { + assert!(matches!(migration(id), Migration::NotApplicable(_)), "{id}"); + } + for id in MigrationId::ALL { + if let Migration::Applies { kind, .. } = migration(id) { + assert_eq!(kind, Kind::Breaking, "{id}"); + } + } +} + +#[test] +fn the_baselines_write_no_setting_except_deleting_the_marker() { + for (store, text) in [("catalog", CATALOG_BASELINE), ("account", ACCOUNT_BASELINE)] { + let statements = super::cql::statements("ks", store, text).expect(store); + for statement in &statements { + let upper = statement.text.to_ascii_uppercase(); + if upper.contains("KS.SETTINGS") && !upper.starts_with("CREATE TABLE") { + assert_eq!( + statement.text, "DELETE FROM ks.settings WHERE key = 'resource_tag_migration'", + "{store}" + ); + } + if upper.starts_with("CREATE") { + assert!(upper.contains("IF NOT EXISTS"), "{}", statement.text); + } + if upper.starts_with("ALTER TABLE") { + assert!(upper.contains(" ADD IF NOT EXISTS "), "{}", statement.text); + } + } + } +} + +#[test] +fn the_baselines_record_every_legacy_file() { + // A release before 1.0 runs no legacy file once every version it knows is + // recorded in its ledger. + for version in 1..=5 { + assert!( + CATALOG_BASELINE.contains(&format!( + "INSERT INTO schema_history (version, description) VALUES ({version}, " + )), + "catalog {version}" + ); + } + for version in 1..=2 { + assert!( + ACCOUNT_BASELINE.contains(&format!( + "INSERT INTO schema_history (version, description) VALUES ({version}, " + )), + "account {version}" + ); + } + let baseline = migration(MigrationId::Baseline); + let Body::File { text, .. } = &baseline.bodies()[0].body else { + panic!("migration 1's catalog body is a file"); + }; + assert!(text.contains("CREATE TABLE IF NOT EXISTS extenddb_servers")); +} + +#[test] +fn a_statement_under_the_lock_is_bounded_inside_the_lease_margin() { + // A statement the holder sent before its deadline is done before its + // row can expire, so a second migrator never runs beside it (the + // module header's rule). CreateAccount holds no lease and gets the + // longer bound a loaded node needs. + let times = super::LeaseTimes::DEFAULT; + assert!(super::backend::ddl_bound(true) < times.margin); + // It is under the stall too, since the runner records progress when a + // statement completes, so a statement that takes the whole bound does + // not stop the renewal. + assert!(super::backend::ddl_bound(true) < times.stall); + assert_eq!( + super::backend::ddl_bound(true), + super::connect::LOCKED_DDL_TIMEOUT + ); + // Measured on one node under ten parallel migration tests: a table drop + // took 14.8 s, which the request timeout cut and this bound covers. + assert!(super::backend::ddl_bound(true) >= std::time::Duration::from_secs(15)); + assert!(super::backend::ddl_bound(true) > super::connect::REQUEST_TIMEOUT); + assert_eq!( + super::backend::ddl_bound(false), + super::connect::UNLOCKED_DDL_TIMEOUT + ); + assert!(super::backend::ddl_bound(false) > super::backend::ddl_bound(true)); +} + +#[test] +fn the_settings_table_is_created_with_the_bound_of_a_schema_statement_outside_the_lock() { + // init and migrate create the settings table before the lock exists, + // because the lock is a row of it, so the CREATE TABLE holds no lease. + // Under the request timeout it failed on a node shared with the direct + // integration suite: "create the settings table: no answer within 10 s". + assert_eq!( + super::SETTINGS_TABLE_BOUND, + super::connect::UNLOCKED_DDL_TIMEOUT + ); + assert!(super::SETTINGS_TABLE_BOUND > super::connect::REQUEST_TIMEOUT); +} diff --git a/crates/storage-cassandra/src/operations.rs b/crates/storage-cassandra/src/operations.rs index e33cd0838..078d3e568 100644 --- a/crates/storage-cassandra/src/operations.rs +++ b/crates/storage-cassandra/src/operations.rs @@ -11,18 +11,16 @@ pub struct CassandraOperationsEngine; impl OperationsEngine for CassandraOperationsEngine { fn parse_connection_string(&self, conn_str: &str) -> Result { - // Cassandra connection string format: "host1:9042,host2:9042/keyspace_prefix" - // Split on '/' to separate hosts from keyspace - let (hosts_str, keyspace) = if let Some((h, k)) = conn_str.split_once('/') { - (h, k.to_string()) - } else { - (conn_str, "extenddb".to_string()) - }; + // Cassandra connection string format: + // "[user[:password]@]host1:9042,host2:9042/keyspace_prefix" + let parsed = crate::config::CassandraStorageConfig::connection_parts(conn_str); + let keyspace = parsed.keyspace_prefix; // Parse the first contact point for compatibility with ConnectionParts - let first_contact = hosts_str - .split(',') - .next() + let first_contact = parsed + .contact_points + .first() + .map(String::as_str) .ok_or_else(|| StorageError::Internal("Empty connection string".to_string()))?; let parts: Vec<&str> = first_contact.split(':').collect(); @@ -42,15 +40,13 @@ impl OperationsEngine for CassandraOperationsEngine { host, port, database: format!("{keyspace}_catalog"), - user: String::new(), - password: String::new(), + user: parsed.username.unwrap_or_default(), + password: parsed.password.unwrap_or_default(), }) } fn redact_connection_string(&self, conn_str: &str) -> String { - // Cassandra connection strings don't contain passwords - // Just return as-is - conn_str.to_string() + crate::config::CassandraStorageConfig::redact_connection_string(conn_str) } fn validate_identifier(&self, name: &str, label: &str) -> Result<(), StorageError> { @@ -89,10 +85,6 @@ impl OperationsEngine for CassandraOperationsEngine { Ok(()) } - fn catalog_version(&self) -> String { - crate::CATALOG_VERSION.to_string() - } - fn setting_default(&self, key: &str) -> Option { use extenddb_core::settings_keys::INDEX_PROPAGATION_DELAY_MS; match key { diff --git a/crates/storage-cassandra/tests/common/mod.rs b/crates/storage-cassandra/tests/common/mod.rs index c1fd1e971..ea97c57ad 100644 --- a/crates/storage-cassandra/tests/common/mod.rs +++ b/crates/storage-cassandra/tests/common/mod.rs @@ -46,10 +46,17 @@ pub async fn flush_pending_drops(session: &Arc) { let _ = session.query(format!("DROP KEYSPACE IF EXISTS {ks}")).await; } } +/// The node the tests use: `EXTENDDB_TEST_CASSANDRA_CONTACT_POINT`, else +/// 127.0.0.1:9042, the port CI's service container publishes. +pub fn test_contact_point() -> String { + std::env::var("EXTENDDB_TEST_CASSANDRA_CONTACT_POINT") + .unwrap_or_else(|_| "127.0.0.1:9042".to_string()) +} + /// Returns a standard test configuration for Cassandra. pub fn test_config() -> CassandraStorageConfig { let mut config = CassandraStorageConfig { - contact_points: vec!["127.0.0.1:9042".to_string()], + contact_points: vec![test_contact_point()], username: Some("cassandra".to_string()), password: Some("cassandra".to_string()), keyspace_prefix: "extenddb_ttl_test".to_string(), @@ -86,13 +93,15 @@ pub async fn ensure_test_account( engine.create_keyspace(catalog_keyspace).await?; } ensure_keyspace_rf(engine, catalog_keyspace).await?; - // Always run migrations — idempotent, guards against concurrent creation races. - extenddb_storage_cassandra::migrations::run_catalog_migrations( - &engine.session_arc(), - catalog_keyspace, + // Always run the migrations: the runner applies only what is missing. + extenddb_storage_cassandra::migrations::migrate_catalog_keyspace( + engine.session_arc(), + "extenddb_ttl_test", + "test", + std::time::Duration::from_secs(120), ) .await - .map_err(|e| format!("Failed to run catalog migrations: {:?}", e))?; + .map_err(|e| format!("Failed to run catalog migrations: {e}"))?; if !engine.keyspace_exists(&account_keyspace).await? { engine.create_keyspace(&account_keyspace).await?; @@ -100,13 +109,15 @@ pub async fn ensure_test_account( } else { ensure_keyspace_rf(engine, &account_keyspace).await?; } - // Always run migrations — idempotent, guards against concurrent creation races. - extenddb_storage_cassandra::migrations::run_data_migrations( - &engine.session_arc(), + // Always run the migrations: the runner applies only what is missing. + extenddb_storage_cassandra::migrations::migrate_account( + engine.session_arc(), + "extenddb_ttl_test", &account_keyspace, + "test", ) .await - .map_err(|e| format!("Failed to run migrations: {:?}", e))?; + .map_err(|e| format!("Failed to run migrations: {e}"))?; let query = format!( "INSERT INTO {}.accounts (account_id, account_name, created_at) VALUES (?, ?, toTimestamp(now())) IF NOT EXISTS", @@ -612,10 +623,12 @@ pub async fn setup_engine() -> CassandraEngine { ensure_keyspace_rf(&engine, &catalog_keyspace) .await .unwrap(); - // Always run migrations — idempotent, guards against concurrent creation races. - extenddb_storage_cassandra::migrations::run_catalog_migrations( - &engine.session_arc(), - &catalog_keyspace, + // Always run the migrations: the runner applies only what is missing. + extenddb_storage_cassandra::migrations::migrate_catalog_keyspace( + engine.session_arc(), + &config.keyspace_prefix, + "test", + std::time::Duration::from_secs(120), ) .await .unwrap(); @@ -725,14 +738,16 @@ pub fn skip_without_cassandra() -> bool { if std::env::var("EXTENDDB_TEST_CASSANDRA").as_deref() == Ok("required") { return false; } - let reachable = std::net::TcpStream::connect_timeout( - &std::net::SocketAddr::from(([127, 0, 0, 1], 9042)), - std::time::Duration::from_secs(2), - ) - .is_ok(); + let contact_point = test_contact_point(); + let reachable = contact_point + .parse::() + .is_ok_and(|address| { + std::net::TcpStream::connect_timeout(&address, std::time::Duration::from_secs(2)) + .is_ok() + }); if !reachable { eprintln!( - "skipping: no Cassandra at 127.0.0.1:9042 (set EXTENDDB_TEST_CASSANDRA=required to fail instead)" + "skipping: no Cassandra at {contact_point} (set EXTENDDB_TEST_CASSANDRA=required to fail instead)" ); } !reachable diff --git a/crates/storage-cassandra/tests/direct/backup_engine.rs b/crates/storage-cassandra/tests/direct/backup_engine.rs index 3f0ef92de..2f2d042c4 100644 --- a/crates/storage-cassandra/tests/direct/backup_engine.rs +++ b/crates/storage-cassandra/tests/direct/backup_engine.rs @@ -1571,34 +1571,10 @@ async fn test_restore_of_deleted_backup_is_not_found() { // ── Isolated catalogs (own keyspace prefix) ──────────────────────────────── -/// Catalog migrations V001-V004: the catalog shape before V005 added -/// `backups_by_arn.definition/active_restores` and `tables.restore_*`. -const CATALOG_THROUGH_V004: &[(&str, &str)] = &[ - ( - "V001__initial_schema.cql", - include_str!("../../migrations/catalog/V001__initial_schema.cql"), - ), - ( - "V002__backup_restore.cql", - include_str!("../../migrations/catalog/V002__backup_restore.cql"), - ), - ( - "V003__account_scoped_idempotency.cql", - include_str!("../../migrations/catalog/V003__account_scoped_idempotency.cql"), - ), - ( - "V004__ttl_backfill_cursor.cql", - include_str!("../../migrations/catalog/V004__ttl_backfill_cursor.cql"), - ), -]; - /// An engine on its own keyspace prefix (so its sweeps and workers touch no -/// other test's tables) with one provisioned account. With `through_v004` -/// the catalog has run only catalog migrations V001-V004, as a catalog not -/// yet migrated by `extenddb migrate --yes`; otherwise every migration. +/// other test's tables) with one provisioned account. async fn isolated_engine( tag: &str, - through_v004: bool, ) -> ( std::sync::Arc, String, @@ -1621,41 +1597,22 @@ async fn isolated_engine( .await .unwrap(); } - if through_v004 { - // Statement by statement, each qualified with the catalog keyspace by - // the function the migration runner uses, recording versions 1-4 in - // schema_history as the runner did for a 0.0.3 catalog. - for (filename, sql) in CATALOG_THROUGH_V004 { - let statements = extenddb_storage_cassandra::migrations::migration_statements( - &catalog_ks, - filename, - sql, - ) - .expect(filename); - for statement in statements { - session.query(statement).await.expect("V001-V004 statement"); - } - } - for version in 1..=4_i32 { - session - .query_with_values( - format!( - "INSERT INTO {catalog_ks}.schema_history (version, description, applied_at) \ - VALUES (?, 'through_v004', toTimestamp(now()))" - ), - cdrs_tokio::query_values!(version), - ) - .await - .expect("record V001-V004"); - } - } else { - extenddb_storage_cassandra::migrations::run_catalog_migrations(&session, &catalog_ks) - .await - .unwrap(); - } - extenddb_storage_cassandra::migrations::run_data_migrations(&session, &account_ks) - .await - .unwrap(); + extenddb_storage_cassandra::migrations::migrate_catalog_keyspace( + session.clone(), + &config.keyspace_prefix, + "test", + std::time::Duration::from_secs(120), + ) + .await + .unwrap(); + extenddb_storage_cassandra::migrations::migrate_account( + session.clone(), + &config.keyspace_prefix, + &account_ks, + "test", + ) + .await + .unwrap(); session .query_with_values( format!( @@ -1669,199 +1626,6 @@ async fn isolated_engine( (engine, account_id, drop_keyspaces) } -/// The columns catalog migration V005 adds are absent from `keyspace`. -async fn assert_catalog_lacks_v005(engine: &CassandraEngine, keyspace: &str) { - let rows = extenddb_storage_cassandra::cassandra_util::query_rows::( - &engine.session_arc(), - "SELECT column_name FROM system_schema.columns WHERE keyspace_name = ?", - cdrs_tokio::query_values!(keyspace), - "columns", - ) - .await - .expect("read schema"); - let columns: Vec = rows - .iter() - .map(|row| { - extenddb_storage_cassandra::cassandra_util::get_column::( - row, - "column_name", - "columns", - ) - .unwrap() - }) - .collect(); - assert!(columns.iter().any(|c| c == "table_status"), "{columns:?}"); - for v005 in [ - "restore_source_backup_arn", - "restore_heartbeat", - "active_restores", - "definition", - ] { - assert!(!columns.iter().any(|c| c == v005), "{v005} present"); - } -} - -/// On a catalog that has not run V005, ordinary table and item operations -/// work; only backup and restore need V005. (CreateTable used to name the -/// V005 restore columns in its INSERT for every table, which CQL rejects.) -#[tokio::test] -async fn test_table_operations_work_on_a_catalog_without_v005() { - if crate::helpers::skip_without_cassandra() { - return; - } - let (engine, account_id, _drop_keyspaces) = isolated_engine("v4", true).await; - assert_catalog_lacks_v005(&engine, &engine.catalog_keyspace()).await; - - let created = engine - .create_table( - &account_id, - CreateTableInput { - table_name: "Unmigrated".to_owned(), - key_schema: vec![ks("id", KeyType::Hash)], - attribute_definitions: vec![ - attr("id", ScalarAttributeType::S), - attr("g", ScalarAttributeType::S), - ], - billing_mode: Some(BillingMode::PayPerRequest), - global_secondary_indexes: Some(vec![GsiInput { - index_name: "ByG".to_owned(), - key_schema: vec![ks("g", KeyType::Hash)], - projection: Projection { - projection_type: ProjectionType::All, - non_key_attributes: None, - }, - provisioned_throughput: None, - }]), - ..Default::default() - }, - ) - .await - .expect("CreateTable on a catalog without V005"); - assert!(created.restore_summary.is_none()); - activate_tables(&engine).await; - let described = describe(&engine, &account_id, "Unmigrated").await; - assert_eq!(described.table_status, TableStatus::Active); - assert!(described.restore_summary.is_none()); - - let key_info = engine - .table_key_info(&account_id, "Unmigrated") - .await - .expect("key info"); - engine - .put_item( - &key_info, - item("a", None, "v"), - false, - None, - &Default::default(), - None, - ) - .await - .expect("PutItem"); - let got = engine - .get_item(&key_info, &key("a", None)) - .await - .expect("GetItem"); - assert!(got.is_some()); - assert_eq!(scan_count(&engine, &key_info).await, 1); - - // Backup needs V005. CreateBackup, DeleteBackup and RestoreTableFromBackup - // are refused with a message naming the file and `migrate` (they failed - // with an internal `Undefined column name ...`); nothing is created. - let assert_refused = |what: &str, result: Result| match result { - Err(StorageError::Unsupported(m)) => { - assert!( - m.contains("V005__backup_definitions_and_restore_registration.cql") - && m.contains("extenddb migrate --yes") - && !m.contains("V004"), - "{what}: {m}" - ); - } - other => panic!("{what}: {other:?}"), - }; - assert_refused( - "CreateBackup", - engine - .create_backup(&account_id, "Unmigrated", "needs-v005") - .await - .map(|d| d.backup_arn), - ); - let some_arn = extenddb_storage::util::backup_arn( - "us-east-1", - &account_id, - "Unmigrated", - &extenddb_storage::util::new_backup_id(), - ); - assert_refused( - "DeleteBackup", - engine - .delete_backup(&account_id, &some_arn) - .await - .map(|d| d.backup_details.backup_arn), - ); - assert_refused( - "RestoreTableFromBackup", - engine - .restore_table_from_backup( - &account_id, - RestoreRequest { - backup_arn: some_arn.clone(), - source_table_arn: described.table_arn.clone(), - restore_date_time: 0.0, - input: CreateTableInput { - table_name: "UnmigratedTarget".to_owned(), - key_schema: vec![ks("id", KeyType::Hash)], - attribute_definitions: vec![attr("id", ScalarAttributeType::S)], - billing_mode: Some(BillingMode::PayPerRequest), - ..Default::default() - }, - }, - ) - .await - .map(|d| d.table_name), - ); - assert!( - engine - .describe_table( - &account_id, - DescribeTableInput { - table_name: "UnmigratedTarget".to_owned() - } - ) - .await - .is_err(), - "a refused restore creates no target" - ); - - // Liveness: once V005 is applied (as `migrate --yes` does, the runner - // skipping the recorded files), the same engine backs up without a - // restart. - extenddb_storage_cassandra::migrations::run_catalog_migrations( - &engine.session_arc(), - &engine.catalog_keyspace(), - ) - .await - .expect("apply V005"); - let backup = engine - .create_backup(&account_id, "Unmigrated", "after-v005") - .await - .expect("CreateBackup after V005"); - engine - .delete_backup(&account_id, &backup.backup_arn) - .await - .expect("DeleteBackup after V005"); - - engine - .delete_table( - &account_id, - DeleteTableInput { - table_name: "Unmigrated".to_owned(), - }, - ) - .await - .expect("DeleteTable on a catalog without V005"); -} - /// Back-date the heartbeat of the CREATING restore target `name` by two /// minutes and give it a foreign owner, as if the process copying into it had /// died; asserts the copy was still running (the LWT applied). @@ -1897,7 +1661,7 @@ async fn test_restore_sweep_worker_sweeps_at_startup_and_stops_on_shutdown() { if crate::helpers::skip_without_cassandra() { return; } - let (engine, account_id, _drop_keyspaces) = isolated_engine("sw", false).await; + let (engine, account_id, _drop_keyspaces) = isolated_engine("sw").await; let source = create_active_table( &engine, &account_id, @@ -1979,7 +1743,7 @@ async fn test_restore_heartbeat_runs_during_target_ddl() { if crate::helpers::skip_without_cassandra() { return; } - let (engine, account_id, _drop_keyspaces) = isolated_engine("hb", false).await; + let (engine, account_id, _drop_keyspaces) = isolated_engine("hb").await; let mut attribute_definitions = vec![ attr("id", ScalarAttributeType::S), attr("sort", ScalarAttributeType::S), diff --git a/crates/storage-cassandra/tests/direct/concurrent_account_creation.rs b/crates/storage-cassandra/tests/direct/concurrent_account_creation.rs index c84192525..77dbac2c4 100644 --- a/crates/storage-cassandra/tests/direct/concurrent_account_creation.rs +++ b/crates/storage-cassandra/tests/direct/concurrent_account_creation.rs @@ -24,7 +24,7 @@ mod tests { use std::sync::Arc; /// Every data table a fresh account keyspace must contain, from - /// migrations/data/V001__initial_data_schema.cql and V002__backup_items.cql. + /// migrations/account/0001_baseline.cql, migration 1's account body. const EXPECTED_DATA_TABLES: &[&str] = &[ "schema_history", "stream_shards", diff --git a/crates/storage-cassandra/tests/direct/keyspace_migrations.rs b/crates/storage-cassandra/tests/direct/keyspace_migrations.rs index eddd3f354..862ee4199 100644 --- a/crates/storage-cassandra/tests/direct/keyspace_migrations.rs +++ b/crates/storage-cassandra/tests/direct/keyspace_migrations.rs @@ -1,8 +1,8 @@ // Copyright 2026 ExtendDB contributors // SPDX-License-Identifier: Apache-2.0 -//! The migration runners create their tables, indexes and added columns in -//! the keyspace they are given, whatever keyspace the session has selected +//! The migration runner creates its tables, indexes and added columns in +//! the keyspace it is given, whatever keyspace the session has selected //! (#632). //! //! cdrs-tokio runs a `USE` on one connection and copies it to the session's @@ -14,6 +14,7 @@ use std::collections::BTreeSet; use std::sync::Arc; +use std::time::Duration; use cdrs_tokio::authenticators::StaticPasswordAuthenticatorProvider; use cdrs_tokio::cluster::NodeTcpConfigBuilder; @@ -23,7 +24,7 @@ use cdrs_tokio::load_balancing::RoundRobinLoadBalancingStrategy; use extenddb_storage::error::StorageError; use extenddb_storage_cassandra::CassandraSession; use extenddb_storage_cassandra::cassandra_util::{get_column, query_rows}; -use extenddb_storage_cassandra::migrations::{run_catalog_migrations, run_data_migrations}; +use extenddb_storage_cassandra::migrations::{migrate_account, migrate_catalog_keyspace}; /// Connections per node, the `max_connections` default. const POOL_SIZE: usize = 10; @@ -31,7 +32,10 @@ const POOL_SIZE: usize = 10; /// Catalog keyspaces, and account keyspaces, the alternating `USE` test migrates. const ROUNDS: usize = 5; -/// What one runner leaves in the keyspace it migrates. +/// How long a catalog run waits for the migration lock. +const LOCK_TIMEOUT: Duration = Duration::from_secs(60); + +/// What migration 1 leaves in a keyspace of one store kind. struct Schema { tables: &'static [&'static str], indexes: &'static [&'static str], @@ -39,7 +43,9 @@ struct Schema { added_columns: &'static [(&'static str, &'static str)], } -/// Catalog migrations V001 to V005. +/// Migration 1's catalog body: the tables, index and columns of the 0.1.x +/// catalog migrations V001 to V005. The runner's ledger and the serving +/// registrations are in neither list. const CATALOG: Schema = Schema { tables: &[ "access_keys", @@ -80,8 +86,9 @@ const CATALOG: Schema = Schema { ], }; -/// Data migrations V001 and V002. -const DATA: Schema = Schema { +/// Migration 1's account body: the tables and indexes of the 0.1.x data +/// migrations V001 and V002. +const ACCOUNT: Schema = Schema { tables: &[ "backup_items", "gsi_pending", @@ -142,7 +149,7 @@ async fn create_keyspaces(session: &Arc, keyspaces: &[String]) } } -/// Drops `keyspaces` now. Each migrated keyspace holds 12 to 24 tables, so +/// Drops `keyspaces` now. Each migrated keyspace holds 13 to 26 tables, so /// leaving them for the harness's queued drops would grow the node's schema /// by hundreds of tables per run. async fn drop_keyspaces(session: &Arc, keyspaces: &[String]) { @@ -188,7 +195,8 @@ async fn tables_in(session: &Arc, keyspace: &str) -> BTreeSet< } /// One line for each table, index or added column of `schema` that `keyspace` -/// lacks, and for each table of `other` (the other runner's schema) it holds. +/// lacks, and for each table of `other` (the other store kind's schema) it +/// holds. async fn problems_in( session: &Arc, keyspace: &str, @@ -223,7 +231,7 @@ async fn problems_in( for table in other.tables.iter().filter(|t| !schema.tables.contains(t)) { if tables.contains(*table) { problems.push(format!( - "{keyspace}: holds {table}, which the other runner creates" + "{keyspace}: holds {table}, which the other store kind's body creates" )); } } @@ -231,52 +239,55 @@ async fn problems_in( } /// Before every migration run the session selects another keyspace with -/// `USE`, as an earlier runner on the same session did before #632: -/// `extenddb init` and `extenddb migrate` run the catalog migrations and then -/// each account's data migrations on the engine's one session, and each -/// runner began with `USE` for its own keyspace. Each keyspace must end up -/// with its own schema and the other keyspace with none. Against runners -/// that rely on `USE`, some runs execute their statements in the other -/// keyspace, so the outcome varies from run to run, and `ROUNDS` runs of -/// each runner give it many chances to show. +/// `USE`, as the 0.1.x runners on the same session did before #632: +/// `extenddb init` and `extenddb migrate` migrate the catalog keyspace and +/// then each account keyspace on one session, and CreateAccount migrates +/// its keyspace on the engine's session, which every other caller shares. +/// Each keyspace must end up with its own schema and the other keyspace +/// with none. Against a runner that relies on `USE`, some runs execute +/// their statements in the other keyspace, so the outcome varies from run +/// to run, and `ROUNDS` runs of each store kind give it many chances to +/// show. Each round is a deployment of its own prefix, since +/// [`migrate_catalog_keyspace`] migrates `_catalog`. #[tokio::test] async fn migrations_ignore_the_keyspace_another_caller_selected() { if crate::helpers::skip_without_cassandra() { return; } let session = pooled_session().await; - let prefix = format!("m632_{}", &crate::helpers::unique_test_id()[..8]); - let decoy = format!("{prefix}_decoy"); - let rounds: Vec<[String; 2]> = (0..ROUNDS) + let base = format!("m632_{}", &crate::helpers::unique_test_id()[..8]); + let decoy = format!("{base}_decoy"); + let rounds: Vec<(String, [String; 2])> = (0..ROUNDS) .map(|i| { - [ - format!("{prefix}_catalog_{i}"), - format!("{prefix}_account_{i}"), - ] + let prefix = format!("{base}_{i}"); + let keyspaces = [format!("{prefix}_catalog"), format!("{prefix}_account")]; + (prefix, keyspaces) }) .collect(); let _drop = crate::helpers::DropKeyspacesOnDrop( std::iter::once(decoy.clone()) - .chain(rounds.iter().flatten().cloned()) + .chain(rounds.iter().flat_map(|(_, keyspaces)| keyspaces.clone())) .collect(), ); create_keyspaces(&session, std::slice::from_ref(&decoy)).await; let use_decoy = format!("USE {decoy}"); let mut problems = Vec::new(); - for round in &rounds { + for (prefix, round) in &rounds { let [catalog, account] = round; create_keyspaces(&session, round).await; session.query(&use_decoy).await.expect("USE decoy"); - if let Err(e) = run_catalog_migrations(&session, catalog).await { - problems.push(format!("{catalog}: run_catalog_migrations: {e:?}")); + if let Err(e) = + migrate_catalog_keyspace(session.clone(), prefix, "test", LOCK_TIMEOUT).await + { + problems.push(format!("{catalog}: migrate_catalog_keyspace: {e:?}")); } session.query(&use_decoy).await.expect("USE decoy"); - if let Err(e) = run_data_migrations(&session, account).await { - problems.push(format!("{account}: run_data_migrations: {e:?}")); + if let Err(e) = migrate_account(session.clone(), prefix, account, "test").await { + problems.push(format!("{account}: migrate_account: {e:?}")); } - problems.extend(problems_in(&session, catalog, &CATALOG, &DATA).await); - problems.extend(problems_in(&session, account, &DATA, &CATALOG).await); + problems.extend(problems_in(&session, catalog, &CATALOG, &ACCOUNT).await); + problems.extend(problems_in(&session, account, &ACCOUNT, &CATALOG).await); drop_keyspaces(&session, round).await; } for table in tables_in(&session, &decoy).await { @@ -300,8 +311,8 @@ async fn migrations_ignore_the_keyspace_another_caller_selected() { /// On a session that never selected a keyspace, any migration statement that /// leaves its keyspace to the session fails ("No keyspace has been -/// specified"). The runners must also leave the session's keyspace unset, so -/// they move no other caller's statements. +/// specified"). The runner must also leave the session's keyspace unset, so +/// it moves no other caller's statements. #[tokio::test] async fn migrations_run_on_a_session_with_no_keyspace_and_leave_none_selected() { if crate::helpers::skip_without_cassandra() { @@ -320,18 +331,19 @@ async fn migrations_run_on_a_session_with_no_keyspace_and_leave_none_selected() let _drop = crate::helpers::DropKeyspacesOnDrop(keyspaces.clone()); create_keyspaces(&session, &keyspaces).await; - let catalog_result = run_catalog_migrations(&session, &catalog).await; - let data_result = run_data_migrations(&session, &account).await; - let mut problems = problems_in(&session, &catalog, &CATALOG, &DATA).await; - problems.extend(problems_in(&session, &account, &DATA, &CATALOG).await); + let catalog_result = + migrate_catalog_keyspace(session.clone(), &prefix, "test", LOCK_TIMEOUT).await; + let account_result = migrate_account(session.clone(), &prefix, &account, "test").await; + let mut problems = problems_in(&session, &catalog, &CATALOG, &ACCOUNT).await; + problems.extend(problems_in(&session, &account, &ACCOUNT, &CATALOG).await); let selected = session.current_keyspace(); drop_keyspaces(&session, &keyspaces).await; catalog_result.expect("catalog migrations on a session with no keyspace"); - data_result.expect("data migrations on a session with no keyspace"); + account_result.expect("account migrations on a session with no keyspace"); assert!(problems.is_empty(), "{}", problems.join("\n")); assert_eq!( selected, None, - "the migration runners selected a keyspace on the session" + "the migration runner selected a keyspace on the session" ); } diff --git a/crates/storage-cassandra/tests/fixtures/golden/0002_resource_tag_condition_keys.txt b/crates/storage-cassandra/tests/fixtures/golden/0002_resource_tag_condition_keys.txt new file mode 100644 index 000000000..8edb18134 --- /dev/null +++ b/crates/storage-cassandra/tests/fixtures/golden/0002_resource_tag_condition_keys.txt @@ -0,0 +1,14 @@ +group devs tagged-group {"Statement":[{"Action":"dynamodb:*","Condition":{"StringEquals":{"aws:ResourceTag/env":"prod"}},"Effect":"Deny","Resource":"*"}],"Version":"2012-10-17"} +role app tagged-role {"Statement":[{"Action":"dynamodb:*","Condition":{"StringEquals":{"aws:ResourceTag/env":"prod"}},"Effect":"Deny","Resource":"*"}],"Version":"2012-10-17"} +user dev SelfServicePolicy {"Statement":[{"Action":["iam:CreateAccessKey","iam:DeleteAccessKey","iam:ListAccessKeys","iam:ChangePassword"],"Effect":"Allow","Resource":"arn:aws:iam::111111111111:user/dev"}],"Version":"2012-10-17"} +user dev tagged-user {"Statement":[{"Action":"dynamodb:*","Condition":{"StringEquals":{"aws:ResourceTag/env":"prod"}},"Effect":"Deny","Resource":"*"}],"Version":"2012-10-17"} +role boundary {"Statement":[{"Action":"dynamodb:*","Condition":{"StringEquals":{"aws:ResourceTag/env":"prod"}},"Effect":"Deny","Resource":"*"}],"Version":"2012-10-17"} +user boundary {"Statement":[{"Action":"dynamodb:*","Condition":{"StringEquals":{"aws:ResourceTag/env":"prod"}},"Effect":"Deny","Resource":"*"}],"Version":"2012-10-17"} +session token-1 app {"Statement":[{"Action":"dynamodb:*","Condition":{"StringEquals":{"aws:ResourceTag/env":"prod"}},"Effect":"Deny","Resource":"*"}],"Version":"2012-10-17"} +-- notes +rewrote account 111111111111 user/dev policy tagged-user (1 key(s)) +rewrote account 111111111111 user/dev policy (1 key(s)) +rewrote account 111111111111 group/devs policy tagged-group (1 key(s)) +rewrote account 111111111111 role/app policy tagged-role (1 key(s)) +rewrote account 111111111111 role/app policy (1 key(s)) +rewrote account 111111111111 assumed-role/app/session policy (1 key(s)) diff --git a/crates/storage-cassandra/tests/migrations.rs b/crates/storage-cassandra/tests/migrations.rs new file mode 100644 index 000000000..34317430b --- /dev/null +++ b/crates/storage-cassandra/tests/migrations.rs @@ -0,0 +1,521 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The catalog migrations against a live node: migration 1's definitions, +//! the status of keyspaces that are not catalogs, the ledger, the migration +//! lock's lease, the serving registration and CreateAccount's account +//! migrations. Each test uses keyspaces of its own prefix and drops them. +//! Skipped without a node unless `EXTENDDB_TEST_CASSANDRA=required`. +//! +//! With `EXTENDDB_BLESS_EXPECTED=1`, +//! `migration_1_creates_the_expected_definitions` writes +//! `migrations/{catalog,account}/0001_baseline.expected.json` instead of +//! comparing. + +#[path = "common/mod.rs"] +mod helpers; +#[path = "migrations/lock.rs"] +mod lock; +mod migrations_support; + +use std::sync::Arc; + +use extenddb_storage::management_store::{ManagementStore, SettingsStore}; +use extenddb_storage::migrations::{ + CatalogMigrations, CatalogStatus, MigrationError, MigrationId, RunKind, StoreKind, Target, + run_locked, +}; +use extenddb_storage_cassandra::CassandraCatalogStore; +use extenddb_storage_cassandra::cassandra_util::CassandraSession; +use extenddb_storage_cassandra::engine::CassandraEngine; +use extenddb_storage_cassandra::migrations::{ + CassandraCatalogMigrations, catalog_status, migrate_account, migrate_catalog_keyspace, + postcheck, run_baseline_unrecorded, +}; +use migrations_support::{TIMEOUT, cql, keyspace, new_prefix, session}; + +/// A catalog keyspace at migration 4 and one account keyspace. +async fn deployment(session: &Arc, prefix: &str) -> String { + keyspace(session, &format!("{prefix}_catalog")).await; + migrate_catalog_keyspace(session.clone(), prefix, "test", TIMEOUT) + .await + .expect("migrate the catalog"); + let account = format!("{prefix}_account_111111111111"); + keyspace(session, &account).await; + cql( + session, + &format!( + "INSERT INTO {prefix}_catalog.accounts (account_id, account_name, created_at) VALUES \ + ('111111111111', 'one', toTimestamp(now()))" + ), + ) + .await; + migrate_account(session.clone(), prefix, &account, "test") + .await + .expect("migrate the account"); + account +} + +fn target_of(prefix: &str) -> Target { + let config = helpers::test_config(); + Target::at(format!( + "{}:{}@{}/{prefix}", + config.username.unwrap_or_default(), + config.password.unwrap_or_default(), + config.contact_points.join(",") + )) +} + +/// A session through the node `EXTENDDB_TEST_CASSANDRA_SECOND_CONTACT_POINT` +/// names, the second node of a two-node cluster, so two migrators reach the +/// catalog through different coordinators; through the test node when it is +/// not set. +async fn second_session() -> Arc { + let mut config = helpers::test_config(); + if let Ok(second) = std::env::var("EXTENDDB_TEST_CASSANDRA_SECOND_CONTACT_POINT") { + config.contact_points = vec![second]; + } + Arc::new( + CassandraEngine::create_session(&config) + .await + .expect("connect to the second node"), + ) +} + +#[tokio::test] +async fn two_migrators_started_together_apply_a_legacy_catalog_once() { + if helpers::skip_without_cassandra() { + return; + } + let session = session().await; + let second = second_session().await; + let (prefix, _drops) = new_prefix("race"); + let catalog = format!("{prefix}_catalog"); + keyspace(&session, &catalog).await; + run_baseline_unrecorded(&session, &catalog, StoreKind::Catalog) + .await + .expect("the legacy tables"); + cql( + &session, + &format!("INSERT INTO {catalog}.settings (key, value) VALUES ('catalog_version', '0.0.4')"), + ) + .await; + let (first, other) = tokio::join!( + migrate_catalog_keyspace(session.clone(), &prefix, "test", TIMEOUT), + migrate_catalog_keyspace(second.clone(), &prefix, "test", TIMEOUT), + ); + let applied = [ + first.expect("the first migrator").applied.len(), + other.expect("the second migrator").applied.len(), + ]; + assert!( + applied.iter().filter(|n| **n == 0).count() == 1 && applied.iter().any(|n| *n > 0), + "one migrator applies and the other finds nothing to do: {applied:?}" + ); + assert_eq!( + catalog_status(second.clone(), &prefix) + .await + .expect("status"), + CatalogStatus::Current { version: 4 } + ); +} + +#[tokio::test] +async fn migration_1_creates_the_expected_definitions() { + if helpers::skip_without_cassandra() { + return; + } + let session = session().await; + let (prefix, _drops) = new_prefix("expected"); + let bless = std::env::var("EXTENDDB_BLESS_EXPECTED").as_deref() == Ok("1"); + let account = format!("{prefix}_account_111111111111"); + if bless { + // Migration 1's bodies on empty keyspaces, without the runner, whose + // post-check reads the file being written. + for (keyspace_name, kind) in [ + (format!("{prefix}_catalog"), StoreKind::Catalog), + (account.clone(), StoreKind::Account), + ] { + keyspace(&session, &keyspace_name).await; + run_baseline_unrecorded(&session, &keyspace_name, kind) + .await + .expect("migration 1's body"); + } + } else { + deployment(&session, &prefix).await; + assert_eq!( + catalog_status(session.clone(), &prefix) + .await + .expect("status"), + CatalogStatus::Current { + version: MigrationId::LATEST.number() + } + ); + } + for (dir, keyspace) in [ + ("catalog", format!("{prefix}_catalog")), + ("account", account.clone()), + ] { + let actual = postcheck::describe(&session, &keyspace) + .await + .expect("describe"); + let path = format!( + "{}/migrations/{dir}/0001_baseline.expected.json", + env!("CARGO_MANIFEST_DIR") + ); + if bless { + let text = serde_json::to_string_pretty(&actual).expect("json") + "\n"; + std::fs::write(&path, text).expect("write the expected file"); + continue; + } + let expected: postcheck::Definitions = + serde_json::from_str(&std::fs::read_to_string(&path).expect("read")).expect("json"); + assert_eq!( + postcheck::compare(&expected, &actual), + Vec::::new(), + "{dir}" + ); + assert_eq!( + expected, actual, + "{dir}: a fresh install names other objects" + ); + } +} + +#[tokio::test] +async fn a_keyspace_that_is_not_a_catalog_is_uninitialized_and_migrate_refuses_it() { + if helpers::skip_without_cassandra() { + return; + } + let session = session().await; + let (prefix, _drops) = new_prefix("uninit"); + let catalog = format!("{prefix}_catalog"); + let status = || catalog_status(session.clone(), &prefix); + // No keyspace at all. + assert_eq!( + status().await.expect("status"), + CatalogStatus::Uninitialized + ); + // An empty keyspace. + keyspace(&session, &catalog).await; + assert_eq!( + status().await.expect("status"), + CatalogStatus::Uninitialized + ); + // A keyspace whose settings table holds only the lock row. + cql( + &session, + &format!("CREATE TABLE {catalog}.settings (key text PRIMARY KEY, value text)"), + ) + .await; + cql( + &session, + &format!("INSERT INTO {catalog}.settings (key, value) VALUES ('migration_lock', 'x')"), + ) + .await; + assert_eq!( + status().await.expect("status"), + CatalogStatus::Uninitialized + ); + // migrate --yes refuses it under the lock, naming init (exit 1). + cql( + &session, + &format!("DELETE FROM {catalog}.settings WHERE key = 'migration_lock'"), + ) + .await; + let migrations = CassandraCatalogMigrations; + let target = target_of(&prefix); + let lock = migrations.lock(&target, TIMEOUT).await.expect("lock"); + let refused = run_locked(&migrations, &target, &lock, "test", RunKind::Migrate).await; + migrations.release(lock).await; + assert_eq!( + refused, + Err(MigrationError::Status(CatalogStatus::Uninitialized)) + ); + assert_eq!( + MigrationError::Status(CatalogStatus::Uninitialized).exit_code(), + 1 + ); + // Another program's keyspace of the same name. + let (other, _other_drops) = new_prefix("foreign"); + let other_catalog = format!("{other}_catalog"); + keyspace(&session, &other_catalog).await; + cql( + &session, + &format!("CREATE TABLE {other_catalog}.settings (name text PRIMARY KEY, val int)"), + ) + .await; + cql( + &session, + &format!("CREATE TABLE {other_catalog}.accounts (id int PRIMARY KEY)"), + ) + .await; + assert_eq!( + catalog_status(session.clone(), &other) + .await + .expect("status"), + CatalogStatus::Uninitialized + ); + // migrate --yes, the preview and verify refuse it and name what it + // holds, as init does; nothing is written to it. The advice for an + // uninitialized catalog, running init, would fail on this keyspace. + for refused in [ + migrations.lock(&target_of(&other), TIMEOUT).await.map(drop), + migrations.status(&target_of(&other)).await.map(drop), + ] { + match refused { + Err(MigrationError::Refused(message)) => assert!( + message.contains("is not an ExtendDB catalog") + && message + .contains("the table settings without the catalog's key and value columns") + && !message.contains("the table accounts"), + "{message}" + ), + other => panic!("{:?}", other.err()), + } + } + assert_eq!( + MigrationError::Refused(String::new()).exit_code(), + 1, + "the script asserts exit 1" + ); + // Another program's keyspace whose settings table has the catalog's two + // columns and rows of its own: refused before the lock row is written. + let (shaped, _shaped_drops) = new_prefix("shaped"); + let shaped_catalog = format!("{shaped}_catalog"); + keyspace(&session, &shaped_catalog).await; + cql( + &session, + &format!("CREATE TABLE {shaped_catalog}.settings (key text PRIMARY KEY, value text)"), + ) + .await; + cql( + &session, + &format!("INSERT INTO {shaped_catalog}.settings (key, value) VALUES ('theme', 'dark')"), + ) + .await; + assert_eq!( + catalog_status(session.clone(), &shaped) + .await + .expect("status"), + CatalogStatus::Uninitialized + ); + match migrations.lock(&target_of(&shaped), TIMEOUT).await { + Err(MigrationError::Refused(message)) => assert!( + message.contains("is not an ExtendDB catalog") + && message.contains("the settings row 'theme'"), + "{message}" + ), + other => panic!("{:?}", other.err()), + } + let keys: Vec = session + .query(format!("SELECT key FROM {shaped_catalog}.settings")) + .await + .expect("read") + .response_body() + .expect("body") + .into_rows() + .unwrap_or_default() + .iter() + .map(|row| { + use cdrs_tokio::types::IntoRustByName; + row.get_r_by_name("key").expect("key") + }) + .collect(); + assert_eq!(keys, ["theme"], "no lock row was written"); +} + +#[tokio::test] +async fn a_ledger_that_cannot_be_read_stops_the_status() { + if helpers::skip_without_cassandra() { + return; + } + let session = session().await; + let (prefix, _drops) = new_prefix("ledger"); + let catalog = format!("{prefix}_catalog"); + keyspace(&session, &catalog).await; + // A ledger whose id this binary cannot read is an error, never "not + // applied" (Cassandra item 2). + cql( + &session, + &format!( + "CREATE TABLE {catalog}.extenddb_migrations (id text PRIMARY KEY, name text, kind \ + text, checksum text, outcome text, notes text, applied_at timestamp, applied_by text)" + ), + ) + .await; + cql( + &session, + &format!( + "INSERT INTO {catalog}.extenddb_migrations (id, name, kind, checksum, outcome, \ + notes, applied_by) VALUES ('1', 'baseline', 'breaking', 'c', 'applied', '', 'x')" + ), + ) + .await; + let error = catalog_status(session.clone(), &prefix) + .await + .expect_err("an unreadable ledger"); + assert!(error.to_string().contains(&catalog), "{error}"); +} + +#[tokio::test] +async fn migration_1_converges_a_legacy_catalog_and_stops_on_a_table_of_another_shape() { + if helpers::skip_without_cassandra() { + return; + } + let session = session().await; + let (prefix, _drops) = new_prefix("legacy"); + let catalog = format!("{prefix}_catalog"); + keyspace(&session, &catalog).await; + // A catalog with the legacy files' tables and no ledger, as a release + // before 1.0 left it, with the traces in its settings: the version row, + // the marker, and a setting an operator changed. + run_baseline_unrecorded(&session, &catalog, StoreKind::Catalog) + .await + .expect("the legacy tables"); + for (key, value) in [ + ("catalog_version", "0.0.4"), + ("resource_tag_migration", "applied"), + ("control_plane_delay_seconds", "1.5"), + ] { + cql( + &session, + &format!("INSERT INTO {catalog}.settings (key, value) VALUES ('{key}', '{value}')"), + ) + .await; + } + let status = catalog_status(session.clone(), &prefix) + .await + .expect("status"); + assert!( + matches!(status, CatalogStatus::Behind { version: None, .. }), + "{status:?}" + ); + migrate_catalog_keyspace(session.clone(), &prefix, "test", TIMEOUT) + .await + .expect("migrate"); + let store = CassandraCatalogStore::catalog_only(session.clone(), prefix.clone()); + assert_eq!( + store.get_setting("catalog_version").await.expect("read"), + Some("4".to_owned()) + ); + assert_eq!( + store + .get_setting("resource_tag_migration") + .await + .expect("read"), + None, + "migration 1 deletes the marker" + ); + assert_eq!( + store + .get_setting("control_plane_delay_seconds") + .await + .expect("read"), + Some("1.5".to_owned()), + "an operator's setting is kept" + ); + + // A table of another shape: migration 1 stops on it and records nothing. + let (other, _other_drops) = new_prefix("shape"); + let other_catalog = format!("{other}_catalog"); + keyspace(&session, &other_catalog).await; + cql( + &session, + &format!("CREATE TABLE {other_catalog}.settings (key text PRIMARY KEY, value text)"), + ) + .await; + cql( + &session, + &format!( + "CREATE TABLE {other_catalog}.accounts (account_id text PRIMARY KEY, account_name int)" + ), + ) + .await; + let error = migrate_catalog_keyspace(session.clone(), &other, "test", TIMEOUT) + .await + .expect_err("a column of another type"); + assert!( + error.to_string().contains("accounts.account_name"), + "{error}" + ); + let status = catalog_status(session.clone(), &other) + .await + .expect("status"); + assert_eq!( + status, + CatalogStatus::Uninitialized, + "no ledger row was written" + ); +} + +#[tokio::test] +async fn create_account_runs_the_account_migrations_through_the_runner() { + if helpers::skip_without_cassandra() { + return; + } + let session = session().await; + let (prefix, _drops) = new_prefix("account"); + keyspace(&session, &format!("{prefix}_catalog")).await; + migrate_catalog_keyspace(session.clone(), &prefix, "test", TIMEOUT) + .await + .expect("migrate the catalog"); + let store = + CassandraCatalogStore::new(session.clone(), prefix.clone(), "datacenter1".to_owned(), 1) + .recording_release("9.9.9"); + // CreateAccount runs the account migrations over one keyspace and leaves + // the deployment's version row as it was: a sentinel shows a write. + cql( + &session, + &format!("UPDATE {prefix}_catalog.settings SET value = '7' WHERE key = 'catalog_version'"), + ) + .await; + store + .create_account("111111111111", "one") + .await + .expect("create the account"); + assert_eq!( + store + .get_setting("catalog_version") + .await + .expect("read the version row"), + Some("7".to_owned()), + "CreateAccount wrote the deployment's catalog_version" + ); + cql( + &session, + &format!("UPDATE {prefix}_catalog.settings SET value = '4' WHERE key = 'catalog_version'"), + ) + .await; + let account = format!("{prefix}_account_111111111111"); + let rows = session + .query(format!( + "SELECT id, outcome, applied_by FROM {account}.extenddb_migrations" + )) + .await + .expect("read") + .response_body() + .expect("body") + .into_rows() + .unwrap_or_default(); + assert_eq!(rows.len(), 1, "the account keyspace records migration 1"); + use cdrs_tokio::types::IntoRustByName; + let id: i32 = rows[0].get_r_by_name("id").expect("id"); + let by: String = rows[0].get_r_by_name("applied_by").expect("applied_by"); + assert_eq!((id, by.as_str()), (1, "9.9.9")); + assert_eq!( + catalog_status(session.clone(), &prefix) + .await + .expect("status"), + CatalogStatus::Current { version: 4 } + ); + // A catalog-only store refuses to create a keyspace with a guessed + // replication. + let catalog_only = CassandraCatalogStore::catalog_only(session.clone(), prefix.clone()); + assert!( + catalog_only + .create_account("222222222222", "two") + .await + .is_err() + ); +} diff --git a/crates/storage-cassandra/tests/migrations/lock.rs b/crates/storage-cassandra/tests/migrations/lock.rs new file mode 100644 index 000000000..56c65643d --- /dev/null +++ b/crates/storage-cassandra/tests/migrations/lock.rs @@ -0,0 +1,299 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The migration lock's lease, the serving registration and the bound on +//! a code body's requests under the lock. + +use std::time::{Duration, Instant}; + +use extenddb_storage::migrations::{CatalogMigrations, CatalogStatus, MigrationError}; +use extenddb_storage_cassandra::migrations::{ + CassandraCatalogMigrations, LeaseTimes, MigrationLock, ServingRegistration, register_server, + schema_statement_bounds, +}; + +use super::{deployment, target_of}; +use crate::helpers; +use crate::migrations_support::{self, TIMEOUT, cql, keyspace, new_prefix, session}; + +#[tokio::test] +async fn the_lock_holds_for_three_ttls_and_a_killed_holder_frees_it_within_one_ttl() { + if helpers::skip_without_cassandra() { + return; + } + let session = session().await; + let (prefix, _drops) = new_prefix("lease"); + let catalog = format!("{prefix}_catalog"); + keyspace(&session, &catalog).await; + cql( + &session, + &format!("CREATE TABLE {catalog}.settings (key text PRIMARY KEY, value text)"), + ) + .await; + let times = LeaseTimes { + ttl: Duration::from_secs(6), + renew_every: Duration::from_millis(1500), + margin: Duration::from_millis(2500), + stall: Duration::from_secs(5), + }; + let holder = MigrationLock::acquire(session.clone(), &catalog, times, TIMEOUT) + .await + .expect("take the lock"); + let other = self::session().await; + let started = Instant::now(); + while started.elapsed() < times.ttl * 3 { + holder.progress(); + let refused = MigrationLock::acquire(other.clone(), &catalog, times, Duration::ZERO).await; + assert!( + matches!(refused, Err(MigrationError::Locked(_))), + "a second holder at {:?}: {refused:?}", + started.elapsed() + ); + holder.check().expect("the holder keeps its lease"); + tokio::time::sleep(Duration::from_millis(700)).await; + } + holder.abandon(); + let killed = Instant::now(); + let next = MigrationLock::acquire(other.clone(), &catalog, times, times.ttl * 3) + .await + .expect("the lock is free once the row expires"); + let waited = killed.elapsed(); + assert!( + waited <= times.ttl + times.renew_every + Duration::from_secs(1), + "waited {waited:?}" + ); + next.release().await; +} + +#[tokio::test] +async fn a_holder_that_makes_no_progress_loses_the_lock_and_writes_nothing() { + if helpers::skip_without_cassandra() { + return; + } + let session = session().await; + let (prefix, _drops) = new_prefix("stall"); + let catalog = format!("{prefix}_catalog"); + keyspace(&session, &catalog).await; + cql( + &session, + &format!("CREATE TABLE {catalog}.settings (key text PRIMARY KEY, value text)"), + ) + .await; + let times = LeaseTimes { + ttl: Duration::from_secs(5), + renew_every: Duration::from_secs(1), + margin: Duration::from_secs(2), + stall: Duration::from_secs(2), + }; + let holder = MigrationLock::acquire(session.clone(), &catalog, times, TIMEOUT) + .await + .expect("take the lock"); + // A body that hangs: the renewal stops after the stall, and the guard + // ends the body once the deadline passes. + let hung = holder + .guard(async { + futures::future::pending::<()>().await; + Ok(()) + }) + .await; + assert!(hung.is_err(), "{hung:?}"); + assert!(holder.check().is_err()); + // Another process takes the lock once the row expires. + let next = MigrationLock::acquire(session.clone(), &catalog, times, times.ttl * 2) + .await + .expect("the lock is free"); + next.release().await; +} + +/// A schema statement sent under the lock, or by `init`'s default-account +/// run under the lock's lease, is bounded by 20 s, inside the lease's 25 s +/// margin; CreateAccount's run, which holds neither, gets 60 s. +#[tokio::test] +async fn a_locked_or_leased_run_bounds_a_schema_statement_inside_the_margin() { + if helpers::skip_without_cassandra() { + return; + } + let session = session().await; + let (prefix, _drops) = new_prefix("bound"); + let catalog = format!("{prefix}_catalog"); + keyspace(&session, &catalog).await; + cql( + &session, + &format!("CREATE TABLE {catalog}.settings (key text PRIMARY KEY, value text)"), + ) + .await; + let lock = MigrationLock::acquire(session.clone(), &catalog, LeaseTimes::DEFAULT, TIMEOUT) + .await + .expect("take the lock"); + let [locked, leased, unlocked] = schema_statement_bounds(&lock, &prefix); + lock.release().await; + assert_eq!(locked, Duration::from_secs(20)); + assert_eq!(leased, locked); + assert!(locked < LeaseTimes::DEFAULT.margin, "{locked:?}"); + assert_eq!(unlocked, Duration::from_secs(60)); +} + +#[tokio::test] +async fn a_server_and_a_migration_refuse_each_other() { + if helpers::skip_without_cassandra() { + return; + } + let session = session().await; + let (prefix, _drops) = new_prefix("serve"); + deployment(&session, &prefix).await; + let catalog = format!("{prefix}_catalog"); + let registration = register_server(session.clone(), &prefix, "127.0.0.1:1", "test") + .await + .expect("register"); + // migrate --yes takes the lock, then waits for the server and exits 6 + // naming it. + let migrations = CassandraCatalogMigrations; + let refused = migrations + .lock(&target_of(&prefix), Duration::from_secs(2)) + .await; + match refused { + Err(MigrationError::Locked(message)) => { + assert!( + message.contains("extenddb serve test on 127.0.0.1:1"), + "{message}" + ); + } + other => panic!("{other:?}"), + } + registration.release().await; + let lock = migrations + .lock(&target_of(&prefix), Duration::from_secs(5)) + .await + .expect("no server is registered"); + // A server that starts while a migration holds the lock exits 6. + let refused = register_server(session.clone(), &prefix, "127.0.0.1:2", "test").await; + assert!( + matches!(&refused, Err(MigrationError::Locked(_))), + "{:?}", + refused.as_ref().err() + ); + migrations.release(lock).await; + // The status read and the preview take no lock beside a server. + let registration: ServingRegistration = + register_server(session.clone(), &prefix, "127.0.0.1:3", "test") + .await + .expect("register once the lock is free"); + assert!(matches!( + migrations.status(&target_of(&prefix)).await, + Ok(CatalogStatus::Current { .. }) + )); + registration.release().await; + let rows = session + .query(format!("SELECT server_id FROM {catalog}.extenddb_servers")) + .await + .expect("read") + .response_body() + .expect("body") + .into_rows() + .unwrap_or_default(); + assert!(rows.is_empty(), "a released server leaves no row"); +} + +/// A code body's request to a node that stops answering fails at the request +/// timeout, through the store the body is handed, rather than when the lease +/// cuts the body 35 to 65 s later (SPEC.md A6: every request under the lock +/// is bounded by 10 s). The node is the test node behind a proxy that stops +/// passing bytes once the session is up. +#[tokio::test] +async fn a_bodys_request_to_a_node_that_stops_answering_fails_at_the_request_timeout() { + use extenddb_storage::migrations::{HasManagementStore, MigrationBackend}; + use extenddb_storage_cassandra::migrations::{CassandraMigrations, Deployment}; + + if helpers::skip_without_cassandra() { + return; + } + let proxy = + migrations_support::SilencingProxy::in_front_of(helpers::test_contact_point()).await; + let config = helpers::test_config(); + let deployment = Deployment { + contact_points: vec![proxy.address.clone()], + keyspace_prefix: "p2m_silent".to_owned(), + username: config.username.clone(), + password: config.password.clone(), + app_role: config.username.clone(), + }; + let session = deployment + .connect() + .await + .expect("connect through the proxy"); + let migrations = CassandraMigrations::for_status(session, &deployment.keyspace_prefix); + let store = migrations.stores().management_store(); + // Through the proxy, a request is answered: the keyspace does not exist. + let answered = store.list_policies("000000000000", "user", "alice").await; + assert!(answered.is_err(), "{answered:?}"); + proxy.go_silent(); + let started = Instant::now(); + let answer = tokio::time::timeout( + Duration::from_secs(15), + store.list_policies("000000000000", "user", "alice"), + ) + .await + .expect("an answer within the request timeout, not the lease"); + let elapsed = started.elapsed(); + let Err(error) = answer else { + panic!("{answer:?}"); + }; + let message = format!("{error:?}"); + assert!( + message.contains("no answer within 10 s"), + "{message} after {elapsed:?}" + ); + assert!(elapsed >= Duration::from_secs(10), "{elapsed:?}"); +} + +/// `init` runs the default account's migrations under the lease of the +/// migration lock it holds, so once that lease has lapsed the account run +/// writes nothing: no table in the account keyspace and no ledger row. +#[tokio::test] +async fn an_account_run_under_a_lapsed_lease_writes_nothing() { + if helpers::skip_without_cassandra() { + return; + } + let session = session().await; + let (prefix, _drops) = new_prefix("lapsed"); + let catalog = format!("{prefix}_catalog"); + keyspace(&session, &catalog).await; + extenddb_storage_cassandra::migrations::migrate_catalog_keyspace( + session.clone(), + &prefix, + "test", + TIMEOUT, + ) + .await + .expect("migrate the catalog"); + let account = format!("{prefix}_account_111111111111"); + keyspace(&session, &account).await; + + let run = extenddb_storage_cassandra::migrations::migrate_account_under_lapsed_lease( + session.clone(), + &prefix, + &account, + ) + .await; + let message = format!("{run:?}"); + assert!( + run.is_err() && message.contains("was not renewed in time"), + "the account run under a lapsed lease: {message}" + ); + let tables = session + .query_with_values( + "SELECT table_name FROM system_schema.tables WHERE keyspace_name = ?", + cdrs_tokio::query_values!(account.as_str()), + ) + .await + .expect("read") + .response_body() + .expect("body") + .into_rows() + .unwrap_or_default(); + assert!( + tables.is_empty(), + "{} tables written to {account} under a lapsed lease", + tables.len() + ); +} diff --git a/crates/storage-cassandra/tests/migrations_stopped.rs b/crates/storage-cassandra/tests/migrations_stopped.rs new file mode 100644 index 000000000..3157ba432 --- /dev/null +++ b/crates/storage-cassandra/tests/migrations_stopped.rs @@ -0,0 +1,516 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Runs stopped part way, as a migrator killed at that point stops, and the +//! run after each, which finishes with the definitions of a fresh install; +//! and migration 2's rewrite of a released catalog's policies, compared with +//! a committed dump. Each test uses keyspaces of its own prefix and drops +//! them. Skipped without a node unless `EXTENDDB_TEST_CASSANDRA=required`. +//! +//! With `EXTENDDB_BLESS_EXPECTED=1`, the migration 2 test writes +//! `tests/fixtures/golden/0002_resource_tag_condition_keys.txt` instead of +//! comparing. + +#[path = "common/mod.rs"] +mod helpers; +mod migrations_support; + +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use cdrs_tokio::types::IntoRustByName; +use extenddb_storage::management_store::{ManagementStore, SettingsStore}; +use extenddb_storage::migrations::{CatalogStatus, MigrationId, Runner, StoreKind}; +use extenddb_storage_cassandra::CassandraCatalogStore; +use extenddb_storage_cassandra::cassandra_util::CassandraSession; +use extenddb_storage_cassandra::migrations::{ + CassandraMigrations, LeaseTimes, MigrationLock, catalog_status, create_lock_table, postcheck, + run_baseline_unrecorded, status_holding, +}; +use migrations_support::{TIMEOUT, cql, keyspace, new_prefix, session}; + +const ACCOUNT: &str = "111111111111"; + +const TAGGED: &str = r#"{"Version":"2012-10-17","Statement":[{"Effect":"Deny","Action":"dynamodb:*","Resource":"*","Condition":{"StringEquals":{"dynamodb:ResourceTag/env":"prod"}}}]}"#; + +/// The catalog keyspace as `init` leaves it before it takes the lock: the +/// keyspace and its `settings` table. +async fn fresh_catalog(session: &Arc, prefix: &str) { + let catalog = format!("{prefix}_catalog"); + keyspace(session, &catalog).await; + create_lock_table(session, &catalog) + .await + .expect("the settings table"); +} + +/// A catalog as a release before 1.0 left it: the tables migration 1 +/// converges and no ledger, `catalog_version` 0.0.4 and the resource-tag +/// marker, one account with its keyspace, and a policy holding a +/// `dynamodb:ResourceTag` key in each of the six places migration 2 +/// rewrites: user, group and role inline policies, user and role +/// permissions boundaries, and a session policy. +async fn released_catalog(session: &Arc, prefix: &str) { + let catalog = format!("{prefix}_catalog"); + let account = format!("{prefix}_account_{ACCOUNT}"); + for (name, kind) in [ + (&catalog, StoreKind::Catalog), + (&account, StoreKind::Account), + ] { + keyspace(session, name).await; + run_baseline_unrecorded(session, name, kind) + .await + .expect("the legacy tables"); + } + for (key, value) in [ + ("catalog_version", "0.0.4"), + ("resource_tag_migration", "applied"), + ] { + cql( + session, + &format!("INSERT INTO {catalog}.settings (key, value) VALUES ('{key}', '{value}')"), + ) + .await; + } + cql( + session, + &format!( + "INSERT INTO {catalog}.accounts (account_id, account_name, created_at) VALUES \ + ('{ACCOUNT}', 'one', toTimestamp(now()))" + ), + ) + .await; + let store = CassandraCatalogStore::catalog_only(session.clone(), prefix.to_owned()); + let tagged: serde_json::Value = serde_json::from_str(TAGGED).expect("json"); + store.create_user(ACCOUNT, "dev", None).await.expect("user"); + store.create_group(ACCOUNT, "devs").await.expect("group"); + store + .create_role(ACCOUNT, "app", &serde_json::json!({})) + .await + .expect("role"); + for (principal_type, name, policy) in [ + ("user", "dev", "tagged-user"), + ("group", "devs", "tagged-group"), + ("role", "app", "tagged-role"), + ] { + store + .put_policy(ACCOUNT, principal_type, name, policy, &tagged) + .await + .expect("policy"); + } + store + .set_user_boundary(ACCOUNT, "dev", &tagged) + .await + .expect("user boundary"); + store + .set_role_boundary(ACCOUNT, "app", &tagged) + .await + .expect("role boundary"); + store + .store_session( + "token-1", + "ASIAFIXTURE0001", + &[0], + ACCOUNT, + "app", + "session", + &None, + &Some(tagged), + time::OffsetDateTime::now_utc() + time::Duration::days(1), + ) + .await + .expect("session"); +} + +/// Add `count` IAM users to [`ACCOUNT`], each with one inline policy holding +/// a `dynamodb:ResourceTag` key, 32 users at a time. +async fn add_tagged_users(store: &CassandraCatalogStore, count: usize) { + use futures::StreamExt; + let tagged: serde_json::Value = serde_json::from_str(TAGGED).expect("json"); + futures::stream::iter(0..count) + .map(|i| { + let tagged = &tagged; + async move { + let name = format!("u{i:05}"); + store.create_user(ACCOUNT, &name, None).await.expect("user"); + store + .put_policy(ACCOUNT, "user", &name, "tagged", tagged) + .await + .expect("policy"); + } + }) + .buffer_unordered(32) + .collect::>() + .await; +} + +/// Take the migration lock and run the pending migrations, as `init` and +/// `migrate --yes` do, with the runner stopped before write `requests` +/// (`usize::MAX` for a run that is not stopped), then release the lock. +/// `catalog_only` leaves out the account keyspaces, and `through` stops +/// after that migration. Returns the run's error, if any, and how many +/// writes its budget had left. A killed holder's lock row stays until it +/// expires, which the lease test in tests/migrations.rs measures; here the +/// lock is released so the next run starts at once. +async fn run_stopped_after( + session: &Arc, + prefix: &str, + requests: usize, + catalog_only: bool, + through: MigrationId, +) -> (Result<(), String>, usize) { + let catalog = format!("{prefix}_catalog"); + let lock = MigrationLock::acquire(session.clone(), &catalog, LeaseTimes::DEFAULT, TIMEOUT) + .await + .expect("take the lock"); + let (result, left) = { + let migrations = if catalog_only { + CassandraMigrations::locked_catalog_only(&lock, prefix) + } else { + CassandraMigrations::locked(&lock, prefix) + } + .stop_after(requests); + let result = Runner::new(&migrations, "test") + .migrate_through(through) + .await + .map(drop) + .map_err(|e| e.to_string()); + (result, migrations.requests_left().unwrap_or_default()) + }; + lock.release().await; + (result, left) +} + +/// The deployment is `Current` at the latest migration, its keyspaces hold +/// the definitions of a fresh install, the version row reads the latest +/// migration and the marker is gone; on a released catalog every policy +/// holds the `aws:ResourceTag` form and none the `dynamodb:` form. +async fn assert_finished(session: &Arc, prefix: &str, released: bool, at: &str) { + assert_eq!( + catalog_status(session.clone(), prefix) + .await + .expect("status"), + CatalogStatus::Current { + version: MigrationId::LATEST.number() + }, + "{at}" + ); + let mut keyspaces = vec![("catalog", format!("{prefix}_catalog"))]; + if released { + keyspaces.push(("account", format!("{prefix}_account_{ACCOUNT}"))); + } + for (dir, keyspace) in keyspaces { + let actual = postcheck::describe(session, &keyspace) + .await + .expect("describe"); + let path = format!( + "{}/migrations/{dir}/0001_baseline.expected.json", + env!("CARGO_MANIFEST_DIR") + ); + let expected: postcheck::Definitions = + serde_json::from_str(&std::fs::read_to_string(&path).expect("read")).expect("json"); + assert_eq!(expected, actual, "{at}: {dir}"); + } + let store = CassandraCatalogStore::catalog_only(session.clone(), prefix.to_owned()); + assert_eq!( + store.get_setting("catalog_version").await.expect("read"), + Some(MigrationId::LATEST.number().to_string()), + "{at}" + ); + assert_eq!( + store + .get_setting("resource_tag_migration") + .await + .expect("read"), + None, + "{at}" + ); + if released { + let text = policies(&store).await; + assert!(!text.contains("dynamodb:ResourceTag"), "{at}: {text}"); + assert_eq!( + text.matches("aws:ResourceTag/env").count(), + 6, + "{at}: {text}" + ); + } +} + +/// Every policy of the released catalog's account, one line each, in a +/// fixed order: inline policies by principal, the two boundaries, then the +/// session policies. +async fn policies(store: &CassandraCatalogStore) -> String { + let mut lines = Vec::new(); + for (principal_type, name) in [("group", "devs"), ("role", "app"), ("user", "dev")] { + let mut found = store + .list_policies(ACCOUNT, principal_type, name) + .await + .expect("policies"); + found.sort_by(|a, b| a.0.cmp(&b.0)); + for (policy, document, _) in found { + lines.push(format!("{principal_type}\t{name}\t{policy}\t{document}")); + } + } + for (principal_type, boundary) in [ + ( + "role", + store.get_role_boundary(ACCOUNT, "app").await.expect("role"), + ), + ( + "user", + store.get_user_boundary(ACCOUNT, "dev").await.expect("user"), + ), + ] { + let boundary = boundary.map(|b| b.to_string()).unwrap_or_default(); + lines.push(format!("{principal_type}\tboundary\t{boundary}")); + } + let mut sessions = store.list_session_policies().await.expect("sessions"); + sessions.sort_by(|a, b| a.session_token.cmp(&b.session_token)); + for s in sessions { + lines.push(format!( + "session\t{}\t{}\t{}", + s.session_token, s.role_name, s.policy + )); + } + lines.join("\n") + "\n" +} + +/// Stop a run before each write in `budgets`, run again with no stop, and +/// check that the second run finished. Returns how many runs stopped. +async fn stop_and_rerun( + session: &Arc, + released: bool, + budgets: &[usize], +) -> usize { + let mut stopped = 0; + for &requests in budgets { + let (prefix, drops) = new_prefix("stopped"); + if released { + released_catalog(session, &prefix).await; + } else { + fresh_catalog(session, &prefix).await; + } + let at = format!("stopped before write {}", requests + 1); + let (first, _) = + run_stopped_after(session, &prefix, requests, false, MigrationId::LATEST).await; + let error = first.expect_err(&format!("{at}: the run is stopped")); + stopped += 1; + let (rerun, _) = + run_stopped_after(session, &prefix, usize::MAX, false, MigrationId::LATEST).await; + if let Err(e) = rerun { + panic!("{at} ({error}): the next run: {e}"); + } + assert_finished(session, &prefix, released, &at).await; + drop(drops); + helpers::flush_pending_drops(session).await; + } + stopped +} + +#[tokio::test] +async fn a_fresh_install_stopped_between_two_statements_of_migration_1_finishes_on_the_next_run() { + if helpers::skip_without_cassandra() { + return; + } + let session = session().await; + assert_eq!(stop_and_rerun(&session, false, &[2]).await, 1); +} + +#[tokio::test] +async fn an_upgrade_stopped_inside_migrations_1_and_2_finishes_on_the_next_run() { + if helpers::skip_without_cassandra() { + return; + } + let session = session().await; + // Count the writes of a run that is not stopped: through migration 1 in + // the catalog keyspace alone, then in the account keyspace, then the + // rest. + let (prefix, drops) = new_prefix("counted"); + released_catalog(&session, &prefix).await; + let mut sent = Vec::new(); + for (catalog_only, through) in [ + (true, MigrationId::Baseline), + (false, MigrationId::Baseline), + (false, MigrationId::LATEST), + ] { + let (done, left) = + run_stopped_after(&session, &prefix, usize::MAX, catalog_only, through).await; + done.expect("a run that is not stopped"); + sent.push(usize::MAX - left); + } + assert_finished(&session, &prefix, true, "not stopped").await; + drop(drops); + helpers::flush_pending_drops(&session).await; + let catalog_1 = sent[0]; + let migration_1 = catalog_1 + sent[1]; + let all = migration_1 + sent[2]; + // Migration 2's body starts with one write and rewrites six policies. + assert!(all >= migration_1 + 7, "{sent:?}"); + let budgets = [ + // Between two statements of migration 1's catalog file. + 2, + // Before migration 1's ledger row in the catalog keyspace, which is + // written in one batch with the version row: as two writes, a stop + // between them left the row beside the 0.1.x version row, and the + // next run refused the catalog as one an older release wrote to. + catalog_1 - 1, + // Between two statements of migration 1's account file. + catalog_1 + 2, + // Inside migration 2, after its first policy rewrite. + migration_1 + 2, + // Before the last write of the run. + all - 1, + ]; + assert_eq!( + stop_and_rerun(&session, true, &budgets).await, + budgets.len() + ); +} + +#[tokio::test] +async fn migration_2_rewrites_a_released_catalogs_policies_in_every_place() { + if helpers::skip_without_cassandra() { + return; + } + let session = session().await; + let (prefix, _drops) = new_prefix("m2"); + released_catalog(&session, &prefix).await; + let (done, _) = + run_stopped_after(&session, &prefix, usize::MAX, false, MigrationId::LATEST).await; + done.expect("migrate"); + assert_finished(&session, &prefix, true, "migrated").await; + let store = CassandraCatalogStore::catalog_only(session.clone(), prefix.clone()); + let mut text = policies(&store).await; + let rows = session + .query(format!( + "SELECT notes FROM {prefix}_catalog.extenddb_migrations WHERE id = 2" + )) + .await + .expect("read") + .response_body() + .expect("body") + .into_rows() + .unwrap_or_default(); + let notes: String = rows + .first() + .expect("migration 2's row") + .get_r_by_name("notes") + .expect("notes"); + assert_eq!(notes.lines().count(), 6, "{notes}"); + text.push_str("-- notes\n"); + text.push_str(¬es); + text.push('\n'); + let path = format!( + "{}/tests/fixtures/golden/0002_resource_tag_condition_keys.txt", + env!("CARGO_MANIFEST_DIR") + ); + if std::env::var("EXTENDDB_BLESS_EXPECTED").as_deref() == Ok("1") { + std::fs::create_dir_all(std::path::Path::new(&path).parent().expect("dir")) + .expect("golden dir"); + std::fs::write(&path, &text).expect("write golden"); + } + let expected = std::fs::read_to_string(&path).unwrap_or_default(); + assert_eq!( + expected, text, + "differs from {path}; regenerate with EXTENDDB_BLESS_EXPECTED=1" + ); +} + +/// Migration 2's ledger row names every policy it rewrote, so on a catalog +/// with many tagged policies the row's notes pass the node's +/// `batch_size_fail_threshold` (50 KB by default), and the row has to be +/// written on its own, not in a batch with the version row. 800 users with +/// one tagged policy each give notes of about 60 KB. +#[tokio::test] +async fn migration_2_records_its_row_when_it_rewrote_many_policies() { + if helpers::skip_without_cassandra() { + return; + } + const USERS: usize = 800; + let session = session().await; + let (prefix, _drops) = new_prefix("m2many"); + released_catalog(&session, &prefix).await; + let store = CassandraCatalogStore::catalog_only(session.clone(), prefix.clone()); + add_tagged_users(&store, USERS).await; + let (done, _) = + run_stopped_after(&session, &prefix, usize::MAX, false, MigrationId::LATEST).await; + done.expect("migrate records migration 2 over many policies"); + assert_eq!( + catalog_status(session.clone(), &prefix) + .await + .expect("status"), + CatalogStatus::Current { + version: MigrationId::LATEST.number() + } + ); + let rows = session + .query(format!( + "SELECT notes FROM {prefix}_catalog.extenddb_migrations WHERE id = 2" + )) + .await + .expect("read") + .response_body() + .expect("body") + .into_rows() + .unwrap_or_default(); + let notes: String = rows + .first() + .expect("migration 2's row") + .get_r_by_name("notes") + .expect("notes"); + assert_eq!(notes.lines().count(), 6 + USERS, "{}", notes.len()); + assert!(notes.len() > 50 * 1024, "{} bytes", notes.len()); + let text = policies(&store).await; + assert!(!text.contains("dynamodb:ResourceTag")); +} + +/// `migrate --yes` and `--forget` read the status under the lock before they +/// act, and on a catalog behind migration 2 that status runs migration 2's +/// `detect` over every principal. Read as the holder, each of its requests +/// is progress, so a read longer than the stall leaves the renewal going and +/// the lock is still held one deadline window later. A read that records +/// nothing stops the renewal at the stall, and the deadline after the last +/// renewal passes inside that window. +#[tokio::test] +async fn the_status_read_under_the_lock_keeps_it_through_a_long_scan() { + if helpers::skip_without_cassandra() { + return; + } + const USERS: usize = 4000; + let session = session().await; + let (prefix, _drops) = new_prefix("m2status"); + released_catalog(&session, &prefix).await; + let store = CassandraCatalogStore::catalog_only(session.clone(), prefix.clone()); + add_tagged_users(&store, USERS).await; + let times = LeaseTimes { + ttl: Duration::from_secs(6), + renew_every: Duration::from_millis(300), + margin: Duration::from_millis(2500), + stall: Duration::from_millis(1500), + }; + let catalog = format!("{prefix}_catalog"); + let lock = MigrationLock::acquire(session.clone(), &catalog, times, TIMEOUT) + .await + .expect("take the lock"); + let started = Instant::now(); + let status = status_holding(&lock, &prefix).await; + let took = started.elapsed(); + let window = times.ttl - times.margin + times.renew_every; + let held = Instant::now(); + while held.elapsed() < window { + lock.progress(); + tokio::time::sleep(Duration::from_millis(100)).await; + } + let kept = lock.check(); + lock.release().await; + assert!( + matches!(status, Ok(CatalogStatus::Behind { .. })), + "{status:?}" + ); + // A read shorter than this keeps the lock without the progress it + // records: the renewal only stops at a tick after the stall. + assert!( + took > times.stall + times.renew_every, + "the status read took {took:?}" + ); + kept.expect("the status read kept the lock"); +} diff --git a/crates/storage-cassandra/tests/migrations_support/mod.rs b/crates/storage-cassandra/tests/migrations_support/mod.rs new file mode 100644 index 000000000..ba6ae08a7 --- /dev/null +++ b/crates/storage-cassandra/tests/migrations_support/mod.rs @@ -0,0 +1,154 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! What the catalog migration tests share: a session to the test node, +//! keyspaces of a test's own prefix, and statements followed by a schema +//! agreement wait. + +use std::sync::Arc; +use std::time::Duration; + +use extenddb_storage_cassandra::cassandra_util::CassandraSession; +use extenddb_storage_cassandra::engine::CassandraEngine; +use extenddb_storage_cassandra::migrations::wait_for_schema_agreement; + +use crate::helpers; + +pub const TIMEOUT: Duration = Duration::from_secs(60); + +pub async fn session() -> Arc { + Arc::new( + CassandraEngine::create_session(&helpers::test_config()) + .await + .expect("connect"), + ) +} + +/// A prefix of this test's own, whose keyspaces are dropped at the end. +pub fn new_prefix(tag: &str) -> (String, helpers::DropKeyspacesOnDrop) { + let prefix = format!("p2m_{tag}_{}", &helpers::unique_test_id()[..8]); + let drops = helpers::DropKeyspacesOnDrop(vec![ + format!("{prefix}_catalog"), + format!("{prefix}_account_111111111111"), + ]); + (prefix, drops) +} + +/// Run `statement`, then wait until every live node agrees on the schema, +/// so the next statement finds what this one created on whichever node it +/// reaches (#637). +pub async fn cql(session: &CassandraSession, statement: &str) { + session + .query(statement) + .await + .unwrap_or_else(|e| panic!("{statement}: {e}")); + wait_for_schema_agreement(session) + .await + .unwrap_or_else(|e| panic!("after {statement}: {e}")); +} + +pub async fn keyspace(session: &CassandraSession, name: &str) { + cql( + session, + &format!( + "CREATE KEYSPACE IF NOT EXISTS {name} WITH replication = {{'class': \ + 'SimpleStrategy', 'replication_factor': 1}}" + ), + ) + .await; +} + +/// A TCP proxy in front of the test node that stops passing bytes on +/// command: a node that never answers, for the tests of the request bounds. +/// Connections through it stay open, so the driver waits for an answer that +/// never comes rather than failing on a closed connection. +// tests/migrations_stopped.rs compiles this module too and uses no proxy. +#[allow(dead_code)] +pub struct SilencingProxy { + pub address: String, + silent: Arc, + accepting: tokio::task::JoinHandle<()>, +} + +#[allow(dead_code)] +impl SilencingProxy { + pub async fn in_front_of(upstream: String) -> Self { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use tokio::net::{TcpListener, TcpStream}; + + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("bind the proxy"); + let address = listener.local_addr().expect("proxy address").to_string(); + let silent = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let flag = Arc::clone(&silent); + let accepting = tokio::spawn(async move { + loop { + let Ok((client, _)) = listener.accept().await else { + return; + }; + let upstream = upstream.clone(); + let flag = Arc::clone(&flag); + tokio::spawn(async move { + let Ok(server) = TcpStream::connect(&upstream).await else { + return; + }; + let (mut from_client, mut to_client) = client.into_split(); + let (mut from_server, mut to_server) = server.into_split(); + // Each direction: pass bytes until the proxy is silent, + // then read and drop them, so neither side sees the + // connection close. + let silent = Arc::clone(&flag); + let up = tokio::spawn(async move { + let mut buf = [0u8; 16384]; + loop { + let n = match from_client.read(&mut buf).await { + Ok(0) | Err(_) => return, + Ok(n) => n, + }; + if silent.load(std::sync::atomic::Ordering::SeqCst) { + continue; + } + if to_server.write_all(&buf[..n]).await.is_err() { + return; + } + } + }); + let silent = Arc::clone(&flag); + let down = tokio::spawn(async move { + let mut buf = [0u8; 16384]; + loop { + let n = match from_server.read(&mut buf).await { + Ok(0) | Err(_) => return, + Ok(n) => n, + }; + if silent.load(std::sync::atomic::Ordering::SeqCst) { + continue; + } + if to_client.write_all(&buf[..n]).await.is_err() { + return; + } + } + }); + let _ = tokio::join!(up, down); + }); + } + }); + Self { + address, + silent, + accepting, + } + } + + /// From now on no byte passes in either direction. + pub fn go_silent(&self) { + self.silent.store(true, std::sync::atomic::Ordering::SeqCst); + } +} + +impl Drop for SilencingProxy { + fn drop(&mut self) { + self.accepting.abort(); + } +} diff --git a/crates/storage-mongodb/Cargo.toml b/crates/storage-mongodb/Cargo.toml index f455971dc..94675749e 100644 --- a/crates/storage-mongodb/Cargo.toml +++ b/crates/storage-mongodb/Cargo.toml @@ -18,6 +18,10 @@ test-hooks = [] # `test-hooks` so the crate's own tests build the production commit and # backfill paths, not the coordination hooks. snapshot-test-hooks = ["extenddb-storage/test-hooks"] +# Test hooks for the migration tests: databases of other names on a shared +# server, a lock that stops renewing as a killed holder does, and the +# post-check's definitions. +test-util = ["extenddb-storage/test-util"] [dependencies] # Internal crates @@ -67,6 +71,12 @@ dashmap.workspace = true # Property-based testing for the filter-pushdown parity harness # (crates/storage-mongodb/tests/pushdown_parity.rs). proptest = "1" -# The crate's snapshot tests need `snapshot-test-hooks`; a dev-dependency on -# itself turns it on for test builds only. -extenddb-storage-mongodb = { path = ".", features = ["snapshot-test-hooks"] } +# The migration tests run on a runtime with the test macro. +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } +# The checksum test checks this backend's migration table with the framework's +# test helper. +extenddb-storage = { workspace = true, features = ["test-util"] } +# The crate's snapshot tests need `snapshot-test-hooks` and its migration tests +# need `test-util`; a dev-dependency on itself turns both on for test builds +# only. +extenddb-storage-mongodb = { path = ".", features = ["snapshot-test-hooks", "test-util"] } diff --git a/crates/storage-mongodb/migrations/catalog/0001_baseline.expected.json b/crates/storage-mongodb/migrations/catalog/0001_baseline.expected.json new file mode 100644 index 000000000..34dde7d10 --- /dev/null +++ b/crates/storage-mongodb/migrations/catalog/0001_baseline.expected.json @@ -0,0 +1,423 @@ +{ + "access_keys": { + "type": "collection", + "options": {}, + "indexes": { + "_id_": { + "key": [ + [ + "_id", + 1 + ] + ], + "options": {} + }, + "account_id_1_user_name_1": { + "key": [ + [ + "account_id", + 1 + ], + [ + "user_name", + 1 + ] + ], + "options": {} + } + } + }, + "accounts": { + "type": "collection", + "options": {}, + "indexes": { + "_id_": { + "key": [ + [ + "_id", + 1 + ] + ], + "options": {} + }, + "account_name_1": { + "key": [ + [ + "account_name", + 1 + ] + ], + "options": { + "unique": true + } + } + } + }, + "admin_users": { + "type": "collection", + "options": {}, + "indexes": { + "_id_": { + "key": [ + [ + "_id", + 1 + ] + ], + "options": {} + } + } + }, + "backups": { + "type": "collection", + "options": {}, + "indexes": { + "_id_": { + "key": [ + [ + "_id", + 1 + ] + ], + "options": {} + }, + "account_id_1_table_name_1": { + "key": [ + [ + "account_id", + 1 + ], + [ + "table_name", + 1 + ] + ], + "options": {} + } + } + }, + "continuous_backups": { + "type": "collection", + "options": {}, + "indexes": { + "_id_": { + "key": [ + [ + "_id", + 1 + ] + ], + "options": {} + } + } + }, + "extenddb_servers": { + "type": "collection", + "options": {}, + "indexes": { + "_id_": { + "key": [ + [ + "_id", + 1 + ] + ], + "options": {} + }, + "expires_at_1": { + "key": [ + [ + "expires_at", + 1 + ] + ], + "options": { + "expireAfterSeconds": 0 + } + } + } + }, + "iam_groups": { + "type": "collection", + "options": {}, + "indexes": { + "_id_": { + "key": [ + [ + "_id", + 1 + ] + ], + "options": {} + }, + "group_arn_1": { + "key": [ + [ + "group_arn", + 1 + ] + ], + "options": { + "unique": true + } + } + } + }, + "iam_permissions_boundaries": { + "type": "collection", + "options": {}, + "indexes": { + "_id_": { + "key": [ + [ + "_id", + 1 + ] + ], + "options": {} + } + } + }, + "iam_policies": { + "type": "collection", + "options": {}, + "indexes": { + "_id_": { + "key": [ + [ + "_id", + 1 + ] + ], + "options": {} + } + } + }, + "iam_roles": { + "type": "collection", + "options": {}, + "indexes": { + "_id_": { + "key": [ + [ + "_id", + 1 + ] + ], + "options": {} + }, + "role_arn_1": { + "key": [ + [ + "role_arn", + 1 + ] + ], + "options": { + "unique": true + } + } + } + }, + "iam_sessions": { + "type": "collection", + "options": {}, + "indexes": { + "_id_": { + "key": [ + [ + "_id", + 1 + ] + ], + "options": {} + }, + "access_key_id_1": { + "key": [ + [ + "access_key_id", + 1 + ] + ], + "options": { + "unique": true + } + }, + "expires_at_1": { + "key": [ + [ + "expires_at", + 1 + ] + ], + "options": { + "expireAfterSeconds": 0 + } + } + } + }, + "iam_users": { + "type": "collection", + "options": {}, + "indexes": { + "_id_": { + "key": [ + [ + "_id", + 1 + ] + ], + "options": {} + }, + "user_arn_1": { + "key": [ + [ + "user_arn", + 1 + ] + ], + "options": { + "unique": true + } + } + } + }, + "indexes": { + "type": "collection", + "options": {}, + "indexes": { + "_id_": { + "key": [ + [ + "_id", + 1 + ] + ], + "options": {} + } + } + }, + "login_attempts": { + "type": "collection", + "options": {}, + "indexes": { + "_id_": { + "key": [ + [ + "_id", + 1 + ] + ], + "options": {} + }, + "principal_1_attempted_at_1": { + "key": [ + [ + "principal", + 1 + ], + [ + "attempted_at", + 1 + ] + ], + "options": {} + } + } + }, + "metrics": { + "type": "collection", + "options": {}, + "indexes": { + "_id.bucket_1": { + "key": [ + [ + "_id.bucket", + 1 + ] + ], + "options": {} + }, + "_id_": { + "key": [ + [ + "_id", + 1 + ] + ], + "options": {} + } + } + }, + "schema_history": { + "type": "collection", + "options": {}, + "indexes": { + "_id_": { + "key": [ + [ + "_id", + 1 + ] + ], + "options": {} + } + } + }, + "settings": { + "type": "collection", + "options": {}, + "indexes": { + "_id_": { + "key": [ + [ + "_id", + 1 + ] + ], + "options": {} + } + } + }, + "tables": { + "type": "collection", + "options": {}, + "indexes": { + "_id_": { + "key": [ + [ + "_id", + 1 + ] + ], + "options": {} + }, + "table_id_1": { + "key": [ + [ + "table_id", + 1 + ] + ], + "options": { + "unique": true + } + } + } + }, + "tags": { + "type": "collection", + "options": {}, + "indexes": { + "_id_": { + "key": [ + [ + "_id", + 1 + ] + ], + "options": {} + } + } + } +} diff --git a/crates/storage-mongodb/migrations/data/0001_baseline.expected.json b/crates/storage-mongodb/migrations/data/0001_baseline.expected.json new file mode 100644 index 000000000..e4c708147 --- /dev/null +++ b/crates/storage-mongodb/migrations/data/0001_baseline.expected.json @@ -0,0 +1,108 @@ +{ + "idempotency_tokens": { + "type": "collection", + "options": {}, + "indexes": { + "_id_": { + "key": [ + [ + "_id", + 1 + ] + ], + "options": {} + }, + "account_id_1_token_1": { + "key": [ + [ + "account_id", + 1 + ], + [ + "token", + 1 + ] + ], + "options": { + "unique": true + } + }, + "created_at_1": { + "key": [ + [ + "created_at", + 1 + ] + ], + "options": { + "expireAfterSeconds": 540 + } + } + } + }, + "stream_records": { + "type": "collection", + "options": {}, + "indexes": { + "_id_": { + "key": [ + [ + "_id", + 1 + ] + ], + "options": {} + }, + "created_at_1": { + "key": [ + [ + "created_at", + 1 + ] + ], + "options": { + "expireAfterSeconds": 86400 + } + }, + "shard_id_1_sequence_number_1": { + "key": [ + [ + "shard_id", + 1 + ], + [ + "sequence_number", + 1 + ] + ], + "options": {} + } + } + }, + "stream_shards": { + "type": "collection", + "options": {}, + "indexes": { + "_id_": { + "key": [ + [ + "_id", + 1 + ] + ], + "options": {} + }, + "shard_id_1": { + "key": [ + [ + "shard_id", + 1 + ] + ], + "options": { + "unique": true + } + } + } + } +} diff --git a/crates/storage-mongodb/src/bootstrapper.rs b/crates/storage-mongodb/src/bootstrapper.rs index c37c3f36f..7719717fd 100644 --- a/crates/storage-mongodb/src/bootstrapper.rs +++ b/crates/storage-mongodb/src/bootstrapper.rs @@ -3,19 +3,26 @@ //! Bootstrapper implementation for `MongoDB`. +use std::sync::{Arc, OnceLock}; + use async_trait::async_trait; use bson::{Document, doc}; -use mongodb::IndexModel; -use mongodb::options::IndexOptions; use extenddb_storage::bootstrapper::{AdminBootstrapResult, Bootstrapper}; use extenddb_storage::error::StorageError; use extenddb_storage::management_store::{OpError, OpResult}; +use extenddb_storage::migrations::HeldLock; + +use crate::migrations::{Lease, REQUEST_TIMEOUT}; /// `MongoDB` bootstrapper for init/destroy/migrate operations. pub struct MongoBootstrapper { client: mongodb::Client, connection_string: String, + /// The lease of the migration lock `init` holds while it writes the + /// bootstrap rows, set by [`Bootstrapper::hold`]; every row's requests + /// check it first. + lease: OnceLock>, } impl MongoBootstrapper { @@ -40,13 +47,18 @@ impl MongoBootstrapper { .unwrap_or("mongodb://localhost:27017") .to_string(); - // Route through the shared guard so init/destroy/migrate also reject - // non-primary read preferences and warn on missing TLS. - let client = crate::connect_guarded(&connection_string, None, false).await?; + // The runner's client: the shared guard's checks (primary reads, a + // write concern no weaker than majority) and its 10 s timeouts to + // connect and to select a server, so `init` and `destroy` give up on + // an unreachable database as `migrate` and `verify` do. + let client = crate::migrations::connect(&connection_string) + .await + .map_err(|e| StorageError::Connection(e.to_string()))?; Ok(Self { client, connection_string, + lease: OnceLock::new(), }) } @@ -54,6 +66,34 @@ impl MongoBootstrapper { self.client.database("extenddb_catalog") } + /// Send one of `init`'s bootstrap requests as the runner sends a + /// migration request: the lock's lease is checked first and nothing is + /// sent once it is past its margin, the answer is waited for at most + /// [`REQUEST_TIMEOUT`], and a completed request counts as progress. + async fn request( + &self, + what: &str, + request: impl std::future::IntoFuture>, + ) -> OpResult { + if let Some(lease) = self.lease.get() { + lease + .check() + .map_err(|e| OpError::Internal(format!("{what}: {e}")))?; + } + let answer = tokio::time::timeout(REQUEST_TIMEOUT, request.into_future()).await; + if let Some(lease) = self.lease.get() { + lease.progressed(); + } + match answer { + Ok(Ok(value)) => Ok(value), + Ok(Err(e)) => Err(OpError::Internal(format!("{what}: {e}"))), + Err(_) => Err(OpError::Internal(format!( + "{what}: no answer within {} s", + REQUEST_TIMEOUT.as_secs() + ))), + } + } + fn data_db(&self) -> mongodb::Database { self.client.database("extenddb_data") } @@ -71,322 +111,25 @@ impl Bootstrapper for MongoBootstrapper { Ok(()) } + /// Nothing: `init` takes the migration lock before its first write, and + /// the lock's claim creates the catalog database. A second `init` is + /// refused under the lock, because the catalog is initialized. async fn create_catalog_db(&self) -> OpResult<()> { - // MongoDB creates databases implicitly on first write. - // We'll create a sentinel collection to materialize the database. - let db = self.catalog_db(); - db.create_collection("schema_history") - .await - .map_err(|e| OpError::Internal(format!("Failed to create catalog db: {e}")))?; Ok(()) } + /// Nothing: migration 1's data body creates the data database's + /// collections under the lock. async fn create_data_db(&self) -> OpResult<()> { - // MongoDB creates databases implicitly on first write. - let db = self.data_db(); - db.create_collection("idempotency_tokens") - .await - .map_err(|e| OpError::Internal(format!("Failed to create data db: {e}")))?; - - let coll = db.collection::("idempotency_tokens"); - - // DDB spec: `ClientRequestToken` dedups retries within a 10-minute - // window. MongoDB's TTL monitor runs on a ~60s cadence, so an - // `expireAfterSeconds = 600` index deletes rows anywhere from - // 10:00 to ~11:00 minutes after `created_at` — retention drifts - // above the spec. Shrink to 540s (9 min) so worst-case retention - // is ≤10 min. The data-plane read path also filters on `created_at` - // to enforce the boundary strictly, independent of TTL monitor - // cadence (see `transact_write_items_impl`). - let ttl_index = IndexModel::builder() - .keys(doc! { "created_at": 1 }) - .options( - IndexOptions::builder() - .expire_after(std::time::Duration::from_secs(540)) - .build(), - ) - .build(); - coll.create_index(ttl_index) - .await - .map_err(|e| OpError::Internal(format!("Failed to create TTL index: {e}")))?; - - // Unique compound index on (account_id, token). Without this, - // two concurrent TransactWriteItems calls with the same token - // both do a snapshot read that misses the other's uncommitted - // insert, and both commit — the operation executes twice. - // With the unique index in place, the second inserter fails - // E11000 and the write path converts that into a retryable - // error, giving the client the read-check path on retry. - coll.create_index( - IndexModel::builder() - .keys(doc! { "account_id": 1, "token": 1 }) - .options(IndexOptions::builder().unique(true).build()) - .build(), - ) - .await - .map_err(|e| OpError::Internal(format!("idempotency_tokens unique index: {e}")))?; - - // stream_shards: unique index on shard_id so a subsequent init/ - // recreate can never insert a duplicate shard document under the - // same shard_id. Combined with `table_id`-derived shard_ids - // (see stream_engine::build_shard_id) this rules out cross-tenant - // shard collisions structurally. - db.create_collection("stream_shards") - .await - .map_err(|e| OpError::Internal(format!("Failed to create stream_shards: {e}")))?; - db.collection::("stream_shards") - .create_index( - IndexModel::builder() - .keys(doc! { "shard_id": 1 }) - .options(IndexOptions::builder().unique(true).build()) - .build(), - ) - .await - .map_err(|e| OpError::Internal(format!("stream_shards shard_id index: {e}")))?; - - // stream_records: TTL index enforcing DDB's 24-hour retention. - // The `expireAfterSeconds` is the delta from the field value, - // not a hard cutoff, so a MongoDB background thread will - // delete records ~1 minute after created_at + 24h. The - // TTL cleanup worker in `ttl_worker.rs` provides a defense- - // in-depth deletion path but the primary enforcement is here. - db.create_collection("stream_records") - .await - .map_err(|e| OpError::Internal(format!("Failed to create stream_records: {e}")))?; - let stream_records = db.collection::("stream_records"); - stream_records - .create_index( - IndexModel::builder() - .keys(doc! { "created_at": 1 }) - .options( - IndexOptions::builder() - .expire_after(std::time::Duration::from_secs(24 * 3600)) - .build(), - ) - .build(), - ) - .await - .map_err(|e| OpError::Internal(format!("stream_records TTL index: {e}")))?; - - // Query-side index on (shard_id, sequence_number). GetRecords - // filters by shard_id and paginates by sequence_number > cursor - // with an ascending sort — the only way this can be efficient - // is if the index prefix matches. Without it, GetRecords does a - // full collection scan every time consumer polls, and cost - // grows linearly in the retained record count. D-m6. - stream_records - .create_index( - IndexModel::builder() - .keys(doc! { "shard_id": 1, "sequence_number": 1 }) - .build(), - ) - .await - .map_err(|e| OpError::Internal(format!("stream_records shard index: {e}")))?; - Ok(()) } - async fn run_catalog_migrations(&self) -> OpResult<()> { - let db = self.catalog_db(); - - // Create all catalog collections - let collections = [ - "accounts", - "tables", - "indexes", - "tags", - "settings", - "admin_users", - "iam_users", - "access_keys", - "iam_groups", - "iam_roles", - "iam_sessions", - "iam_policies", - "iam_permissions_boundaries", - "metrics", - "login_attempts", - "backups", - "continuous_backups", - ]; - - for coll_name in collections { - // create_collection is idempotent in recent MongoDB versions - let _ = db.create_collection(coll_name).await; + /// Hold the lease of the lock `init` took, so each bootstrap row's + /// requests check it; another backend's lock holds nothing. + fn hold(&self, lock: &HeldLock) { + if let Some(lease) = crate::migrations::lease_of(lock) { + let _ = self.lease.set(lease); } - - // Create indexes for catalog collections - - // accounts: unique index on account_name - let accounts = db.collection::("accounts"); - accounts - .create_index( - IndexModel::builder() - .keys(doc! { "account_name": 1 }) - .options(IndexOptions::builder().unique(true).build()) - .build(), - ) - .await - .map_err(|e| OpError::Internal(format!("accounts index: {e}")))?; - - // tables: unique index on table_id - let tables = db.collection::("tables"); - tables - .create_index( - IndexModel::builder() - .keys(doc! { "table_id": 1 }) - .options(IndexOptions::builder().unique(true).build()) - .build(), - ) - .await - .map_err(|e| OpError::Internal(format!("tables table_id index: {e}")))?; - - // iam_users: unique index on user_arn - let iam_users = db.collection::("iam_users"); - iam_users - .create_index( - IndexModel::builder() - .keys(doc! { "user_arn": 1 }) - .options(IndexOptions::builder().unique(true).build()) - .build(), - ) - .await - .map_err(|e| OpError::Internal(format!("iam_users user_arn index: {e}")))?; - - // access_keys: index on (account_id, user_name) - let access_keys = db.collection::("access_keys"); - access_keys - .create_index( - IndexModel::builder() - .keys(doc! { "account_id": 1, "user_name": 1 }) - .build(), - ) - .await - .map_err(|e| OpError::Internal(format!("access_keys index: {e}")))?; - - // iam_groups: unique index on group_arn - let iam_groups = db.collection::("iam_groups"); - iam_groups - .create_index( - IndexModel::builder() - .keys(doc! { "group_arn": 1 }) - .options(IndexOptions::builder().unique(true).build()) - .build(), - ) - .await - .map_err(|e| OpError::Internal(format!("iam_groups index: {e}")))?; - - // iam_roles: unique index on role_arn - let iam_roles = db.collection::("iam_roles"); - iam_roles - .create_index( - IndexModel::builder() - .keys(doc! { "role_arn": 1 }) - .options(IndexOptions::builder().unique(true).build()) - .build(), - ) - .await - .map_err(|e| OpError::Internal(format!("iam_roles index: {e}")))?; - - // iam_sessions: unique index on access_key_id, TTL on expires_at - let iam_sessions = db.collection::("iam_sessions"); - iam_sessions - .create_index( - IndexModel::builder() - .keys(doc! { "access_key_id": 1 }) - .options(IndexOptions::builder().unique(true).build()) - .build(), - ) - .await - .map_err(|e| OpError::Internal(format!("iam_sessions access_key index: {e}")))?; - iam_sessions - .create_index( - IndexModel::builder() - .keys(doc! { "expires_at": 1 }) - .options( - IndexOptions::builder() - .expire_after(std::time::Duration::from_secs(0)) - .build(), - ) - .build(), - ) - .await - .map_err(|e| OpError::Internal(format!("iam_sessions TTL index: {e}")))?; - - // metrics: index on bucket - let metrics = db.collection::("metrics"); - metrics - .create_index(IndexModel::builder().keys(doc! { "_id.bucket": 1 }).build()) - .await - .map_err(|e| OpError::Internal(format!("metrics bucket index: {e}")))?; - - // login_attempts: compound index - let login_attempts = db.collection::("login_attempts"); - login_attempts - .create_index( - IndexModel::builder() - .keys(doc! { "principal": 1, "attempted_at": 1 }) - .build(), - ) - .await - .map_err(|e| OpError::Internal(format!("login_attempts index: {e}")))?; - - // backups: index on (account_id, table_name) - let backups = db.collection::("backups"); - backups - .create_index( - IndexModel::builder() - .keys(doc! { "account_id": 1, "table_name": 1 }) - .build(), - ) - .await - .map_err(|e| OpError::Internal(format!("backups index: {e}")))?; - - // Seed catalog_version in settings - let settings = db.collection::("settings"); - let _ = settings - .update_one( - doc! { "_id": "catalog_version" }, - doc! { "$setOnInsert": { "value": "0.0.2" } }, - ) - .upsert(true) - .await; - - // Seed the resource-tag migration marker on this fresh catalog so the - // server's startup guard passes. An existing catalog created before - // this change has no such row, so serve refuses it until - // `extenddb migrate --yes` records it. - let _ = settings - .update_one( - doc! { "_id": "resource_tag_migration" }, - doc! { "$setOnInsert": { "value": "applied" } }, - ) - .upsert(true) - .await; - - // Record migration - let schema_history = db.collection::("schema_history"); - let _ = schema_history - .insert_one(doc! { - "_id": "001_initial", - "applied_at": bson::DateTime::now(), - }) - .await; // Ignore E11000 (already applied) - - Ok(()) - } - - async fn run_data_migrations(&self) -> OpResult<()> { - // Data database schema is minimal for MongoDB (just idempotency_tokens) - // Table collections are created on-demand - Ok(()) - } - - async fn pending_data_migrations(&self) -> OpResult> { - // MongoDB has no versioned data-database migrations — table - // collections are created on-demand, and the idempotency_tokens - // collection is created in create_data_db. Nothing to apply. - Ok(Vec::new()) } async fn record_data_connection(&self) -> OpResult<()> { @@ -394,24 +137,28 @@ impl Bootstrapper for MongoBootstrapper { let settings = db.collection::("settings"); let data_db_name = "extenddb_data".to_string(); - settings - .update_one( - doc! { "_id": "data_database_name" }, - doc! { "$set": { "value": &data_db_name } }, - ) - .upsert(true) - .await - .map_err(|e| OpError::Internal(format!("record_data_connection: {e}")))?; + self.request( + "record_data_connection", + settings + .update_one( + doc! { "_id": "data_database_name" }, + doc! { "$set": { "value": &data_db_name } }, + ) + .upsert(true), + ) + .await?; let redacted = redact_connection_string(&self.connection_string); - settings - .update_one( - doc! { "_id": "data_connection_string" }, - doc! { "$set": { "value": &redacted } }, - ) - .upsert(true) - .await - .map_err(|e| OpError::Internal(format!("record data_connection_string: {e}")))?; + self.request( + "record data_connection_string", + settings + .update_one( + doc! { "_id": "data_connection_string" }, + doc! { "$set": { "value": &redacted } }, + ) + .upsert(true), + ) + .await?; Ok(()) } @@ -424,10 +171,12 @@ impl Bootstrapper for MongoBootstrapper { let settings = db.collection::("settings"); // Check if already exists - let existing = settings - .find_one(doc! { "_id": "encryption_key" }) - .await - .map_err(|e| OpError::Internal(format!("check encryption key: {e}")))?; + let existing = self + .request( + "check encryption key", + settings.find_one(doc! { "_id": "encryption_key" }), + ) + .await?; if existing.is_some() { return Ok(()); @@ -438,15 +187,22 @@ impl Bootstrapper for MongoBootstrapper { rand::rng().fill_bytes(&mut key_bytes); let key_b64 = base64::engine::general_purpose::STANDARD.encode(key_bytes); - // Ignore E11000 (race: someone else created it first) - let _ = settings - .insert_one(doc! { - "_id": "encryption_key", - "value": &key_b64, - }) - .await; - - Ok(()) + // A duplicate `_id` (code 11000) means another process wrote the key + // first, which keeps one key; every other error stops `init`. + self.request("write the encryption key", async { + match settings + .insert_one(doc! { + "_id": "encryption_key", + "value": &key_b64, + }) + .await + { + Ok(_) => Ok(()), + Err(e) if crate::migrations::is_duplicate_key(&e) => Ok(()), + Err(e) => Err(e), + } + }) + .await } async fn bootstrap_default_account(&self) -> OpResult<()> { @@ -454,24 +210,24 @@ impl Bootstrapper for MongoBootstrapper { let accounts = db.collection::("accounts"); // Check if any account exists - let count = accounts - .count_documents(doc! {}) - .await - .map_err(|e| OpError::Internal(format!("count accounts: {e}")))?; + let count = self + .request("count accounts", accounts.count_documents(doc! {})) + .await?; if count > 0 { return Ok(()); } let account_id = uuid::Uuid::new_v4().to_string(); - accounts - .insert_one(doc! { + self.request( + "create default account", + accounts.insert_one(doc! { "_id": &account_id, "account_name": "default", "created_at": bson::DateTime::now(), - }) - .await - .map_err(|e| OpError::Internal(format!("create default account: {e}")))?; + }), + ) + .await?; Ok(()) } @@ -487,10 +243,12 @@ impl Bootstrapper for MongoBootstrapper { let admin_users = db.collection::("admin_users"); // Check if admin already exists - let existing = admin_users - .find_one(doc! { "_id": &username }) - .await - .map_err(|e| OpError::Internal(format!("check admin: {e}")))?; + let existing = self + .request( + "check admin", + admin_users.find_one(doc! { "_id": &username }), + ) + .await?; if existing.is_some() { return Ok(AdminBootstrapResult { @@ -517,14 +275,15 @@ impl Bootstrapper for MongoBootstrapper { let password_hash = bcrypt::hash(&password, bcrypt::DEFAULT_COST) .map_err(|e| OpError::Internal(format!("bcrypt hash: {e}")))?; - admin_users - .insert_one(doc! { + self.request( + "create admin", + admin_users.insert_one(doc! { "_id": &username, "password_hash": &password_hash, "created_at": bson::DateTime::now(), - }) - .await - .map_err(|e| OpError::Internal(format!("create admin: {e}")))?; + }), + ) + .await?; Ok(AdminBootstrapResult { username, @@ -534,15 +293,6 @@ impl Bootstrapper for MongoBootstrapper { }) } - async fn is_catalog_initialized(&self) -> OpResult { - let db = self.catalog_db(); - let collections = db - .list_collection_names() - .await - .map_err(|e| OpError::Internal(format!("list collections: {e}")))?; - Ok(collections.contains(&"settings".to_string())) - } - async fn list_table_names(&self) -> OpResult> { use futures::TryStreamExt; @@ -599,24 +349,6 @@ impl Bootstrapper for MongoBootstrapper { Ok(()) } - async fn read_catalog_version(&self) -> OpResult> { - let db = self.catalog_db(); - let settings = db.collection::("settings"); - let doc = settings - .find_one(doc! { "_id": "catalog_version" }) - .await - .map_err(|e| OpError::Internal(format!("read catalog_version: {e}")))?; - Ok(doc.and_then(|d| { - d.get_str("value") - .ok() - .map(std::string::ToString::to_string) - })) - } - - fn expected_catalog_version(&self) -> String { - "0.0.2".to_string() - } - fn catalog_database_name(&self) -> String { "extenddb_catalog".to_string() } @@ -629,6 +361,16 @@ impl Bootstrapper for MongoBootstrapper { format!("{}/extenddb_catalog", self.connection_string) } + /// The connection string the config file names, which the server uses + /// too, and the redacted one `record_data_connection` records, so + /// taking the lock knows it runs for `init` and may create the catalog. + fn init_target(&self) -> extenddb_storage::migrations::Target { + extenddb_storage::migrations::Target { + data_connection: Some(redact_connection_string(&self.connection_string)), + ..extenddb_storage::migrations::Target::at(self.connection_string.clone()) + } + } + fn generate_backend_config_section(&self) -> String { format!( r#"[storage.mongodb] @@ -670,6 +412,124 @@ fn redact_connection_string(uri: &str) -> String { mod tests { use super::redact_connection_string; + /// `init` and `destroy` reach the database through this client. Against + /// a port nothing listens on, a request fails after the runner's 10 s + /// server selection timeout, not the driver's 30 s default; the outer + /// timeout turns a hang into a failure instead of a blocked test run. + #[tokio::test] + async fn a_request_to_an_unreachable_database_fails_within_the_runner_timeout() { + use extenddb_storage::bootstrapper::Bootstrapper; + use std::time::{Duration, Instant}; + + let port = std::net::TcpListener::bind("127.0.0.1:0") + .and_then(|listener| listener.local_addr()) + .expect("a free port") + .port(); + let config = std::env::temp_dir().join(format!( + "extenddb-unreachable-{}-{port}.toml", + std::process::id() + )); + std::fs::write( + &config, + format!( + "[storage]\nbackend = \"mongodb\"\n\n[storage.mongodb]\nconnection_string = \ + \"mongodb://127.0.0.1:{port}/?directConnection=true\"\n" + ), + ) + .expect("write the config"); + let bootstrapper = + super::MongoBootstrapper::from_config(config.to_str().expect("a UTF-8 path"), &[]) + .await + .expect("the client is built without connecting"); + let started = Instant::now(); + let answer = tokio::time::timeout(Duration::from_secs(40), bootstrapper.get_data_db_name()) + .await + .expect("the request ended instead of waiting"); + let took = started.elapsed(); + let _ = std::fs::remove_file(&config); + assert!(answer.is_err(), "{answer:?}"); + assert!( + took < Duration::from_secs(15), + "the request failed {took:?} after it was sent" + ); + } + + /// `init` writes the bootstrap rows under the migration lock, and each + /// row's requests check the lock's lease first, as the runner does before + /// a migration request. Past the lease's margin nothing is sent: the + /// write to a port nothing listens on fails at once with the lease's + /// reason, not after the 10 s server selection timeout. + #[tokio::test] + async fn a_bootstrap_row_is_refused_once_the_held_lease_lapses() { + use crate::migrations::{Lease, Timing}; + use extenddb_storage::bootstrapper::Bootstrapper; + use std::time::{Duration, Instant}; + + let port = std::net::TcpListener::bind("127.0.0.1:0") + .and_then(|listener| listener.local_addr()) + .expect("a free port") + .port(); + let config = std::env::temp_dir().join(format!( + "extenddb-lapsed-{}-{port}.toml", + std::process::id() + )); + std::fs::write( + &config, + format!( + "[storage]\nbackend = \"mongodb\"\n\n[storage.mongodb]\nconnection_string = \ + \"mongodb://127.0.0.1:{port}/?directConnection=true\"\n" + ), + ) + .expect("write the config"); + let bootstrapper = + super::MongoBootstrapper::from_config(config.to_str().expect("a UTF-8 path"), &[]) + .await + .expect("the client is built without connecting"); + let _ = std::fs::remove_file(&config); + // A lease last renewed a full lease ago, with no renewal task. + let renewed = Instant::now() + .checked_sub(Timing::PRODUCTION.lease) + .expect("the clock has run for a lease"); + bootstrapper + .lease + .set(Lease::new(renewed, Timing::PRODUCTION)) + .ok() + .expect("no lease was held yet"); + + for (what, write) in [ + ("encryption key", bootstrapper.bootstrap_encryption_key()), + ("default account", bootstrapper.bootstrap_default_account()), + ("data connection", bootstrapper.record_data_connection()), + ] { + let started = Instant::now(); + let error = tokio::time::timeout(Duration::from_secs(5), write) + .await + .unwrap_or_else(|_| panic!("the {what} write waited for an answer")) + .expect_err("refused"); + assert!( + format!("{error:?}").contains("the migration lock was last renewed"), + "{what}: {error:?}" + ); + assert!( + started.elapsed() < Duration::from_secs(2), + "the {what} write was sent" + ); + } + let admin = tokio::time::timeout( + Duration::from_secs(5), + bootstrapper.bootstrap_admin_user(None, None), + ) + .await + .expect("the admin write waited for an answer"); + match admin { + Err(error) => assert!( + format!("{error:?}").contains("the migration lock was last renewed"), + "admin: {error:?}" + ), + Ok(result) => panic!("the admin row was written: {}", result.username), + } + } + #[test] fn redact_userinfo_with_password() { assert_eq!( diff --git a/crates/storage-mongodb/src/catalog_store.rs b/crates/storage-mongodb/src/catalog_store.rs index e5f8158c6..b33530908 100644 --- a/crates/storage-mongodb/src/catalog_store.rs +++ b/crates/storage-mongodb/src/catalog_store.rs @@ -34,6 +34,17 @@ impl MongoCatalogStore { } } + /// A store on `catalog_db`, a handle the caller opened with its own + /// options and name (the migration runner's). + #[must_use] + pub(crate) fn on_database(client: mongodb::Client, catalog_db: mongodb::Database) -> Self { + Self { + client, + catalog_db, + encryption_key: None, + } + } + /// Get a reference to the catalog database. pub(crate) fn catalog_db(&self) -> &mongodb::Database { &self.catalog_db diff --git a/crates/storage-mongodb/src/lib.rs b/crates/storage-mongodb/src/lib.rs index 3d0c619c5..88e524fe2 100644 --- a/crates/storage-mongodb/src/lib.rs +++ b/crates/storage-mongodb/src/lib.rs @@ -19,6 +19,7 @@ mod data; mod data_engine; mod management_store; mod metadata_engine; +pub mod migrations; mod operations; pub mod pushdown; mod snapshot_engine; @@ -46,11 +47,28 @@ use extenddb_storage::server_components::{BackendError, ServerComponents}; /// Backend-specific runtime hooks for `MongoDB`. struct MongoRuntimeHooks { engine: Arc, + /// The serving registration, released when the server shuts down. + registration: Arc>>, } #[async_trait::async_trait] impl ServerRuntimeHooks for MongoRuntimeHooks { async fn spawn_workers(&self, ctx: &WorkerContext) -> Vec> { + let registration = self + .registration + .lock() + .unwrap_or_else(|e| e.into_inner()) + .take(); + let token = ctx.shutdown.clone(); + // The drain after the listener closes waits for this task, so the + // registration is deleted before the process ends and a migration + // can start at once. + let release_registration = tokio::spawn(async move { + token.cancelled().await; + if let Some(registration) = registration { + registration.release().await; + } + }); let storage_for_ttl = self.engine.clone(); let metrics = ctx.metrics.clone(); let ttl = tokio::spawn(async move { @@ -71,7 +89,7 @@ impl ServerRuntimeHooks for MongoRuntimeHooks { tracing::info!( "MongoDB backend: TTL, stream cleanup, GSI backfill, and control-plane workers spawned" ); - vec![ttl, stream, backfill, control_plane] + vec![ttl, stream, backfill, control_plane, release_registration] } fn backend_info(&self) -> Option { @@ -79,6 +97,25 @@ impl ServerRuntimeHooks for MongoRuntimeHooks { } } +/// End the process with exit status 1 once the serving registration is +/// lost: it has not been renewed for 30 s, so `migrate --yes` may no longer +/// see this server. The PID file is removed at exit. A registration released +/// at shutdown ends this task without exiting. +async fn stop_when_registration_lost(mut lost: tokio::sync::watch::Receiver>) { + while lost.changed().await.is_ok() { + let reason = lost.borrow().clone(); + if let Some(reason) = reason { + let message = format!( + "extenddb fatal: {reason}; the server stops, because a migration can start on \ + the catalog once the serving registration is gone" + ); + tracing::error!("{message}"); + eprintln!("{message}"); + std::process::exit(1); + } + } +} + /// Build the assembled server components for the mongo backend (`serve`). fn server_components_factory( config: &(dyn extenddb_storage::config::StorageConfig + 'static), @@ -86,7 +123,7 @@ fn server_components_factory( // MongoDB bootstrap needs operator input (databases, admin credentials), so // `bootstrap_if_uninitialized` is not honored here: an uninitialized catalog // fails with the explicit-`init` guidance, matching the PostgreSQL backend. - _options: extenddb_storage::server_components::ServerComponentsOptions, + options: extenddb_storage::server_components::ServerComponentsOptions, ) -> std::pin::Pin< Box> + Send>, > { @@ -113,79 +150,118 @@ fn server_components_factory( let tx_read_concern = config::parse_transaction_read_concern(&raw_read_concern) .map_err(BackendError::InitializationFailed)?; - // Create MongoEngine - let engine = MongoEngine::new( + // Register as the catalog's server, then read the migration lock and + // the catalog's status, before the engine writes anything: a server + // that refuses has deleted its registration and written nothing else. + let registration = migrations::register_and_check( + &migrations::MongoTarget::new(&connection_string), + options.release, + migrations::Timing::PRODUCTION, + ) + .await + .map_err(BackendError::Catalog)?; + tokio::spawn(stop_when_registration_lost(registration.lost())); + let registration = Arc::new(std::sync::Mutex::new(Some(registration))); + let built = build_components( &connection_string, ®ion, max_connections, + max_catalog_connections, tx_read_concern, + Arc::clone(®istration), ) + .await; + if built.is_err() { + let taken = registration + .lock() + .unwrap_or_else(|e| e.into_inner()) + .take(); + if let Some(registration) = taken { + registration.release().await; + } + } + built + }) +} + +/// The engine and stores `serve` runs on, built once the server holds its +/// registration. +async fn build_components( + connection_string: &str, + region: &str, + max_connections: Option, + max_catalog_connections: Option, + tx_read_concern: mongodb::options::ReadConcern, + registration: Arc>>, +) -> Result { + // Create MongoEngine + let engine = MongoEngine::new(connection_string, region, max_connections, tx_read_concern) .await .map_err(|e| BackendError::ConnectionFailed { backend: "mongodb".to_string(), details: e.to_string(), })?; - // A crash can leave a backup's metadata claimed by a restore or a - // delete that will never finish, or a restore target nothing will - // finish. The sweep removes only state whose heartbeat has stopped, so - // it is safe while another process shares this catalog; the GSI - // backfill worker repeats it periodically. - match engine.sweep_stale_backup_claims().await { - Ok(sweep) if sweep.is_empty() => {} - Ok(sweep) => tracing::warn!("startup backup sweep: {sweep}"), - Err(e) => tracing::error!("Failed to sweep stale backup claims: {e}"), - } + // A crash can leave a backup's metadata claimed by a restore or a + // delete that will never finish, or a restore target nothing will + // finish. The sweep removes only state whose heartbeat has stopped, so + // it is safe while another process shares this catalog; the GSI + // backfill worker repeats it periodically. + match engine.sweep_stale_backup_claims().await { + Ok(sweep) if sweep.is_empty() => {} + Ok(sweep) => tracing::warn!("startup backup sweep: {sweep}"), + Err(e) => tracing::error!("Failed to sweep stale backup claims: {e}"), + } - let engine = Arc::new(engine); + let engine = Arc::new(engine); - // Create one shared catalog/management client. MongoDB Client clones - // share the underlying pool, so max_catalog_connections limits the - // combined catalog and auth traffic rather than creating two - // independent pools with twice the configured ceiling. - let catalog_client = connect_guarded(&connection_string, max_catalog_connections, false) - .await - .map_err(|e| BackendError::ConnectionFailed { - backend: "mongodb".to_string(), - details: format!("Failed to create catalog client: {e}"), - })?; - - // Load encryption key from settings collection. A missing key must be - // a hard failure: an empty key would base64-decode to zero bytes and - // panic in aes_gcm (`Key::from_slice` requires 32 bytes). Refuse to - // start instead, matching the postgres backend. - let catalog_db = catalog_client.database("extenddb_catalog"); - let settings_coll = catalog_db.collection::("settings"); - let enc_key = settings_coll - .find_one(mongodb::bson::doc! { "_id": "encryption_key" }) - .await - .map_err(|e| BackendError::InitializationFailed(format!("Load encryption key: {e}")))? - .and_then(|d| d.get_str("value").ok().map(std::borrow::ToOwned::to_owned)) - .ok_or(BackendError::MissingEncryptionKey)?; - - let auth_client = catalog_client.clone(); - let catalog_store = Arc::new(MongoCatalogStore::with_encryption_key( - catalog_client, - enc_key.clone(), - )) as Arc; - - // Create credential store. The bin layer wraps this in - // CachedCredentialStore using the operator-configured TTL - // before constructing the auth provider. - let cred_store: Arc = - Arc::new(MongoCredentialStore::new(auth_client, enc_key)); - - // Create runtime hooks - let runtime_hooks = Box::new(MongoRuntimeHooks { - engine: engine.clone(), - }); + // Create one shared catalog/management client. MongoDB Client clones + // share the underlying pool, so max_catalog_connections limits the + // combined catalog and auth traffic rather than creating two + // independent pools with twice the configured ceiling. + let catalog_client = connect_guarded(connection_string, max_catalog_connections, false) + .await + .map_err(|e| BackendError::ConnectionFailed { + backend: "mongodb".to_string(), + details: format!("Failed to create catalog client: {e}"), + })?; - Ok(ServerComponents { - engine, - catalog_store, - credential_store: cred_store, - runtime_hooks: Some(runtime_hooks), - }) + // Load encryption key from settings collection. A missing key must be + // a hard failure: an empty key would base64-decode to zero bytes and + // panic in aes_gcm (`Key::from_slice` requires 32 bytes). Refuse to + // start instead, matching the postgres backend. + let catalog_db = catalog_client.database("extenddb_catalog"); + let settings_coll = catalog_db.collection::("settings"); + let enc_key = settings_coll + .find_one(mongodb::bson::doc! { "_id": "encryption_key" }) + .await + .map_err(|e| BackendError::InitializationFailed(format!("Load encryption key: {e}")))? + .and_then(|d| d.get_str("value").ok().map(std::borrow::ToOwned::to_owned)) + .ok_or(BackendError::MissingEncryptionKey)?; + + let auth_client = catalog_client.clone(); + let catalog_store = Arc::new(MongoCatalogStore::with_encryption_key( + catalog_client, + enc_key.clone(), + )) as Arc; + + // Create credential store. The bin layer wraps this in + // CachedCredentialStore using the operator-configured TTL + // before constructing the auth provider. + let cred_store: Arc = + Arc::new(MongoCredentialStore::new(auth_client, enc_key)); + + // Create runtime hooks + let runtime_hooks = Box::new(MongoRuntimeHooks { + engine: engine.clone(), + registration, + }); + + Ok(ServerComponents { + engine, + catalog_store, + credential_store: cred_store, + runtime_hooks: Some(runtime_hooks), }) } @@ -255,8 +331,7 @@ pub fn backend() -> extenddb_storage::Backend { }) }, server_components: server_components_factory, - // Migrates through its bootstrapper until it moves onto the runner. - catalog_migrations: None, + catalog_migrations: &migrations::MongoCatalogMigrations, } } @@ -398,7 +473,7 @@ pub(crate) async fn connect_guarded( /// The checked [`mongodb::options::ClientOptions`] behind [`connect_guarded`], /// for a caller that needs to read an effective option (the pool size) before /// building the client. -async fn guarded_client_options( +pub(crate) async fn guarded_client_options( connection_string: &str, max_pool_size: Option, warn_on_no_tls: bool, @@ -539,11 +614,12 @@ pub(crate) fn majority_write_concern( /// concern other than `majority` in the connection string; this does not rely /// on it. fn catalog_database(client: &mongodb::Client) -> mongodb::Database { - catalog_database_named(client, "extenddb_catalog") + catalog_database_named(client, migrations::CATALOG_DATABASE) } -/// [`catalog_database`] under another name, for test fixtures that must run -/// the production option set on a scratch database. +/// [`catalog_database`] for the database `name`: the migration runner uses +/// the same options on both databases, and the migration and snapshot test +/// fixtures run the production option set on databases of other names. pub(crate) fn catalog_database_named(client: &mongodb::Client, name: &str) -> mongodb::Database { use mongodb::options::{DatabaseOptions, ReadConcern, ReadPreference, SelectionCriteria}; client.database_with_options( diff --git a/crates/storage-mongodb/src/migrations/backend.rs b/crates/storage-mongodb/src/migrations/backend.rs new file mode 100644 index 000000000..31f4c36d9 --- /dev/null +++ b/crates/storage-mongodb/src/migrations/backend.rs @@ -0,0 +1,497 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The runner's MongoDB parts. +//! +//! Per body, in its store: the lease check, the body, the post-check the +//! first time migration 1 is applied to the store, then the ledger row and +//! (in the catalog) the version row, written in one transaction with write +//! concern majority after the body completes. Every request goes through +//! [`MongoStores::request`], which checks the migration lock's lease first +//! and waits at most [`REQUEST_TIMEOUT`] for the answer, so a holder past +//! its deadline stops before its next request and records nothing. + +use std::sync::Arc; + +use async_trait::async_trait; +use bson::{Bson, Document, doc}; +use extenddb_storage::management_store::ManagementStore; +use extenddb_storage::migrations::{ + CatalogTraces, Done, HasManagementStore, LedgerColumns, LedgerRow, LedgerWrites, Migration, + MigrationBackend, MigrationError, MigrationId, Released, Step, StoreKind, StoreRef, + in_transaction, record_rows, +}; +use futures::TryStreamExt; + +use super::guarded::LeasedManagementStore; +use super::locks::{Lease, MigrationLock}; +use super::{ + LEDGER, MongoTarget, REQUEST_TIMEOUT, catalog_handle, connect, driver_error, postcheck, +}; +use crate::MongoCatalogStore; + +/// NamespaceExists: `create` on a collection that exists with other options. +const NAMESPACE_EXISTS: i32 = 48; + +/// The handle code bodies receive: both databases, and the management store +/// migration 2 runs through. +pub struct MongoStores { + pub(crate) catalog: mongodb::Database, + pub(crate) data: mongodb::Database, + lease: Option>, + management: LeasedManagementStore, + /// Requests left before the runner stops as a killed migrator does. + #[cfg(feature = "test-util")] + stop_after: Option>, +} + +impl MongoStores { + /// Send one request: check the lease, then wait at most + /// [`REQUEST_TIMEOUT`] for the answer. + /// + /// # Errors + /// + /// The lease is past its deadline (nothing is sent), the request failed + /// or it had no answer in time. + pub(crate) async fn request( + &self, + what: impl std::fmt::Display, + request: impl std::future::Future>, + ) -> Result { + self.check()?; + #[cfg(feature = "test-util")] + if super::guarded::stopped(self.stop_after.as_deref()) { + return Err(MigrationError::Store(format!( + "{what}: stopped by the test" + ))); + } + let answer = tokio::time::timeout(REQUEST_TIMEOUT, request).await; + if let Some(lease) = &self.lease { + lease.progressed(); + } + match answer { + Ok(Ok(value)) => Ok(value), + Ok(Err(e)) => Err(driver_error(what, &e)), + Err(_) => Err(MigrationError::Store(format!( + "{what}: no answer within {} s", + REQUEST_TIMEOUT.as_secs() + ))), + } + } + + /// Whether this process may still act under the migration lock; always + /// for a read that holds none. + /// + /// # Errors + /// + /// The lock was lost or its lease is past the margin. + pub(crate) fn check(&self) -> Result<(), MigrationError> { + match &self.lease { + Some(lease) => lease.check(), + None => Ok(()), + } + } + + fn db(&self, kind: StoreKind) -> &mongodb::Database { + match kind { + StoreKind::Catalog => &self.catalog, + StoreKind::Data | StoreKind::Account => &self.data, + } + } +} + +impl HasManagementStore for MongoStores { + fn management_store(&self) -> &dyn ManagementStore { + &self.management + } +} + +/// The runner's view of one MongoDB deployment. +pub struct MongoMigrations { + client: mongodb::Client, + stores: MongoStores, +} + +impl MongoMigrations { + /// Open the deployment `target` names. With `lock`, every request checks + /// its lease; without one, the handle is for reads (the status). + /// + /// # Errors + /// + /// The client cannot be built from the connection string. + pub async fn open( + target: &MongoTarget, + lock: Option<&MigrationLock>, + ) -> Result { + let client = connect(&target.connection_string).await?; + let catalog = catalog_handle(&client, &target.catalog); + let data = catalog_handle(&client, &target.data); + let lease = lock.map(MigrationLock::lease); + let management = LeasedManagementStore::new( + MongoCatalogStore::on_database(client.clone(), catalog.clone()), + lease.clone(), + ); + Ok(Self { + client, + stores: MongoStores { + catalog, + data, + lease, + management, + #[cfg(feature = "test-util")] + stop_after: None, + }, + }) + } + + /// Stop before the request after the first `requests`, as a migrator + /// killed at that point stops: nothing after it is sent. + #[cfg(feature = "test-util")] + #[must_use] + pub fn stop_after(mut self, requests: usize) -> Self { + let left = Arc::new(std::sync::atomic::AtomicUsize::new(requests)); + self.stores.management.stop_after(Arc::clone(&left)); + self.stores.stop_after = Some(left); + self + } + + /// Close the client's connections. + pub async fn close(self) { + self.client.shutdown().await; + } + + fn row_error(store: &str, id: &Bson, what: &str) -> MigrationError { + MigrationError::Store(format!( + "{LEDGER} document {id} in {store} has no {what}, which this binary cannot read" + )) + } + + fn columns(store: &str, d: &Document) -> Result, MigrationError> { + let id = d.get("_id").cloned().unwrap_or(Bson::Null); + let number = match &id { + Bson::Int32(n) => i64::from(*n), + Bson::Int64(n) => *n, + _ => return Err(Self::row_error(store, &id, "integer _id")), + }; + let text = |field: &str| -> Result { + d.get_str(field) + .map(str::to_owned) + .map_err(|_| Self::row_error(store, &id, field)) + }; + let applied_at = match d.get("applied_at") { + Some(Bson::DateTime(t)) => t.try_to_rfc3339_string().unwrap_or_else(|_| t.to_string()), + _ => return Err(Self::row_error(store, &id, "applied_at date")), + }; + Ok(( + number, + text("name")?, + text("kind")?, + text("checksum")?, + text("outcome")?, + text("notes")?, + applied_at, + text("applied_by")?, + )) + } +} + +/// The writes that record a body in one database: the ledger row, and in +/// the catalog the version row, in one transaction, so a run stopped between +/// them cannot leave a ledger row for migration 1 beside a released +/// release's dotted version, which reads as written by an older release. +/// The ledger collection is created before the transaction starts. +struct Writes<'a> { + stores: &'a MongoStores, + db: &'a mongodb::Database, + session: tokio::sync::Mutex, +} + +impl Writes<'_> { + async fn begin(&self) -> Result<(), MigrationError> { + let mut session = self.session.lock().await; + self.stores + .request("start the transaction that records the migration", async { + session + .start_transaction() + .read_concern(mongodb::options::ReadConcern::local()) + .write_concern(mongodb::options::WriteConcern::majority()) + .await + }) + .await + } + + async fn commit(&self) -> Result<(), MigrationError> { + let mut session = self.session.lock().await; + self.stores + .request("commit the ledger and version rows", async { + session.commit_transaction().await + }) + .await + } + + async fn abort(&self) { + let mut session = self.session.lock().await; + let _ = tokio::time::timeout(REQUEST_TIMEOUT, session.abort_transaction()).await; + } +} + +/// Create a database's ledger collection, accepting one that exists. +async fn create_ledger(stores: &MongoStores, db: &mongodb::Database) -> Result<(), MigrationError> { + stores + .request(format!("create {}.{LEDGER}", db.name()), async { + match db.create_collection(LEDGER).await { + Err(e) if command_code(&e) == Some(NAMESPACE_EXISTS) => Ok(()), + other => other, + } + }) + .await +} + +#[async_trait] +impl LedgerWrites for Writes<'_> { + type Error = MigrationError; + + /// Done before the transaction starts: [`MongoMigrations::apply`] + /// creates the collection first. + async fn create_ledger(&self) -> Result<(), MigrationError> { + Ok(()) + } + + async fn insert_row(&self, row: &LedgerRow) -> Result<(), MigrationError> { + // `$literal` keeps a value that starts with `$` from being read as + // a field path; `applied_at` is the server's clock. + let literal = |value: &str| doc! { "$literal": value }; + let ledger = self.db.collection::(LEDGER); + let update = vec![doc! { "$set": { + "name": literal(&row.name), + "kind": literal(row.kind.as_str()), + "checksum": literal(&row.checksum), + "outcome": literal(row.outcome.as_str()), + "notes": literal(&row.notes), + "applied_at": "$$NOW", + "applied_by": literal(&row.applied_by), + } }]; + let mut session = self.session.lock().await; + self.stores + .request( + format!("record migration {} in {}.{LEDGER}", row.id, self.db.name()), + async { + ledger + .update_one(doc! { "_id": i64::from(row.id) }, update) + .upsert(true) + .session(&mut *session) + .await + .map(|_| ()) + }, + ) + .await + } + + async fn write_version(&self, version: u32) -> Result<(), MigrationError> { + let mut session = self.session.lock().await; + write_version(self.stores, version, Some(&mut session)).await + } +} + +async fn write_version( + stores: &MongoStores, + version: u32, + session: Option<&mut mongodb::ClientSession>, +) -> Result<(), MigrationError> { + let settings = stores.catalog.collection::("settings"); + stores + .request("write the catalog_version settings row", async { + let update = settings + .update_one( + doc! { "_id": "catalog_version" }, + doc! { "$set": { "value": version.to_string() } }, + ) + .upsert(true); + match session { + Some(session) => update.session(session).await, + None => update.await, + } + .map(|_| ()) + }) + .await +} + +fn command_code(error: &mongodb::error::Error) -> Option { + match error.kind.as_ref() { + mongodb::error::ErrorKind::Command(command) => Some(command.code), + _ => None, + } +} + +#[async_trait] +impl MigrationBackend for MongoMigrations { + type Stores = MongoStores; + + fn backend_name(&self) -> &'static str { + "mongodb" + } + + fn migration(&self, id: MigrationId) -> Migration { + super::migration(id) + } + + fn released(&self) -> &'static [Released] { + super::RELEASED + } + + fn stores(&self) -> &MongoStores { + &self.stores + } + + async fn store_names(&self, kind: StoreKind) -> Result, MigrationError> { + Ok(match kind { + StoreKind::Catalog => vec![self.stores.catalog.name().to_owned()], + StoreKind::Data => vec![self.stores.data.name().to_owned()], + StoreKind::Account => Vec::new(), + }) + } + + /// A ledger collection that does not exist reads as no documents; any + /// other failure is an error. + async fn read_ledger(&self, store: &StoreRef) -> Result, MigrationError> { + let ledger = self.stores.db(store.kind).collection::(LEDGER); + let docs = self + .stores + .request(format!("read {}.{LEDGER}", store.name), async { + let cursor = ledger.find(doc! {}).sort(doc! { "_id": 1 }).await?; + cursor.try_collect::>().await + }) + .await?; + docs.iter() + .map(|d| LedgerRow::from_columns(Some(&store.name), Self::columns(&store.name, d)?)) + .collect() + } + + async fn read_traces(&self) -> Result { + let settings = self.stores.catalog.collection::("settings"); + let docs = self + .stores + .request("read the catalog's settings", async { + let cursor = settings + .find(doc! { "_id": { "$in": ["catalog_version", "resource_tag_migration"] } }) + .await?; + cursor.try_collect::>().await + }) + .await?; + let rows: Vec<(String, String)> = docs + .iter() + .filter_map(|d| { + let key = d.get_str("_id").ok()?.to_owned(); + let value = match d.get("value") { + Some(Bson::String(s)) => s.clone(), + Some(other) => other.to_string(), + None => String::new(), + }; + Some((key, value)) + }) + .collect(); + Ok(CatalogTraces::from_settings(&rows)) + } + + /// The claim's upsert creates `settings` in an empty catalog, so a + /// catalog holding that collection alone holds nothing to detect. + async fn catalog_is_empty(&self) -> Result { + let db = &self.stores.catalog; + let names = self + .stores + .request(format!("list the collections of {}", db.name()), async { + db.list_collection_names().await + }) + .await?; + Ok(names.iter().all(|n| n == "settings")) + } + + async fn before_first_write( + &self, + _from: Option, + _to: u32, + ) -> Result, MigrationError> { + Ok(Vec::new()) + } + + async fn apply( + &self, + store: &StoreRef, + id: MigrationId, + step: Step<'_, MongoStores>, + row: Option, + catalog_version: Option, + ) -> Result { + self.stores.check()?; + let done = match step { + Step::File { name, .. } => { + return Err(MigrationError::Store(format!( + "migration {id} has a file body ({name}), and MongoDB runs code bodies only" + ))); + } + // A body stopped by the lease reports the lease, which says why. + Step::Code(body) => body + .run(&self.stores) + .await + .map_err(|e| self.stores.check().err().unwrap_or(e))?, + Step::Record => Done::default(), + }; + // The first application only: on a store at a later migration the + // objects migration 1 creates may carry what later migrations added. + if id == MigrationId::Baseline && row.is_some() { + let expected = match store.kind { + StoreKind::Catalog => super::CATALOG_EXPECTED, + StoreKind::Data | StoreKind::Account => super::DATA_EXPECTED, + }; + postcheck::check( + &self.stores, + self.stores.db(store.kind), + expected, + &store.name, + ) + .await?; + } + let db = self.stores.db(store.kind); + if row.is_some() { + create_ledger(&self.stores, db).await?; + } + let session = self + .stores + .request("start a session", async { + self.client.start_session().await + }) + .await?; + let writes = Writes { + stores: &self.stores, + db, + session: tokio::sync::Mutex::new(session), + }; + in_transaction( + writes.begin(), + record_rows(&writes, row, &done.notes, catalog_version), + writes.commit(), + writes.abort(), + ) + .await?; + Ok(done) + } + + async fn write_catalog_version(&self, version: u32) -> Result<(), MigrationError> { + write_version(&self.stores, version, None).await + } + + async fn delete_ledger_row(&self, store: &StoreRef, id: u32) -> Result<(), MigrationError> { + let ledger = self.stores.db(store.kind).collection::(LEDGER); + self.stores + .request( + format!("forget migration {id} in {}.{LEDGER}", store.name), + async { + ledger + .delete_one(doc! { "_id": i64::from(id) }) + .await + .map(|_| ()) + }, + ) + .await + } +} diff --git a/crates/storage-mongodb/src/migrations/commands.rs b/crates/storage-mongodb/src/migrations/commands.rs new file mode 100644 index 000000000..446c16358 --- /dev/null +++ b/crates/storage-mongodb/src/migrations/commands.rs @@ -0,0 +1,176 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The MongoDB backend's [`CatalogMigrations`], which `init`, `migrate` and +//! `verify` call through the installed backend. +//! +//! A command's [`Target`] gives the connection string the config file +//! names, which the server uses too; the databases are always +//! `extenddb_catalog` and `extenddb_data`. `migrate --yes` and `--forget` +//! read the status before they take the lock, and a store that is not a +//! catalog is refused before anything is written to it: taking the lock +//! writes the lock document into the catalog's `settings` collection. Only +//! `init`, whose target names the data connection it records, takes the +//! lock on a store that is not a catalog yet. + +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use async_trait::async_trait; +use extenddb_storage::migrations::{ + CatalogMigrations, CatalogStatus, HeldLock, LedgerRow, Listed, MigrationError, MigrationId, + RunKind, RunReport, Runner, Target, +}; + +use super::{ + CATALOG_DATABASE, DATA_DATABASE, Lease, MigrationLock, MongoMigrations, MongoTarget, Timing, + catalog_status, status_holding, +}; + +/// The MongoDB backend's catalog migrations for the commands. +pub struct MongoCatalogMigrations; + +/// What the migration lock carries for [`CatalogMigrations::run`] and +/// [`CatalogMigrations::forget`]. +struct Held { + lock: MigrationLock, + target: MongoTarget, +} + +fn held(lock: &HeldLock) -> Result<&Held, MigrationError> { + lock.get::().ok_or_else(|| { + MigrationError::Refused("the migration lock was not taken by the MongoDB backend".into()) + }) +} + +/// The lease of a lock this backend took, which `init`'s bootstrapper checks +/// before each row it writes; `None` for another backend's lock. +pub(crate) fn lease_of(lock: &HeldLock) -> Option> { + held(lock).ok().map(|held| held.lock.lease()) +} + +#[async_trait] +impl CatalogMigrations for MongoCatalogMigrations { + fn listed(&self, id: MigrationId) -> Listed { + Listed::of(&super::migration(id)) + } + + fn backup_advice(&self, _target: &Target) -> String { + format!( + "back up both MongoDB databases, the catalog database {CATALOG_DATABASE} and the data \ + database {DATA_DATABASE}" + ) + } + + async fn status(&self, target: &Target) -> Result { + catalog_status(&MongoTarget::new(&target.location)).await + } + + /// Read as the holder, so the requests of a long status read are + /// progress that keeps the lock. + async fn status_under_lock( + &self, + lock: &HeldLock, + _target: &Target, + ) -> Result { + let held = held(lock)?; + status_holding(&held.target, &held.lock).await + } + + async fn catalog_ledger(&self, target: &Target) -> Result, MigrationError> { + let migrations = MongoMigrations::open(&MongoTarget::new(&target.location), None).await?; + let rows = Runner::new(&migrations, "").catalog_ledger().await; + migrations.close().await; + rows + } + + async fn lock(&self, target: &Target, timeout: Duration) -> Result { + lock( + &MongoTarget::new(&target.location), + target.data_connection.is_some(), + timeout, + Timing::PRODUCTION, + ) + .await + .map(|(lock, target)| { + HeldLock::new(target.connection_string.clone(), Held { lock, target }) + }) + } + + async fn run( + &self, + lock: &HeldLock, + release: &str, + kind: RunKind, + ) -> Result { + let held = held(lock)?; + let migrations = MongoMigrations::open(&held.target, Some(&held.lock)).await?; + let runner = Runner::new(&migrations, release); + let result = match kind { + RunKind::Create | RunKind::Migrate => runner.migrate().await, + RunKind::Reconverge => runner.reconverge().await, + }; + migrations.close().await; + result + } + + async fn forget(&self, lock: &HeldLock, id: u32) -> Result<(), MigrationError> { + let held = held(lock)?; + let migrations = MongoMigrations::open(&held.target, Some(&held.lock)).await?; + let result = Runner::new(&migrations, "").forget(id).await; + migrations.close().await; + result + } +} + +/// Take the migration lock on `target`, waiting up to `timeout` for another +/// migrator and then, within the same timeout, for running servers to stop. +/// Unless `creating` (`init`), a store that is not a catalog is refused +/// first, with nothing written. +/// +/// # Errors +/// +/// The store is not a catalog (`Uninitialized`, exit 1), the lock or a +/// server outlasted `timeout` ([`MigrationError::Locked`], exit 6), or a +/// request failed. +pub async fn lock( + target: &MongoTarget, + creating: bool, + timeout: Duration, + timing: Timing, +) -> Result<(MigrationLock, MongoTarget), MigrationError> { + if !creating { + let status = catalog_status(target).await?; + if status == CatalogStatus::Uninitialized { + return Err(MigrationError::Status(status)); + } + } + let started = Instant::now(); + let lock = MigrationLock::acquire(target, timeout, timing).await?; + if let Err(e) = lock + .wait_for_servers(timeout.saturating_sub(started.elapsed())) + .await + { + lock.release().await; + return Err(e); + } + Ok((lock, target.clone())) +} + +#[cfg(test)] +mod tests { + use extenddb_storage::migrations::{CatalogMigrations, Target}; + + use super::MongoCatalogMigrations; + + #[test] + fn the_backup_advice_names_both_databases_and_no_password() { + let target = Target::at("mongodb://app:s3cret@localhost:27017/?replicaSet=rs0"); + let advice = MongoCatalogMigrations.backup_advice(&target); + assert_eq!( + advice, + "back up both MongoDB databases, the catalog database extenddb_catalog and the data \ + database extenddb_data" + ); + } +} diff --git a/crates/storage-mongodb/src/migrations/guarded.rs b/crates/storage-mongodb/src/migrations/guarded.rs new file mode 100644 index 000000000..cef14f729 --- /dev/null +++ b/crates/storage-mongodb/src/migrations/guarded.rs @@ -0,0 +1,255 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The management store migration 2's shared body runs through on MongoDB. +//! +//! Every call checks the migration lock's lease first, so a holder that was +//! paused past its deadline sends no further request, as the runner checks +//! before each of its own, and waits at most +//! [`REQUEST_TIMEOUT`](super::REQUEST_TIMEOUT) for its answer, as each of +//! the runner's requests does. A call that completes is progress, as a +//! request of the runner's is, so the renewal task keeps the lock through a +//! scan that lasts longer than the idle limit. +//! +//! `list_all_accounts` reads every document in `accounts`, including the +//! default account a released `init` wrote with only `_id`, which the +//! store's own listing skips because it has no `account_id` field; its +//! `_id` is the id its principals would carry. + +use std::sync::Arc; + +use bson::{Document, doc}; +use extenddb_storage::management_store::{ + AccessKeyCreated, AccountDetail, GroupDetail, GroupListEntry, ManagementStore, OpError, + OpResult, RoleDetail, RoleListEntry, SessionPolicyEntry, UserDetail, UserListEntry, +}; +use futures::future::BoxFuture; + +use super::REQUEST_TIMEOUT; +use super::locks::Lease; +use crate::MongoCatalogStore; + +/// [`MongoCatalogStore`] with the lease checked before every call, a timeout +/// on every call, and progress recorded when a call completes. +pub(crate) struct LeasedManagementStore { + inner: MongoCatalogStore, + accounts: mongodb::Collection, + lease: Option>, + #[cfg(feature = "test-util")] + stop_after: Option>, +} + +/// Whether the test hook's request budget is spent; takes one request. +#[cfg(feature = "test-util")] +pub(crate) fn stopped(left: Option<&std::sync::atomic::AtomicUsize>) -> bool { + use std::sync::atomic::Ordering; + // A compare-exchange loop: `fetch_update` is deprecated on current + // stable and its replacement does not exist on 1.88. + let Some(left) = left else { + return false; + }; + let mut n = left.load(Ordering::SeqCst); + loop { + let Some(next) = n.checked_sub(1) else { + return true; + }; + match left.compare_exchange(n, next, Ordering::SeqCst, Ordering::SeqCst) { + Ok(_) => return false, + Err(actual) => n = actual, + } + } +} + +impl LeasedManagementStore { + pub(crate) fn new(inner: MongoCatalogStore, lease: Option>) -> Self { + let accounts = inner.catalog_db().collection::("accounts"); + Self { + inner, + accounts, + lease, + #[cfg(feature = "test-util")] + stop_after: None, + } + } + + #[cfg(feature = "test-util")] + pub(crate) fn stop_after(&mut self, left: Arc) { + self.stop_after = Some(left); + } + + fn lease_ok(&self) -> OpResult<()> { + #[cfg(feature = "test-util")] + if stopped(self.stop_after.as_deref()) { + return Err(OpError::Internal("stopped by the test".to_owned())); + } + match &self.lease { + Some(lease) => lease.check().map_err(|e| OpError::Internal(e.to_string())), + None => Ok(()), + } + } + + /// Record that a call completed, as `MongoStores::request` does after + /// each request, so the renewal task keeps the lock while a scan runs. + fn progressed(&self) { + if let Some(lease) = &self.lease { + lease.progressed(); + } + } +} + +/// Delegate each method to the inner store after checking the lease, wait +/// at most [`REQUEST_TIMEOUT`] for its answer, and record progress when the +/// call completes. +macro_rules! checked { + ($( fn $name:ident(&self $(, $arg:ident: $ty:ty)* $(,)?) -> $ret:ty; )+) => { + $( + fn $name(&self $(, $arg: $ty)*) -> $ret { + if let Err(e) = self.lease_ok() { + return Box::pin(async move { Err(e) }); + } + let call = self.inner.$name($($arg),*); + Box::pin(async move { + let answer = tokio::time::timeout(REQUEST_TIMEOUT, call).await; + self.progressed(); + answer.unwrap_or_else(|_| Err(no_answer(stringify!($name)))) + }) + } + )+ + }; +} + +/// A call that had no answer within [`REQUEST_TIMEOUT`]. +fn no_answer(what: &str) -> OpError { + OpError::Internal(format!( + "{what}: no answer within {} s", + REQUEST_TIMEOUT.as_secs() + )) +} + +impl ManagementStore for LeasedManagementStore { + fn list_all_accounts(&self) -> BoxFuture<'_, OpResult>> { + Box::pin(async move { + use futures::TryStreamExt; + self.lease_ok()?; + let read = async { + let cursor = self.accounts.find(doc! {}).sort(doc! { "_id": 1 }).await?; + cursor.try_collect::>().await + }; + let answer = tokio::time::timeout(REQUEST_TIMEOUT, read).await; + self.progressed(); + let docs = match answer { + Ok(Ok(docs)) => docs, + Ok(Err(e)) => return Err(OpError::Internal(format!("list accounts: {e}"))), + Err(_) => return Err(no_answer("list accounts")), + }; + Ok(docs + .iter() + .filter_map(|d| { + let id = d.get_str("account_id").or_else(|_| d.get_str("_id")).ok()?; + Some(( + id.to_owned(), + d.get_str("account_name").unwrap_or_default().to_owned(), + )) + }) + .collect()) + }) + } + + checked! { + fn create_account(&self, account_id: &str, account_name: &str) -> BoxFuture<'_, OpResult<()>>; + fn delete_account(&self, account_id: &str) -> BoxFuture<'_, OpResult<()>>; + fn default_account_id(&self) -> BoxFuture<'_, OpResult>>; + fn list_all_accounts_full(&self) -> BoxFuture<'_, OpResult>>; + fn list_accounts_for(&self, account_id: &str) -> BoxFuture<'_, OpResult>>; + fn get_account_detail(&self, account_id: &str) -> BoxFuture<'_, OpResult>>; + fn dashboard_counts(&self) -> BoxFuture<'_, OpResult<(i64, i64)>>; + fn create_user(&self, account_id: &str, user_name: &str, password_hash: Option<&str>) -> BoxFuture<'_, OpResult<()>>; + fn delete_user(&self, account_id: &str, user_name: &str) -> BoxFuture<'_, OpResult<()>>; + fn list_users(&self, account_id: &str) -> BoxFuture<'_, OpResult>>; + fn get_user_detail(&self, account_id: &str, user_name: &str) -> BoxFuture<'_, OpResult>>; + fn verify_iam_user_password(&self, account_id: &str, user_name: &str, password: &str) -> BoxFuture<'_, OpResult>; + fn change_user_password(&self, account_id: &str, user_name: &str, password_hash: &str) -> BoxFuture<'_, OpResult<()>>; + fn tag_user(&self, account_id: &str, user_name: &str, tags: &[(String, String)]) -> BoxFuture<'_, OpResult<()>>; + fn untag_user(&self, account_id: &str, user_name: &str, tag_keys: &[String]) -> BoxFuture<'_, OpResult<()>>; + fn list_user_tags(&self, account_id: &str, user_name: &str) -> BoxFuture<'_, OpResult>>; + fn create_group(&self, account_id: &str, group_name: &str) -> BoxFuture<'_, OpResult<()>>; + fn delete_group(&self, account_id: &str, group_name: &str) -> BoxFuture<'_, OpResult<()>>; + fn list_groups(&self, account_id: &str) -> BoxFuture<'_, OpResult>>; + fn get_group_detail(&self, account_id: &str, group_name: &str) -> BoxFuture<'_, OpResult>>; + fn add_group_member(&self, account_id: &str, group_name: &str, user_name: &str) -> BoxFuture<'_, OpResult<()>>; + fn remove_group_member(&self, account_id: &str, group_name: &str, user_name: &str) -> BoxFuture<'_, OpResult<()>>; + fn create_role(&self, account_id: &str, role_name: &str, trust_policy: &serde_json::Value) -> BoxFuture<'_, OpResult<()>>; + fn delete_role(&self, account_id: &str, role_name: &str) -> BoxFuture<'_, OpResult<()>>; + fn list_roles(&self, account_id: &str) -> BoxFuture<'_, OpResult>>; + fn get_role_detail(&self, account_id: &str, role_name: &str) -> BoxFuture<'_, OpResult>>; + fn get_role_trust_policy(&self, account_id: &str, role_name: &str) -> BoxFuture<'_, OpResult>>; + fn tag_role(&self, account_id: &str, role_name: &str, tags: &[(String, String)]) -> BoxFuture<'_, OpResult<()>>; + fn untag_role(&self, account_id: &str, role_name: &str, tag_keys: &[String]) -> BoxFuture<'_, OpResult<()>>; + fn list_role_tags(&self, account_id: &str, role_name: &str) -> BoxFuture<'_, OpResult>>; + fn put_policy(&self, account_id: &str, principal_type: &str, principal_name: &str, policy_name: &str, document: &serde_json::Value) -> BoxFuture<'_, OpResult<()>>; + fn delete_policy(&self, account_id: &str, principal_type: &str, principal_name: &str, policy_name: &str) -> BoxFuture<'_, OpResult<()>>; + fn list_policies(&self, account_id: &str, principal_type: &str, principal_name: &str) -> BoxFuture<'_, OpResult>>; + fn set_user_boundary(&self, account_id: &str, user_name: &str, document: &serde_json::Value) -> BoxFuture<'_, OpResult<()>>; + fn get_user_boundary(&self, account_id: &str, user_name: &str) -> BoxFuture<'_, OpResult>>; + fn delete_user_boundary(&self, account_id: &str, user_name: &str) -> BoxFuture<'_, OpResult<()>>; + fn set_role_boundary(&self, account_id: &str, role_name: &str, document: &serde_json::Value) -> BoxFuture<'_, OpResult<()>>; + fn get_role_boundary(&self, account_id: &str, role_name: &str) -> BoxFuture<'_, OpResult>>; + fn delete_role_boundary(&self, account_id: &str, role_name: &str) -> BoxFuture<'_, OpResult<()>>; + fn create_access_key(&self, account_id: &str, user_name: &str) -> BoxFuture<'_, OpResult>; + fn delete_access_key(&self, account_id: &str, user_name: &str, key_id: &str) -> BoxFuture<'_, OpResult<()>>; + fn list_access_keys(&self, account_id: &str, user_name: &str) -> BoxFuture<'_, OpResult>>; + fn import_access_key(&self, account_id: &str, user_name: &str, access_key_id: &str, secret_access_key: &str) -> BoxFuture<'_, OpResult<()>>; + fn store_session(&self, session_token: &str, access_key_id: &str, secret_key_encrypted: &[u8], account_id: &str, role_name: &str, session_name: &str, session_tags: &Option, session_policy: &Option, expires_at: time::OffsetDateTime) -> BoxFuture<'_, OpResult<()>>; + fn list_session_policies(&self) -> BoxFuture<'_, OpResult>>; + fn update_session_policy(&self, session_token: &str, document: &serde_json::Value) -> BoxFuture<'_, OpResult<()>>; + fn fetch_caller_tags(&self, account_id: &str, resource: &str) -> BoxFuture<'_, OpResult>>; + } +} + +#[cfg(test)] +mod tests { + use std::time::{Duration, Instant}; + + use extenddb_storage::management_store::{ManagementStore, OpError}; + + use super::super::REQUEST_TIMEOUT; + use super::super::locks::tests::silent_server; + use super::LeasedManagementStore; + use crate::MongoCatalogStore; + + /// A delegated call to a database that stopped answering fails at the + /// request timeout, not when the driver gives up on the connection. + #[tokio::test] + async fn a_delegated_call_with_no_answer_fails_at_the_request_timeout() { + let (address, accepting) = silent_server().await; + let mut options = mongodb::options::ClientOptions::parse(format!( + "mongodb://{address}/?directConnection=true" + )) + .await + .expect("options"); + options.connect_timeout = Some(Duration::from_secs(60)); + options.server_selection_timeout = Some(Duration::from_secs(60)); + let client = mongodb::Client::with_options(options).expect("client"); + let store = LeasedManagementStore::new( + MongoCatalogStore::on_database(client.clone(), client.database("silent")), + None, + ); + let started = Instant::now(); + let answer = tokio::time::timeout( + REQUEST_TIMEOUT + Duration::from_secs(5), + store.list_policies("000000000000", "user", "alice"), + ) + .await + .expect("an answer within the request timeout"); + let elapsed = started.elapsed(); + let Err(OpError::Internal(message)) = answer else { + panic!("{answer:?}"); + }; + assert!( + message.contains("no answer within 10 s"), + "{message} after {elapsed:?}" + ); + assert!(elapsed >= REQUEST_TIMEOUT, "{elapsed:?}"); + accepting.abort(); + } +} diff --git a/crates/storage-mongodb/src/migrations/locks.rs b/crates/storage-mongodb/src/migrations/locks.rs new file mode 100644 index 000000000..92b4be65e --- /dev/null +++ b/crates/storage-mongodb/src/migrations/locks.rs @@ -0,0 +1,485 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The migration lock and the serving registration. +//! +//! The lock is the document `_id: "migration_lock"` in the catalog's +//! `settings` collection, which exists before the first migrator runs (the +//! claim's upsert creates it in an empty database). A claim is one +//! `findOneAndUpdate` with the filter on `_id` alone and the condition in a +//! pipeline update, because MongoDB refuses `$expr` in an upsert's filter: +//! it sets `owner` to the claimant's token and `expires_at` to the server's +//! `$$NOW` plus the lease, only when the document has no `expires_at` or one +//! that has passed. The claimant holds the lock when the returned `owner` is +//! its token. A renewal moves `expires_at` with `updateOne({_id, owner})`, +//! and a match count of 0 means the lock was lost; the release is +//! `deleteOne({_id, owner})`. Every write uses write concern majority and +//! every request waits at most [`REQUEST_TIMEOUT`](super::REQUEST_TIMEOUT). +//! +//! The holder keeps a deadline on the monotonic clock: the send time of its +//! last successful claim or renewal plus the lease minus a margin of 25 s, +//! which is longer than a request may take. The runner checks it before +//! every request, so a holder that was paused past it stops without +//! recording anything. The renewal task stops renewing when the holder has +//! made no progress for 30 s, so a hung holder loses the lock. +//! +//! A server registers with one document in `extenddb_servers` whose +//! `expires_at` is the server's `$$NOW` plus 45 s, renewed every 15 s, and +//! stops (exit 1) once it has not renewed for 30 s. `migrate --yes` takes +//! the lock and then waits for every registration whose `expires_at` has +//! not passed; a server registers and then reads the lock. Each takes its +//! own first and then looks for the other, so two that start together +//! cannot both miss each other. + +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use bson::{Bson, Document, doc}; +use extenddb_storage::migrations::MigrationError; +use mongodb::options::ReturnDocument; +use tokio::task::JoinHandle; + +use super::registrations::live_registrations; +use super::{LOCK_ID, MongoTarget, REQUEST_TIMEOUT, SERVERS, connect, driver_error}; + +/// How long the lock, its renewals and the registration last. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Timing { + /// A claim or renewal sets `expires_at` this far after the server's + /// clock. + pub lease: Duration, + /// How often the holder renews. + pub renew_every: Duration, + /// The holder stops acting this long before its lease ends. + pub margin: Duration, + /// The renewal task stops renewing when the holder has made no progress + /// for this long. + pub idle_limit: Duration, + /// A registration's `expires_at` after its server's last renewal. + pub registration: Duration, + /// A server that has not renewed its registration for this long stops. + pub registration_deadline: Duration, + /// How often a claimant tries again while another process holds the + /// lock, and `migrate --yes` reads the registrations again. + pub retry_every: Duration, +} + +impl Timing { + /// The production timings: a 60 s lease renewed every 15 s with a 25 s + /// margin, and 45 s registrations renewed every 15 s. + pub const PRODUCTION: Self = Self { + lease: Duration::from_secs(60), + renew_every: Duration::from_secs(15), + margin: Duration::from_secs(25), + idle_limit: Duration::from_secs(30), + registration: Duration::from_secs(45), + registration_deadline: Duration::from_secs(30), + retry_every: Duration::from_millis(500), + }; + + /// How long after a successful claim or renewal the holder may act. + fn valid_for(self) -> Duration { + self.lease.saturating_sub(self.margin) + } +} + +/// Milliseconds of a duration, for `$add` with `$$NOW`. +pub(super) fn millis(d: Duration) -> i64 { + i64::try_from(d.as_millis()).unwrap_or(i64::MAX) +} + +/// A random owner token, so a restarted process cannot inherit a claim. +pub(super) fn token() -> String { + uuid::Uuid::new_v4().to_string() +} + +/// The holder's view of its lease: when it last renewed, when it last made +/// progress, and why it lost the lock. +pub(crate) struct Lease { + renewed: Mutex, + progress: Mutex, + lost: Mutex>, + timing: Timing, +} + +impl Lease { + pub(crate) fn new(renewed: Instant, timing: Timing) -> Arc { + Arc::new(Self { + renewed: Mutex::new(renewed), + progress: Mutex::new(Instant::now()), + lost: Mutex::new(None), + timing, + }) + } + + /// Whether the holder may still act: the lock was not lost, and its last + /// successful renewal was sent less than the lease minus the margin ago. + pub(crate) fn check(&self) -> Result<(), MigrationError> { + if let Some(reason) = self.lost.lock().unwrap_or_else(|e| e.into_inner()).clone() { + return Err(MigrationError::Store(reason)); + } + let since = self + .renewed + .lock() + .unwrap_or_else(|e| e.into_inner()) + .elapsed(); + if since >= self.timing.valid_for() { + return Err(self.mark_lost(format!( + "the migration lock was last renewed {} s ago, and this process acts on it only \ + for {} s after a renewal; another process may hold it now, so nothing more is \ + applied", + since.as_secs(), + self.timing.valid_for().as_secs() + ))); + } + Ok(()) + } + + /// Record that the holder made progress (a request completed). + pub(crate) fn progressed(&self) { + *self.progress.lock().unwrap_or_else(|e| e.into_inner()) = Instant::now(); + } + + fn idle_for(&self) -> Duration { + self.progress + .lock() + .unwrap_or_else(|e| e.into_inner()) + .elapsed() + } + + fn renewed_at(&self, sent: Instant) { + *self.renewed.lock().unwrap_or_else(|e| e.into_inner()) = sent; + } + + fn mark_lost(&self, reason: String) -> MigrationError { + let mut lost = self.lost.lock().unwrap_or_else(|e| e.into_inner()); + MigrationError::Store(lost.get_or_insert(reason).clone()) + } +} + +/// The lock document's state, as a claim returns it. +fn owner_of(lock: Option<&Document>) -> Option<&str> { + lock.and_then(|d| d.get_str("owner").ok()) +} + +/// Claim the lock for `token`: one `findOneAndUpdate` that takes a missing +/// or expired lock and leaves a live one as it is. Returns whether `token` +/// holds it, and the document after the update. +async fn claim( + settings: &mongodb::Collection, + token: &str, + lease: Duration, +) -> Result<(bool, Option), MigrationError> { + let pipeline = vec![ + doc! { "$set": { "free": { "$or": [ + { "$eq": [{ "$type": "$expires_at" }, "missing"] }, + { "$lt": ["$expires_at", "$$NOW"] }, + ] } } }, + doc! { "$set": { + "owner": { "$cond": ["$free", token, "$owner"] }, + "expires_at": { "$cond": ["$free", { "$add": ["$$NOW", millis(lease)] }, "$expires_at"] }, + } }, + doc! { "$unset": "free" }, + ]; + let request = settings + .find_one_and_update(doc! { "_id": LOCK_ID }, pipeline) + .upsert(true) + .return_document(ReturnDocument::After); + match tokio::time::timeout(REQUEST_TIMEOUT, request).await { + Ok(Ok(after)) => Ok((owner_of(after.as_ref()) == Some(token), after)), + // Two claims upserting a missing document at once: the second one + // fails on the `_id`, and claims again. + Ok(Err(e)) if is_duplicate_key(&e) => Ok((false, None)), + Ok(Err(e)) => Err(driver_error("take the migration lock", &e)), + Err(_) => Err(timed_out("take the migration lock")), + } +} + +pub(crate) fn is_duplicate_key(error: &mongodb::error::Error) -> bool { + use mongodb::error::{ErrorKind, WriteFailure}; + match error.kind.as_ref() { + ErrorKind::Write(WriteFailure::WriteError(e)) => e.code == 11000, + ErrorKind::Command(e) => e.code == 11000, + _ => false, + } +} + +pub(super) fn timed_out(what: &str) -> MigrationError { + MigrationError::Store(format!( + "{what}: no answer within {} s", + REQUEST_TIMEOUT.as_secs() + )) +} + +/// Who holds the lock, for a message. +pub(super) fn holder(lock: Option<&Document>) -> String { + match lock { + Some(d) => format!( + "another migration (owner {}, expires {})", + d.get_str("owner").unwrap_or("unknown"), + d.get("expires_at") + .map_or_else(|| "never".to_owned(), describe_time) + ), + None => "another migration".to_owned(), + } +} + +pub(super) fn describe_time(value: &Bson) -> String { + match value { + Bson::DateTime(t) => t.try_to_rfc3339_string().unwrap_or_else(|_| t.to_string()), + other => other.to_string(), + } +} + +/// The migration lock, held for a run. Dropping it releases it; a process +/// that dies without releasing it leaves it to expire with its lease. +pub struct MigrationLock { + target: MongoTarget, + settings: mongodb::Collection, + servers: mongodb::Collection, + token: String, + lease: Arc, + renewal: JoinHandle<()>, + released: bool, +} + +impl MigrationLock { + /// Take the migration lock, waiting up to `timeout` while another + /// process holds it, and renew it every [`Timing::renew_every`]. + /// + /// # Errors + /// + /// [`MigrationError::Locked`] (exit 6) naming the holder when it + /// outlasts `timeout`, or a request failed. + pub async fn acquire( + target: &MongoTarget, + timeout: Duration, + timing: Timing, + ) -> Result { + let client = connect(&target.connection_string).await?; + let db = super::catalog_handle(&client, &target.catalog); + let settings = db.collection::("settings"); + let token = token(); + let started = Instant::now(); + loop { + let sent = Instant::now(); + let (held, lock) = claim(&settings, &token, timing.lease).await?; + if held { + let lease = Lease::new(sent, timing); + let renewal = + spawn_renewal(settings.clone(), token.clone(), Arc::clone(&lease), timing); + return Ok(Self { + target: target.clone(), + settings, + servers: db.collection::(SERVERS), + token, + lease, + renewal, + released: false, + }); + } + if started.elapsed() >= timeout { + return Err(MigrationError::Locked(format!( + "{} held the migration lock for the whole {} s lock timeout; nothing was \ + changed. Run the command again once it finishes, or with a longer \ + --lock-timeout", + holder(lock.as_ref()), + timeout.as_secs() + ))); + } + tokio::time::sleep(timing.retry_every).await; + } + } + + /// The lease the runner checks before every request. + pub(crate) fn lease(&self) -> Arc { + Arc::clone(&self.lease) + } + + /// Whether this process may still act under the lock. + /// + /// # Errors + /// + /// The lock was lost, or its lease is past the margin. + pub fn check(&self) -> Result<(), MigrationError> { + self.lease.check() + } + + /// Wait up to `timeout` for every running server's registration to end. + /// + /// # Errors + /// + /// [`MigrationError::Locked`] naming the servers when one outlasts + /// `timeout`, or a request failed. + pub async fn wait_for_servers(&self, timeout: Duration) -> Result<(), MigrationError> { + let started = Instant::now(); + loop { + self.check()?; + let live = live_registrations(&self.servers).await?; + self.lease.progressed(); + if live.is_empty() { + return Ok(()); + } + if started.elapsed() >= timeout { + return Err(MigrationError::Locked(format!( + "a server is serving this catalog: {}; migrate --yes waited {} s for it to \ + stop and changed nothing. Stop every server that uses the catalog, then run \ + the command again", + live.join(", "), + timeout.as_secs() + ))); + } + tokio::time::sleep(self.lease.timing.retry_every).await; + } + } + + /// Delete the lock document if this process still owns it. + pub async fn release(mut self) { + self.renewal.abort(); + self.released = true; + let _ = tokio::time::timeout( + REQUEST_TIMEOUT, + self.settings + .delete_one(doc! { "_id": LOCK_ID, "owner": &self.token }), + ) + .await; + } + + /// Stop renewing and keep the document, as a process that was killed + /// does: the lock then ends when its lease does. + #[cfg(feature = "test-util")] + pub fn abandon(mut self) { + self.renewal.abort(); + self.released = true; + } +} + +impl Drop for MigrationLock { + /// A lock the caller did not release is released here. The commands run + /// on a single-threaded runtime that this call blocks, and the driver's + /// connections are served by tasks on that runtime, so the release runs + /// on a thread of its own with a client of its own. + fn drop(&mut self) { + self.renewal.abort(); + if self.released { + return; + } + let target = self.target.clone(); + let token = self.token.clone(); + let released = std::thread::spawn(move || { + let Ok(runtime) = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + else { + return; + }; + runtime.block_on(async move { + let Ok(client) = connect(&target.connection_string).await else { + return; + }; + let settings = super::catalog_handle(&client, &target.catalog) + .collection::("settings"); + let _ = tokio::time::timeout( + REQUEST_TIMEOUT, + settings.delete_one(doc! { "_id": LOCK_ID, "owner": &token }), + ) + .await; + }); + }); + let _ = released.join(); + } +} + +/// Renew the lock every `renew_every` until the lease is lost: a renewal +/// that matches no document, or no successful renewal before the holder's +/// deadline, or a holder with no progress for `idle_limit`. A renewal with +/// no answer is retried at once with the same token. +fn spawn_renewal( + settings: mongodb::Collection, + token: String, + lease: Arc, + timing: Timing, +) -> JoinHandle<()> { + tokio::spawn(async move { + let mut wait = timing.renew_every; + loop { + tokio::time::sleep(wait).await; + if lease.idle_for() >= timing.idle_limit { + lease.mark_lost(format!( + "the migration made no progress for {} s, so this process stopped renewing \ + the migration lock; nothing more is applied", + lease.idle_for().as_secs() + )); + return; + } + if lease.check().is_err() { + return; + } + let sent = Instant::now(); + let renewal = settings.update_one( + doc! { "_id": LOCK_ID, "owner": &token }, + vec![ + doc! { "$set": { "expires_at": { "$add": ["$$NOW", millis(timing.lease)] } } }, + ], + ); + match tokio::time::timeout(REQUEST_TIMEOUT, renewal).await { + Ok(Ok(result)) if result.matched_count == 1 => { + lease.renewed_at(sent); + wait = timing.renew_every; + } + Ok(Ok(_)) => { + lease.mark_lost( + "the migration lock document no longer names this process, so another \ + process may hold it; nothing more is applied" + .to_owned(), + ); + return; + } + // No answer, or an error: try again at once until the + // holder's deadline passes, which `check` then reports. + Ok(Err(_)) | Err(_) => wait = timing.retry_every, + } + } + }) +} + +#[cfg(test)] +pub(super) mod tests { + use std::time::Duration; + + use super::Timing; + + /// The production timings are the ones ADR-0023's lock table and the + /// upgrade manual give, which the Cassandra lock module carries under + /// its own names: a 60 s lease renewed every 15 s and acted on until + /// 25 s before it ends, renewal stopped after 30 s without progress, + /// 45 s registrations renewed every 15 s, and a server that has not + /// renewed for 30 s stops. A change here is a change to both documents. + #[test] + fn the_production_timings_are_the_documented_ones() { + let timing = Timing::PRODUCTION; + assert_eq!(timing.lease, Duration::from_secs(60)); + assert_eq!(timing.renew_every, Duration::from_secs(15)); + assert_eq!(timing.margin, Duration::from_secs(25)); + assert_eq!(timing.valid_for(), Duration::from_secs(35)); + assert_eq!(timing.idle_limit, Duration::from_secs(30)); + assert_eq!(timing.registration, Duration::from_secs(45)); + assert_eq!(timing.registration_deadline, Duration::from_secs(30)); + } + + /// A database that stopped answering: it accepts connections and never + /// replies, so every request waits until its timeout. + pub(crate) async fn silent_server() -> (std::net::SocketAddr, tokio::task::JoinHandle<()>) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind"); + let address = listener.local_addr().expect("address"); + let accepting = tokio::spawn(async move { + let mut open = Vec::new(); + while let Ok((socket, _)) = listener.accept().await { + open.push(socket); + } + }); + (address, accepting) + } +} diff --git a/crates/storage-mongodb/src/migrations/m0001_baseline.rs b/crates/storage-mongodb/src/migrations/m0001_baseline.rs new file mode 100644 index 000000000..6b900a47a --- /dev/null +++ b/crates/storage-mongodb/src/migrations/m0001_baseline.rs @@ -0,0 +1,235 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Migration 1, `baseline`, MongoDB: create the v1.0 collections and indexes +//! in an empty catalog and data database, and bring every released +//! deployment to them. +//! +//! The catalog body creates the catalog collections and their indexes, the +//! `extenddb_servers` collection the serving registration lives in, and the +//! legacy ledger document `001_initial` that every released `init` wrote, +//! and deletes the `resource_tag_migration` settings row every released +//! `init` seeded. The data body creates the data database's collections and +//! indexes, which before 1.0 only `init` created. Every request is safe to +//! send again: creating a collection that exists with the same options +//! succeeds, one that exists with other options fails with NamespaceExists +//! (code 48), which is accepted by its code and left to the post-check, and +//! creating an index that exists with the same key and options succeeds. +//! Every other error stops the body, so the runner records nothing. +//! +//! The checksum test pins this file once the migration is released. The +//! module is excluded from rustfmt and clippy so neither can require an +//! edit, and keeps its index definitions here instead of calling the +//! crate's, which may change. + +use std::time::Duration; + +use async_trait::async_trait; +use bson::{Document, doc}; +use extenddb_storage::migrations::{CodeBody, Done, MigrationError}; +use mongodb::IndexModel; +use mongodb::options::IndexOptions; + +use super::MongoStores; + +/// This module, whose SHA-256 the checksum test pins. +pub(crate) const SOURCE: &str = include_str!("m0001_baseline.rs"); + +/// Recorded as this migration's checksum. Changes only with a new migration. +const FINGERPRINT: &str = "0001_baseline/mongodb/1"; + +/// NamespaceExists: `create` on a collection that exists with other options. +const NAMESPACE_EXISTS: i32 = 48; + +/// The catalog collections every released `init` created, and the +/// collection the serving registration lives in. +const CATALOG_COLLECTIONS: [&str; 19] = [ + "schema_history", + "accounts", + "tables", + "indexes", + "tags", + "settings", + "admin_users", + "iam_users", + "access_keys", + "iam_groups", + "iam_roles", + "iam_sessions", + "iam_policies", + "iam_permissions_boundaries", + "metrics", + "login_attempts", + "backups", + "continuous_backups", + "extenddb_servers", +]; + +/// The data collections every released `init` created. +const DATA_COLLECTIONS: [&str; 3] = ["idempotency_tokens", "stream_shards", "stream_records"]; + +/// One index: its collection, key and options. +struct Index { + collection: &'static str, + key: Document, + options: IndexOptions, +} + +fn plain(collection: &'static str, key: Document) -> Index { + Index { collection, key, options: IndexOptions::default() } +} + +fn unique(collection: &'static str, key: Document) -> Index { + Index { collection, key, options: IndexOptions::builder().unique(true).build() } +} + +fn expiring(collection: &'static str, key: Document, after: Duration) -> Index { + Index { collection, key, options: IndexOptions::builder().expire_after(after).build() } +} + +/// The catalog indexes, as every released `init` created them, and the +/// expiry of the serving registrations. +fn catalog_indexes() -> Vec { + vec![ + unique("accounts", doc! { "account_name": 1 }), + unique("tables", doc! { "table_id": 1 }), + unique("iam_users", doc! { "user_arn": 1 }), + plain("access_keys", doc! { "account_id": 1, "user_name": 1 }), + unique("iam_groups", doc! { "group_arn": 1 }), + unique("iam_roles", doc! { "role_arn": 1 }), + unique("iam_sessions", doc! { "access_key_id": 1 }), + expiring("iam_sessions", doc! { "expires_at": 1 }, Duration::from_secs(0)), + plain("metrics", doc! { "_id.bucket": 1 }), + plain("login_attempts", doc! { "principal": 1, "attempted_at": 1 }), + plain("backups", doc! { "account_id": 1, "table_name": 1 }), + // A server that stopped without deleting its registration leaves a + // document whose `expires_at` has passed; the server's TTL monitor + // removes it. Nothing reads an expired registration as live. + expiring("extenddb_servers", doc! { "expires_at": 1 }, Duration::from_secs(0)), + ] +} + +/// The data indexes, as every released `init` created them. +fn data_indexes() -> Vec { + vec![ + // 540 s, so a token is kept for at most 10 minutes with the TTL + // monitor's 60 s cadence. + expiring("idempotency_tokens", doc! { "created_at": 1 }, Duration::from_secs(540)), + // One TransactWriteItems token runs once per account. + unique("idempotency_tokens", doc! { "account_id": 1, "token": 1 }), + // Two stream enables cannot insert the same shard. + unique("stream_shards", doc! { "shard_id": 1 }), + // DynamoDB's 24-hour stream retention. + expiring("stream_records", doc! { "created_at": 1 }, Duration::from_secs(24 * 3600)), + plain("stream_records", doc! { "shard_id": 1, "sequence_number": 1 }), + ] +} + +/// Create `names` in `db`, accepting a collection that exists. +async fn create_collections( + stores: &MongoStores, + db: &mongodb::Database, + names: &[&str], +) -> Result<(), MigrationError> { + for name in names { + let created = stores + .request(format!("create the collection {}.{name}", db.name()), async { + match db.create_collection(*name).await { + Err(e) if command_code(&e) == Some(NAMESPACE_EXISTS) => Ok(()), + other => other, + } + }) + .await; + created?; + } + Ok(()) +} + +/// Create every index in `db`. +async fn create_indexes( + stores: &MongoStores, + db: &mongodb::Database, + indexes: Vec, +) -> Result<(), MigrationError> { + for index in indexes { + let collection = db.collection::(index.collection); + let what = format!("create the index {:?} on {}.{}", index.key, db.name(), index.collection); + let model = IndexModel::builder().keys(index.key).options(index.options).build(); + stores.request(what, async { collection.create_index(model).await.map(|_| ()) }).await?; + } + Ok(()) +} + +/// The server's error code of a failed command, if it reported one. +fn command_code(error: &mongodb::error::Error) -> Option { + match error.kind.as_ref() { + mongodb::error::ErrorKind::Command(command) => Some(command.code), + _ => None, + } +} + +/// Migration 1's catalog body. +pub(crate) struct CatalogBaseline; + +#[async_trait] +impl CodeBody for CatalogBaseline { + fn fingerprint(&self) -> &'static str { + FINGERPRINT + } + + async fn detect(&self, _stores: &MongoStores) -> Result, MigrationError> { + Ok(Vec::new()) + } + + async fn run(&self, stores: &MongoStores) -> Result { + let db = &stores.catalog; + create_collections(stores, db, &CATALOG_COLLECTIONS).await?; + create_indexes(stores, db, catalog_indexes()).await?; + // The legacy ledger: a released `migrate --yes` that finds it runs + // no legacy step. `$setOnInsert` keeps the date a released `init` + // wrote. + let history = db.collection::("schema_history"); + stores + .request("record 001_initial in schema_history", async { + history + .update_one( + doc! { "_id": "001_initial" }, + doc! { "$setOnInsert": { "applied_at": bson::DateTime::now() } }, + ) + .upsert(true) + .await + .map(|_| ()) + }) + .await?; + // Every released `serve` from v0.1.14 requires this row, so deleting + // it makes them refuse this catalog. + let settings = db.collection::("settings"); + stores + .request("delete the resource_tag_migration settings row", async { + settings.delete_one(doc! { "_id": "resource_tag_migration" }).await.map(|_| ()) + }) + .await?; + Ok(Done::default()) + } +} + +/// Migration 1's data body. +pub(crate) struct DataBaseline; + +#[async_trait] +impl CodeBody for DataBaseline { + fn fingerprint(&self) -> &'static str { + FINGERPRINT + } + + async fn detect(&self, _stores: &MongoStores) -> Result, MigrationError> { + Ok(Vec::new()) + } + + async fn run(&self, stores: &MongoStores) -> Result { + let db = &stores.data; + create_collections(stores, db, &DATA_COLLECTIONS).await?; + create_indexes(stores, db, data_indexes()).await?; + Ok(Done::default()) + } +} diff --git a/crates/storage-mongodb/src/migrations/mod.rs b/crates/storage-mongodb/src/migrations/mod.rs new file mode 100644 index 000000000..bd765980f --- /dev/null +++ b/crates/storage-mongodb/src/migrations/mod.rs @@ -0,0 +1,290 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The MongoDB backend's catalog migrations. +//! +//! Two stores: the catalog database ([`StoreKind::Catalog`]) and the data +//! database ([`StoreKind::Data`]). [`migration`] is the table, an exhaustive +//! `match` from every [`MigrationId`] to this backend's bodies. [`RELEASED`] +//! pins the released bodies; the release PR that raises `RELEASED_THROUGH` +//! pastes the lines the checksum test prints. +//! +//! Entry points: [`catalog_status`] reads with no lock and no write; +//! [`MigrationLock`] serializes migrators and waits for running servers; +//! [`migrate_holding`] applies under it; [`ServingRegistration`] is what a +//! server holds while it serves, and [`register_and_check`] takes it and +//! reads the status before a server opens its engine. +//! [`MongoCatalogMigrations`] is how the commands reach them. +//! +//! MongoDB has no transaction around a body and its ledger row here: every +//! request a body sends is safe to send again, the row is written after the +//! body completes with write concern majority, and a crash runs the body +//! again. Every request waits at most [`REQUEST_TIMEOUT`], and every +//! connection is opened with the same timeout and the connection string the +//! server uses. + +mod backend; +mod commands; +mod guarded; +mod locks; +mod registrations; +// Frozen once released: neither rustfmt nor a later clippy lint may require +// an edit that changes its checksum. +#[rustfmt::skip] +#[allow(clippy::all, deprecated)] +mod m0001_baseline; +mod postcheck; + +use std::time::Duration; + +use extenddb_storage::migrations::m0002_resource_tag_condition_keys::{ + self, ResourceTagConditionKeys, +}; +use extenddb_storage::migrations::{ + CatalogStatus, Kind, Migration, MigrationError, MigrationId, Released, RunReport, Runner, + StoreBody, StoreKind, +}; + +pub use backend::{MongoMigrations, MongoStores}; +pub(crate) use commands::lease_of; +pub use commands::{MongoCatalogMigrations, lock}; +pub(crate) use locks::{Lease, is_duplicate_key}; +pub use locks::{MigrationLock, Timing}; +#[cfg(feature = "test-util")] +pub use postcheck::{Collection, Definitions, Index, describe_store}; +pub use registrations::ServingRegistration; + +/// The catalog database every MongoDB deployment uses. +pub const CATALOG_DATABASE: &str = "extenddb_catalog"; +/// The data database every MongoDB deployment uses. +pub const DATA_DATABASE: &str = "extenddb_data"; +/// The ledger collection in each database. +pub const LEDGER: &str = "extenddb_migrations"; +/// The collection the serving registrations live in, created by migration 1. +pub const SERVERS: &str = "extenddb_servers"; +/// The `_id` of the migration lock in the catalog's `settings` collection. +pub const LOCK_ID: &str = "migration_lock"; +/// How long any one request waits for its answer, and how long a connection +/// may take to open: shorter than the lock's 25 s margin. +pub const REQUEST_TIMEOUT: Duration = Duration::from_secs(10); + +/// The definitions migration 1 creates in an empty catalog database, which +/// the post-check compares the database with the first time migration 1 is +/// applied to it. +pub(crate) const CATALOG_EXPECTED: &str = + include_str!("../../migrations/catalog/0001_baseline.expected.json"); + +/// The same for the data database. +pub(crate) const DATA_EXPECTED: &str = + include_str!("../../migrations/data/0001_baseline.expected.json"); + +/// Checksums of the released bodies (none until v1.0). +pub(crate) const RELEASED: &[Released] = &[]; + +/// This backend's entry for every id. Public so `migrate --list` can print +/// each migration's kind without connecting. +#[must_use] +pub fn migration(id: MigrationId) -> Migration { + match id { + MigrationId::Baseline => Migration::Applies { + kind: Kind::Breaking, + convergent: true, + bodies: vec![ + StoreBody::code( + StoreKind::Catalog, + &m0001_baseline::CatalogBaseline, + m0001_baseline::SOURCE, + ), + StoreBody::code( + StoreKind::Data, + &m0001_baseline::DataBaseline, + m0001_baseline::SOURCE, + ), + ], + }, + MigrationId::ResourceTagConditionKeys => Migration::Applies { + kind: Kind::Breaking, + convergent: true, + bodies: vec![StoreBody::code( + StoreKind::Catalog, + &ResourceTagConditionKeys, + m0002_resource_tag_condition_keys::SOURCE, + )], + }, + MigrationId::SortKeyDefinitions => Migration::NotApplicable( + "MongoDB's #259 defect differed: the attribute definitions of index keys were not \ + stored, which issue #648 covers", + ), + MigrationId::IndexTableBaseKey => Migration::NotApplicable( + "index collections get their compound index when they are created", + ), + } +} + +/// The deployment the runner works on: the connection string the server +/// uses, and the two databases. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MongoTarget { + pub connection_string: String, + pub catalog: String, + pub data: String, +} + +impl MongoTarget { + /// The deployment at `connection_string`, with its two databases. + #[must_use] + pub fn new(connection_string: impl Into) -> Self { + Self { + connection_string: connection_string.into(), + catalog: CATALOG_DATABASE.to_owned(), + data: DATA_DATABASE.to_owned(), + } + } + + /// A deployment in databases of other names, for tests that share one + /// server. + #[cfg(feature = "test-util")] + #[must_use] + pub fn with_databases( + connection_string: impl Into, + catalog: impl Into, + data: impl Into, + ) -> Self { + Self { + connection_string: connection_string.into(), + catalog: catalog.into(), + data: data.into(), + } + } +} + +/// A client for the runner: the connection string's checks (primary reads, +/// a write concern no weaker than majority), with [`REQUEST_TIMEOUT`] to +/// connect and to select a server. +pub(crate) async fn connect(connection_string: &str) -> Result { + let mut options = crate::guarded_client_options(connection_string, None, false) + .await + .map_err(|e| MigrationError::store("connect to MongoDB", e))?; + options.connect_timeout = Some(REQUEST_TIMEOUT); + options.server_selection_timeout = Some(REQUEST_TIMEOUT); + mongodb::Client::with_options(options) + .map_err(|e| MigrationError::store("connect to MongoDB", e)) +} + +/// A database with the engine's catalog options: primary reads, read +/// concern `local`, write concern majority. +pub(crate) fn catalog_handle(client: &mongodb::Client, name: &str) -> mongodb::Database { + crate::catalog_database_named(client, name) +} + +/// A driver error, with the server's error code when it reported one. +pub(crate) fn driver_error( + what: impl std::fmt::Display, + error: &mongodb::error::Error, +) -> MigrationError { + use mongodb::error::{ErrorKind, WriteFailure}; + let code = match error.kind.as_ref() { + ErrorKind::Command(e) => Some((e.code, e.code_name.clone())), + ErrorKind::Write(WriteFailure::WriteError(e)) => Some((e.code, String::new())), + ErrorKind::Write(WriteFailure::WriteConcernError(e)) => Some((e.code, e.code_name.clone())), + _ => None, + }; + match code { + Some((code, name)) if !name.is_empty() => { + MigrationError::Store(format!("{what}: {error} (code {code}, {name})")) + } + Some((code, _)) => MigrationError::Store(format!("{what}: {error} (code {code})")), + None => MigrationError::store(what, error), + } +} + +/// The catalog status. Read-only: no lock is taken and nothing is written, +/// so it runs beside a server. A database that holds no ledger and no +/// `catalog_version` row, such as an empty one, one holding only the +/// migration lock, or another program's, is `Uninitialized`. +/// +/// # Errors +/// +/// A connection fails, or a ledger cannot be read. +pub async fn catalog_status(target: &MongoTarget) -> Result { + let migrations = MongoMigrations::open(target, None).await?; + // A read records nothing, so no release is named. + let status = Runner::new(&migrations, "").status().await; + migrations.close().await; + status +} + +/// The catalog status, read as the holder of `lock`: every request checks +/// its lease and is progress, so a status that runs migration 2's `detect` +/// on a catalog with many principals keeps the lock. +/// +/// # Errors +/// +/// As [`catalog_status`], or the lock was lost. +pub async fn status_holding( + target: &MongoTarget, + lock: &MigrationLock, +) -> Result { + let migrations = MongoMigrations::open(target, Some(lock)).await?; + let status = Runner::new(&migrations, "").status().await; + migrations.close().await; + status +} + +/// Apply every pending migration under `lock`, which this process holds. +/// +/// # Errors +/// +/// As [`Runner::migrate`]. +pub async fn migrate_holding( + target: &MongoTarget, + release: &str, + lock: &MigrationLock, +) -> Result { + let migrations = MongoMigrations::open(target, Some(lock)).await?; + let result = Runner::new(&migrations, release).migrate().await; + migrations.close().await; + result +} + +/// Register as the catalog's server and read its status, before anything +/// opens the engine: a server that refuses has written nothing but its +/// registration, which it deletes. The registration is written before the +/// lock and the status are read, so a migration that starts in between +/// waits for this server. +/// +/// # Errors +/// +/// [`MigrationError::Locked`] while a migration holds the lock, the status +/// when it is not `Current` or `AheadCompatible`, or a request failed. +pub async fn register_and_check( + target: &MongoTarget, + release: &str, + timing: Timing, +) -> Result { + let registration = ServingRegistration::register(target, release, timing).await?; + let status = match catalog_status(target).await { + Ok(status) => status, + Err(e) => { + registration.release().await; + return Err(e); + } + }; + match status { + CatalogStatus::Current { .. } => Ok(registration), + CatalogStatus::AheadCompatible { .. } => { + tracing::warn!( + "{status}; serving it, because a binary may serve migrations it does not know \ + when every one of them is additive" + ); + Ok(registration) + } + status => { + registration.release().await; + Err(MigrationError::Status(status)) + } + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/storage-mongodb/src/migrations/postcheck.rs b/crates/storage-mongodb/src/migrations/postcheck.rs new file mode 100644 index 000000000..70ff7746b --- /dev/null +++ b/crates/storage-mongodb/src/migrations/postcheck.rs @@ -0,0 +1,347 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The check after migration 1: the collections and indexes a store has, +//! compared with the definitions migration 1 creates in an empty store +//! (`migrations//0001_baseline.expected.json`). +//! +//! A collection is described by its type and its `listCollections` options, +//! an index by its key, in order, and every option `listIndexes` reports +//! except the index version `v` (unique, TTL seconds, sparse, partial +//! filter, collation). Numbers compare by value, so an Int32 and a Double 1 +//! are the same direction. Only the collections the expected file names +//! are read, and only the indexes it names on them: a store also holds +//! per-table collections and indexes that no migration creates. + +use std::collections::BTreeMap; + +use bson::{Bson, Document, doc}; +use extenddb_storage::migrations::MigrationError; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use super::MongoStores; + +/// One collection's definition. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Collection { + /// `collection`, `view` or `timeseries`. + #[serde(rename = "type")] + pub kind: String, + pub options: Value, + pub indexes: BTreeMap, +} + +/// One index's definition. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Index { + /// The key's fields and directions, in order. + pub key: Vec<(String, Value)>, + /// Every other option, by name. + pub options: BTreeMap, +} + +/// A store's definitions, by collection name. +pub type Definitions = BTreeMap; + +/// A BSON value as JSON, with every integral number as an integer. +fn json(value: &Bson) -> Value { + match value { + Bson::Int32(i) => Value::from(i64::from(*i)), + Bson::Int64(i) => Value::from(*i), + #[allow(clippy::cast_possible_truncation)] + Bson::Double(f) if f.fract() == 0.0 && f.abs() < 9.0e15 => Value::from(*f as i64), + Bson::Double(f) => serde_json::json!(f), + Bson::String(s) => Value::from(s.clone()), + Bson::Boolean(b) => Value::from(*b), + Bson::Null => Value::Null, + Bson::Document(d) => object(d), + Bson::Array(items) => Value::Array(items.iter().map(json).collect()), + other => Value::from(other.to_string()), + } +} + +fn object(d: &Document) -> Value { + Value::Object(d.iter().map(|(k, v)| (k.clone(), json(v))).collect()) +} + +/// The definitions of the named collections that exist in `db`. +/// +/// # Errors +/// +/// A request failed. +pub async fn describe( + stores: &MongoStores, + db: &mongodb::Database, + names: &[String], +) -> Result { + let listed = stores + .request(format!("list the collections of {}", db.name()), async { + db.run_command(doc! { + "listCollections": 1, + "filter": { "name": { "$in": names } }, + "cursor": { "batchSize": 1000 }, + }) + .await + }) + .await?; + let mut definitions = Definitions::new(); + for spec in first_batch(&listed, db.name())? { + let name = spec.get_str("name").unwrap_or_default().to_owned(); + let kind = spec.get_str("type").unwrap_or("collection").to_owned(); + let options = spec + .get_document("options") + .map_or(Value::Object(serde_json::Map::new()), object); + let indexes = if kind == "collection" { + list_indexes(stores, db, &name).await? + } else { + BTreeMap::new() + }; + definitions.insert( + name, + Collection { + kind, + options, + indexes, + }, + ); + } + Ok(definitions) +} + +/// Every collection of `db` except the per-table ones (`_ddb_*` and +/// `_backup_*`) and the ledger, which the runner creates: what migration 1 +/// leaves in an empty store, for the expected files and the schema dump. +/// +/// # Errors +/// +/// A request failed. +#[cfg(feature = "test-util")] +pub async fn describe_store( + stores: &MongoStores, + db: &mongodb::Database, +) -> Result { + let names = stores + .request(format!("list the collections of {}", db.name()), async { + db.list_collection_names().await + }) + .await?; + let names: Vec = names + .into_iter() + .filter(|n| !n.starts_with("_ddb_") && !n.starts_with("_backup_") && n != super::LEDGER) + .collect(); + describe(stores, db, &names).await +} + +async fn list_indexes( + stores: &MongoStores, + db: &mongodb::Database, + collection: &str, +) -> Result, MigrationError> { + let listed = stores + .request( + format!("list the indexes of {}.{collection}", db.name()), + async { + db.run_command(doc! { "listIndexes": collection, "cursor": { "batchSize": 1000 } }) + .await + }, + ) + .await?; + let mut indexes = BTreeMap::new(); + for spec in first_batch(&listed, db.name())? { + let name = spec.get_str("name").unwrap_or_default().to_owned(); + let key = spec + .get_document("key") + .map(|k| k.iter().map(|(f, d)| (f.clone(), json(d))).collect()) + .unwrap_or_default(); + let options = spec + .iter() + .filter(|(k, _)| !matches!(k.as_str(), "v" | "key" | "name" | "ns")) + .map(|(k, v)| (k.clone(), json(v))) + .collect(); + indexes.insert(name, Index { key, options }); + } + Ok(indexes) +} + +/// The documents of a command's cursor, which must be complete in its first +/// batch: the lists read here are a few dozen entries long. +fn first_batch<'a>(reply: &'a Document, db: &str) -> Result, MigrationError> { + let cursor = reply + .get_document("cursor") + .map_err(|e| MigrationError::Store(format!("read the definitions in {db}: {e}")))?; + if cursor.get_i64("id").unwrap_or(0) != 0 { + return Err(MigrationError::Store(format!( + "read the definitions in {db}: the server returned them in more than one batch" + ))); + } + Ok(cursor + .get_array("firstBatch") + .map(|batch| batch.iter().filter_map(Bson::as_document).collect()) + .unwrap_or_default()) +} + +/// Compare `actual` with the definitions `expected` names. Returns the +/// first difference, naming the object. +fn compare(expected: &Definitions, actual: &Definitions, store: &str) -> Option { + for (name, want) in expected { + let Some(have) = actual.get(name) else { + return Some(format!("{store}: the collection {name} is missing")); + }; + if have.kind != want.kind || have.options != want.options { + return Some(format!( + "{store}: the collection {name} is a {} with options {}; migration 1 creates a {} \ + with options {}", + have.kind, have.options, want.kind, want.options + )); + } + for (index, want_index) in &want.indexes { + match have.indexes.get(index) { + None => return Some(format!("{store}: the index {index} on {name} is missing")), + Some(have_index) if have_index != want_index => { + return Some(format!( + "{store}: the index {index} on {name} has key {:?} and options {:?}; \ + migration 1 creates key {:?} and options {:?}", + have_index.key, have_index.options, want_index.key, want_index.options + )); + } + Some(_) => {} + } + } + } + None +} + +/// Check `db` against the expected definitions, after migration 1's body +/// ran in it for the first time. +/// +/// # Errors +/// +/// A difference, naming the object, or a request failed. +pub(crate) async fn check( + stores: &MongoStores, + db: &mongodb::Database, + expected: &str, + store: &str, +) -> Result<(), MigrationError> { + let expected: Definitions = serde_json::from_str(expected).map_err(|e| { + MigrationError::Store(format!( + "the expected definitions of {store} cannot be read: {e}" + )) + })?; + let names: Vec = expected.keys().cloned().collect(); + let actual = describe(stores, db, &names).await?; + match compare(&expected, &actual, store) { + None => Ok(()), + Some(difference) => Err(MigrationError::Store(format!( + "after migration 1, {difference}; nothing was recorded for this store" + ))), + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + fn index(key: &[(&str, i64)], options: &[(&str, Value)]) -> Index { + Index { + key: key + .iter() + .map(|(f, d)| ((*f).to_owned(), json!(d))) + .collect(), + options: options + .iter() + .map(|(k, v)| ((*k).to_owned(), v.clone())) + .collect(), + } + } + + fn collection(indexes: &[(&str, Index)]) -> Collection { + Collection { + kind: "collection".to_owned(), + options: json!({}), + indexes: indexes + .iter() + .map(|(n, i)| ((*n).to_owned(), i.clone())) + .collect(), + } + } + + #[test] + fn numbers_compare_by_value_and_keys_keep_their_order() { + assert_eq!(json(&Bson::Double(1.0)), json(&Bson::Int32(1))); + assert_eq!(json(&Bson::Int64(540)), json!(540)); + let d = doc! { "b": 1, "a": -1 }; + let key: Vec<(String, Value)> = d.iter().map(|(f, v)| (f.clone(), json(v))).collect(); + assert_eq!( + key, + vec![("b".to_owned(), json!(1)), ("a".to_owned(), json!(-1))] + ); + } + + #[test] + fn a_difference_names_the_object_and_unnamed_objects_are_ignored() { + let unique = index(&[("a", 1)], &[("unique", json!(true))]); + let expected: Definitions = + [("c".to_owned(), collection(&[("a_1", unique.clone())]))].into(); + let mut actual = expected.clone(); + actual.insert("_ddb_x".to_owned(), collection(&[])); + actual + .get_mut("c") + .unwrap() + .indexes + .insert("extra_1".to_owned(), index(&[("extra", 1)], &[])); + assert_eq!(compare(&expected, &actual, "s"), None); + + let plain = index(&[("a", 1)], &[]); + actual + .get_mut("c") + .unwrap() + .indexes + .insert("a_1".to_owned(), plain); + let difference = compare(&expected, &actual, "s").unwrap(); + assert!(difference.contains("the index a_1 on c"), "{difference}"); + + actual.get_mut("c").unwrap().indexes.remove("a_1"); + assert_eq!( + compare(&expected, &actual, "s").unwrap(), + "s: the index a_1 on c is missing" + ); + let swapped = index(&[("b", 1), ("a", 1)], &[]); + let ordered = index(&[("a", 1), ("b", 1)], &[]); + let expected: Definitions = [("c".to_owned(), collection(&[("i", ordered)]))].into(); + let actual: Definitions = [("c".to_owned(), collection(&[("i", swapped)]))].into(); + assert!(compare(&expected, &actual, "s").is_some()); + + let mut capped = collection(&[]); + capped.options = json!({ "capped": true }); + let expected: Definitions = [("c".to_owned(), collection(&[]))].into(); + let actual: Definitions = [("c".to_owned(), capped)].into(); + assert!( + compare(&expected, &actual, "s") + .unwrap() + .contains("the collection c is a") + ); + assert_eq!( + compare(&expected, &Definitions::new(), "s").unwrap(), + "s: the collection c is missing" + ); + } + + #[test] + fn the_expected_files_name_every_collection_migration_1_creates() { + let catalog: Definitions = serde_json::from_str(super::super::CATALOG_EXPECTED).unwrap(); + let data: Definitions = serde_json::from_str(super::super::DATA_EXPECTED).unwrap(); + assert_eq!(catalog.len(), 19); + assert!(catalog.contains_key("extenddb_servers")); + assert!(!catalog.contains_key(super::super::LEDGER)); + assert_eq!( + data.keys().map(String::as_str).collect::>(), + ["idempotency_tokens", "stream_records", "stream_shards"] + ); + let unique = &data["idempotency_tokens"].indexes["account_id_1_token_1"]; + assert_eq!(unique.options.get("unique"), Some(&json!(true))); + } +} diff --git a/crates/storage-mongodb/src/migrations/registrations.rs b/crates/storage-mongodb/src/migrations/registrations.rs new file mode 100644 index 000000000..ab994b10d --- /dev/null +++ b/crates/storage-mongodb/src/migrations/registrations.rs @@ -0,0 +1,277 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The serving registration: the document a server holds in +//! `extenddb_servers` while it serves, and the read of every live one that +//! `migrate --yes` waits on. The lock it is paired with is in `locks.rs`, +//! whose module doc describes both. + +use std::time::Instant; + +use bson::{Document, doc}; +use extenddb_storage::migrations::MigrationError; +use tokio::sync::watch; +use tokio::task::JoinHandle; + +use super::locks::{Timing, describe_time, holder, millis, timed_out, token}; +use super::{LOCK_ID, MongoTarget, REQUEST_TIMEOUT, SERVERS, connect, driver_error}; + +/// The registrations whose `expires_at` has not passed, by the server's +/// clock, each named for a message. +pub(crate) async fn live_registrations( + servers: &mongodb::Collection, +) -> Result, MigrationError> { + use futures::TryStreamExt; + let read = async { + let cursor = servers + .find(doc! { "$expr": { "$gt": ["$expires_at", "$$NOW"] } }) + .await?; + cursor.try_collect::>().await + }; + let docs = match tokio::time::timeout(REQUEST_TIMEOUT, read).await { + Ok(Ok(docs)) => docs, + Ok(Err(e)) => return Err(driver_error("read the serving registrations", &e)), + Err(_) => return Err(timed_out("read the serving registrations")), + }; + Ok(docs + .iter() + .map(|d| { + format!( + "pid {} on {} (release {}, registered {}, expires {})", + d.get("pid") + .map_or_else(|| "unknown".to_owned(), ToString::to_string), + d.get_str("host").unwrap_or("an unknown host"), + d.get_str("release").unwrap_or("unknown"), + d.get("registered_at") + .map_or_else(|| "unknown".to_owned(), describe_time), + d.get("expires_at") + .map_or_else(|| "unknown".to_owned(), describe_time), + ) + }) + .collect()) +} + +/// This host's name, for the registration. +fn host_name() -> String { + std::fs::read_to_string("/proc/sys/kernel/hostname") + .ok() + .map(|s| s.trim().to_owned()) + .filter(|s| !s.is_empty()) + .or_else(|| std::env::var("HOSTNAME").ok()) + .or_else(|| std::env::var("COMPUTERNAME").ok()) + .unwrap_or_else(|| "an unknown host".to_owned()) +} + +/// What a server holds while it serves: its document in `extenddb_servers`, +/// renewed every [`Timing::renew_every`]. +pub struct ServingRegistration { + servers: mongodb::Collection, + token: String, + renewal: JoinHandle<()>, + lost: watch::Receiver>, +} + +impl ServingRegistration { + /// Register as a server of `target`'s catalog, then read the migration + /// lock: a migration that holds it makes this server refuse with exit 6. + /// + /// # Errors + /// + /// [`MigrationError::Locked`] while a migration holds the lock, or a + /// request failed. The registration is deleted on every error. + pub async fn register( + target: &MongoTarget, + release: &str, + timing: Timing, + ) -> Result { + let client = connect(&target.connection_string).await?; + let db = super::catalog_handle(&client, &target.catalog); + let servers = db.collection::(SERVERS); + let token = token(); + let insert = servers + .update_one( + doc! { "_id": &token }, + vec![doc! { "$set": { + "host": host_name(), + "pid": i64::from(std::process::id()), + "release": release, + "registered_at": "$$NOW", + "expires_at": { "$add": ["$$NOW", millis(timing.registration)] }, + } }], + ) + .upsert(true); + let sent = Instant::now(); + match tokio::time::timeout(REQUEST_TIMEOUT, insert).await { + Ok(Ok(_)) => {} + Ok(Err(e)) => return Err(driver_error("register this server", &e)), + Err(_) => return Err(timed_out("register this server")), + } + let (tx, lost) = watch::channel(None); + let renewal = spawn_registration_renewal(servers.clone(), token.clone(), timing, sent, tx); + let registration = Self { + servers, + token, + renewal, + lost, + }; + // The lock holds while its `expires_at` has not passed by the + // database server's clock, as the claim decides. + let settings = db.collection::("settings"); + let live = settings + .find_one(doc! { "_id": LOCK_ID, "$expr": { "$gt": ["$expires_at", "$$NOW"] } }); + let lock = match tokio::time::timeout(REQUEST_TIMEOUT, live).await { + Ok(Ok(lock)) => lock, + Ok(Err(e)) => { + registration.release().await; + return Err(driver_error("read the migration lock", &e)); + } + Err(_) => { + registration.release().await; + return Err(timed_out("read the migration lock")); + } + }; + if let Some(lock) = lock { + registration.release().await; + return Err(MigrationError::Locked(format!( + "{} holds the migration lock on this catalog, so the server does not start; \ + start it again once the migration finishes", + holder(Some(&lock)) + ))); + } + Ok(registration) + } + + /// Changes to `Some(reason)` once the registration is lost. + #[must_use] + pub fn lost(&self) -> watch::Receiver> { + self.lost.clone() + } + + /// Stop renewing and delete the registration. + pub async fn release(self) { + self.renewal.abort(); + let _ = tokio::time::timeout( + REQUEST_TIMEOUT, + self.servers.delete_one(doc! { "_id": &self.token }), + ) + .await; + } +} + +impl Drop for ServingRegistration { + fn drop(&mut self) { + self.renewal.abort(); + } +} + +/// Renew a registration every `renew_every`. A renewal that matches no +/// document, or no successful renewal for `registration_deadline`, reports +/// the registration lost on `lost` and ends. No wait and no renewal outlasts +/// the deadline, so the loss is reported `registration_deadline` after the +/// send time of the last successful renewal, not a request timeout later. +fn spawn_registration_renewal( + servers: mongodb::Collection, + token: String, + timing: Timing, + registered: Instant, + lost: watch::Sender>, +) -> JoinHandle<()> { + tokio::spawn(async move { + let mut renewed = registered; + let mut wait = timing.renew_every; + let left = |renewed: Instant| { + timing + .registration_deadline + .saturating_sub(renewed.elapsed()) + }; + loop { + tokio::time::sleep(wait.min(left(renewed))).await; + let sent = Instant::now(); + let renewal = servers.update_one( + doc! { "_id": &token }, + vec![doc! { "$set": { "expires_at": { "$add": ["$$NOW", millis(timing.registration)] } } }], + ); + let answer = tokio::time::timeout(REQUEST_TIMEOUT.min(left(renewed)), renewal).await; + let reason = match answer { + Ok(Ok(result)) if result.matched_count == 1 => { + renewed = sent; + wait = timing.renew_every; + continue; + } + Ok(Ok(_)) => { + "this server's registration in extenddb_servers was deleted".to_owned() + } + Ok(Err(_)) | Err(_) if renewed.elapsed() < timing.registration_deadline => { + wait = timing.retry_every; + continue; + } + Ok(Err(e)) => format!( + "this server could not renew its registration for {} s ({e})", + renewed.elapsed().as_secs() + ), + Err(_) => format!( + "this server could not renew its registration for {} s (no answer)", + renewed.elapsed().as_secs() + ), + }; + let _ = lost.send(Some(reason)); + return; + } + }) +} + +#[cfg(test)] +mod tests { + use std::time::{Duration, Instant}; + + use bson::Document; + use tokio::sync::watch; + + use super::super::locks::tests::silent_server; + use super::{Timing, spawn_registration_renewal}; + + #[tokio::test] + async fn a_registration_whose_renewals_get_no_answer_is_lost_at_its_deadline() { + let (address, accepting) = silent_server().await; + let mut options = mongodb::options::ClientOptions::parse(format!( + "mongodb://{address}/?directConnection=true" + )) + .await + .expect("options"); + options.connect_timeout = Some(Duration::from_secs(60)); + options.server_selection_timeout = Some(Duration::from_secs(60)); + let client = mongodb::Client::with_options(options).expect("client"); + let servers = client + .database("silent") + .collection::("extenddb_servers"); + let timing = Timing { + renew_every: Duration::from_millis(200), + registration_deadline: Duration::from_secs(1), + retry_every: Duration::from_millis(50), + ..Timing::PRODUCTION + }; + let (tx, mut lost) = watch::channel(None); + let registered = Instant::now(); + let renewal = + spawn_registration_renewal(servers, "token".to_owned(), timing, registered, tx); + // Without the bound, the first renewal would wait the whole 10 s + // request timeout before the loss could be reported. + tokio::time::timeout(Duration::from_secs(5), lost.changed()) + .await + .expect("reported well before the request timeout") + .expect("changed"); + let after = registered.elapsed(); + assert!( + after >= timing.registration_deadline + && after < timing.registration_deadline + Duration::from_millis(500), + "the loss was reported {after:?} after the registration" + ); + let reason = lost.borrow().clone().expect("a reason"); + assert!( + reason.contains("could not renew its registration for 1 s"), + "{reason}" + ); + renewal.abort(); + accepting.abort(); + } +} diff --git a/crates/storage-mongodb/src/migrations/tests.rs b/crates/storage-mongodb/src/migrations/tests.rs new file mode 100644 index 000000000..97dfbe2d1 --- /dev/null +++ b/crates/storage-mongodb/src/migrations/tests.rs @@ -0,0 +1,80 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Tests of the MongoDB migration table that need no database. + +use extenddb_storage::migrations::{ + Body, Migration, MigrationId, RELEASED_THROUGH, StoreKind, check_released, refuse_additive, +}; + +use super::{RELEASED, migration}; + +#[test] +fn the_released_bodies_match_their_checksums() { + let lines = check_released("mongodb", migration, RELEASED, RELEASED_THROUGH) + .unwrap_or_else(|e| panic!("{e}")); + println!("{lines}"); +} + +#[test] +fn no_migration_is_declared_additive_before_the_old_binary_job_exists() { + refuse_additive("mongodb", migration).unwrap_or_else(|e| panic!("{e}")); +} + +#[test] +fn every_migration_records_a_row_in_the_catalog_and_names_its_stores() { + for id in MigrationId::ALL { + let kinds = migration(id).store_kinds(); + assert_eq!(kinds.first(), Some(&StoreKind::Catalog), "{id}"); + assert!(!kinds.contains(&StoreKind::Account), "{id}"); + } + assert_eq!( + migration(MigrationId::Baseline).store_kinds(), + [StoreKind::Catalog, StoreKind::Data] + ); + assert_eq!( + migration(MigrationId::ResourceTagConditionKeys).store_kinds(), + [StoreKind::Catalog] + ); +} + +#[test] +fn every_body_is_code_and_three_and_four_do_not_apply() { + for id in MigrationId::ALL { + for body in migration(id).bodies() { + assert!(matches!(body.body, Body::Code { .. }), "{id}"); + } + } + for id in [ + MigrationId::SortKeyDefinitions, + MigrationId::IndexTableBaseKey, + ] { + let Migration::NotApplicable(reason) = migration(id) else { + panic!("{id} applies on MongoDB"); + }; + assert!(!reason.is_empty(), "{id}"); + } + assert!(migration(MigrationId::Baseline).convergent()); + assert!(migration(MigrationId::ResourceTagConditionKeys).convergent()); +} + +/// The lock, the registrations and the ledgers are read and written through +/// this handle: primary reads, and write concern majority whatever the +/// connection string says. +#[tokio::test] +async fn the_runner_reads_the_primary_and_writes_with_write_concern_majority() { + use mongodb::options::{Acknowledgment, ReadPreference, SelectionCriteria}; + // The client connects lazily, so no server is contacted. + let client = mongodb::Client::with_uri_str("mongodb://127.0.0.1:1/?directConnection=true&w=1") + .await + .expect("client"); + let db = super::catalog_handle(&client, "any"); + assert_eq!( + db.write_concern().and_then(|w| w.w.clone()), + Some(Acknowledgment::Majority) + ); + assert!(matches!( + db.selection_criteria(), + Some(SelectionCriteria::ReadPreference(ReadPreference::Primary)) + )); +} diff --git a/crates/storage-mongodb/src/operations.rs b/crates/storage-mongodb/src/operations.rs index 4257fdbdf..0afbe6547 100644 --- a/crates/storage-mongodb/src/operations.rs +++ b/crates/storage-mongodb/src/operations.rs @@ -108,12 +108,6 @@ impl OperationsEngine for MongoOperationsEngine { Ok(()) } - fn catalog_version(&self) -> String { - // Mongo catalog version — matches the constant enforced by - // MongoBootstrapper::expected_catalog_version. - "0.0.2".to_owned() - } - fn setting_default(&self, key: &str) -> Option { // The engine reads no index propagation delay. match key { diff --git a/crates/storage-mongodb/tests/fixtures/golden/0001_baseline.txt b/crates/storage-mongodb/tests/fixtures/golden/0001_baseline.txt new file mode 100644 index 000000000..01656c16b --- /dev/null +++ b/crates/storage-mongodb/tests/fixtures/golden/0001_baseline.txt @@ -0,0 +1,557 @@ +{ + "catalog": { + "access_keys": { + "type": "collection", + "options": {}, + "indexes": { + "_id_": { + "key": [ + [ + "_id", + 1 + ] + ], + "options": {} + }, + "account_id_1_user_name_1": { + "key": [ + [ + "account_id", + 1 + ], + [ + "user_name", + 1 + ] + ], + "options": {} + } + } + }, + "accounts": { + "type": "collection", + "options": {}, + "indexes": { + "_id_": { + "key": [ + [ + "_id", + 1 + ] + ], + "options": {} + }, + "account_name_1": { + "key": [ + [ + "account_name", + 1 + ] + ], + "options": { + "unique": true + } + } + } + }, + "admin_users": { + "type": "collection", + "options": {}, + "indexes": { + "_id_": { + "key": [ + [ + "_id", + 1 + ] + ], + "options": {} + } + } + }, + "backups": { + "type": "collection", + "options": {}, + "indexes": { + "_id_": { + "key": [ + [ + "_id", + 1 + ] + ], + "options": {} + }, + "account_id_1_table_name_1": { + "key": [ + [ + "account_id", + 1 + ], + [ + "table_name", + 1 + ] + ], + "options": {} + } + } + }, + "continuous_backups": { + "type": "collection", + "options": {}, + "indexes": { + "_id_": { + "key": [ + [ + "_id", + 1 + ] + ], + "options": {} + } + } + }, + "extenddb_servers": { + "type": "collection", + "options": {}, + "indexes": { + "_id_": { + "key": [ + [ + "_id", + 1 + ] + ], + "options": {} + }, + "expires_at_1": { + "key": [ + [ + "expires_at", + 1 + ] + ], + "options": { + "expireAfterSeconds": 0 + } + } + } + }, + "iam_groups": { + "type": "collection", + "options": {}, + "indexes": { + "_id_": { + "key": [ + [ + "_id", + 1 + ] + ], + "options": {} + }, + "group_arn_1": { + "key": [ + [ + "group_arn", + 1 + ] + ], + "options": { + "unique": true + } + } + } + }, + "iam_permissions_boundaries": { + "type": "collection", + "options": {}, + "indexes": { + "_id_": { + "key": [ + [ + "_id", + 1 + ] + ], + "options": {} + } + } + }, + "iam_policies": { + "type": "collection", + "options": {}, + "indexes": { + "_id_": { + "key": [ + [ + "_id", + 1 + ] + ], + "options": {} + } + } + }, + "iam_roles": { + "type": "collection", + "options": {}, + "indexes": { + "_id_": { + "key": [ + [ + "_id", + 1 + ] + ], + "options": {} + }, + "role_arn_1": { + "key": [ + [ + "role_arn", + 1 + ] + ], + "options": { + "unique": true + } + } + } + }, + "iam_sessions": { + "type": "collection", + "options": {}, + "indexes": { + "_id_": { + "key": [ + [ + "_id", + 1 + ] + ], + "options": {} + }, + "access_key_id_1": { + "key": [ + [ + "access_key_id", + 1 + ] + ], + "options": { + "unique": true + } + }, + "expires_at_1": { + "key": [ + [ + "expires_at", + 1 + ] + ], + "options": { + "expireAfterSeconds": 0 + } + } + } + }, + "iam_users": { + "type": "collection", + "options": {}, + "indexes": { + "_id_": { + "key": [ + [ + "_id", + 1 + ] + ], + "options": {} + }, + "user_arn_1": { + "key": [ + [ + "user_arn", + 1 + ] + ], + "options": { + "unique": true + } + } + } + }, + "indexes": { + "type": "collection", + "options": {}, + "indexes": { + "_id_": { + "key": [ + [ + "_id", + 1 + ] + ], + "options": {} + } + } + }, + "login_attempts": { + "type": "collection", + "options": {}, + "indexes": { + "_id_": { + "key": [ + [ + "_id", + 1 + ] + ], + "options": {} + }, + "principal_1_attempted_at_1": { + "key": [ + [ + "principal", + 1 + ], + [ + "attempted_at", + 1 + ] + ], + "options": {} + } + } + }, + "metrics": { + "type": "collection", + "options": {}, + "indexes": { + "_id.bucket_1": { + "key": [ + [ + "_id.bucket", + 1 + ] + ], + "options": {} + }, + "_id_": { + "key": [ + [ + "_id", + 1 + ] + ], + "options": {} + } + } + }, + "schema_history": { + "type": "collection", + "options": {}, + "indexes": { + "_id_": { + "key": [ + [ + "_id", + 1 + ] + ], + "options": {} + } + } + }, + "settings": { + "type": "collection", + "options": {}, + "indexes": { + "_id_": { + "key": [ + [ + "_id", + 1 + ] + ], + "options": {} + } + } + }, + "tables": { + "type": "collection", + "options": {}, + "indexes": { + "_id_": { + "key": [ + [ + "_id", + 1 + ] + ], + "options": {} + }, + "table_id_1": { + "key": [ + [ + "table_id", + 1 + ] + ], + "options": { + "unique": true + } + } + } + }, + "tags": { + "type": "collection", + "options": {}, + "indexes": { + "_id_": { + "key": [ + [ + "_id", + 1 + ] + ], + "options": {} + } + } + } + }, + "data": { + "counters": { + "type": "collection", + "options": {}, + "indexes": { + "_id_": { + "key": [ + [ + "_id", + 1 + ] + ], + "options": {} + } + } + }, + "idempotency_tokens": { + "type": "collection", + "options": {}, + "indexes": { + "_id_": { + "key": [ + [ + "_id", + 1 + ] + ], + "options": {} + }, + "account_id_1_token_1": { + "key": [ + [ + "account_id", + 1 + ], + [ + "token", + 1 + ] + ], + "options": { + "unique": true + } + }, + "created_at_1": { + "key": [ + [ + "created_at", + 1 + ] + ], + "options": { + "expireAfterSeconds": 540 + } + } + } + }, + "stream_records": { + "type": "collection", + "options": {}, + "indexes": { + "_id_": { + "key": [ + [ + "_id", + 1 + ] + ], + "options": {} + }, + "created_at_1": { + "key": [ + [ + "created_at", + 1 + ] + ], + "options": { + "expireAfterSeconds": 86400 + } + }, + "shard_id_1_sequence_number_1": { + "key": [ + [ + "shard_id", + 1 + ], + [ + "sequence_number", + 1 + ] + ], + "options": {} + } + } + }, + "stream_shards": { + "type": "collection", + "options": {}, + "indexes": { + "_id_": { + "key": [ + [ + "_id", + 1 + ] + ], + "options": {} + }, + "shard_id_1": { + "key": [ + [ + "shard_id", + 1 + ] + ], + "options": { + "unique": true + } + } + } + } + }, + "settings": [ + "catalog_version", + "data_connection_string", + "data_database_name", + "encryption_key" + ], + "schema_history": [ + "001_initial" + ] +} diff --git a/crates/storage-mongodb/tests/fixtures/golden/0002_resource_tag_condition_keys.txt b/crates/storage-mongodb/tests/fixtures/golden/0002_resource_tag_condition_keys.txt new file mode 100644 index 000000000..75964327a --- /dev/null +++ b/crates/storage-mongodb/tests/fixtures/golden/0002_resource_tag_condition_keys.txt @@ -0,0 +1,83 @@ +{ + "policies": [ + { + "account": "123456789012", + "principal": "user cu-tester", + "policy": "SelfServicePolicy", + "document": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "iam:CreateAccessKey", + "iam:DeleteAccessKey", + "iam:ListAccessKeys", + "iam:ChangePassword" + ], + "Resource": "arn:aws:iam::123456789012:user/cu-tester" + } + ] + } + }, + { + "account": "123456789012", + "principal": "user cu-tester", + "policy": "dynamodb-full", + "document": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": "dynamodb:*", + "Resource": "*" + } + ] + } + }, + { + "account": "123456789012", + "principal": "user cu-tester", + "policy": "deny-prod", + "document": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Deny", + "Action": "dynamodb:DeleteTable", + "Resource": "*", + "Condition": { + "StringEquals": { + "aws:ResourceTag/env": "prod" + } + } + } + ] + } + }, + { + "account": "", + "principal": "user ops", + "policy": "deny-prod-default", + "document": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Deny", + "Action": "dynamodb:*", + "Resource": "*", + "Condition": { + "StringEquals": { + "aws:ResourceTag/env": "prod" + } + } + } + ] + } + } + ], + "notes": [ + "rewrote account user/ops policy deny-prod-default (1 key(s))", + "rewrote account 123456789012 user/cu-tester policy deny-prod (1 key(s))" + ] +} diff --git a/crates/storage-mongodb/tests/fixtures/released/0.1.12.json b/crates/storage-mongodb/tests/fixtures/released/0.1.12.json new file mode 100644 index 000000000..a493f4ea5 --- /dev/null +++ b/crates/storage-mongodb/tests/fixtures/released/0.1.12.json @@ -0,0 +1,2683 @@ +{ + "databases": { + "extenddb_catalog": { + "collections": [ + { + "name": "access_keys", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "account_id": { + "$numberInt": "1" + }, + "user_name": { + "$numberInt": "1" + } + }, + "name": "account_id_1_user_name_1" + } + ], + "documents": [ + { + "_id": { + "$oid": "6ae3dbec99e7d94eaa5c300d" + }, + "access_key_id": "AKIAEXTENDDBAIQC4XUG", + "account_id": "123456789012", + "user_name": "cu-tester", + "secret_key_encrypted": { + "$binary": { + "base64": "bEUrlxat7IKKvNcCYy9+9aSxZkDSlXqocFR2plh9pY+fYo6v3wezbrAueKVuy33h2+aOj+ZpFWAfvceoVOsTM+MQ8X0=", + "subType": "00" + } + }, + "is_active": true, + "created_at": { + "$date": { + "$numberLong": "1793317868287" + } + } + } + ] + }, + { + "name": "accounts", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "account_name": { + "$numberInt": "1" + } + }, + "name": "account_name_1", + "unique": true + } + ], + "documents": [ + { + "_id": "75757ae3-e10b-4acc-a56b-bd992a52e5c0", + "account_name": "default", + "created_at": { + "$date": { + "$numberLong": "1793317865722" + } + } + }, + { + "_id": { + "$oid": "6ae3dbea99e7d94eaa5c300b" + }, + "account_id": "123456789012", + "account_name": "cu-seed", + "created_at": { + "$date": { + "$numberLong": "1793317866824" + } + } + } + ] + }, + { + "name": "admin_users", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + } + ], + "documents": [ + { + "_id": "admin", + "password_hash": "$2b$12$S2CbgWxOZkPl1rEJDhfn1uy/SgRInCPu9LDmwnLXG/4K1pIikyS7G", + "created_at": { + "$date": { + "$numberLong": "1793317865956" + } + } + } + ] + }, + { + "name": "backups", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "account_id": { + "$numberInt": "1" + }, + "table_name": { + "$numberInt": "1" + } + }, + "name": "account_id_1_table_name_1" + } + ], + "documents": [] + }, + { + "name": "continuous_backups", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + } + ], + "documents": [] + }, + { + "name": "iam_groups", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "group_arn": { + "$numberInt": "1" + } + }, + "name": "group_arn_1", + "unique": true + } + ], + "documents": [] + }, + { + "name": "iam_permissions_boundaries", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + } + ], + "documents": [] + }, + { + "name": "iam_policies", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + } + ], + "documents": [ + { + "_id": { + "$oid": "6ae3dbeb6f866ae74b02c900" + }, + "policy_name": "SelfServicePolicy", + "principal_name": "cu-tester", + "principal_type": "user", + "account_id": "123456789012", + "created_at": { + "$date": { + "$numberLong": "1793317867190" + } + }, + "policy_document": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "iam:CreateAccessKey", + "iam:DeleteAccessKey", + "iam:ListAccessKeys", + "iam:ChangePassword" + ], + "Resource": "arn:aws:iam::123456789012:user/cu-tester" + } + ] + } + }, + { + "_id": { + "$oid": "6ae3dbeb6f866ae74b02c905" + }, + "policy_name": "dynamodb-full", + "principal_name": "cu-tester", + "principal_type": "user", + "account_id": "123456789012", + "created_at": { + "$date": { + "$numberLong": "1793317867566" + } + }, + "policy_document": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": "dynamodb:*", + "Resource": "*" + } + ] + } + }, + { + "_id": { + "$oid": "6ae3dbeb6f866ae74b02c90b" + }, + "policy_name": "deny-prod", + "principal_name": "cu-tester", + "principal_type": "user", + "account_id": "123456789012", + "created_at": { + "$date": { + "$numberLong": "1793317867958" + } + }, + "policy_document": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Deny", + "Action": "dynamodb:DeleteTable", + "Resource": "*", + "Condition": { + "StringEquals": { + "dynamodb:ResourceTag/env": "prod" + } + } + } + ] + } + } + ] + }, + { + "name": "iam_roles", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "role_arn": { + "$numberInt": "1" + } + }, + "name": "role_arn_1", + "unique": true + } + ], + "documents": [] + }, + { + "name": "iam_sessions", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "access_key_id": { + "$numberInt": "1" + } + }, + "name": "access_key_id_1", + "unique": true + }, + { + "key": { + "expires_at": { + "$numberInt": "1" + } + }, + "name": "expires_at_1", + "expireAfterSeconds": { + "$numberInt": "0" + } + } + ], + "documents": [] + }, + { + "name": "iam_users", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "user_arn": { + "$numberInt": "1" + } + }, + "name": "user_arn_1", + "unique": true + } + ], + "documents": [ + { + "_id": { + "$oid": "6ae3dbeb99e7d94eaa5c300c" + }, + "account_id": "123456789012", + "user_name": "cu-tester", + "user_arn": "arn:aws:iam::123456789012:user/cu-tester", + "created_at": { + "$date": { + "$numberLong": "1793317867188" + } + } + } + ] + }, + { + "name": "indexes", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + } + ], + "documents": [ + { + "_id": { + "table_id": "0be1f21a-1085-4d7a-a3f3-2eea27890f1a", + "index_name": "by_gpk" + }, + "index_id": "0f252ac6-e551-4f4f-9b6c-493a50ba5fad", + "index_type": "GSI", + "key_schema": [ + { + "AttributeName": "gpk", + "KeyType": "HASH" + }, + { + "AttributeName": "gsk", + "KeyType": "RANGE" + } + ], + "projection": { + "ProjectionType": "ALL" + }, + "index_status": "ACTIVE", + "provisioned_throughput": null + }, + { + "_id": { + "table_id": "0be1f21a-1085-4d7a-a3f3-2eea27890f1a", + "index_name": "by_lsk" + }, + "index_id": "21390ee1-34fa-4121-bfd9-aa9102a03d64", + "index_type": "LSI", + "key_schema": [ + { + "AttributeName": "pk", + "KeyType": "HASH" + }, + { + "AttributeName": "lsk", + "KeyType": "RANGE" + } + ], + "projection": { + "ProjectionType": "KEYS_ONLY" + }, + "index_status": "ACTIVE", + "provisioned_throughput": null + } + ] + }, + { + "name": "login_attempts", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "principal": { + "$numberInt": "1" + }, + "attempted_at": { + "$numberInt": "1" + } + }, + "name": "principal_1_attempted_at_1" + } + ], + "documents": [] + }, + { + "name": "metrics", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "_id.bucket": { + "$numberInt": "1" + } + }, + "name": "_id.bucket_1" + } + ], + "documents": [ + { + "_id": { + "$oid": "6ae3dbee99e7d94eaa5c301c" + }, + "bucket": { + "$date": { + "$numberLong": "1793317860000" + } + }, + "metric": "ConsumedWriteCapacityUnits", + "sum": { + "$numberInt": "10" + }, + "count": { + "$numberLong": "10" + }, + "min": { + "$numberInt": "1" + }, + "max": { + "$numberInt": "1" + }, + "table_name": "cu_orders", + "operation": "PutItem" + }, + { + "_id": { + "$oid": "6ae3dbee99e7d94eaa5c301d" + }, + "bucket": { + "$date": { + "$numberLong": "1793317860000" + } + }, + "metric": "SuccessfulRequestLatency", + "sum": { + "$numberInt": "2033" + }, + "count": { + "$numberLong": "2" + }, + "min": { + "$numberInt": "996" + }, + "max": { + "$numberInt": "1037" + }, + "table_name": "cu_orders", + "operation": "DescribeTable" + }, + { + "_id": { + "$oid": "6ae3dbee99e7d94eaa5c301e" + }, + "bucket": { + "$date": { + "$numberLong": "1793317860000" + } + }, + "metric": "RequestCount", + "sum": { + "$numberInt": "2" + }, + "count": { + "$numberLong": "2" + }, + "min": { + "$numberInt": "1" + }, + "max": { + "$numberInt": "1" + }, + "operation": "DescribeTable" + }, + { + "_id": { + "$oid": "6ae3dbee99e7d94eaa5c301f" + }, + "bucket": { + "$date": { + "$numberLong": "1793317860000" + } + }, + "metric": "RequestCount", + "sum": { + "$numberInt": "10" + }, + "count": { + "$numberLong": "10" + }, + "min": { + "$numberInt": "1" + }, + "max": { + "$numberInt": "1" + }, + "operation": "PutItem" + }, + { + "_id": { + "$oid": "6ae3dbee99e7d94eaa5c3020" + }, + "bucket": { + "$date": { + "$numberLong": "1793317860000" + } + }, + "metric": "StorageQueryCount", + "sum": { + "$numberInt": "4" + }, + "count": { + "$numberLong": "4" + }, + "min": { + "$numberInt": "1" + }, + "max": { + "$numberInt": "1" + }, + "index_name": "Request", + "operation": "Catalog" + }, + { + "_id": { + "$oid": "6ae3dbee99e7d94eaa5c3021" + }, + "bucket": { + "$date": { + "$numberLong": "1793317860000" + } + }, + "metric": "SuccessfulRequestLatency", + "sum": { + "$numberInt": "60064" + }, + "count": { + "$numberLong": "10" + }, + "min": { + "$numberInt": "3600" + }, + "max": { + "$numberInt": "19384" + }, + "table_name": "cu_orders", + "operation": "PutItem" + }, + { + "_id": { + "$oid": "6ae3dbee99e7d94eaa5c3022" + }, + "bucket": { + "$date": { + "$numberLong": "1793317860000" + } + }, + "metric": "RequestCount", + "sum": { + "$numberInt": "1" + }, + "count": { + "$numberLong": "1" + }, + "min": { + "$numberInt": "1" + }, + "max": { + "$numberInt": "1" + }, + "operation": "UpdateTimeToLive" + }, + { + "_id": { + "$oid": "6ae3dbee99e7d94eaa5c3023" + }, + "bucket": { + "$date": { + "$numberLong": "1793317860000" + } + }, + "metric": "SuccessfulRequestLatency", + "sum": { + "$numberInt": "84238" + }, + "count": { + "$numberLong": "1" + }, + "min": { + "$numberInt": "84238" + }, + "max": { + "$numberInt": "84238" + }, + "table_name": "cu_orders", + "operation": "CreateTable" + }, + { + "_id": { + "$oid": "6ae3dbee99e7d94eaa5c3024" + }, + "bucket": { + "$date": { + "$numberLong": "1793317860000" + } + }, + "metric": "StorageQueryLatency", + "sum": { + "$numberInt": "93815" + }, + "count": { + "$numberLong": "4" + }, + "min": { + "$numberInt": "542" + }, + "max": { + "$numberInt": "80851" + }, + "index_name": "Request", + "operation": "Catalog" + }, + { + "_id": { + "$oid": "6ae3dbee99e7d94eaa5c3025" + }, + "bucket": { + "$date": { + "$numberLong": "1793317860000" + } + }, + "metric": "StorageQueryCount", + "sum": { + "$numberInt": "10" + }, + "count": { + "$numberLong": "10" + }, + "min": { + "$numberInt": "1" + }, + "max": { + "$numberInt": "1" + }, + "index_name": "Request", + "operation": "Data" + }, + { + "_id": { + "$oid": "6ae3dbee99e7d94eaa5c3026" + }, + "bucket": { + "$date": { + "$numberLong": "1793317860000" + } + }, + "metric": "RequestCount", + "sum": { + "$numberInt": "1" + }, + "count": { + "$numberLong": "1" + }, + "min": { + "$numberInt": "1" + }, + "max": { + "$numberInt": "1" + }, + "operation": "CreateTable" + }, + { + "_id": { + "$oid": "6ae3dbee99e7d94eaa5c3027" + }, + "bucket": { + "$date": { + "$numberLong": "1793317860000" + } + }, + "metric": "SuccessfulRequestLatency", + "sum": { + "$numberInt": "11641" + }, + "count": { + "$numberLong": "1" + }, + "min": { + "$numberInt": "11641" + }, + "max": { + "$numberInt": "11641" + }, + "table_name": "cu_orders", + "operation": "UpdateTimeToLive" + }, + { + "_id": { + "$oid": "6ae3dbee99e7d94eaa5c3028" + }, + "bucket": { + "$date": { + "$numberLong": "1793317860000" + } + }, + "metric": "StorageQueryLatency", + "sum": { + "$numberInt": "58444" + }, + "count": { + "$numberLong": "10" + }, + "min": { + "$numberInt": "3506" + }, + "max": { + "$numberInt": "18557" + }, + "index_name": "Request", + "operation": "Data" + } + ] + }, + { + "name": "schema_history", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + } + ], + "documents": [ + { + "_id": "001_initial", + "applied_at": { + "$date": { + "$numberLong": "1793317865714" + } + } + } + ] + }, + { + "name": "settings", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + } + ], + "documents": [ + { + "_id": "catalog_version", + "value": "0.0.2" + }, + { + "_id": "data_connection_string", + "value": "mongodb://localhost:22411/?replicaSet=rs0" + }, + { + "_id": "data_database_name", + "value": "extenddb_data" + }, + { + "_id": "encryption_key", + "value": "O3/7nP92zj2RE/tBGBtvnvdWIzb7BLZg9RrSYQGEaP0=" + } + ] + }, + { + "name": "tables", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "table_id": { + "$numberInt": "1" + } + }, + "name": "table_id_1", + "unique": true + } + ], + "documents": [ + { + "_id": { + "account_id": "123456789012", + "table_name": "cu_orders" + }, + "key_schema": [ + { + "AttributeName": "pk", + "KeyType": "HASH" + }, + { + "AttributeName": "sk", + "KeyType": "RANGE" + } + ], + "attribute_definitions": [ + { + "AttributeName": "pk", + "AttributeType": "S" + }, + { + "AttributeName": "sk", + "AttributeType": "S" + }, + { + "AttributeName": "gpk", + "AttributeType": "S" + }, + { + "AttributeName": "gsk", + "AttributeType": "N" + }, + { + "AttributeName": "lsk", + "AttributeType": "S" + } + ], + "billing_mode": "PAY_PER_REQUEST", + "provisioned_throughput": null, + "stream_specification": { + "StreamEnabled": true, + "StreamViewType": "NEW_AND_OLD_IMAGES" + }, + "table_status": "ACTIVE", + "creation_date_time": { + "$date": { + "$numberLong": "1793317869000" + } + }, + "table_size_bytes": { + "$numberLong": "0" + }, + "item_count": { + "$numberLong": "0" + }, + "table_arn": "arn:aws:dynamodb:us-east-1:123456789012:table/cu_orders", + "table_id": "0be1f21a-1085-4d7a-a3f3-2eea27890f1a", + "deletion_protection_enabled": false, + "ttl_attribute": "expires", + "stream_label": "2026-10-29T23:51:09", + "table_class": null, + "sse_specification": null, + "on_demand_throughput": null, + "ttl_index_ready": true + } + ] + }, + { + "name": "tags", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + } + ], + "documents": [] + } + ] + }, + "extenddb_data": { + "collections": [ + { + "name": "_ddb_0be1f21a-1085-4d7a-a3f3-2eea27890f1a", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "pk": { + "$numberInt": "1" + }, + "sk_s": { + "$numberInt": "1" + } + }, + "name": "pk_1_sk_s_1", + "unique": true + }, + { + "key": { + "item_data.expires.N": { + "$numberInt": "1" + } + }, + "name": "idx_ttl_expires", + "sparse": true + } + ], + "documents": [ + { + "_id": "2:p0,3:s00,", + "pk": "p0", + "sk_s": "s00", + "item_data": { + "body": { + "S": "item 0" + }, + "expires": { + "N": "4102444800" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "0" + }, + "lsk": { + "S": "l9" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s00" + } + } + }, + { + "_id": "2:p0,3:s03,", + "pk": "p0", + "sk_s": "s03", + "item_data": { + "body": { + "S": "item 3" + }, + "expires": { + "N": "4102444803" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "3" + }, + "lsk": { + "S": "l6" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s03" + } + } + }, + { + "_id": "2:p0,3:s06,", + "pk": "p0", + "sk_s": "s06", + "item_data": { + "body": { + "S": "item 6" + }, + "expires": { + "N": "4102444806" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "6" + }, + "lsk": { + "S": "l3" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s06" + } + } + }, + { + "_id": "2:p0,3:s09,", + "pk": "p0", + "sk_s": "s09", + "item_data": { + "body": { + "S": "item 9" + }, + "expires": { + "N": "4102444809" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "9" + }, + "lsk": { + "S": "l0" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s09" + } + } + }, + { + "_id": "2:p1,3:s01,", + "pk": "p1", + "sk_s": "s01", + "item_data": { + "body": { + "S": "item 1" + }, + "expires": { + "N": "4102444801" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "1" + }, + "lsk": { + "S": "l8" + }, + "pk": { + "S": "p1" + }, + "sk": { + "S": "s01" + } + } + }, + { + "_id": "2:p1,3:s04,", + "pk": "p1", + "sk_s": "s04", + "item_data": { + "body": { + "S": "item 4" + }, + "expires": { + "N": "4102444804" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "4" + }, + "lsk": { + "S": "l5" + }, + "pk": { + "S": "p1" + }, + "sk": { + "S": "s04" + } + } + }, + { + "_id": "2:p1,3:s07,", + "pk": "p1", + "sk_s": "s07", + "item_data": { + "body": { + "S": "item 7" + }, + "expires": { + "N": "4102444807" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "7" + }, + "lsk": { + "S": "l2" + }, + "pk": { + "S": "p1" + }, + "sk": { + "S": "s07" + } + } + }, + { + "_id": "2:p2,3:s02,", + "pk": "p2", + "sk_s": "s02", + "item_data": { + "body": { + "S": "item 2" + }, + "expires": { + "N": "4102444802" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "2" + }, + "lsk": { + "S": "l7" + }, + "pk": { + "S": "p2" + }, + "sk": { + "S": "s02" + } + } + }, + { + "_id": "2:p2,3:s05,", + "pk": "p2", + "sk_s": "s05", + "item_data": { + "body": { + "S": "item 5" + }, + "expires": { + "N": "4102444805" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "5" + }, + "lsk": { + "S": "l4" + }, + "pk": { + "S": "p2" + }, + "sk": { + "S": "s05" + } + } + }, + { + "_id": "2:p2,3:s08,", + "pk": "p2", + "sk_s": "s08", + "item_data": { + "body": { + "S": "item 8" + }, + "expires": { + "N": "4102444808" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "8" + }, + "lsk": { + "S": "l1" + }, + "pk": { + "S": "p2" + }, + "sk": { + "S": "s08" + } + } + } + ] + }, + { + "name": "_ddb_0f252ac6-e551-4f4f-9b6c-493a50ba5fad", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "pk": { + "$numberInt": "1" + }, + "sk_n": { + "$numberInt": "1" + }, + "base_pk": { + "$numberInt": "1" + }, + "base_sk_s": { + "$numberInt": "1" + } + }, + "name": "pk_1_sk_n_1_base_pk_1_base_sk_s_1" + } + ], + "documents": [ + { + "_id": "2:g0,1:0,2:p0,3:s00,", + "pk": "g0", + "base_pk": "p0", + "sk_n": { + "$numberDecimal": "0" + }, + "base_sk_s": "s00", + "item_data": { + "body": { + "S": "item 0" + }, + "expires": { + "N": "4102444800" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "0" + }, + "lsk": { + "S": "l9" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s00" + } + } + }, + { + "_id": "2:g0,1:2,2:p2,3:s02,", + "pk": "g0", + "base_pk": "p2", + "sk_n": { + "$numberDecimal": "2" + }, + "base_sk_s": "s02", + "item_data": { + "body": { + "S": "item 2" + }, + "expires": { + "N": "4102444802" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "2" + }, + "lsk": { + "S": "l7" + }, + "pk": { + "S": "p2" + }, + "sk": { + "S": "s02" + } + } + }, + { + "_id": "2:g0,1:4,2:p1,3:s04,", + "pk": "g0", + "base_pk": "p1", + "sk_n": { + "$numberDecimal": "4" + }, + "base_sk_s": "s04", + "item_data": { + "body": { + "S": "item 4" + }, + "expires": { + "N": "4102444804" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "4" + }, + "lsk": { + "S": "l5" + }, + "pk": { + "S": "p1" + }, + "sk": { + "S": "s04" + } + } + }, + { + "_id": "2:g0,1:6,2:p0,3:s06,", + "pk": "g0", + "base_pk": "p0", + "sk_n": { + "$numberDecimal": "6" + }, + "base_sk_s": "s06", + "item_data": { + "body": { + "S": "item 6" + }, + "expires": { + "N": "4102444806" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "6" + }, + "lsk": { + "S": "l3" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s06" + } + } + }, + { + "_id": "2:g0,1:8,2:p2,3:s08,", + "pk": "g0", + "base_pk": "p2", + "sk_n": { + "$numberDecimal": "8" + }, + "base_sk_s": "s08", + "item_data": { + "body": { + "S": "item 8" + }, + "expires": { + "N": "4102444808" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "8" + }, + "lsk": { + "S": "l1" + }, + "pk": { + "S": "p2" + }, + "sk": { + "S": "s08" + } + } + }, + { + "_id": "2:g1,1:1,2:p1,3:s01,", + "pk": "g1", + "base_pk": "p1", + "sk_n": { + "$numberDecimal": "1" + }, + "base_sk_s": "s01", + "item_data": { + "body": { + "S": "item 1" + }, + "expires": { + "N": "4102444801" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "1" + }, + "lsk": { + "S": "l8" + }, + "pk": { + "S": "p1" + }, + "sk": { + "S": "s01" + } + } + }, + { + "_id": "2:g1,1:3,2:p0,3:s03,", + "pk": "g1", + "base_pk": "p0", + "sk_n": { + "$numberDecimal": "3" + }, + "base_sk_s": "s03", + "item_data": { + "body": { + "S": "item 3" + }, + "expires": { + "N": "4102444803" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "3" + }, + "lsk": { + "S": "l6" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s03" + } + } + }, + { + "_id": "2:g1,1:5,2:p2,3:s05,", + "pk": "g1", + "base_pk": "p2", + "sk_n": { + "$numberDecimal": "5" + }, + "base_sk_s": "s05", + "item_data": { + "body": { + "S": "item 5" + }, + "expires": { + "N": "4102444805" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "5" + }, + "lsk": { + "S": "l4" + }, + "pk": { + "S": "p2" + }, + "sk": { + "S": "s05" + } + } + }, + { + "_id": "2:g1,1:7,2:p1,3:s07,", + "pk": "g1", + "base_pk": "p1", + "sk_n": { + "$numberDecimal": "7" + }, + "base_sk_s": "s07", + "item_data": { + "body": { + "S": "item 7" + }, + "expires": { + "N": "4102444807" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "7" + }, + "lsk": { + "S": "l2" + }, + "pk": { + "S": "p1" + }, + "sk": { + "S": "s07" + } + } + }, + { + "_id": "2:g1,1:9,2:p0,3:s09,", + "pk": "g1", + "base_pk": "p0", + "sk_n": { + "$numberDecimal": "9" + }, + "base_sk_s": "s09", + "item_data": { + "body": { + "S": "item 9" + }, + "expires": { + "N": "4102444809" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "9" + }, + "lsk": { + "S": "l0" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s09" + } + } + } + ] + }, + { + "name": "_ddb_21390ee1-34fa-4121-bfd9-aa9102a03d64", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "pk": { + "$numberInt": "1" + }, + "sk_s": { + "$numberInt": "1" + }, + "base_pk": { + "$numberInt": "1" + }, + "base_sk_s": { + "$numberInt": "1" + } + }, + "name": "pk_1_sk_s_1_base_pk_1_base_sk_s_1" + } + ], + "documents": [ + { + "_id": "2:p0,2:l0,2:p0,3:s09,", + "pk": "p0", + "base_pk": "p0", + "sk_s": "l0", + "base_sk_s": "s09", + "item_data": { + "lsk": { + "S": "l0" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s09" + } + } + }, + { + "_id": "2:p0,2:l3,2:p0,3:s06,", + "pk": "p0", + "base_pk": "p0", + "sk_s": "l3", + "base_sk_s": "s06", + "item_data": { + "lsk": { + "S": "l3" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s06" + } + } + }, + { + "_id": "2:p0,2:l6,2:p0,3:s03,", + "pk": "p0", + "base_pk": "p0", + "sk_s": "l6", + "base_sk_s": "s03", + "item_data": { + "lsk": { + "S": "l6" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s03" + } + } + }, + { + "_id": "2:p0,2:l9,2:p0,3:s00,", + "pk": "p0", + "base_pk": "p0", + "sk_s": "l9", + "base_sk_s": "s00", + "item_data": { + "lsk": { + "S": "l9" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s00" + } + } + }, + { + "_id": "2:p1,2:l2,2:p1,3:s07,", + "pk": "p1", + "base_pk": "p1", + "sk_s": "l2", + "base_sk_s": "s07", + "item_data": { + "lsk": { + "S": "l2" + }, + "pk": { + "S": "p1" + }, + "sk": { + "S": "s07" + } + } + }, + { + "_id": "2:p1,2:l5,2:p1,3:s04,", + "pk": "p1", + "base_pk": "p1", + "sk_s": "l5", + "base_sk_s": "s04", + "item_data": { + "lsk": { + "S": "l5" + }, + "pk": { + "S": "p1" + }, + "sk": { + "S": "s04" + } + } + }, + { + "_id": "2:p1,2:l8,2:p1,3:s01,", + "pk": "p1", + "base_pk": "p1", + "sk_s": "l8", + "base_sk_s": "s01", + "item_data": { + "lsk": { + "S": "l8" + }, + "pk": { + "S": "p1" + }, + "sk": { + "S": "s01" + } + } + }, + { + "_id": "2:p2,2:l1,2:p2,3:s08,", + "pk": "p2", + "base_pk": "p2", + "sk_s": "l1", + "base_sk_s": "s08", + "item_data": { + "lsk": { + "S": "l1" + }, + "pk": { + "S": "p2" + }, + "sk": { + "S": "s08" + } + } + }, + { + "_id": "2:p2,2:l4,2:p2,3:s05,", + "pk": "p2", + "base_pk": "p2", + "sk_s": "l4", + "base_sk_s": "s05", + "item_data": { + "lsk": { + "S": "l4" + }, + "pk": { + "S": "p2" + }, + "sk": { + "S": "s05" + } + } + }, + { + "_id": "2:p2,2:l7,2:p2,3:s02,", + "pk": "p2", + "base_pk": "p2", + "sk_s": "l7", + "base_sk_s": "s02", + "item_data": { + "lsk": { + "S": "l7" + }, + "pk": { + "S": "p2" + }, + "sk": { + "S": "s02" + } + } + } + ] + }, + { + "name": "counters", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + } + ], + "documents": [ + { + "_id": "stream_seq:shardId-0be1f21a-1085-4d7a-a3f3-2eea27890f1a-000000000001", + "value": { + "$numberLong": "7" + } + }, + { + "_id": "stream_seq:shardId-0be1f21a-1085-4d7a-a3f3-2eea27890f1a-000000000003", + "value": { + "$numberLong": "3" + } + } + ] + }, + { + "name": "idempotency_tokens", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "created_at": { + "$numberInt": "1" + } + }, + "name": "created_at_1", + "expireAfterSeconds": { + "$numberInt": "540" + } + }, + { + "key": { + "account_id": { + "$numberInt": "1" + }, + "token": { + "$numberInt": "1" + } + }, + "name": "account_id_1_token_1", + "unique": true + } + ], + "documents": [] + }, + { + "name": "stream_records", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "created_at": { + "$numberInt": "1" + } + }, + "name": "created_at_1", + "expireAfterSeconds": { + "$numberInt": "86400" + } + }, + { + "key": { + "shard_id": { + "$numberInt": "1" + }, + "sequence_number": { + "$numberInt": "1" + } + }, + "name": "shard_id_1_sequence_number_1" + } + ], + "documents": [ + { + "_id": { + "$oid": "6ae3dbed99e7d94eaa5c3012" + }, + "sequence_number": "000000000000000000001", + "shard_id": "shardId-0be1f21a-1085-4d7a-a3f3-2eea27890f1a-000000000001", + "table_id": "0be1f21a-1085-4d7a-a3f3-2eea27890f1a", + "event_name": "INSERT", + "record_data": { + "eventID": "1fadf95a-b41b-462f-a65e-0ef30981b2d5", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "us-east-1", + "dynamodb": { + "ApproximateCreationDateTime": { + "$numberLong": "1793317869" + }, + "Keys": { + "pk": { + "S": "p0" + }, + "sk": { + "S": "s00" + } + }, + "NewImage": { + "body": { + "S": "item 0" + }, + "expires": { + "N": "4102444800" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "0" + }, + "lsk": { + "S": "l9" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s00" + } + }, + "SequenceNumber": "000000000000000000001", + "SizeBytes": { + "$numberLong": "45" + }, + "StreamViewType": "NEW_AND_OLD_IMAGES" + } + }, + "created_at": { + "$date": { + "$numberLong": "1793317869879" + } + } + }, + { + "_id": { + "$oid": "6ae3dbed99e7d94eaa5c3013" + }, + "sequence_number": "000000000000000000001", + "shard_id": "shardId-0be1f21a-1085-4d7a-a3f3-2eea27890f1a-000000000003", + "table_id": "0be1f21a-1085-4d7a-a3f3-2eea27890f1a", + "event_name": "INSERT", + "record_data": { + "eventID": "4109c1e3-9648-448e-a11c-7ddbc90a468e", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "us-east-1", + "dynamodb": { + "ApproximateCreationDateTime": { + "$numberLong": "1793317869" + }, + "Keys": { + "pk": { + "S": "p1" + }, + "sk": { + "S": "s01" + } + }, + "NewImage": { + "body": { + "S": "item 1" + }, + "expires": { + "N": "4102444801" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "1" + }, + "lsk": { + "S": "l8" + }, + "pk": { + "S": "p1" + }, + "sk": { + "S": "s01" + } + }, + "SequenceNumber": "000000000000000000001", + "SizeBytes": { + "$numberLong": "47" + }, + "StreamViewType": "NEW_AND_OLD_IMAGES" + } + }, + "created_at": { + "$date": { + "$numberLong": "1793317869886" + } + } + }, + { + "_id": { + "$oid": "6ae3dbed99e7d94eaa5c3014" + }, + "sequence_number": "000000000000000000002", + "shard_id": "shardId-0be1f21a-1085-4d7a-a3f3-2eea27890f1a-000000000001", + "table_id": "0be1f21a-1085-4d7a-a3f3-2eea27890f1a", + "event_name": "INSERT", + "record_data": { + "eventID": "ddea44f1-b287-42b9-a75c-e2929c0a5c4f", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "us-east-1", + "dynamodb": { + "ApproximateCreationDateTime": { + "$numberLong": "1793317869" + }, + "Keys": { + "pk": { + "S": "p2" + }, + "sk": { + "S": "s02" + } + }, + "NewImage": { + "body": { + "S": "item 2" + }, + "expires": { + "N": "4102444802" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "2" + }, + "lsk": { + "S": "l7" + }, + "pk": { + "S": "p2" + }, + "sk": { + "S": "s02" + } + }, + "SequenceNumber": "000000000000000000002", + "SizeBytes": { + "$numberLong": "47" + }, + "StreamViewType": "NEW_AND_OLD_IMAGES" + } + }, + "created_at": { + "$date": { + "$numberLong": "1793317869891" + } + } + }, + { + "_id": { + "$oid": "6ae3dbed99e7d94eaa5c3015" + }, + "sequence_number": "000000000000000000003", + "shard_id": "shardId-0be1f21a-1085-4d7a-a3f3-2eea27890f1a-000000000001", + "table_id": "0be1f21a-1085-4d7a-a3f3-2eea27890f1a", + "event_name": "INSERT", + "record_data": { + "eventID": "3f9a9027-f7f0-4473-b5f8-88790fe33c5b", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "us-east-1", + "dynamodb": { + "ApproximateCreationDateTime": { + "$numberLong": "1793317869" + }, + "Keys": { + "pk": { + "S": "p0" + }, + "sk": { + "S": "s03" + } + }, + "NewImage": { + "body": { + "S": "item 3" + }, + "expires": { + "N": "4102444803" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "3" + }, + "lsk": { + "S": "l6" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s03" + } + }, + "SequenceNumber": "000000000000000000003", + "SizeBytes": { + "$numberLong": "47" + }, + "StreamViewType": "NEW_AND_OLD_IMAGES" + } + }, + "created_at": { + "$date": { + "$numberLong": "1793317869897" + } + } + }, + { + "_id": { + "$oid": "6ae3dbed99e7d94eaa5c3016" + }, + "sequence_number": "000000000000000000002", + "shard_id": "shardId-0be1f21a-1085-4d7a-a3f3-2eea27890f1a-000000000003", + "table_id": "0be1f21a-1085-4d7a-a3f3-2eea27890f1a", + "event_name": "INSERT", + "record_data": { + "eventID": "b15da506-5eff-4398-b183-4f400b27d60b", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "us-east-1", + "dynamodb": { + "ApproximateCreationDateTime": { + "$numberLong": "1793317869" + }, + "Keys": { + "pk": { + "S": "p1" + }, + "sk": { + "S": "s04" + } + }, + "NewImage": { + "body": { + "S": "item 4" + }, + "expires": { + "N": "4102444804" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "4" + }, + "lsk": { + "S": "l5" + }, + "pk": { + "S": "p1" + }, + "sk": { + "S": "s04" + } + }, + "SequenceNumber": "000000000000000000002", + "SizeBytes": { + "$numberLong": "47" + }, + "StreamViewType": "NEW_AND_OLD_IMAGES" + } + }, + "created_at": { + "$date": { + "$numberLong": "1793317869903" + } + } + }, + { + "_id": { + "$oid": "6ae3dbed99e7d94eaa5c3017" + }, + "sequence_number": "000000000000000000004", + "shard_id": "shardId-0be1f21a-1085-4d7a-a3f3-2eea27890f1a-000000000001", + "table_id": "0be1f21a-1085-4d7a-a3f3-2eea27890f1a", + "event_name": "INSERT", + "record_data": { + "eventID": "52a7c1ab-d7cc-4661-8887-795d75653b90", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "us-east-1", + "dynamodb": { + "ApproximateCreationDateTime": { + "$numberLong": "1793317869" + }, + "Keys": { + "pk": { + "S": "p2" + }, + "sk": { + "S": "s05" + } + }, + "NewImage": { + "body": { + "S": "item 5" + }, + "expires": { + "N": "4102444805" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "5" + }, + "lsk": { + "S": "l4" + }, + "pk": { + "S": "p2" + }, + "sk": { + "S": "s05" + } + }, + "SequenceNumber": "000000000000000000004", + "SizeBytes": { + "$numberLong": "47" + }, + "StreamViewType": "NEW_AND_OLD_IMAGES" + } + }, + "created_at": { + "$date": { + "$numberLong": "1793317869910" + } + } + }, + { + "_id": { + "$oid": "6ae3dbed99e7d94eaa5c3018" + }, + "sequence_number": "000000000000000000005", + "shard_id": "shardId-0be1f21a-1085-4d7a-a3f3-2eea27890f1a-000000000001", + "table_id": "0be1f21a-1085-4d7a-a3f3-2eea27890f1a", + "event_name": "INSERT", + "record_data": { + "eventID": "393afb09-b343-4e34-bd03-4dbeb487a351", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "us-east-1", + "dynamodb": { + "ApproximateCreationDateTime": { + "$numberLong": "1793317869" + }, + "Keys": { + "pk": { + "S": "p0" + }, + "sk": { + "S": "s06" + } + }, + "NewImage": { + "body": { + "S": "item 6" + }, + "expires": { + "N": "4102444806" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "6" + }, + "lsk": { + "S": "l3" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s06" + } + }, + "SequenceNumber": "000000000000000000005", + "SizeBytes": { + "$numberLong": "47" + }, + "StreamViewType": "NEW_AND_OLD_IMAGES" + } + }, + "created_at": { + "$date": { + "$numberLong": "1793317869916" + } + } + }, + { + "_id": { + "$oid": "6ae3dbed99e7d94eaa5c3019" + }, + "sequence_number": "000000000000000000003", + "shard_id": "shardId-0be1f21a-1085-4d7a-a3f3-2eea27890f1a-000000000003", + "table_id": "0be1f21a-1085-4d7a-a3f3-2eea27890f1a", + "event_name": "INSERT", + "record_data": { + "eventID": "fc47dda6-1b4a-4ca9-90f2-94d066630fc0", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "us-east-1", + "dynamodb": { + "ApproximateCreationDateTime": { + "$numberLong": "1793317869" + }, + "Keys": { + "pk": { + "S": "p1" + }, + "sk": { + "S": "s07" + } + }, + "NewImage": { + "body": { + "S": "item 7" + }, + "expires": { + "N": "4102444807" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "7" + }, + "lsk": { + "S": "l2" + }, + "pk": { + "S": "p1" + }, + "sk": { + "S": "s07" + } + }, + "SequenceNumber": "000000000000000000003", + "SizeBytes": { + "$numberLong": "47" + }, + "StreamViewType": "NEW_AND_OLD_IMAGES" + } + }, + "created_at": { + "$date": { + "$numberLong": "1793317869922" + } + } + }, + { + "_id": { + "$oid": "6ae3dbed99e7d94eaa5c301a" + }, + "sequence_number": "000000000000000000006", + "shard_id": "shardId-0be1f21a-1085-4d7a-a3f3-2eea27890f1a-000000000001", + "table_id": "0be1f21a-1085-4d7a-a3f3-2eea27890f1a", + "event_name": "INSERT", + "record_data": { + "eventID": "0c8f0f25-6c2b-4702-aff6-934ea8257097", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "us-east-1", + "dynamodb": { + "ApproximateCreationDateTime": { + "$numberLong": "1793317869" + }, + "Keys": { + "pk": { + "S": "p2" + }, + "sk": { + "S": "s08" + } + }, + "NewImage": { + "body": { + "S": "item 8" + }, + "expires": { + "N": "4102444808" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "8" + }, + "lsk": { + "S": "l1" + }, + "pk": { + "S": "p2" + }, + "sk": { + "S": "s08" + } + }, + "SequenceNumber": "000000000000000000006", + "SizeBytes": { + "$numberLong": "47" + }, + "StreamViewType": "NEW_AND_OLD_IMAGES" + } + }, + "created_at": { + "$date": { + "$numberLong": "1793317869928" + } + } + }, + { + "_id": { + "$oid": "6ae3dbed99e7d94eaa5c301b" + }, + "sequence_number": "000000000000000000007", + "shard_id": "shardId-0be1f21a-1085-4d7a-a3f3-2eea27890f1a-000000000001", + "table_id": "0be1f21a-1085-4d7a-a3f3-2eea27890f1a", + "event_name": "INSERT", + "record_data": { + "eventID": "b7b41cc0-2ef9-443a-9f95-7911c0128af2", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "us-east-1", + "dynamodb": { + "ApproximateCreationDateTime": { + "$numberLong": "1793317869" + }, + "Keys": { + "pk": { + "S": "p0" + }, + "sk": { + "S": "s09" + } + }, + "NewImage": { + "body": { + "S": "item 9" + }, + "expires": { + "N": "4102444809" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "9" + }, + "lsk": { + "S": "l0" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s09" + } + }, + "SequenceNumber": "000000000000000000007", + "SizeBytes": { + "$numberLong": "47" + }, + "StreamViewType": "NEW_AND_OLD_IMAGES" + } + }, + "created_at": { + "$date": { + "$numberLong": "1793317869935" + } + } + } + ] + }, + { + "name": "stream_shards", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "shard_id": { + "$numberInt": "1" + } + }, + "name": "shard_id_1", + "unique": true + } + ], + "documents": [ + { + "_id": { + "$oid": "6ae3dbed99e7d94eaa5c300e" + }, + "shard_id": "shardId-0be1f21a-1085-4d7a-a3f3-2eea27890f1a-000000000000", + "table_id": "0be1f21a-1085-4d7a-a3f3-2eea27890f1a", + "starting_sequence_number": "000000000000000000000", + "created_at": { + "$date": { + "$numberLong": "1793317869284" + } + } + }, + { + "_id": { + "$oid": "6ae3dbed99e7d94eaa5c300f" + }, + "shard_id": "shardId-0be1f21a-1085-4d7a-a3f3-2eea27890f1a-000000000001", + "table_id": "0be1f21a-1085-4d7a-a3f3-2eea27890f1a", + "starting_sequence_number": "000000000000000000000", + "created_at": { + "$date": { + "$numberLong": "1793317869285" + } + } + }, + { + "_id": { + "$oid": "6ae3dbed99e7d94eaa5c3010" + }, + "shard_id": "shardId-0be1f21a-1085-4d7a-a3f3-2eea27890f1a-000000000002", + "table_id": "0be1f21a-1085-4d7a-a3f3-2eea27890f1a", + "starting_sequence_number": "000000000000000000000", + "created_at": { + "$date": { + "$numberLong": "1793317869286" + } + } + }, + { + "_id": { + "$oid": "6ae3dbed99e7d94eaa5c3011" + }, + "shard_id": "shardId-0be1f21a-1085-4d7a-a3f3-2eea27890f1a-000000000003", + "table_id": "0be1f21a-1085-4d7a-a3f3-2eea27890f1a", + "starting_sequence_number": "000000000000000000000", + "created_at": { + "$date": { + "$numberLong": "1793317869287" + } + } + } + ] + } + ] + } + } +} diff --git a/crates/storage-mongodb/tests/fixtures/released/0.1.13.json b/crates/storage-mongodb/tests/fixtures/released/0.1.13.json new file mode 100644 index 000000000..51b20abd1 --- /dev/null +++ b/crates/storage-mongodb/tests/fixtures/released/0.1.13.json @@ -0,0 +1,2683 @@ +{ + "databases": { + "extenddb_catalog": { + "collections": [ + { + "name": "access_keys", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "account_id": { + "$numberInt": "1" + }, + "user_name": { + "$numberInt": "1" + } + }, + "name": "account_id_1_user_name_1" + } + ], + "documents": [ + { + "_id": { + "$oid": "6ae3dbeccb8ec258b3a46a71" + }, + "access_key_id": "AKIAEXTENDDBMUNO9P5D", + "account_id": "123456789012", + "user_name": "cu-tester", + "secret_key_encrypted": { + "$binary": { + "base64": "bWotK71OxipZvGtKEjh4YGv2Rxm6wmSPIYDyOSXfVxDAKlSsydWusXP9LNvJXlVwSjpynFVtA2uvJ7LbeiP71BJB5N8=", + "subType": "00" + } + }, + "is_active": true, + "created_at": { + "$date": { + "$numberLong": "1793317868345" + } + } + } + ] + }, + { + "name": "accounts", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "account_name": { + "$numberInt": "1" + } + }, + "name": "account_name_1", + "unique": true + } + ], + "documents": [ + { + "_id": "ccc47e7d-ef4b-4d03-b575-0bb0383e8040", + "account_name": "default", + "created_at": { + "$date": { + "$numberLong": "1793317865745" + } + } + }, + { + "_id": { + "$oid": "6ae3dbeacb8ec258b3a46a6f" + }, + "account_id": "123456789012", + "account_name": "cu-seed", + "created_at": { + "$date": { + "$numberLong": "1793317866845" + } + } + } + ] + }, + { + "name": "admin_users", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + } + ], + "documents": [ + { + "_id": "admin", + "password_hash": "$2b$12$S7BAnxLb5pJ/Y.peUIPI9.6fgXv9hXSSis9uN/VLOM2pLECCrp.6u", + "created_at": { + "$date": { + "$numberLong": "1793317865980" + } + } + } + ] + }, + { + "name": "backups", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "account_id": { + "$numberInt": "1" + }, + "table_name": { + "$numberInt": "1" + } + }, + "name": "account_id_1_table_name_1" + } + ], + "documents": [] + }, + { + "name": "continuous_backups", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + } + ], + "documents": [] + }, + { + "name": "iam_groups", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "group_arn": { + "$numberInt": "1" + } + }, + "name": "group_arn_1", + "unique": true + } + ], + "documents": [] + }, + { + "name": "iam_permissions_boundaries", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + } + ], + "documents": [] + }, + { + "name": "iam_policies", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + } + ], + "documents": [ + { + "_id": { + "$oid": "6ae3dbeb658fd80e808ade76" + }, + "principal_type": "user", + "policy_name": "SelfServicePolicy", + "principal_name": "cu-tester", + "account_id": "123456789012", + "created_at": { + "$date": { + "$numberLong": "1793317867273" + } + }, + "policy_document": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "iam:CreateAccessKey", + "iam:DeleteAccessKey", + "iam:ListAccessKeys", + "iam:ChangePassword" + ], + "Resource": "arn:aws:iam::123456789012:user/cu-tester" + } + ] + } + }, + { + "_id": { + "$oid": "6ae3dbeb658fd80e808ade7b" + }, + "principal_type": "user", + "policy_name": "dynamodb-full", + "principal_name": "cu-tester", + "account_id": "123456789012", + "created_at": { + "$date": { + "$numberLong": "1793317867621" + } + }, + "policy_document": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": "dynamodb:*", + "Resource": "*" + } + ] + } + }, + { + "_id": { + "$oid": "6ae3dbec658fd80e808ade81" + }, + "principal_type": "user", + "policy_name": "deny-prod", + "principal_name": "cu-tester", + "account_id": "123456789012", + "created_at": { + "$date": { + "$numberLong": "1793317868008" + } + }, + "policy_document": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Deny", + "Action": "dynamodb:DeleteTable", + "Resource": "*", + "Condition": { + "StringEquals": { + "dynamodb:ResourceTag/env": "prod" + } + } + } + ] + } + } + ] + }, + { + "name": "iam_roles", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "role_arn": { + "$numberInt": "1" + } + }, + "name": "role_arn_1", + "unique": true + } + ], + "documents": [] + }, + { + "name": "iam_sessions", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "access_key_id": { + "$numberInt": "1" + } + }, + "name": "access_key_id_1", + "unique": true + }, + { + "key": { + "expires_at": { + "$numberInt": "1" + } + }, + "name": "expires_at_1", + "expireAfterSeconds": { + "$numberInt": "0" + } + } + ], + "documents": [] + }, + { + "name": "iam_users", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "user_arn": { + "$numberInt": "1" + } + }, + "name": "user_arn_1", + "unique": true + } + ], + "documents": [ + { + "_id": { + "$oid": "6ae3dbebcb8ec258b3a46a70" + }, + "account_id": "123456789012", + "user_name": "cu-tester", + "user_arn": "arn:aws:iam::123456789012:user/cu-tester", + "created_at": { + "$date": { + "$numberLong": "1793317867270" + } + } + } + ] + }, + { + "name": "indexes", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + } + ], + "documents": [ + { + "_id": { + "table_id": "c675a942-95bb-48c8-8474-eac51bdb57f5", + "index_name": "by_gpk" + }, + "index_id": "1a8261b2-1a28-4ece-b098-b253bbee3c6c", + "index_type": "GSI", + "key_schema": [ + { + "AttributeName": "gpk", + "KeyType": "HASH" + }, + { + "AttributeName": "gsk", + "KeyType": "RANGE" + } + ], + "projection": { + "ProjectionType": "ALL" + }, + "index_status": "ACTIVE", + "provisioned_throughput": null + }, + { + "_id": { + "table_id": "c675a942-95bb-48c8-8474-eac51bdb57f5", + "index_name": "by_lsk" + }, + "index_id": "04f9d8f6-a3e1-4c22-825e-dd27223f9b0f", + "index_type": "LSI", + "key_schema": [ + { + "AttributeName": "pk", + "KeyType": "HASH" + }, + { + "AttributeName": "lsk", + "KeyType": "RANGE" + } + ], + "projection": { + "ProjectionType": "KEYS_ONLY" + }, + "index_status": "ACTIVE", + "provisioned_throughput": null + } + ] + }, + { + "name": "login_attempts", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "principal": { + "$numberInt": "1" + }, + "attempted_at": { + "$numberInt": "1" + } + }, + "name": "principal_1_attempted_at_1" + } + ], + "documents": [] + }, + { + "name": "metrics", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "_id.bucket": { + "$numberInt": "1" + } + }, + "name": "_id.bucket_1" + } + ], + "documents": [ + { + "_id": { + "$oid": "6ae3dbeecb8ec258b3a46a80" + }, + "bucket": { + "$date": { + "$numberLong": "1793317860000" + } + }, + "metric": "StorageQueryLatency", + "sum": { + "$numberInt": "86363" + }, + "count": { + "$numberLong": "4" + }, + "min": { + "$numberInt": "588" + }, + "max": { + "$numberInt": "72531" + }, + "index_name": "Request", + "operation": "Catalog" + }, + { + "_id": { + "$oid": "6ae3dbeecb8ec258b3a46a81" + }, + "bucket": { + "$date": { + "$numberLong": "1793317860000" + } + }, + "metric": "SuccessfulRequestLatency", + "sum": { + "$numberInt": "2313" + }, + "count": { + "$numberLong": "2" + }, + "min": { + "$numberInt": "1141" + }, + "max": { + "$numberInt": "1172" + }, + "table_name": "cu_orders", + "operation": "DescribeTable" + }, + { + "_id": { + "$oid": "6ae3dbeecb8ec258b3a46a82" + }, + "bucket": { + "$date": { + "$numberLong": "1793317860000" + } + }, + "metric": "SuccessfulRequestLatency", + "sum": { + "$numberInt": "59153" + }, + "count": { + "$numberLong": "10" + }, + "min": { + "$numberInt": "3659" + }, + "max": { + "$numberInt": "18091" + }, + "table_name": "cu_orders", + "operation": "PutItem" + }, + { + "_id": { + "$oid": "6ae3dbeecb8ec258b3a46a83" + }, + "bucket": { + "$date": { + "$numberLong": "1793317860000" + } + }, + "metric": "StorageQueryCount", + "sum": { + "$numberInt": "10" + }, + "count": { + "$numberLong": "10" + }, + "min": { + "$numberInt": "1" + }, + "max": { + "$numberInt": "1" + }, + "index_name": "Request", + "operation": "Data" + }, + { + "_id": { + "$oid": "6ae3dbeecb8ec258b3a46a84" + }, + "bucket": { + "$date": { + "$numberLong": "1793317860000" + } + }, + "metric": "RequestCount", + "sum": { + "$numberInt": "1" + }, + "count": { + "$numberLong": "1" + }, + "min": { + "$numberInt": "1" + }, + "max": { + "$numberInt": "1" + }, + "operation": "CreateTable" + }, + { + "_id": { + "$oid": "6ae3dbeecb8ec258b3a46a85" + }, + "bucket": { + "$date": { + "$numberLong": "1793317860000" + } + }, + "metric": "ConsumedWriteCapacityUnits", + "sum": { + "$numberInt": "10" + }, + "count": { + "$numberLong": "10" + }, + "min": { + "$numberInt": "1" + }, + "max": { + "$numberInt": "1" + }, + "table_name": "cu_orders", + "operation": "PutItem" + }, + { + "_id": { + "$oid": "6ae3dbeecb8ec258b3a46a86" + }, + "bucket": { + "$date": { + "$numberLong": "1793317860000" + } + }, + "metric": "RequestCount", + "sum": { + "$numberInt": "10" + }, + "count": { + "$numberLong": "10" + }, + "min": { + "$numberInt": "1" + }, + "max": { + "$numberInt": "1" + }, + "operation": "PutItem" + }, + { + "_id": { + "$oid": "6ae3dbeecb8ec258b3a46a87" + }, + "bucket": { + "$date": { + "$numberLong": "1793317860000" + } + }, + "metric": "SuccessfulRequestLatency", + "sum": { + "$numberInt": "12291" + }, + "count": { + "$numberLong": "1" + }, + "min": { + "$numberInt": "12291" + }, + "max": { + "$numberInt": "12291" + }, + "table_name": "cu_orders", + "operation": "UpdateTimeToLive" + }, + { + "_id": { + "$oid": "6ae3dbeecb8ec258b3a46a88" + }, + "bucket": { + "$date": { + "$numberLong": "1793317860000" + } + }, + "metric": "SuccessfulRequestLatency", + "sum": { + "$numberInt": "75512" + }, + "count": { + "$numberLong": "1" + }, + "min": { + "$numberInt": "75512" + }, + "max": { + "$numberInt": "75512" + }, + "table_name": "cu_orders", + "operation": "CreateTable" + }, + { + "_id": { + "$oid": "6ae3dbeecb8ec258b3a46a89" + }, + "bucket": { + "$date": { + "$numberLong": "1793317860000" + } + }, + "metric": "StorageQueryLatency", + "sum": { + "$numberInt": "57333" + }, + "count": { + "$numberLong": "10" + }, + "min": { + "$numberInt": "3611" + }, + "max": { + "$numberInt": "17217" + }, + "index_name": "Request", + "operation": "Data" + }, + { + "_id": { + "$oid": "6ae3dbeecb8ec258b3a46a8a" + }, + "bucket": { + "$date": { + "$numberLong": "1793317860000" + } + }, + "metric": "RequestCount", + "sum": { + "$numberInt": "1" + }, + "count": { + "$numberLong": "1" + }, + "min": { + "$numberInt": "1" + }, + "max": { + "$numberInt": "1" + }, + "operation": "UpdateTimeToLive" + }, + { + "_id": { + "$oid": "6ae3dbeecb8ec258b3a46a8b" + }, + "bucket": { + "$date": { + "$numberLong": "1793317860000" + } + }, + "metric": "RequestCount", + "sum": { + "$numberInt": "2" + }, + "count": { + "$numberLong": "2" + }, + "min": { + "$numberInt": "1" + }, + "max": { + "$numberInt": "1" + }, + "operation": "DescribeTable" + }, + { + "_id": { + "$oid": "6ae3dbeecb8ec258b3a46a8c" + }, + "bucket": { + "$date": { + "$numberLong": "1793317860000" + } + }, + "metric": "StorageQueryCount", + "sum": { + "$numberInt": "4" + }, + "count": { + "$numberLong": "4" + }, + "min": { + "$numberInt": "1" + }, + "max": { + "$numberInt": "1" + }, + "index_name": "Request", + "operation": "Catalog" + } + ] + }, + { + "name": "schema_history", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + } + ], + "documents": [ + { + "_id": "001_initial", + "applied_at": { + "$date": { + "$numberLong": "1793317865735" + } + } + } + ] + }, + { + "name": "settings", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + } + ], + "documents": [ + { + "_id": "catalog_version", + "value": "0.0.2" + }, + { + "_id": "data_connection_string", + "value": "mongodb://localhost:22412/?replicaSet=rs0" + }, + { + "_id": "data_database_name", + "value": "extenddb_data" + }, + { + "_id": "encryption_key", + "value": "OsAIlXzKiPGUvbEH2fNMs7HG8mc7JRJ/pkpMDVDEorY=" + } + ] + }, + { + "name": "tables", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "table_id": { + "$numberInt": "1" + } + }, + "name": "table_id_1", + "unique": true + } + ], + "documents": [ + { + "_id": { + "account_id": "123456789012", + "table_name": "cu_orders" + }, + "key_schema": [ + { + "AttributeName": "pk", + "KeyType": "HASH" + }, + { + "AttributeName": "sk", + "KeyType": "RANGE" + } + ], + "attribute_definitions": [ + { + "AttributeName": "pk", + "AttributeType": "S" + }, + { + "AttributeName": "sk", + "AttributeType": "S" + }, + { + "AttributeName": "gpk", + "AttributeType": "S" + }, + { + "AttributeName": "gsk", + "AttributeType": "N" + }, + { + "AttributeName": "lsk", + "AttributeType": "S" + } + ], + "billing_mode": "PAY_PER_REQUEST", + "provisioned_throughput": null, + "stream_specification": { + "StreamEnabled": true, + "StreamViewType": "NEW_AND_OLD_IMAGES" + }, + "table_status": "ACTIVE", + "creation_date_time": { + "$date": { + "$numberLong": "1793317869000" + } + }, + "table_size_bytes": { + "$numberLong": "0" + }, + "item_count": { + "$numberLong": "0" + }, + "table_arn": "arn:aws:dynamodb:us-east-1:123456789012:table/cu_orders", + "table_id": "c675a942-95bb-48c8-8474-eac51bdb57f5", + "deletion_protection_enabled": false, + "ttl_attribute": "expires", + "stream_label": "2026-10-29T23:51:09", + "table_class": null, + "sse_specification": null, + "on_demand_throughput": null, + "ttl_index_ready": true + } + ] + }, + { + "name": "tags", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + } + ], + "documents": [] + } + ] + }, + "extenddb_data": { + "collections": [ + { + "name": "_ddb_04f9d8f6-a3e1-4c22-825e-dd27223f9b0f", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "pk": { + "$numberInt": "1" + }, + "sk_s": { + "$numberInt": "1" + }, + "base_pk": { + "$numberInt": "1" + }, + "base_sk_s": { + "$numberInt": "1" + } + }, + "name": "pk_1_sk_s_1_base_pk_1_base_sk_s_1" + } + ], + "documents": [ + { + "_id": "2:p0,2:l0,2:p0,3:s09,", + "pk": "p0", + "base_pk": "p0", + "sk_s": "l0", + "base_sk_s": "s09", + "item_data": { + "lsk": { + "S": "l0" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s09" + } + } + }, + { + "_id": "2:p0,2:l3,2:p0,3:s06,", + "pk": "p0", + "base_pk": "p0", + "sk_s": "l3", + "base_sk_s": "s06", + "item_data": { + "lsk": { + "S": "l3" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s06" + } + } + }, + { + "_id": "2:p0,2:l6,2:p0,3:s03,", + "pk": "p0", + "base_pk": "p0", + "sk_s": "l6", + "base_sk_s": "s03", + "item_data": { + "lsk": { + "S": "l6" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s03" + } + } + }, + { + "_id": "2:p0,2:l9,2:p0,3:s00,", + "pk": "p0", + "base_pk": "p0", + "sk_s": "l9", + "base_sk_s": "s00", + "item_data": { + "lsk": { + "S": "l9" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s00" + } + } + }, + { + "_id": "2:p1,2:l2,2:p1,3:s07,", + "pk": "p1", + "base_pk": "p1", + "sk_s": "l2", + "base_sk_s": "s07", + "item_data": { + "lsk": { + "S": "l2" + }, + "pk": { + "S": "p1" + }, + "sk": { + "S": "s07" + } + } + }, + { + "_id": "2:p1,2:l5,2:p1,3:s04,", + "pk": "p1", + "base_pk": "p1", + "sk_s": "l5", + "base_sk_s": "s04", + "item_data": { + "lsk": { + "S": "l5" + }, + "pk": { + "S": "p1" + }, + "sk": { + "S": "s04" + } + } + }, + { + "_id": "2:p1,2:l8,2:p1,3:s01,", + "pk": "p1", + "base_pk": "p1", + "sk_s": "l8", + "base_sk_s": "s01", + "item_data": { + "lsk": { + "S": "l8" + }, + "pk": { + "S": "p1" + }, + "sk": { + "S": "s01" + } + } + }, + { + "_id": "2:p2,2:l1,2:p2,3:s08,", + "pk": "p2", + "base_pk": "p2", + "sk_s": "l1", + "base_sk_s": "s08", + "item_data": { + "lsk": { + "S": "l1" + }, + "pk": { + "S": "p2" + }, + "sk": { + "S": "s08" + } + } + }, + { + "_id": "2:p2,2:l4,2:p2,3:s05,", + "pk": "p2", + "base_pk": "p2", + "sk_s": "l4", + "base_sk_s": "s05", + "item_data": { + "lsk": { + "S": "l4" + }, + "pk": { + "S": "p2" + }, + "sk": { + "S": "s05" + } + } + }, + { + "_id": "2:p2,2:l7,2:p2,3:s02,", + "pk": "p2", + "base_pk": "p2", + "sk_s": "l7", + "base_sk_s": "s02", + "item_data": { + "lsk": { + "S": "l7" + }, + "pk": { + "S": "p2" + }, + "sk": { + "S": "s02" + } + } + } + ] + }, + { + "name": "_ddb_1a8261b2-1a28-4ece-b098-b253bbee3c6c", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "pk": { + "$numberInt": "1" + }, + "sk_n": { + "$numberInt": "1" + }, + "base_pk": { + "$numberInt": "1" + }, + "base_sk_s": { + "$numberInt": "1" + } + }, + "name": "pk_1_sk_n_1_base_pk_1_base_sk_s_1" + } + ], + "documents": [ + { + "_id": "2:g0,1:0,2:p0,3:s00,", + "pk": "g0", + "base_pk": "p0", + "sk_n": { + "$numberDecimal": "0" + }, + "base_sk_s": "s00", + "item_data": { + "body": { + "S": "item 0" + }, + "expires": { + "N": "4102444800" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "0" + }, + "lsk": { + "S": "l9" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s00" + } + } + }, + { + "_id": "2:g0,1:2,2:p2,3:s02,", + "pk": "g0", + "base_pk": "p2", + "sk_n": { + "$numberDecimal": "2" + }, + "base_sk_s": "s02", + "item_data": { + "body": { + "S": "item 2" + }, + "expires": { + "N": "4102444802" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "2" + }, + "lsk": { + "S": "l7" + }, + "pk": { + "S": "p2" + }, + "sk": { + "S": "s02" + } + } + }, + { + "_id": "2:g0,1:4,2:p1,3:s04,", + "pk": "g0", + "base_pk": "p1", + "sk_n": { + "$numberDecimal": "4" + }, + "base_sk_s": "s04", + "item_data": { + "body": { + "S": "item 4" + }, + "expires": { + "N": "4102444804" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "4" + }, + "lsk": { + "S": "l5" + }, + "pk": { + "S": "p1" + }, + "sk": { + "S": "s04" + } + } + }, + { + "_id": "2:g0,1:6,2:p0,3:s06,", + "pk": "g0", + "base_pk": "p0", + "sk_n": { + "$numberDecimal": "6" + }, + "base_sk_s": "s06", + "item_data": { + "body": { + "S": "item 6" + }, + "expires": { + "N": "4102444806" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "6" + }, + "lsk": { + "S": "l3" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s06" + } + } + }, + { + "_id": "2:g0,1:8,2:p2,3:s08,", + "pk": "g0", + "base_pk": "p2", + "sk_n": { + "$numberDecimal": "8" + }, + "base_sk_s": "s08", + "item_data": { + "body": { + "S": "item 8" + }, + "expires": { + "N": "4102444808" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "8" + }, + "lsk": { + "S": "l1" + }, + "pk": { + "S": "p2" + }, + "sk": { + "S": "s08" + } + } + }, + { + "_id": "2:g1,1:1,2:p1,3:s01,", + "pk": "g1", + "base_pk": "p1", + "sk_n": { + "$numberDecimal": "1" + }, + "base_sk_s": "s01", + "item_data": { + "body": { + "S": "item 1" + }, + "expires": { + "N": "4102444801" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "1" + }, + "lsk": { + "S": "l8" + }, + "pk": { + "S": "p1" + }, + "sk": { + "S": "s01" + } + } + }, + { + "_id": "2:g1,1:3,2:p0,3:s03,", + "pk": "g1", + "base_pk": "p0", + "sk_n": { + "$numberDecimal": "3" + }, + "base_sk_s": "s03", + "item_data": { + "body": { + "S": "item 3" + }, + "expires": { + "N": "4102444803" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "3" + }, + "lsk": { + "S": "l6" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s03" + } + } + }, + { + "_id": "2:g1,1:5,2:p2,3:s05,", + "pk": "g1", + "base_pk": "p2", + "sk_n": { + "$numberDecimal": "5" + }, + "base_sk_s": "s05", + "item_data": { + "body": { + "S": "item 5" + }, + "expires": { + "N": "4102444805" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "5" + }, + "lsk": { + "S": "l4" + }, + "pk": { + "S": "p2" + }, + "sk": { + "S": "s05" + } + } + }, + { + "_id": "2:g1,1:7,2:p1,3:s07,", + "pk": "g1", + "base_pk": "p1", + "sk_n": { + "$numberDecimal": "7" + }, + "base_sk_s": "s07", + "item_data": { + "body": { + "S": "item 7" + }, + "expires": { + "N": "4102444807" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "7" + }, + "lsk": { + "S": "l2" + }, + "pk": { + "S": "p1" + }, + "sk": { + "S": "s07" + } + } + }, + { + "_id": "2:g1,1:9,2:p0,3:s09,", + "pk": "g1", + "base_pk": "p0", + "sk_n": { + "$numberDecimal": "9" + }, + "base_sk_s": "s09", + "item_data": { + "body": { + "S": "item 9" + }, + "expires": { + "N": "4102444809" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "9" + }, + "lsk": { + "S": "l0" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s09" + } + } + } + ] + }, + { + "name": "_ddb_c675a942-95bb-48c8-8474-eac51bdb57f5", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "pk": { + "$numberInt": "1" + }, + "sk_s": { + "$numberInt": "1" + } + }, + "name": "pk_1_sk_s_1", + "unique": true + }, + { + "key": { + "item_data.expires.N": { + "$numberInt": "1" + } + }, + "name": "idx_ttl_expires", + "sparse": true + } + ], + "documents": [ + { + "_id": "2:p0,3:s00,", + "pk": "p0", + "sk_s": "s00", + "item_data": { + "body": { + "S": "item 0" + }, + "expires": { + "N": "4102444800" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "0" + }, + "lsk": { + "S": "l9" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s00" + } + } + }, + { + "_id": "2:p0,3:s03,", + "pk": "p0", + "sk_s": "s03", + "item_data": { + "body": { + "S": "item 3" + }, + "expires": { + "N": "4102444803" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "3" + }, + "lsk": { + "S": "l6" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s03" + } + } + }, + { + "_id": "2:p0,3:s06,", + "pk": "p0", + "sk_s": "s06", + "item_data": { + "body": { + "S": "item 6" + }, + "expires": { + "N": "4102444806" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "6" + }, + "lsk": { + "S": "l3" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s06" + } + } + }, + { + "_id": "2:p0,3:s09,", + "pk": "p0", + "sk_s": "s09", + "item_data": { + "body": { + "S": "item 9" + }, + "expires": { + "N": "4102444809" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "9" + }, + "lsk": { + "S": "l0" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s09" + } + } + }, + { + "_id": "2:p1,3:s01,", + "pk": "p1", + "sk_s": "s01", + "item_data": { + "body": { + "S": "item 1" + }, + "expires": { + "N": "4102444801" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "1" + }, + "lsk": { + "S": "l8" + }, + "pk": { + "S": "p1" + }, + "sk": { + "S": "s01" + } + } + }, + { + "_id": "2:p1,3:s04,", + "pk": "p1", + "sk_s": "s04", + "item_data": { + "body": { + "S": "item 4" + }, + "expires": { + "N": "4102444804" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "4" + }, + "lsk": { + "S": "l5" + }, + "pk": { + "S": "p1" + }, + "sk": { + "S": "s04" + } + } + }, + { + "_id": "2:p1,3:s07,", + "pk": "p1", + "sk_s": "s07", + "item_data": { + "body": { + "S": "item 7" + }, + "expires": { + "N": "4102444807" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "7" + }, + "lsk": { + "S": "l2" + }, + "pk": { + "S": "p1" + }, + "sk": { + "S": "s07" + } + } + }, + { + "_id": "2:p2,3:s02,", + "pk": "p2", + "sk_s": "s02", + "item_data": { + "body": { + "S": "item 2" + }, + "expires": { + "N": "4102444802" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "2" + }, + "lsk": { + "S": "l7" + }, + "pk": { + "S": "p2" + }, + "sk": { + "S": "s02" + } + } + }, + { + "_id": "2:p2,3:s05,", + "pk": "p2", + "sk_s": "s05", + "item_data": { + "body": { + "S": "item 5" + }, + "expires": { + "N": "4102444805" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "5" + }, + "lsk": { + "S": "l4" + }, + "pk": { + "S": "p2" + }, + "sk": { + "S": "s05" + } + } + }, + { + "_id": "2:p2,3:s08,", + "pk": "p2", + "sk_s": "s08", + "item_data": { + "body": { + "S": "item 8" + }, + "expires": { + "N": "4102444808" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "8" + }, + "lsk": { + "S": "l1" + }, + "pk": { + "S": "p2" + }, + "sk": { + "S": "s08" + } + } + } + ] + }, + { + "name": "counters", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + } + ], + "documents": [ + { + "_id": "stream_seq:shardId-c675a942-95bb-48c8-8474-eac51bdb57f5-000000000001", + "value": { + "$numberLong": "7" + } + }, + { + "_id": "stream_seq:shardId-c675a942-95bb-48c8-8474-eac51bdb57f5-000000000003", + "value": { + "$numberLong": "3" + } + } + ] + }, + { + "name": "idempotency_tokens", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "created_at": { + "$numberInt": "1" + } + }, + "name": "created_at_1", + "expireAfterSeconds": { + "$numberInt": "540" + } + }, + { + "key": { + "account_id": { + "$numberInt": "1" + }, + "token": { + "$numberInt": "1" + } + }, + "name": "account_id_1_token_1", + "unique": true + } + ], + "documents": [] + }, + { + "name": "stream_records", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "created_at": { + "$numberInt": "1" + } + }, + "name": "created_at_1", + "expireAfterSeconds": { + "$numberInt": "86400" + } + }, + { + "key": { + "shard_id": { + "$numberInt": "1" + }, + "sequence_number": { + "$numberInt": "1" + } + }, + "name": "shard_id_1_sequence_number_1" + } + ], + "documents": [ + { + "_id": { + "$oid": "6ae3dbedcb8ec258b3a46a76" + }, + "sequence_number": "000000000000000000001", + "shard_id": "shardId-c675a942-95bb-48c8-8474-eac51bdb57f5-000000000001", + "table_id": "c675a942-95bb-48c8-8474-eac51bdb57f5", + "event_name": "INSERT", + "record_data": { + "eventID": "225f2cd5-07d3-472e-a825-533c3068b094", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "us-east-1", + "dynamodb": { + "ApproximateCreationDateTime": { + "$numberLong": "1793317869" + }, + "Keys": { + "pk": { + "S": "p0" + }, + "sk": { + "S": "s00" + } + }, + "NewImage": { + "body": { + "S": "item 0" + }, + "expires": { + "N": "4102444800" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "0" + }, + "lsk": { + "S": "l9" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s00" + } + }, + "SequenceNumber": "000000000000000000001", + "SizeBytes": { + "$numberLong": "45" + }, + "StreamViewType": "NEW_AND_OLD_IMAGES" + } + }, + "created_at": { + "$date": { + "$numberLong": "1793317869871" + } + } + }, + { + "_id": { + "$oid": "6ae3dbedcb8ec258b3a46a77" + }, + "sequence_number": "000000000000000000001", + "shard_id": "shardId-c675a942-95bb-48c8-8474-eac51bdb57f5-000000000003", + "table_id": "c675a942-95bb-48c8-8474-eac51bdb57f5", + "event_name": "INSERT", + "record_data": { + "eventID": "d27cc271-e051-4b61-95cb-de0727811a27", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "us-east-1", + "dynamodb": { + "ApproximateCreationDateTime": { + "$numberLong": "1793317869" + }, + "Keys": { + "pk": { + "S": "p1" + }, + "sk": { + "S": "s01" + } + }, + "NewImage": { + "body": { + "S": "item 1" + }, + "expires": { + "N": "4102444801" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "1" + }, + "lsk": { + "S": "l8" + }, + "pk": { + "S": "p1" + }, + "sk": { + "S": "s01" + } + }, + "SequenceNumber": "000000000000000000001", + "SizeBytes": { + "$numberLong": "47" + }, + "StreamViewType": "NEW_AND_OLD_IMAGES" + } + }, + "created_at": { + "$date": { + "$numberLong": "1793317869878" + } + } + }, + { + "_id": { + "$oid": "6ae3dbedcb8ec258b3a46a78" + }, + "sequence_number": "000000000000000000002", + "shard_id": "shardId-c675a942-95bb-48c8-8474-eac51bdb57f5-000000000001", + "table_id": "c675a942-95bb-48c8-8474-eac51bdb57f5", + "event_name": "INSERT", + "record_data": { + "eventID": "712b7d15-d24c-47f4-aa86-25828e09196e", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "us-east-1", + "dynamodb": { + "ApproximateCreationDateTime": { + "$numberLong": "1793317869" + }, + "Keys": { + "pk": { + "S": "p2" + }, + "sk": { + "S": "s02" + } + }, + "NewImage": { + "body": { + "S": "item 2" + }, + "expires": { + "N": "4102444802" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "2" + }, + "lsk": { + "S": "l7" + }, + "pk": { + "S": "p2" + }, + "sk": { + "S": "s02" + } + }, + "SequenceNumber": "000000000000000000002", + "SizeBytes": { + "$numberLong": "47" + }, + "StreamViewType": "NEW_AND_OLD_IMAGES" + } + }, + "created_at": { + "$date": { + "$numberLong": "1793317869886" + } + } + }, + { + "_id": { + "$oid": "6ae3dbedcb8ec258b3a46a79" + }, + "sequence_number": "000000000000000000003", + "shard_id": "shardId-c675a942-95bb-48c8-8474-eac51bdb57f5-000000000001", + "table_id": "c675a942-95bb-48c8-8474-eac51bdb57f5", + "event_name": "INSERT", + "record_data": { + "eventID": "b7a711b1-0cfc-4f36-b761-e6d0cac080c3", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "us-east-1", + "dynamodb": { + "ApproximateCreationDateTime": { + "$numberLong": "1793317869" + }, + "Keys": { + "pk": { + "S": "p0" + }, + "sk": { + "S": "s03" + } + }, + "NewImage": { + "body": { + "S": "item 3" + }, + "expires": { + "N": "4102444803" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "3" + }, + "lsk": { + "S": "l6" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s03" + } + }, + "SequenceNumber": "000000000000000000003", + "SizeBytes": { + "$numberLong": "47" + }, + "StreamViewType": "NEW_AND_OLD_IMAGES" + } + }, + "created_at": { + "$date": { + "$numberLong": "1793317869891" + } + } + }, + { + "_id": { + "$oid": "6ae3dbedcb8ec258b3a46a7a" + }, + "sequence_number": "000000000000000000002", + "shard_id": "shardId-c675a942-95bb-48c8-8474-eac51bdb57f5-000000000003", + "table_id": "c675a942-95bb-48c8-8474-eac51bdb57f5", + "event_name": "INSERT", + "record_data": { + "eventID": "01892bc2-6ca6-4d81-8a8e-6e7ffe89eb9f", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "us-east-1", + "dynamodb": { + "ApproximateCreationDateTime": { + "$numberLong": "1793317869" + }, + "Keys": { + "pk": { + "S": "p1" + }, + "sk": { + "S": "s04" + } + }, + "NewImage": { + "body": { + "S": "item 4" + }, + "expires": { + "N": "4102444804" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "4" + }, + "lsk": { + "S": "l5" + }, + "pk": { + "S": "p1" + }, + "sk": { + "S": "s04" + } + }, + "SequenceNumber": "000000000000000000002", + "SizeBytes": { + "$numberLong": "47" + }, + "StreamViewType": "NEW_AND_OLD_IMAGES" + } + }, + "created_at": { + "$date": { + "$numberLong": "1793317869897" + } + } + }, + { + "_id": { + "$oid": "6ae3dbedcb8ec258b3a46a7b" + }, + "sequence_number": "000000000000000000004", + "shard_id": "shardId-c675a942-95bb-48c8-8474-eac51bdb57f5-000000000001", + "table_id": "c675a942-95bb-48c8-8474-eac51bdb57f5", + "event_name": "INSERT", + "record_data": { + "eventID": "d213e159-aa61-47a6-b5f0-1c6a9eca48ae", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "us-east-1", + "dynamodb": { + "ApproximateCreationDateTime": { + "$numberLong": "1793317869" + }, + "Keys": { + "pk": { + "S": "p2" + }, + "sk": { + "S": "s05" + } + }, + "NewImage": { + "body": { + "S": "item 5" + }, + "expires": { + "N": "4102444805" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "5" + }, + "lsk": { + "S": "l4" + }, + "pk": { + "S": "p2" + }, + "sk": { + "S": "s05" + } + }, + "SequenceNumber": "000000000000000000004", + "SizeBytes": { + "$numberLong": "47" + }, + "StreamViewType": "NEW_AND_OLD_IMAGES" + } + }, + "created_at": { + "$date": { + "$numberLong": "1793317869904" + } + } + }, + { + "_id": { + "$oid": "6ae3dbedcb8ec258b3a46a7c" + }, + "sequence_number": "000000000000000000005", + "shard_id": "shardId-c675a942-95bb-48c8-8474-eac51bdb57f5-000000000001", + "table_id": "c675a942-95bb-48c8-8474-eac51bdb57f5", + "event_name": "INSERT", + "record_data": { + "eventID": "8f144011-c367-4ab3-9b01-9bc3b45749a0", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "us-east-1", + "dynamodb": { + "ApproximateCreationDateTime": { + "$numberLong": "1793317869" + }, + "Keys": { + "pk": { + "S": "p0" + }, + "sk": { + "S": "s06" + } + }, + "NewImage": { + "body": { + "S": "item 6" + }, + "expires": { + "N": "4102444806" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "6" + }, + "lsk": { + "S": "l3" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s06" + } + }, + "SequenceNumber": "000000000000000000005", + "SizeBytes": { + "$numberLong": "47" + }, + "StreamViewType": "NEW_AND_OLD_IMAGES" + } + }, + "created_at": { + "$date": { + "$numberLong": "1793317869910" + } + } + }, + { + "_id": { + "$oid": "6ae3dbedcb8ec258b3a46a7d" + }, + "sequence_number": "000000000000000000003", + "shard_id": "shardId-c675a942-95bb-48c8-8474-eac51bdb57f5-000000000003", + "table_id": "c675a942-95bb-48c8-8474-eac51bdb57f5", + "event_name": "INSERT", + "record_data": { + "eventID": "058495ae-8227-4560-8569-284cbdce48f0", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "us-east-1", + "dynamodb": { + "ApproximateCreationDateTime": { + "$numberLong": "1793317869" + }, + "Keys": { + "pk": { + "S": "p1" + }, + "sk": { + "S": "s07" + } + }, + "NewImage": { + "body": { + "S": "item 7" + }, + "expires": { + "N": "4102444807" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "7" + }, + "lsk": { + "S": "l2" + }, + "pk": { + "S": "p1" + }, + "sk": { + "S": "s07" + } + }, + "SequenceNumber": "000000000000000000003", + "SizeBytes": { + "$numberLong": "47" + }, + "StreamViewType": "NEW_AND_OLD_IMAGES" + } + }, + "created_at": { + "$date": { + "$numberLong": "1793317869918" + } + } + }, + { + "_id": { + "$oid": "6ae3dbedcb8ec258b3a46a7e" + }, + "sequence_number": "000000000000000000006", + "shard_id": "shardId-c675a942-95bb-48c8-8474-eac51bdb57f5-000000000001", + "table_id": "c675a942-95bb-48c8-8474-eac51bdb57f5", + "event_name": "INSERT", + "record_data": { + "eventID": "01d3b39f-841c-4981-b022-ec7bbd012adc", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "us-east-1", + "dynamodb": { + "ApproximateCreationDateTime": { + "$numberLong": "1793317869" + }, + "Keys": { + "pk": { + "S": "p2" + }, + "sk": { + "S": "s08" + } + }, + "NewImage": { + "body": { + "S": "item 8" + }, + "expires": { + "N": "4102444808" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "8" + }, + "lsk": { + "S": "l1" + }, + "pk": { + "S": "p2" + }, + "sk": { + "S": "s08" + } + }, + "SequenceNumber": "000000000000000000006", + "SizeBytes": { + "$numberLong": "47" + }, + "StreamViewType": "NEW_AND_OLD_IMAGES" + } + }, + "created_at": { + "$date": { + "$numberLong": "1793317869923" + } + } + }, + { + "_id": { + "$oid": "6ae3dbedcb8ec258b3a46a7f" + }, + "sequence_number": "000000000000000000007", + "shard_id": "shardId-c675a942-95bb-48c8-8474-eac51bdb57f5-000000000001", + "table_id": "c675a942-95bb-48c8-8474-eac51bdb57f5", + "event_name": "INSERT", + "record_data": { + "eventID": "98895d7c-76c4-4ef9-aeeb-b205a50b1fa3", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "us-east-1", + "dynamodb": { + "ApproximateCreationDateTime": { + "$numberLong": "1793317869" + }, + "Keys": { + "pk": { + "S": "p0" + }, + "sk": { + "S": "s09" + } + }, + "NewImage": { + "body": { + "S": "item 9" + }, + "expires": { + "N": "4102444809" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "9" + }, + "lsk": { + "S": "l0" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s09" + } + }, + "SequenceNumber": "000000000000000000007", + "SizeBytes": { + "$numberLong": "47" + }, + "StreamViewType": "NEW_AND_OLD_IMAGES" + } + }, + "created_at": { + "$date": { + "$numberLong": "1793317869929" + } + } + } + ] + }, + { + "name": "stream_shards", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "shard_id": { + "$numberInt": "1" + } + }, + "name": "shard_id_1", + "unique": true + } + ], + "documents": [ + { + "_id": { + "$oid": "6ae3dbedcb8ec258b3a46a72" + }, + "shard_id": "shardId-c675a942-95bb-48c8-8474-eac51bdb57f5-000000000000", + "table_id": "c675a942-95bb-48c8-8474-eac51bdb57f5", + "starting_sequence_number": "000000000000000000000", + "created_at": { + "$date": { + "$numberLong": "1793317869281" + } + } + }, + { + "_id": { + "$oid": "6ae3dbedcb8ec258b3a46a73" + }, + "shard_id": "shardId-c675a942-95bb-48c8-8474-eac51bdb57f5-000000000001", + "table_id": "c675a942-95bb-48c8-8474-eac51bdb57f5", + "starting_sequence_number": "000000000000000000000", + "created_at": { + "$date": { + "$numberLong": "1793317869282" + } + } + }, + { + "_id": { + "$oid": "6ae3dbedcb8ec258b3a46a74" + }, + "shard_id": "shardId-c675a942-95bb-48c8-8474-eac51bdb57f5-000000000002", + "table_id": "c675a942-95bb-48c8-8474-eac51bdb57f5", + "starting_sequence_number": "000000000000000000000", + "created_at": { + "$date": { + "$numberLong": "1793317869283" + } + } + }, + { + "_id": { + "$oid": "6ae3dbedcb8ec258b3a46a75" + }, + "shard_id": "shardId-c675a942-95bb-48c8-8474-eac51bdb57f5-000000000003", + "table_id": "c675a942-95bb-48c8-8474-eac51bdb57f5", + "starting_sequence_number": "000000000000000000000", + "created_at": { + "$date": { + "$numberLong": "1793317869284" + } + } + } + ] + } + ] + } + } +} diff --git a/crates/storage-mongodb/tests/fixtures/released/0.1.14.json b/crates/storage-mongodb/tests/fixtures/released/0.1.14.json new file mode 100644 index 000000000..f10b9c457 --- /dev/null +++ b/crates/storage-mongodb/tests/fixtures/released/0.1.14.json @@ -0,0 +1,2687 @@ +{ + "databases": { + "extenddb_catalog": { + "collections": [ + { + "name": "access_keys", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "account_id": { + "$numberInt": "1" + }, + "user_name": { + "$numberInt": "1" + } + }, + "name": "account_id_1_user_name_1" + } + ], + "documents": [ + { + "_id": { + "$oid": "6ae3dbec9036b684fdf55c5a" + }, + "access_key_id": "AKIAEXTENDDBEAT40RLC", + "account_id": "123456789012", + "user_name": "cu-tester", + "secret_key_encrypted": { + "$binary": { + "base64": "d2suZTVYf0N8J/1KS5y0HabRQpzkPtlYxCtrjfJGUeKwId6YuBTC4AP6B7Ysy+FBUvTqsIU+TTjuVmMGw2fYHayvRo8=", + "subType": "00" + } + }, + "is_active": true, + "created_at": { + "$date": { + "$numberLong": "1793317868367" + } + } + } + ] + }, + { + "name": "accounts", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "account_name": { + "$numberInt": "1" + } + }, + "name": "account_name_1", + "unique": true + } + ], + "documents": [ + { + "_id": "e70f0015-38e5-40c1-9e15-bdbcbedd92bb", + "account_name": "default", + "created_at": { + "$date": { + "$numberLong": "1793317865754" + } + } + }, + { + "_id": { + "$oid": "6ae3dbea9036b684fdf55c58" + }, + "account_id": "123456789012", + "account_name": "cu-seed", + "created_at": { + "$date": { + "$numberLong": "1793317866898" + } + } + } + ] + }, + { + "name": "admin_users", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + } + ], + "documents": [ + { + "_id": "admin", + "password_hash": "$2b$12$PVYZnUEcitmORDUcNuTEqO25TyitWU20dCRHutNERVJQlve96W9rW", + "created_at": { + "$date": { + "$numberLong": "1793317865991" + } + } + } + ] + }, + { + "name": "backups", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "account_id": { + "$numberInt": "1" + }, + "table_name": { + "$numberInt": "1" + } + }, + "name": "account_id_1_table_name_1" + } + ], + "documents": [] + }, + { + "name": "continuous_backups", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + } + ], + "documents": [] + }, + { + "name": "iam_groups", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "group_arn": { + "$numberInt": "1" + } + }, + "name": "group_arn_1", + "unique": true + } + ], + "documents": [] + }, + { + "name": "iam_permissions_boundaries", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + } + ], + "documents": [] + }, + { + "name": "iam_policies", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + } + ], + "documents": [ + { + "_id": { + "$oid": "6ae3dbeb53f0a8048c714186" + }, + "principal_name": "cu-tester", + "principal_type": "user", + "account_id": "123456789012", + "policy_name": "SelfServicePolicy", + "created_at": { + "$date": { + "$numberLong": "1793317867265" + } + }, + "policy_document": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "iam:CreateAccessKey", + "iam:DeleteAccessKey", + "iam:ListAccessKeys", + "iam:ChangePassword" + ], + "Resource": "arn:aws:iam::123456789012:user/cu-tester" + } + ] + } + }, + { + "_id": { + "$oid": "6ae3dbeb53f0a8048c71418b" + }, + "principal_name": "cu-tester", + "principal_type": "user", + "account_id": "123456789012", + "policy_name": "dynamodb-full", + "created_at": { + "$date": { + "$numberLong": "1793317867648" + } + }, + "policy_document": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": "dynamodb:*", + "Resource": "*" + } + ] + } + }, + { + "_id": { + "$oid": "6ae3dbeb53f0a8048c714191" + }, + "principal_name": "cu-tester", + "principal_type": "user", + "account_id": "123456789012", + "policy_name": "deny-prod", + "created_at": { + "$date": { + "$numberLong": "1793317867995" + } + }, + "policy_document": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Deny", + "Action": "dynamodb:DeleteTable", + "Resource": "*", + "Condition": { + "StringEquals": { + "dynamodb:ResourceTag/env": "prod" + } + } + } + ] + } + } + ] + }, + { + "name": "iam_roles", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "role_arn": { + "$numberInt": "1" + } + }, + "name": "role_arn_1", + "unique": true + } + ], + "documents": [] + }, + { + "name": "iam_sessions", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "access_key_id": { + "$numberInt": "1" + } + }, + "name": "access_key_id_1", + "unique": true + }, + { + "key": { + "expires_at": { + "$numberInt": "1" + } + }, + "name": "expires_at_1", + "expireAfterSeconds": { + "$numberInt": "0" + } + } + ], + "documents": [] + }, + { + "name": "iam_users", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "user_arn": { + "$numberInt": "1" + } + }, + "name": "user_arn_1", + "unique": true + } + ], + "documents": [ + { + "_id": { + "$oid": "6ae3dbeb9036b684fdf55c59" + }, + "account_id": "123456789012", + "user_name": "cu-tester", + "user_arn": "arn:aws:iam::123456789012:user/cu-tester", + "created_at": { + "$date": { + "$numberLong": "1793317867262" + } + } + } + ] + }, + { + "name": "indexes", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + } + ], + "documents": [ + { + "_id": { + "table_id": "c993823b-a7c2-4362-969d-4e0479f50459", + "index_name": "by_gpk" + }, + "index_id": "fa00be17-b0a8-48ad-966b-cf52d0bee871", + "index_type": "GSI", + "key_schema": [ + { + "AttributeName": "gpk", + "KeyType": "HASH" + }, + { + "AttributeName": "gsk", + "KeyType": "RANGE" + } + ], + "projection": { + "ProjectionType": "ALL" + }, + "index_status": "ACTIVE", + "provisioned_throughput": null + }, + { + "_id": { + "table_id": "c993823b-a7c2-4362-969d-4e0479f50459", + "index_name": "by_lsk" + }, + "index_id": "68b1d050-a43d-423b-b3bb-711d756c3073", + "index_type": "LSI", + "key_schema": [ + { + "AttributeName": "pk", + "KeyType": "HASH" + }, + { + "AttributeName": "lsk", + "KeyType": "RANGE" + } + ], + "projection": { + "ProjectionType": "KEYS_ONLY" + }, + "index_status": "ACTIVE", + "provisioned_throughput": null + } + ] + }, + { + "name": "login_attempts", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "principal": { + "$numberInt": "1" + }, + "attempted_at": { + "$numberInt": "1" + } + }, + "name": "principal_1_attempted_at_1" + } + ], + "documents": [] + }, + { + "name": "metrics", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "_id.bucket": { + "$numberInt": "1" + } + }, + "name": "_id.bucket_1" + } + ], + "documents": [ + { + "_id": { + "$oid": "6ae3dbee9036b684fdf55c69" + }, + "bucket": { + "$date": { + "$numberLong": "1793317860000" + } + }, + "metric": "StorageQueryLatency", + "sum": { + "$numberInt": "98912" + }, + "count": { + "$numberLong": "4" + }, + "min": { + "$numberInt": "540" + }, + "max": { + "$numberInt": "84302" + }, + "index_name": "Request", + "operation": "Catalog" + }, + { + "_id": { + "$oid": "6ae3dbee9036b684fdf55c6a" + }, + "bucket": { + "$date": { + "$numberLong": "1793317860000" + } + }, + "metric": "SuccessfulRequestLatency", + "sum": { + "$numberInt": "13224" + }, + "count": { + "$numberLong": "1" + }, + "min": { + "$numberInt": "13224" + }, + "max": { + "$numberInt": "13224" + }, + "table_name": "cu_orders", + "operation": "UpdateTimeToLive" + }, + { + "_id": { + "$oid": "6ae3dbee9036b684fdf55c6b" + }, + "bucket": { + "$date": { + "$numberLong": "1793317860000" + } + }, + "metric": "StorageQueryLatency", + "sum": { + "$numberInt": "59218" + }, + "count": { + "$numberLong": "10" + }, + "min": { + "$numberInt": "3531" + }, + "max": { + "$numberInt": "17527" + }, + "index_name": "Request", + "operation": "Data" + }, + { + "_id": { + "$oid": "6ae3dbee9036b684fdf55c6c" + }, + "bucket": { + "$date": { + "$numberLong": "1793317860000" + } + }, + "metric": "ConsumedWriteCapacityUnits", + "sum": { + "$numberInt": "10" + }, + "count": { + "$numberLong": "10" + }, + "min": { + "$numberInt": "1" + }, + "max": { + "$numberInt": "1" + }, + "table_name": "cu_orders", + "operation": "PutItem" + }, + { + "_id": { + "$oid": "6ae3dbee9036b684fdf55c6d" + }, + "bucket": { + "$date": { + "$numberLong": "1793317860000" + } + }, + "metric": "RequestCount", + "sum": { + "$numberInt": "1" + }, + "count": { + "$numberLong": "1" + }, + "min": { + "$numberInt": "1" + }, + "max": { + "$numberInt": "1" + }, + "operation": "UpdateTimeToLive" + }, + { + "_id": { + "$oid": "6ae3dbee9036b684fdf55c6e" + }, + "bucket": { + "$date": { + "$numberLong": "1793317860000" + } + }, + "metric": "RequestCount", + "sum": { + "$numberInt": "10" + }, + "count": { + "$numberLong": "10" + }, + "min": { + "$numberInt": "1" + }, + "max": { + "$numberInt": "1" + }, + "operation": "PutItem" + }, + { + "_id": { + "$oid": "6ae3dbee9036b684fdf55c6f" + }, + "bucket": { + "$date": { + "$numberLong": "1793317860000" + } + }, + "metric": "SuccessfulRequestLatency", + "sum": { + "$numberInt": "60996" + }, + "count": { + "$numberLong": "10" + }, + "min": { + "$numberInt": "3612" + }, + "max": { + "$numberInt": "18334" + }, + "table_name": "cu_orders", + "operation": "PutItem" + }, + { + "_id": { + "$oid": "6ae3dbee9036b684fdf55c70" + }, + "bucket": { + "$date": { + "$numberLong": "1793317860000" + } + }, + "metric": "SuccessfulRequestLatency", + "sum": { + "$numberInt": "87223" + }, + "count": { + "$numberLong": "1" + }, + "min": { + "$numberInt": "87223" + }, + "max": { + "$numberInt": "87223" + }, + "table_name": "cu_orders", + "operation": "CreateTable" + }, + { + "_id": { + "$oid": "6ae3dbee9036b684fdf55c71" + }, + "bucket": { + "$date": { + "$numberLong": "1793317860000" + } + }, + "metric": "StorageQueryCount", + "sum": { + "$numberInt": "4" + }, + "count": { + "$numberLong": "4" + }, + "min": { + "$numberInt": "1" + }, + "max": { + "$numberInt": "1" + }, + "index_name": "Request", + "operation": "Catalog" + }, + { + "_id": { + "$oid": "6ae3dbee9036b684fdf55c72" + }, + "bucket": { + "$date": { + "$numberLong": "1793317860000" + } + }, + "metric": "RequestCount", + "sum": { + "$numberInt": "2" + }, + "count": { + "$numberLong": "2" + }, + "min": { + "$numberInt": "1" + }, + "max": { + "$numberInt": "1" + }, + "operation": "DescribeTable" + }, + { + "_id": { + "$oid": "6ae3dbee9036b684fdf55c73" + }, + "bucket": { + "$date": { + "$numberLong": "1793317860000" + } + }, + "metric": "RequestCount", + "sum": { + "$numberInt": "1" + }, + "count": { + "$numberLong": "1" + }, + "min": { + "$numberInt": "1" + }, + "max": { + "$numberInt": "1" + }, + "operation": "CreateTable" + }, + { + "_id": { + "$oid": "6ae3dbee9036b684fdf55c74" + }, + "bucket": { + "$date": { + "$numberLong": "1793317860000" + } + }, + "metric": "StorageQueryCount", + "sum": { + "$numberInt": "10" + }, + "count": { + "$numberLong": "10" + }, + "min": { + "$numberInt": "1" + }, + "max": { + "$numberInt": "1" + }, + "index_name": "Request", + "operation": "Data" + }, + { + "_id": { + "$oid": "6ae3dbee9036b684fdf55c75" + }, + "bucket": { + "$date": { + "$numberLong": "1793317860000" + } + }, + "metric": "SuccessfulRequestLatency", + "sum": { + "$numberInt": "2243" + }, + "count": { + "$numberLong": "2" + }, + "min": { + "$numberInt": "1106" + }, + "max": { + "$numberInt": "1137" + }, + "table_name": "cu_orders", + "operation": "DescribeTable" + } + ] + }, + { + "name": "schema_history", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + } + ], + "documents": [ + { + "_id": "001_initial", + "applied_at": { + "$date": { + "$numberLong": "1793317865746" + } + } + } + ] + }, + { + "name": "settings", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + } + ], + "documents": [ + { + "_id": "catalog_version", + "value": "0.0.2" + }, + { + "_id": "data_connection_string", + "value": "mongodb://localhost:22413/?replicaSet=rs0" + }, + { + "_id": "data_database_name", + "value": "extenddb_data" + }, + { + "_id": "encryption_key", + "value": "fDE8azYPDmQortu3b+HIGpfOIwUBlJ3+vxQYBnfXmnQ=" + }, + { + "_id": "resource_tag_migration", + "value": "applied" + } + ] + }, + { + "name": "tables", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "table_id": { + "$numberInt": "1" + } + }, + "name": "table_id_1", + "unique": true + } + ], + "documents": [ + { + "_id": { + "account_id": "123456789012", + "table_name": "cu_orders" + }, + "key_schema": [ + { + "AttributeName": "pk", + "KeyType": "HASH" + }, + { + "AttributeName": "sk", + "KeyType": "RANGE" + } + ], + "attribute_definitions": [ + { + "AttributeName": "pk", + "AttributeType": "S" + }, + { + "AttributeName": "sk", + "AttributeType": "S" + }, + { + "AttributeName": "gpk", + "AttributeType": "S" + }, + { + "AttributeName": "gsk", + "AttributeType": "N" + }, + { + "AttributeName": "lsk", + "AttributeType": "S" + } + ], + "billing_mode": "PAY_PER_REQUEST", + "provisioned_throughput": null, + "stream_specification": { + "StreamEnabled": true, + "StreamViewType": "NEW_AND_OLD_IMAGES" + }, + "table_status": "ACTIVE", + "creation_date_time": { + "$date": { + "$numberLong": "1793317869000" + } + }, + "table_size_bytes": { + "$numberLong": "0" + }, + "item_count": { + "$numberLong": "0" + }, + "table_arn": "arn:aws:dynamodb:us-east-1:123456789012:table/cu_orders", + "table_id": "c993823b-a7c2-4362-969d-4e0479f50459", + "deletion_protection_enabled": false, + "ttl_attribute": "expires", + "stream_label": "2026-10-29T23:51:09.256", + "table_class": null, + "sse_specification": null, + "on_demand_throughput": null, + "ttl_index_ready": true + } + ] + }, + { + "name": "tags", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + } + ], + "documents": [] + } + ] + }, + "extenddb_data": { + "collections": [ + { + "name": "_ddb_68b1d050-a43d-423b-b3bb-711d756c3073", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "pk": { + "$numberInt": "1" + }, + "sk_s": { + "$numberInt": "1" + }, + "base_pk": { + "$numberInt": "1" + }, + "base_sk_s": { + "$numberInt": "1" + } + }, + "name": "pk_1_sk_s_1_base_pk_1_base_sk_s_1" + } + ], + "documents": [ + { + "_id": "2:p0,2:l0,2:p0,3:s09,", + "pk": "p0", + "base_pk": "p0", + "sk_s": "l0", + "base_sk_s": "s09", + "item_data": { + "lsk": { + "S": "l0" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s09" + } + } + }, + { + "_id": "2:p0,2:l3,2:p0,3:s06,", + "pk": "p0", + "base_pk": "p0", + "sk_s": "l3", + "base_sk_s": "s06", + "item_data": { + "lsk": { + "S": "l3" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s06" + } + } + }, + { + "_id": "2:p0,2:l6,2:p0,3:s03,", + "pk": "p0", + "base_pk": "p0", + "sk_s": "l6", + "base_sk_s": "s03", + "item_data": { + "lsk": { + "S": "l6" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s03" + } + } + }, + { + "_id": "2:p0,2:l9,2:p0,3:s00,", + "pk": "p0", + "base_pk": "p0", + "sk_s": "l9", + "base_sk_s": "s00", + "item_data": { + "lsk": { + "S": "l9" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s00" + } + } + }, + { + "_id": "2:p1,2:l2,2:p1,3:s07,", + "pk": "p1", + "base_pk": "p1", + "sk_s": "l2", + "base_sk_s": "s07", + "item_data": { + "lsk": { + "S": "l2" + }, + "pk": { + "S": "p1" + }, + "sk": { + "S": "s07" + } + } + }, + { + "_id": "2:p1,2:l5,2:p1,3:s04,", + "pk": "p1", + "base_pk": "p1", + "sk_s": "l5", + "base_sk_s": "s04", + "item_data": { + "lsk": { + "S": "l5" + }, + "pk": { + "S": "p1" + }, + "sk": { + "S": "s04" + } + } + }, + { + "_id": "2:p1,2:l8,2:p1,3:s01,", + "pk": "p1", + "base_pk": "p1", + "sk_s": "l8", + "base_sk_s": "s01", + "item_data": { + "lsk": { + "S": "l8" + }, + "pk": { + "S": "p1" + }, + "sk": { + "S": "s01" + } + } + }, + { + "_id": "2:p2,2:l1,2:p2,3:s08,", + "pk": "p2", + "base_pk": "p2", + "sk_s": "l1", + "base_sk_s": "s08", + "item_data": { + "lsk": { + "S": "l1" + }, + "pk": { + "S": "p2" + }, + "sk": { + "S": "s08" + } + } + }, + { + "_id": "2:p2,2:l4,2:p2,3:s05,", + "pk": "p2", + "base_pk": "p2", + "sk_s": "l4", + "base_sk_s": "s05", + "item_data": { + "lsk": { + "S": "l4" + }, + "pk": { + "S": "p2" + }, + "sk": { + "S": "s05" + } + } + }, + { + "_id": "2:p2,2:l7,2:p2,3:s02,", + "pk": "p2", + "base_pk": "p2", + "sk_s": "l7", + "base_sk_s": "s02", + "item_data": { + "lsk": { + "S": "l7" + }, + "pk": { + "S": "p2" + }, + "sk": { + "S": "s02" + } + } + } + ] + }, + { + "name": "_ddb_c993823b-a7c2-4362-969d-4e0479f50459", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "pk": { + "$numberInt": "1" + }, + "sk_s": { + "$numberInt": "1" + } + }, + "name": "pk_1_sk_s_1", + "unique": true + }, + { + "key": { + "item_data.expires.N": { + "$numberInt": "1" + } + }, + "name": "idx_ttl_expires", + "sparse": true + } + ], + "documents": [ + { + "_id": "2:p0,3:s00,", + "pk": "p0", + "sk_s": "s00", + "item_data": { + "body": { + "S": "item 0" + }, + "expires": { + "N": "4102444800" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "0" + }, + "lsk": { + "S": "l9" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s00" + } + } + }, + { + "_id": "2:p0,3:s03,", + "pk": "p0", + "sk_s": "s03", + "item_data": { + "body": { + "S": "item 3" + }, + "expires": { + "N": "4102444803" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "3" + }, + "lsk": { + "S": "l6" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s03" + } + } + }, + { + "_id": "2:p0,3:s06,", + "pk": "p0", + "sk_s": "s06", + "item_data": { + "body": { + "S": "item 6" + }, + "expires": { + "N": "4102444806" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "6" + }, + "lsk": { + "S": "l3" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s06" + } + } + }, + { + "_id": "2:p0,3:s09,", + "pk": "p0", + "sk_s": "s09", + "item_data": { + "body": { + "S": "item 9" + }, + "expires": { + "N": "4102444809" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "9" + }, + "lsk": { + "S": "l0" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s09" + } + } + }, + { + "_id": "2:p1,3:s01,", + "pk": "p1", + "sk_s": "s01", + "item_data": { + "body": { + "S": "item 1" + }, + "expires": { + "N": "4102444801" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "1" + }, + "lsk": { + "S": "l8" + }, + "pk": { + "S": "p1" + }, + "sk": { + "S": "s01" + } + } + }, + { + "_id": "2:p1,3:s04,", + "pk": "p1", + "sk_s": "s04", + "item_data": { + "body": { + "S": "item 4" + }, + "expires": { + "N": "4102444804" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "4" + }, + "lsk": { + "S": "l5" + }, + "pk": { + "S": "p1" + }, + "sk": { + "S": "s04" + } + } + }, + { + "_id": "2:p1,3:s07,", + "pk": "p1", + "sk_s": "s07", + "item_data": { + "body": { + "S": "item 7" + }, + "expires": { + "N": "4102444807" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "7" + }, + "lsk": { + "S": "l2" + }, + "pk": { + "S": "p1" + }, + "sk": { + "S": "s07" + } + } + }, + { + "_id": "2:p2,3:s02,", + "pk": "p2", + "sk_s": "s02", + "item_data": { + "body": { + "S": "item 2" + }, + "expires": { + "N": "4102444802" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "2" + }, + "lsk": { + "S": "l7" + }, + "pk": { + "S": "p2" + }, + "sk": { + "S": "s02" + } + } + }, + { + "_id": "2:p2,3:s05,", + "pk": "p2", + "sk_s": "s05", + "item_data": { + "body": { + "S": "item 5" + }, + "expires": { + "N": "4102444805" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "5" + }, + "lsk": { + "S": "l4" + }, + "pk": { + "S": "p2" + }, + "sk": { + "S": "s05" + } + } + }, + { + "_id": "2:p2,3:s08,", + "pk": "p2", + "sk_s": "s08", + "item_data": { + "body": { + "S": "item 8" + }, + "expires": { + "N": "4102444808" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "8" + }, + "lsk": { + "S": "l1" + }, + "pk": { + "S": "p2" + }, + "sk": { + "S": "s08" + } + } + } + ] + }, + { + "name": "_ddb_fa00be17-b0a8-48ad-966b-cf52d0bee871", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "pk": { + "$numberInt": "1" + }, + "sk_n": { + "$numberInt": "1" + }, + "base_pk": { + "$numberInt": "1" + }, + "base_sk_s": { + "$numberInt": "1" + } + }, + "name": "pk_1_sk_n_1_base_pk_1_base_sk_s_1" + } + ], + "documents": [ + { + "_id": "2:g0,1:0,2:p0,3:s00,", + "pk": "g0", + "base_pk": "p0", + "sk_n": { + "$numberDecimal": "0" + }, + "base_sk_s": "s00", + "item_data": { + "body": { + "S": "item 0" + }, + "expires": { + "N": "4102444800" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "0" + }, + "lsk": { + "S": "l9" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s00" + } + } + }, + { + "_id": "2:g0,1:2,2:p2,3:s02,", + "pk": "g0", + "base_pk": "p2", + "sk_n": { + "$numberDecimal": "2" + }, + "base_sk_s": "s02", + "item_data": { + "body": { + "S": "item 2" + }, + "expires": { + "N": "4102444802" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "2" + }, + "lsk": { + "S": "l7" + }, + "pk": { + "S": "p2" + }, + "sk": { + "S": "s02" + } + } + }, + { + "_id": "2:g0,1:4,2:p1,3:s04,", + "pk": "g0", + "base_pk": "p1", + "sk_n": { + "$numberDecimal": "4" + }, + "base_sk_s": "s04", + "item_data": { + "body": { + "S": "item 4" + }, + "expires": { + "N": "4102444804" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "4" + }, + "lsk": { + "S": "l5" + }, + "pk": { + "S": "p1" + }, + "sk": { + "S": "s04" + } + } + }, + { + "_id": "2:g0,1:6,2:p0,3:s06,", + "pk": "g0", + "base_pk": "p0", + "sk_n": { + "$numberDecimal": "6" + }, + "base_sk_s": "s06", + "item_data": { + "body": { + "S": "item 6" + }, + "expires": { + "N": "4102444806" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "6" + }, + "lsk": { + "S": "l3" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s06" + } + } + }, + { + "_id": "2:g0,1:8,2:p2,3:s08,", + "pk": "g0", + "base_pk": "p2", + "sk_n": { + "$numberDecimal": "8" + }, + "base_sk_s": "s08", + "item_data": { + "body": { + "S": "item 8" + }, + "expires": { + "N": "4102444808" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "8" + }, + "lsk": { + "S": "l1" + }, + "pk": { + "S": "p2" + }, + "sk": { + "S": "s08" + } + } + }, + { + "_id": "2:g1,1:1,2:p1,3:s01,", + "pk": "g1", + "base_pk": "p1", + "sk_n": { + "$numberDecimal": "1" + }, + "base_sk_s": "s01", + "item_data": { + "body": { + "S": "item 1" + }, + "expires": { + "N": "4102444801" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "1" + }, + "lsk": { + "S": "l8" + }, + "pk": { + "S": "p1" + }, + "sk": { + "S": "s01" + } + } + }, + { + "_id": "2:g1,1:3,2:p0,3:s03,", + "pk": "g1", + "base_pk": "p0", + "sk_n": { + "$numberDecimal": "3" + }, + "base_sk_s": "s03", + "item_data": { + "body": { + "S": "item 3" + }, + "expires": { + "N": "4102444803" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "3" + }, + "lsk": { + "S": "l6" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s03" + } + } + }, + { + "_id": "2:g1,1:5,2:p2,3:s05,", + "pk": "g1", + "base_pk": "p2", + "sk_n": { + "$numberDecimal": "5" + }, + "base_sk_s": "s05", + "item_data": { + "body": { + "S": "item 5" + }, + "expires": { + "N": "4102444805" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "5" + }, + "lsk": { + "S": "l4" + }, + "pk": { + "S": "p2" + }, + "sk": { + "S": "s05" + } + } + }, + { + "_id": "2:g1,1:7,2:p1,3:s07,", + "pk": "g1", + "base_pk": "p1", + "sk_n": { + "$numberDecimal": "7" + }, + "base_sk_s": "s07", + "item_data": { + "body": { + "S": "item 7" + }, + "expires": { + "N": "4102444807" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "7" + }, + "lsk": { + "S": "l2" + }, + "pk": { + "S": "p1" + }, + "sk": { + "S": "s07" + } + } + }, + { + "_id": "2:g1,1:9,2:p0,3:s09,", + "pk": "g1", + "base_pk": "p0", + "sk_n": { + "$numberDecimal": "9" + }, + "base_sk_s": "s09", + "item_data": { + "body": { + "S": "item 9" + }, + "expires": { + "N": "4102444809" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "9" + }, + "lsk": { + "S": "l0" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s09" + } + } + } + ] + }, + { + "name": "counters", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + } + ], + "documents": [ + { + "_id": "stream_seq:shardId-c993823b-a7c2-4362-969d-4e0479f50459-000000000001", + "value": { + "$numberLong": "7" + } + }, + { + "_id": "stream_seq:shardId-c993823b-a7c2-4362-969d-4e0479f50459-000000000003", + "value": { + "$numberLong": "3" + } + } + ] + }, + { + "name": "idempotency_tokens", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "created_at": { + "$numberInt": "1" + } + }, + "name": "created_at_1", + "expireAfterSeconds": { + "$numberInt": "540" + } + }, + { + "key": { + "account_id": { + "$numberInt": "1" + }, + "token": { + "$numberInt": "1" + } + }, + "name": "account_id_1_token_1", + "unique": true + } + ], + "documents": [] + }, + { + "name": "stream_records", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "created_at": { + "$numberInt": "1" + } + }, + "name": "created_at_1", + "expireAfterSeconds": { + "$numberInt": "86400" + } + }, + { + "key": { + "shard_id": { + "$numberInt": "1" + }, + "sequence_number": { + "$numberInt": "1" + } + }, + "name": "shard_id_1_sequence_number_1" + } + ], + "documents": [ + { + "_id": { + "$oid": "6ae3dbed9036b684fdf55c5f" + }, + "sequence_number": "000000000000000000001", + "shard_id": "shardId-c993823b-a7c2-4362-969d-4e0479f50459-000000000001", + "table_id": "c993823b-a7c2-4362-969d-4e0479f50459", + "event_name": "INSERT", + "record_data": { + "eventID": "d3758dc5-a111-4771-a511-5ff3446519be", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "us-east-1", + "dynamodb": { + "ApproximateCreationDateTime": { + "$numberLong": "1793317869" + }, + "Keys": { + "pk": { + "S": "p0" + }, + "sk": { + "S": "s00" + } + }, + "NewImage": { + "body": { + "S": "item 0" + }, + "expires": { + "N": "4102444800" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "0" + }, + "lsk": { + "S": "l9" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s00" + } + }, + "SequenceNumber": "000000000000000000001", + "SizeBytes": { + "$numberLong": "45" + }, + "StreamViewType": "NEW_AND_OLD_IMAGES" + } + }, + "created_at": { + "$date": { + "$numberLong": "1793317869883" + } + } + }, + { + "_id": { + "$oid": "6ae3dbed9036b684fdf55c60" + }, + "sequence_number": "000000000000000000001", + "shard_id": "shardId-c993823b-a7c2-4362-969d-4e0479f50459-000000000003", + "table_id": "c993823b-a7c2-4362-969d-4e0479f50459", + "event_name": "INSERT", + "record_data": { + "eventID": "5c6fc395-6cee-4f7c-8652-ca69e0779b0d", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "us-east-1", + "dynamodb": { + "ApproximateCreationDateTime": { + "$numberLong": "1793317869" + }, + "Keys": { + "pk": { + "S": "p1" + }, + "sk": { + "S": "s01" + } + }, + "NewImage": { + "body": { + "S": "item 1" + }, + "expires": { + "N": "4102444801" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "1" + }, + "lsk": { + "S": "l8" + }, + "pk": { + "S": "p1" + }, + "sk": { + "S": "s01" + } + }, + "SequenceNumber": "000000000000000000001", + "SizeBytes": { + "$numberLong": "47" + }, + "StreamViewType": "NEW_AND_OLD_IMAGES" + } + }, + "created_at": { + "$date": { + "$numberLong": "1793317869890" + } + } + }, + { + "_id": { + "$oid": "6ae3dbed9036b684fdf55c61" + }, + "sequence_number": "000000000000000000002", + "shard_id": "shardId-c993823b-a7c2-4362-969d-4e0479f50459-000000000001", + "table_id": "c993823b-a7c2-4362-969d-4e0479f50459", + "event_name": "INSERT", + "record_data": { + "eventID": "b68ae262-b1a7-4c4d-84b6-efb157cfc8f2", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "us-east-1", + "dynamodb": { + "ApproximateCreationDateTime": { + "$numberLong": "1793317869" + }, + "Keys": { + "pk": { + "S": "p2" + }, + "sk": { + "S": "s02" + } + }, + "NewImage": { + "body": { + "S": "item 2" + }, + "expires": { + "N": "4102444802" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "2" + }, + "lsk": { + "S": "l7" + }, + "pk": { + "S": "p2" + }, + "sk": { + "S": "s02" + } + }, + "SequenceNumber": "000000000000000000002", + "SizeBytes": { + "$numberLong": "47" + }, + "StreamViewType": "NEW_AND_OLD_IMAGES" + } + }, + "created_at": { + "$date": { + "$numberLong": "1793317869896" + } + } + }, + { + "_id": { + "$oid": "6ae3dbed9036b684fdf55c62" + }, + "sequence_number": "000000000000000000003", + "shard_id": "shardId-c993823b-a7c2-4362-969d-4e0479f50459-000000000001", + "table_id": "c993823b-a7c2-4362-969d-4e0479f50459", + "event_name": "INSERT", + "record_data": { + "eventID": "1b4db8e5-af1b-4e40-aa76-755d8ec29587", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "us-east-1", + "dynamodb": { + "ApproximateCreationDateTime": { + "$numberLong": "1793317869" + }, + "Keys": { + "pk": { + "S": "p0" + }, + "sk": { + "S": "s03" + } + }, + "NewImage": { + "body": { + "S": "item 3" + }, + "expires": { + "N": "4102444803" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "3" + }, + "lsk": { + "S": "l6" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s03" + } + }, + "SequenceNumber": "000000000000000000003", + "SizeBytes": { + "$numberLong": "47" + }, + "StreamViewType": "NEW_AND_OLD_IMAGES" + } + }, + "created_at": { + "$date": { + "$numberLong": "1793317869901" + } + } + }, + { + "_id": { + "$oid": "6ae3dbed9036b684fdf55c63" + }, + "sequence_number": "000000000000000000002", + "shard_id": "shardId-c993823b-a7c2-4362-969d-4e0479f50459-000000000003", + "table_id": "c993823b-a7c2-4362-969d-4e0479f50459", + "event_name": "INSERT", + "record_data": { + "eventID": "c21a83bb-7644-4078-9fa0-02170f1e2bb4", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "us-east-1", + "dynamodb": { + "ApproximateCreationDateTime": { + "$numberLong": "1793317869" + }, + "Keys": { + "pk": { + "S": "p1" + }, + "sk": { + "S": "s04" + } + }, + "NewImage": { + "body": { + "S": "item 4" + }, + "expires": { + "N": "4102444804" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "4" + }, + "lsk": { + "S": "l5" + }, + "pk": { + "S": "p1" + }, + "sk": { + "S": "s04" + } + }, + "SequenceNumber": "000000000000000000002", + "SizeBytes": { + "$numberLong": "47" + }, + "StreamViewType": "NEW_AND_OLD_IMAGES" + } + }, + "created_at": { + "$date": { + "$numberLong": "1793317869909" + } + } + }, + { + "_id": { + "$oid": "6ae3dbed9036b684fdf55c64" + }, + "sequence_number": "000000000000000000004", + "shard_id": "shardId-c993823b-a7c2-4362-969d-4e0479f50459-000000000001", + "table_id": "c993823b-a7c2-4362-969d-4e0479f50459", + "event_name": "INSERT", + "record_data": { + "eventID": "504df5e0-4454-495e-9dbe-5dfcdec65412", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "us-east-1", + "dynamodb": { + "ApproximateCreationDateTime": { + "$numberLong": "1793317869" + }, + "Keys": { + "pk": { + "S": "p2" + }, + "sk": { + "S": "s05" + } + }, + "NewImage": { + "body": { + "S": "item 5" + }, + "expires": { + "N": "4102444805" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "5" + }, + "lsk": { + "S": "l4" + }, + "pk": { + "S": "p2" + }, + "sk": { + "S": "s05" + } + }, + "SequenceNumber": "000000000000000000004", + "SizeBytes": { + "$numberLong": "47" + }, + "StreamViewType": "NEW_AND_OLD_IMAGES" + } + }, + "created_at": { + "$date": { + "$numberLong": "1793317869917" + } + } + }, + { + "_id": { + "$oid": "6ae3dbed9036b684fdf55c65" + }, + "sequence_number": "000000000000000000005", + "shard_id": "shardId-c993823b-a7c2-4362-969d-4e0479f50459-000000000001", + "table_id": "c993823b-a7c2-4362-969d-4e0479f50459", + "event_name": "INSERT", + "record_data": { + "eventID": "38cdd547-b41c-427b-a973-79bfaa10097a", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "us-east-1", + "dynamodb": { + "ApproximateCreationDateTime": { + "$numberLong": "1793317869" + }, + "Keys": { + "pk": { + "S": "p0" + }, + "sk": { + "S": "s06" + } + }, + "NewImage": { + "body": { + "S": "item 6" + }, + "expires": { + "N": "4102444806" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "6" + }, + "lsk": { + "S": "l3" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s06" + } + }, + "SequenceNumber": "000000000000000000005", + "SizeBytes": { + "$numberLong": "47" + }, + "StreamViewType": "NEW_AND_OLD_IMAGES" + } + }, + "created_at": { + "$date": { + "$numberLong": "1793317869923" + } + } + }, + { + "_id": { + "$oid": "6ae3dbed9036b684fdf55c66" + }, + "sequence_number": "000000000000000000003", + "shard_id": "shardId-c993823b-a7c2-4362-969d-4e0479f50459-000000000003", + "table_id": "c993823b-a7c2-4362-969d-4e0479f50459", + "event_name": "INSERT", + "record_data": { + "eventID": "fd4250a8-6a99-4084-8cda-8d2ff3cc3dc0", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "us-east-1", + "dynamodb": { + "ApproximateCreationDateTime": { + "$numberLong": "1793317869" + }, + "Keys": { + "pk": { + "S": "p1" + }, + "sk": { + "S": "s07" + } + }, + "NewImage": { + "body": { + "S": "item 7" + }, + "expires": { + "N": "4102444807" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "7" + }, + "lsk": { + "S": "l2" + }, + "pk": { + "S": "p1" + }, + "sk": { + "S": "s07" + } + }, + "SequenceNumber": "000000000000000000003", + "SizeBytes": { + "$numberLong": "47" + }, + "StreamViewType": "NEW_AND_OLD_IMAGES" + } + }, + "created_at": { + "$date": { + "$numberLong": "1793317869930" + } + } + }, + { + "_id": { + "$oid": "6ae3dbed9036b684fdf55c67" + }, + "sequence_number": "000000000000000000006", + "shard_id": "shardId-c993823b-a7c2-4362-969d-4e0479f50459-000000000001", + "table_id": "c993823b-a7c2-4362-969d-4e0479f50459", + "event_name": "INSERT", + "record_data": { + "eventID": "995c9b0e-6477-4829-9ce3-c33203c6da9b", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "us-east-1", + "dynamodb": { + "ApproximateCreationDateTime": { + "$numberLong": "1793317869" + }, + "Keys": { + "pk": { + "S": "p2" + }, + "sk": { + "S": "s08" + } + }, + "NewImage": { + "body": { + "S": "item 8" + }, + "expires": { + "N": "4102444808" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "8" + }, + "lsk": { + "S": "l1" + }, + "pk": { + "S": "p2" + }, + "sk": { + "S": "s08" + } + }, + "SequenceNumber": "000000000000000000006", + "SizeBytes": { + "$numberLong": "47" + }, + "StreamViewType": "NEW_AND_OLD_IMAGES" + } + }, + "created_at": { + "$date": { + "$numberLong": "1793317869935" + } + } + }, + { + "_id": { + "$oid": "6ae3dbed9036b684fdf55c68" + }, + "sequence_number": "000000000000000000007", + "shard_id": "shardId-c993823b-a7c2-4362-969d-4e0479f50459-000000000001", + "table_id": "c993823b-a7c2-4362-969d-4e0479f50459", + "event_name": "INSERT", + "record_data": { + "eventID": "2c81d6b4-f6df-4f94-be85-a23a786a40c6", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "us-east-1", + "dynamodb": { + "ApproximateCreationDateTime": { + "$numberLong": "1793317869" + }, + "Keys": { + "pk": { + "S": "p0" + }, + "sk": { + "S": "s09" + } + }, + "NewImage": { + "body": { + "S": "item 9" + }, + "expires": { + "N": "4102444809" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "9" + }, + "lsk": { + "S": "l0" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s09" + } + }, + "SequenceNumber": "000000000000000000007", + "SizeBytes": { + "$numberLong": "47" + }, + "StreamViewType": "NEW_AND_OLD_IMAGES" + } + }, + "created_at": { + "$date": { + "$numberLong": "1793317869943" + } + } + } + ] + }, + { + "name": "stream_shards", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "shard_id": { + "$numberInt": "1" + } + }, + "name": "shard_id_1", + "unique": true + } + ], + "documents": [ + { + "_id": { + "$oid": "6ae3dbed9036b684fdf55c5b" + }, + "shard_id": "shardId-c993823b-a7c2-4362-969d-4e0479f50459-000000000000", + "table_id": "c993823b-a7c2-4362-969d-4e0479f50459", + "starting_sequence_number": "000000000000000000000", + "created_at": { + "$date": { + "$numberLong": "1793317869288" + } + } + }, + { + "_id": { + "$oid": "6ae3dbed9036b684fdf55c5c" + }, + "shard_id": "shardId-c993823b-a7c2-4362-969d-4e0479f50459-000000000001", + "table_id": "c993823b-a7c2-4362-969d-4e0479f50459", + "starting_sequence_number": "000000000000000000000", + "created_at": { + "$date": { + "$numberLong": "1793317869290" + } + } + }, + { + "_id": { + "$oid": "6ae3dbed9036b684fdf55c5d" + }, + "shard_id": "shardId-c993823b-a7c2-4362-969d-4e0479f50459-000000000002", + "table_id": "c993823b-a7c2-4362-969d-4e0479f50459", + "starting_sequence_number": "000000000000000000000", + "created_at": { + "$date": { + "$numberLong": "1793317869291" + } + } + }, + { + "_id": { + "$oid": "6ae3dbed9036b684fdf55c5e" + }, + "shard_id": "shardId-c993823b-a7c2-4362-969d-4e0479f50459-000000000003", + "table_id": "c993823b-a7c2-4362-969d-4e0479f50459", + "starting_sequence_number": "000000000000000000000", + "created_at": { + "$date": { + "$numberLong": "1793317869292" + } + } + } + ] + } + ] + } + } +} diff --git a/crates/storage-mongodb/tests/fixtures/released/README.md b/crates/storage-mongodb/tests/fixtures/released/README.md new file mode 100644 index 000000000..682632778 --- /dev/null +++ b/crates/storage-mongodb/tests/fixtures/released/README.md @@ -0,0 +1,30 @@ +# Released MongoDB catalogs + +Each `.json` is a deployment made by a released image, printed by +`dump.js` in canonical Extended JSON with every object's key order kept: +for both databases, every collection with its options, its indexes and its +documents. `load.js` loads one into an empty server with `mongosh`, and the +Rust tests load them with `tests/migrations_support/fixture.rs`. + +| File | Made by | +|---|---| +| `0.1.12.json` | `make-fixture.sh 0.1.12 ...` with `extenddb/extenddb-mongodb:0.1.12`. It stands for v0.1.4 to v0.1.11 as well: their bootstrapper is the same file (`git rev-parse v0.1.4:crates/storage-mongodb/src/bootstrapper.rs` equals the v0.1.12 one), and no image older than 0.1.12 exists | +| `0.1.13.json` | `make-fixture.sh 0.1.13 ...` | +| `0.1.14.json` | `make-fixture.sh 0.1.14 ...`; the first release that seeds the `resource_tag_migration` row | +| `dotted-ttl.json` | `make-fixture.sh 0.1.12 ... dotted-ttl`: a second table, `cu_dotted`, with TTL on `exp.at`, and the index releases before v0.1.12 created for that attribute (`idx_ttl_exp.at` on `item_data.exp.at.N`, sparse), which v0.1.12 and later do not create | + +Every state was made on 2026-10-29 against `mongo:7` (server 7.0.45) as a +one-node replica set: `init`, then the image's server, then through +`extenddb manage` the account 123456789012 with the IAM user `cu-tester`, the +inline policies `dynamodb-full` and `deny-prod` (a Deny whose condition +names `dynamodb:ResourceTag/env`) and an access key, then `seed.py`: the +table `cu_orders` with a GSI, an LSI, a stream, TTL on `expires` and ten +items. The deployments were throwaway; their encryption keys and password +hashes are kept so a loaded fixture can be served. + +To make one again (Docker, `mongo:7`, Python with boto3; the ports are +published on 127.0.0.1 only): + +``` +crates/storage-mongodb/tests/fixtures/released/make-fixture.sh 0.1.14 fixture-mongo 27031 18491 +``` diff --git a/crates/storage-mongodb/tests/fixtures/released/accepted-differences.json b/crates/storage-mongodb/tests/fixtures/released/accepted-differences.json new file mode 100644 index 000000000..7ed5f7ebc --- /dev/null +++ b/crates/storage-mongodb/tests/fixtures/released/accepted-differences.json @@ -0,0 +1,8 @@ +{ + "dotted-ttl": [ + { + "line": "index extenddb_data.T_cu_dotted {\"key\":{\"item_data.exp.at.N\":1},\"name\":\"idx_ttl_exp.at\",\"sparse\":true}", + "reason": "Releases before v0.1.12 created this index for a TTL attribute with a dot in its name, and v0.1.12 and later create none. No migration removes it: the TTL sweep does not use it, and disabling TTL on the table drops it (crates/storage-mongodb/src/metadata_engine.rs, the legacy dotted-name index)." + } + ] +} diff --git a/crates/storage-mongodb/tests/fixtures/released/dotted-ttl.json b/crates/storage-mongodb/tests/fixtures/released/dotted-ttl.json new file mode 100644 index 000000000..ecc4ccb43 --- /dev/null +++ b/crates/storage-mongodb/tests/fixtures/released/dotted-ttl.json @@ -0,0 +1,4486 @@ +{ + "databases": { + "extenddb_catalog": { + "collections": [ + { + "name": "access_keys", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "account_id": { + "$numberInt": "1" + }, + "user_name": { + "$numberInt": "1" + } + }, + "name": "account_id_1_user_name_1" + } + ], + "documents": [ + { + "_id": { + "$oid": "6ae3dca1833a09338997bbc1" + }, + "access_key_id": "AKIAEXTENDDB5M4PDNOX", + "account_id": "123456789012", + "user_name": "cu-tester", + "secret_key_encrypted": { + "$binary": { + "base64": "nntyZBigJK1x58RbGSI5jQgJtn1tX7EY5t56b2F15pwwoTOFjdLn0Nya6OMOy2rcr5kro1c1wFCF24JpCoZW6bfP8hc=", + "subType": "00" + } + }, + "is_active": true, + "created_at": { + "$date": { + "$numberLong": "1793318049565" + } + } + } + ] + }, + { + "name": "accounts", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "account_name": { + "$numberInt": "1" + } + }, + "name": "account_name_1", + "unique": true + } + ], + "documents": [ + { + "_id": "3cfa7480-a2b0-4906-a5d3-f4b71f00578b", + "account_name": "default", + "created_at": { + "$date": { + "$numberLong": "1793318047302" + } + } + }, + { + "_id": { + "$oid": "6ae3dca0833a09338997bbbf" + }, + "account_id": "123456789012", + "account_name": "cu-seed", + "created_at": { + "$date": { + "$numberLong": "1793318048185" + } + } + } + ] + }, + { + "name": "admin_users", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + } + ], + "documents": [ + { + "_id": "admin", + "password_hash": "$2b$12$8xouI6YV4yUND7UF/.zHQemMPDUbYjXSeQhwNDa511p5.kb4Ury52", + "created_at": { + "$date": { + "$numberLong": "1793318047537" + } + } + } + ] + }, + { + "name": "backups", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "account_id": { + "$numberInt": "1" + }, + "table_name": { + "$numberInt": "1" + } + }, + "name": "account_id_1_table_name_1" + } + ], + "documents": [] + }, + { + "name": "continuous_backups", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + } + ], + "documents": [] + }, + { + "name": "iam_groups", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "group_arn": { + "$numberInt": "1" + } + }, + "name": "group_arn_1", + "unique": true + } + ], + "documents": [] + }, + { + "name": "iam_permissions_boundaries", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + } + ], + "documents": [] + }, + { + "name": "iam_policies", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + } + ], + "documents": [ + { + "_id": { + "$oid": "6ae3dca00bdff5a74355460c" + }, + "account_id": "123456789012", + "policy_name": "SelfServicePolicy", + "principal_name": "cu-tester", + "principal_type": "user", + "created_at": { + "$date": { + "$numberLong": "1793318048531" + } + }, + "policy_document": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "iam:CreateAccessKey", + "iam:DeleteAccessKey", + "iam:ListAccessKeys", + "iam:ChangePassword" + ], + "Resource": "arn:aws:iam::123456789012:user/cu-tester" + } + ] + } + }, + { + "_id": { + "$oid": "6ae3dca00bdff5a743554612" + }, + "account_id": "123456789012", + "policy_name": "dynamodb-full", + "principal_name": "cu-tester", + "principal_type": "user", + "created_at": { + "$date": { + "$numberLong": "1793318048856" + } + }, + "policy_document": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": "dynamodb:*", + "Resource": "*" + } + ] + } + }, + { + "_id": { + "$oid": "6ae3dca10bdff5a743554617" + }, + "account_id": "123456789012", + "policy_name": "deny-prod", + "principal_name": "cu-tester", + "principal_type": "user", + "created_at": { + "$date": { + "$numberLong": "1793318049231" + } + }, + "policy_document": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Deny", + "Action": "dynamodb:DeleteTable", + "Resource": "*", + "Condition": { + "StringEquals": { + "dynamodb:ResourceTag/env": "prod" + } + } + } + ] + } + } + ] + }, + { + "name": "iam_roles", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "role_arn": { + "$numberInt": "1" + } + }, + "name": "role_arn_1", + "unique": true + } + ], + "documents": [] + }, + { + "name": "iam_sessions", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "access_key_id": { + "$numberInt": "1" + } + }, + "name": "access_key_id_1", + "unique": true + }, + { + "key": { + "expires_at": { + "$numberInt": "1" + } + }, + "name": "expires_at_1", + "expireAfterSeconds": { + "$numberInt": "0" + } + } + ], + "documents": [] + }, + { + "name": "iam_users", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "user_arn": { + "$numberInt": "1" + } + }, + "name": "user_arn_1", + "unique": true + } + ], + "documents": [ + { + "_id": { + "$oid": "6ae3dca0833a09338997bbc0" + }, + "account_id": "123456789012", + "user_name": "cu-tester", + "user_arn": "arn:aws:iam::123456789012:user/cu-tester", + "created_at": { + "$date": { + "$numberLong": "1793318048528" + } + } + } + ] + }, + { + "name": "indexes", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + } + ], + "documents": [ + { + "_id": { + "table_id": "86d32a95-5f27-4c2c-9e40-61c0b6acb9fb", + "index_name": "by_gpk" + }, + "index_id": "34ff4a88-57e0-4cba-b717-034946600988", + "index_type": "GSI", + "key_schema": [ + { + "AttributeName": "gpk", + "KeyType": "HASH" + }, + { + "AttributeName": "gsk", + "KeyType": "RANGE" + } + ], + "projection": { + "ProjectionType": "ALL" + }, + "index_status": "ACTIVE", + "provisioned_throughput": null + }, + { + "_id": { + "table_id": "86d32a95-5f27-4c2c-9e40-61c0b6acb9fb", + "index_name": "by_lsk" + }, + "index_id": "ed7f4b33-6603-4360-9e00-50a4b4174133", + "index_type": "LSI", + "key_schema": [ + { + "AttributeName": "pk", + "KeyType": "HASH" + }, + { + "AttributeName": "lsk", + "KeyType": "RANGE" + } + ], + "projection": { + "ProjectionType": "KEYS_ONLY" + }, + "index_status": "ACTIVE", + "provisioned_throughput": null + }, + { + "_id": { + "table_id": "be6507b1-cd5b-4663-abdb-d8b7c41c951f", + "index_name": "by_gpk" + }, + "index_id": "67717d79-ede4-4f85-8eb9-824afbc78d2e", + "index_type": "GSI", + "key_schema": [ + { + "AttributeName": "gpk", + "KeyType": "HASH" + }, + { + "AttributeName": "gsk", + "KeyType": "RANGE" + } + ], + "projection": { + "ProjectionType": "ALL" + }, + "index_status": "ACTIVE", + "provisioned_throughput": null + }, + { + "_id": { + "table_id": "be6507b1-cd5b-4663-abdb-d8b7c41c951f", + "index_name": "by_lsk" + }, + "index_id": "30eb80a0-57fe-48fa-b556-6f937ce19201", + "index_type": "LSI", + "key_schema": [ + { + "AttributeName": "pk", + "KeyType": "HASH" + }, + { + "AttributeName": "lsk", + "KeyType": "RANGE" + } + ], + "projection": { + "ProjectionType": "KEYS_ONLY" + }, + "index_status": "ACTIVE", + "provisioned_throughput": null + } + ] + }, + { + "name": "login_attempts", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "principal": { + "$numberInt": "1" + }, + "attempted_at": { + "$numberInt": "1" + } + }, + "name": "principal_1_attempted_at_1" + } + ], + "documents": [] + }, + { + "name": "metrics", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "_id.bucket": { + "$numberInt": "1" + } + }, + "name": "_id.bucket_1" + } + ], + "documents": [ + { + "_id": { + "$oid": "6ae3dca4833a09338997bbde" + }, + "bucket": { + "$date": { + "$numberLong": "1793318040000" + } + }, + "metric": "SuccessfulRequestLatency", + "sum": { + "$numberInt": "10812" + }, + "count": { + "$numberLong": "1" + }, + "min": { + "$numberInt": "10812" + }, + "max": { + "$numberInt": "10812" + }, + "table_name": "cu_orders", + "operation": "UpdateTimeToLive" + }, + { + "_id": { + "$oid": "6ae3dca4833a09338997bbdf" + }, + "bucket": { + "$date": { + "$numberLong": "1793318040000" + } + }, + "metric": "ConsumedWriteCapacityUnits", + "sum": { + "$numberInt": "10" + }, + "count": { + "$numberLong": "10" + }, + "min": { + "$numberInt": "1" + }, + "max": { + "$numberInt": "1" + }, + "table_name": "cu_orders", + "operation": "PutItem" + }, + { + "_id": { + "$oid": "6ae3dca4833a09338997bbe0" + }, + "bucket": { + "$date": { + "$numberLong": "1793318040000" + } + }, + "metric": "StorageQueryCount", + "sum": { + "$numberInt": "20" + }, + "count": { + "$numberLong": "20" + }, + "min": { + "$numberInt": "1" + }, + "max": { + "$numberInt": "1" + }, + "index_name": "Request", + "operation": "Data" + }, + { + "_id": { + "$oid": "6ae3dca4833a09338997bbe1" + }, + "bucket": { + "$date": { + "$numberLong": "1793318040000" + } + }, + "metric": "SuccessfulRequestLatency", + "sum": { + "$numberInt": "2114" + }, + "count": { + "$numberLong": "2" + }, + "min": { + "$numberInt": "967" + }, + "max": { + "$numberInt": "1147" + }, + "table_name": "cu_dotted", + "operation": "DescribeTable" + }, + { + "_id": { + "$oid": "6ae3dca4833a09338997bbe2" + }, + "bucket": { + "$date": { + "$numberLong": "1793318040000" + } + }, + "metric": "RequestCount", + "sum": { + "$numberInt": "20" + }, + "count": { + "$numberLong": "20" + }, + "min": { + "$numberInt": "1" + }, + "max": { + "$numberInt": "1" + }, + "operation": "PutItem" + }, + { + "_id": { + "$oid": "6ae3dca4833a09338997bbe3" + }, + "bucket": { + "$date": { + "$numberLong": "1793318040000" + } + }, + "metric": "RequestCount", + "sum": { + "$numberInt": "4" + }, + "count": { + "$numberLong": "4" + }, + "min": { + "$numberInt": "1" + }, + "max": { + "$numberInt": "1" + }, + "operation": "DescribeTable" + }, + { + "_id": { + "$oid": "6ae3dca4833a09338997bbe4" + }, + "bucket": { + "$date": { + "$numberLong": "1793318040000" + } + }, + "metric": "SuccessfulRequestLatency", + "sum": { + "$numberInt": "71677" + }, + "count": { + "$numberLong": "1" + }, + "min": { + "$numberInt": "71677" + }, + "max": { + "$numberInt": "71677" + }, + "table_name": "cu_orders", + "operation": "CreateTable" + }, + { + "_id": { + "$oid": "6ae3dca4833a09338997bbe5" + }, + "bucket": { + "$date": { + "$numberLong": "1793318040000" + } + }, + "metric": "StorageQueryLatency", + "sum": { + "$numberInt": "153283" + }, + "count": { + "$numberLong": "8" + }, + "min": { + "$numberInt": "538" + }, + "max": { + "$numberInt": "68203" + }, + "index_name": "Request", + "operation": "Catalog" + }, + { + "_id": { + "$oid": "6ae3dca4833a09338997bbe6" + }, + "bucket": { + "$date": { + "$numberLong": "1793318040000" + } + }, + "metric": "SuccessfulRequestLatency", + "sum": { + "$numberInt": "2281" + }, + "count": { + "$numberLong": "2" + }, + "min": { + "$numberInt": "1087" + }, + "max": { + "$numberInt": "1194" + }, + "table_name": "cu_orders", + "operation": "DescribeTable" + }, + { + "_id": { + "$oid": "6ae3dca4833a09338997bbe7" + }, + "bucket": { + "$date": { + "$numberLong": "1793318040000" + } + }, + "metric": "StorageQueryLatency", + "sum": { + "$numberInt": "105492" + }, + "count": { + "$numberLong": "20" + }, + "min": { + "$numberInt": "3473" + }, + "max": { + "$numberInt": "13767" + }, + "index_name": "Request", + "operation": "Data" + }, + { + "_id": { + "$oid": "6ae3dca4833a09338997bbe8" + }, + "bucket": { + "$date": { + "$numberLong": "1793318040000" + } + }, + "metric": "RequestCount", + "sum": { + "$numberInt": "2" + }, + "count": { + "$numberLong": "2" + }, + "min": { + "$numberInt": "1" + }, + "max": { + "$numberInt": "1" + }, + "operation": "CreateTable" + }, + { + "_id": { + "$oid": "6ae3dca4833a09338997bbe9" + }, + "bucket": { + "$date": { + "$numberLong": "1793318040000" + } + }, + "metric": "StorageQueryCount", + "sum": { + "$numberInt": "8" + }, + "count": { + "$numberLong": "8" + }, + "min": { + "$numberInt": "1" + }, + "max": { + "$numberInt": "1" + }, + "index_name": "Request", + "operation": "Catalog" + }, + { + "_id": { + "$oid": "6ae3dca4833a09338997bbea" + }, + "bucket": { + "$date": { + "$numberLong": "1793318040000" + } + }, + "metric": "SuccessfulRequestLatency", + "sum": { + "$numberInt": "56010" + }, + "count": { + "$numberLong": "10" + }, + "min": { + "$numberInt": "4405" + }, + "max": { + "$numberInt": "7034" + }, + "table_name": "cu_dotted", + "operation": "PutItem" + }, + { + "_id": { + "$oid": "6ae3dca4833a09338997bbeb" + }, + "bucket": { + "$date": { + "$numberLong": "1793318040000" + } + }, + "metric": "SuccessfulRequestLatency", + "sum": { + "$numberInt": "67779" + }, + "count": { + "$numberLong": "1" + }, + "min": { + "$numberInt": "67779" + }, + "max": { + "$numberInt": "67779" + }, + "table_name": "cu_dotted", + "operation": "CreateTable" + }, + { + "_id": { + "$oid": "6ae3dca4833a09338997bbec" + }, + "bucket": { + "$date": { + "$numberLong": "1793318040000" + } + }, + "metric": "ConsumedWriteCapacityUnits", + "sum": { + "$numberInt": "10" + }, + "count": { + "$numberLong": "10" + }, + "min": { + "$numberInt": "1" + }, + "max": { + "$numberInt": "1" + }, + "table_name": "cu_dotted", + "operation": "PutItem" + }, + { + "_id": { + "$oid": "6ae3dca4833a09338997bbed" + }, + "bucket": { + "$date": { + "$numberLong": "1793318040000" + } + }, + "metric": "SuccessfulRequestLatency", + "sum": { + "$numberInt": "4350" + }, + "count": { + "$numberLong": "1" + }, + "min": { + "$numberInt": "4350" + }, + "max": { + "$numberInt": "4350" + }, + "table_name": "cu_dotted", + "operation": "UpdateTimeToLive" + }, + { + "_id": { + "$oid": "6ae3dca4833a09338997bbee" + }, + "bucket": { + "$date": { + "$numberLong": "1793318040000" + } + }, + "metric": "SuccessfulRequestLatency", + "sum": { + "$numberInt": "52333" + }, + "count": { + "$numberLong": "10" + }, + "min": { + "$numberInt": "3523" + }, + "max": { + "$numberInt": "14448" + }, + "table_name": "cu_orders", + "operation": "PutItem" + }, + { + "_id": { + "$oid": "6ae3dca4833a09338997bbef" + }, + "bucket": { + "$date": { + "$numberLong": "1793318040000" + } + }, + "metric": "RequestCount", + "sum": { + "$numberInt": "2" + }, + "count": { + "$numberLong": "2" + }, + "min": { + "$numberInt": "1" + }, + "max": { + "$numberInt": "1" + }, + "operation": "UpdateTimeToLive" + } + ] + }, + { + "name": "schema_history", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + } + ], + "documents": [ + { + "_id": "001_initial", + "applied_at": { + "$date": { + "$numberLong": "1793318047295" + } + } + } + ] + }, + { + "name": "settings", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + } + ], + "documents": [ + { + "_id": "catalog_version", + "value": "0.0.2" + }, + { + "_id": "data_connection_string", + "value": "mongodb://localhost:22414/?replicaSet=rs0" + }, + { + "_id": "data_database_name", + "value": "extenddb_data" + }, + { + "_id": "encryption_key", + "value": "csFcLe6avIQnlK3UFdHDRPGop6M/ssPCHhTpuG3gliU=" + } + ] + }, + { + "name": "tables", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "table_id": { + "$numberInt": "1" + } + }, + "name": "table_id_1", + "unique": true + } + ], + "documents": [ + { + "_id": { + "account_id": "123456789012", + "table_name": "cu_dotted" + }, + "key_schema": [ + { + "AttributeName": "pk", + "KeyType": "HASH" + }, + { + "AttributeName": "sk", + "KeyType": "RANGE" + } + ], + "attribute_definitions": [ + { + "AttributeName": "pk", + "AttributeType": "S" + }, + { + "AttributeName": "sk", + "AttributeType": "S" + }, + { + "AttributeName": "gpk", + "AttributeType": "S" + }, + { + "AttributeName": "gsk", + "AttributeType": "N" + }, + { + "AttributeName": "lsk", + "AttributeType": "S" + } + ], + "billing_mode": "PAY_PER_REQUEST", + "provisioned_throughput": null, + "stream_specification": { + "StreamEnabled": true, + "StreamViewType": "NEW_AND_OLD_IMAGES" + }, + "table_status": "ACTIVE", + "creation_date_time": { + "$date": { + "$numberLong": "1793318051000" + } + }, + "table_size_bytes": { + "$numberLong": "0" + }, + "item_count": { + "$numberLong": "0" + }, + "table_arn": "arn:aws:dynamodb:us-east-1:123456789012:table/cu_dotted", + "table_id": "be6507b1-cd5b-4663-abdb-d8b7c41c951f", + "deletion_protection_enabled": false, + "ttl_attribute": "exp.at", + "stream_label": "2026-10-29T23:54:11", + "table_class": null, + "sse_specification": null, + "on_demand_throughput": null, + "ttl_index_ready": true + }, + { + "_id": { + "account_id": "123456789012", + "table_name": "cu_orders" + }, + "key_schema": [ + { + "AttributeName": "pk", + "KeyType": "HASH" + }, + { + "AttributeName": "sk", + "KeyType": "RANGE" + } + ], + "attribute_definitions": [ + { + "AttributeName": "pk", + "AttributeType": "S" + }, + { + "AttributeName": "sk", + "AttributeType": "S" + }, + { + "AttributeName": "gpk", + "AttributeType": "S" + }, + { + "AttributeName": "gsk", + "AttributeType": "N" + }, + { + "AttributeName": "lsk", + "AttributeType": "S" + } + ], + "billing_mode": "PAY_PER_REQUEST", + "provisioned_throughput": null, + "stream_specification": { + "StreamEnabled": true, + "StreamViewType": "NEW_AND_OLD_IMAGES" + }, + "table_status": "ACTIVE", + "creation_date_time": { + "$date": { + "$numberLong": "1793318050000" + } + }, + "table_size_bytes": { + "$numberLong": "0" + }, + "item_count": { + "$numberLong": "0" + }, + "table_arn": "arn:aws:dynamodb:us-east-1:123456789012:table/cu_orders", + "table_id": "86d32a95-5f27-4c2c-9e40-61c0b6acb9fb", + "deletion_protection_enabled": false, + "ttl_attribute": "expires", + "stream_label": "2026-10-29T23:54:10", + "table_class": null, + "sse_specification": null, + "on_demand_throughput": null, + "ttl_index_ready": true + } + ] + }, + { + "name": "tags", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + } + ], + "documents": [] + } + ] + }, + "extenddb_data": { + "collections": [ + { + "name": "_ddb_30eb80a0-57fe-48fa-b556-6f937ce19201", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "pk": { + "$numberInt": "1" + }, + "sk_s": { + "$numberInt": "1" + }, + "base_pk": { + "$numberInt": "1" + }, + "base_sk_s": { + "$numberInt": "1" + } + }, + "name": "pk_1_sk_s_1_base_pk_1_base_sk_s_1" + } + ], + "documents": [ + { + "_id": "2:p0,2:l0,2:p0,3:s09,", + "pk": "p0", + "base_pk": "p0", + "sk_s": "l0", + "base_sk_s": "s09", + "item_data": { + "lsk": { + "S": "l0" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s09" + } + } + }, + { + "_id": "2:p0,2:l3,2:p0,3:s06,", + "pk": "p0", + "base_pk": "p0", + "sk_s": "l3", + "base_sk_s": "s06", + "item_data": { + "lsk": { + "S": "l3" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s06" + } + } + }, + { + "_id": "2:p0,2:l6,2:p0,3:s03,", + "pk": "p0", + "base_pk": "p0", + "sk_s": "l6", + "base_sk_s": "s03", + "item_data": { + "lsk": { + "S": "l6" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s03" + } + } + }, + { + "_id": "2:p0,2:l9,2:p0,3:s00,", + "pk": "p0", + "base_pk": "p0", + "sk_s": "l9", + "base_sk_s": "s00", + "item_data": { + "lsk": { + "S": "l9" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s00" + } + } + }, + { + "_id": "2:p1,2:l2,2:p1,3:s07,", + "pk": "p1", + "base_pk": "p1", + "sk_s": "l2", + "base_sk_s": "s07", + "item_data": { + "lsk": { + "S": "l2" + }, + "pk": { + "S": "p1" + }, + "sk": { + "S": "s07" + } + } + }, + { + "_id": "2:p1,2:l5,2:p1,3:s04,", + "pk": "p1", + "base_pk": "p1", + "sk_s": "l5", + "base_sk_s": "s04", + "item_data": { + "lsk": { + "S": "l5" + }, + "pk": { + "S": "p1" + }, + "sk": { + "S": "s04" + } + } + }, + { + "_id": "2:p1,2:l8,2:p1,3:s01,", + "pk": "p1", + "base_pk": "p1", + "sk_s": "l8", + "base_sk_s": "s01", + "item_data": { + "lsk": { + "S": "l8" + }, + "pk": { + "S": "p1" + }, + "sk": { + "S": "s01" + } + } + }, + { + "_id": "2:p2,2:l1,2:p2,3:s08,", + "pk": "p2", + "base_pk": "p2", + "sk_s": "l1", + "base_sk_s": "s08", + "item_data": { + "lsk": { + "S": "l1" + }, + "pk": { + "S": "p2" + }, + "sk": { + "S": "s08" + } + } + }, + { + "_id": "2:p2,2:l4,2:p2,3:s05,", + "pk": "p2", + "base_pk": "p2", + "sk_s": "l4", + "base_sk_s": "s05", + "item_data": { + "lsk": { + "S": "l4" + }, + "pk": { + "S": "p2" + }, + "sk": { + "S": "s05" + } + } + }, + { + "_id": "2:p2,2:l7,2:p2,3:s02,", + "pk": "p2", + "base_pk": "p2", + "sk_s": "l7", + "base_sk_s": "s02", + "item_data": { + "lsk": { + "S": "l7" + }, + "pk": { + "S": "p2" + }, + "sk": { + "S": "s02" + } + } + } + ] + }, + { + "name": "_ddb_34ff4a88-57e0-4cba-b717-034946600988", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "pk": { + "$numberInt": "1" + }, + "sk_n": { + "$numberInt": "1" + }, + "base_pk": { + "$numberInt": "1" + }, + "base_sk_s": { + "$numberInt": "1" + } + }, + "name": "pk_1_sk_n_1_base_pk_1_base_sk_s_1" + } + ], + "documents": [ + { + "_id": "2:g0,1:0,2:p0,3:s00,", + "pk": "g0", + "base_pk": "p0", + "sk_n": { + "$numberDecimal": "0" + }, + "base_sk_s": "s00", + "item_data": { + "body": { + "S": "item 0" + }, + "expires": { + "N": "4102444800" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "0" + }, + "lsk": { + "S": "l9" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s00" + } + } + }, + { + "_id": "2:g0,1:2,2:p2,3:s02,", + "pk": "g0", + "base_pk": "p2", + "sk_n": { + "$numberDecimal": "2" + }, + "base_sk_s": "s02", + "item_data": { + "body": { + "S": "item 2" + }, + "expires": { + "N": "4102444802" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "2" + }, + "lsk": { + "S": "l7" + }, + "pk": { + "S": "p2" + }, + "sk": { + "S": "s02" + } + } + }, + { + "_id": "2:g0,1:4,2:p1,3:s04,", + "pk": "g0", + "base_pk": "p1", + "sk_n": { + "$numberDecimal": "4" + }, + "base_sk_s": "s04", + "item_data": { + "body": { + "S": "item 4" + }, + "expires": { + "N": "4102444804" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "4" + }, + "lsk": { + "S": "l5" + }, + "pk": { + "S": "p1" + }, + "sk": { + "S": "s04" + } + } + }, + { + "_id": "2:g0,1:6,2:p0,3:s06,", + "pk": "g0", + "base_pk": "p0", + "sk_n": { + "$numberDecimal": "6" + }, + "base_sk_s": "s06", + "item_data": { + "body": { + "S": "item 6" + }, + "expires": { + "N": "4102444806" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "6" + }, + "lsk": { + "S": "l3" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s06" + } + } + }, + { + "_id": "2:g0,1:8,2:p2,3:s08,", + "pk": "g0", + "base_pk": "p2", + "sk_n": { + "$numberDecimal": "8" + }, + "base_sk_s": "s08", + "item_data": { + "body": { + "S": "item 8" + }, + "expires": { + "N": "4102444808" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "8" + }, + "lsk": { + "S": "l1" + }, + "pk": { + "S": "p2" + }, + "sk": { + "S": "s08" + } + } + }, + { + "_id": "2:g1,1:1,2:p1,3:s01,", + "pk": "g1", + "base_pk": "p1", + "sk_n": { + "$numberDecimal": "1" + }, + "base_sk_s": "s01", + "item_data": { + "body": { + "S": "item 1" + }, + "expires": { + "N": "4102444801" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "1" + }, + "lsk": { + "S": "l8" + }, + "pk": { + "S": "p1" + }, + "sk": { + "S": "s01" + } + } + }, + { + "_id": "2:g1,1:3,2:p0,3:s03,", + "pk": "g1", + "base_pk": "p0", + "sk_n": { + "$numberDecimal": "3" + }, + "base_sk_s": "s03", + "item_data": { + "body": { + "S": "item 3" + }, + "expires": { + "N": "4102444803" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "3" + }, + "lsk": { + "S": "l6" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s03" + } + } + }, + { + "_id": "2:g1,1:5,2:p2,3:s05,", + "pk": "g1", + "base_pk": "p2", + "sk_n": { + "$numberDecimal": "5" + }, + "base_sk_s": "s05", + "item_data": { + "body": { + "S": "item 5" + }, + "expires": { + "N": "4102444805" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "5" + }, + "lsk": { + "S": "l4" + }, + "pk": { + "S": "p2" + }, + "sk": { + "S": "s05" + } + } + }, + { + "_id": "2:g1,1:7,2:p1,3:s07,", + "pk": "g1", + "base_pk": "p1", + "sk_n": { + "$numberDecimal": "7" + }, + "base_sk_s": "s07", + "item_data": { + "body": { + "S": "item 7" + }, + "expires": { + "N": "4102444807" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "7" + }, + "lsk": { + "S": "l2" + }, + "pk": { + "S": "p1" + }, + "sk": { + "S": "s07" + } + } + }, + { + "_id": "2:g1,1:9,2:p0,3:s09,", + "pk": "g1", + "base_pk": "p0", + "sk_n": { + "$numberDecimal": "9" + }, + "base_sk_s": "s09", + "item_data": { + "body": { + "S": "item 9" + }, + "expires": { + "N": "4102444809" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "9" + }, + "lsk": { + "S": "l0" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s09" + } + } + } + ] + }, + { + "name": "_ddb_67717d79-ede4-4f85-8eb9-824afbc78d2e", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "pk": { + "$numberInt": "1" + }, + "sk_n": { + "$numberInt": "1" + }, + "base_pk": { + "$numberInt": "1" + }, + "base_sk_s": { + "$numberInt": "1" + } + }, + "name": "pk_1_sk_n_1_base_pk_1_base_sk_s_1" + } + ], + "documents": [ + { + "_id": "2:g0,1:0,2:p0,3:s00,", + "pk": "g0", + "base_pk": "p0", + "sk_n": { + "$numberDecimal": "0" + }, + "base_sk_s": "s00", + "item_data": { + "body": { + "S": "item 0" + }, + "exp.at": { + "N": "4102444800" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "0" + }, + "lsk": { + "S": "l9" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s00" + } + } + }, + { + "_id": "2:g0,1:2,2:p2,3:s02,", + "pk": "g0", + "base_pk": "p2", + "sk_n": { + "$numberDecimal": "2" + }, + "base_sk_s": "s02", + "item_data": { + "body": { + "S": "item 2" + }, + "exp.at": { + "N": "4102444802" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "2" + }, + "lsk": { + "S": "l7" + }, + "pk": { + "S": "p2" + }, + "sk": { + "S": "s02" + } + } + }, + { + "_id": "2:g0,1:4,2:p1,3:s04,", + "pk": "g0", + "base_pk": "p1", + "sk_n": { + "$numberDecimal": "4" + }, + "base_sk_s": "s04", + "item_data": { + "body": { + "S": "item 4" + }, + "exp.at": { + "N": "4102444804" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "4" + }, + "lsk": { + "S": "l5" + }, + "pk": { + "S": "p1" + }, + "sk": { + "S": "s04" + } + } + }, + { + "_id": "2:g0,1:6,2:p0,3:s06,", + "pk": "g0", + "base_pk": "p0", + "sk_n": { + "$numberDecimal": "6" + }, + "base_sk_s": "s06", + "item_data": { + "body": { + "S": "item 6" + }, + "exp.at": { + "N": "4102444806" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "6" + }, + "lsk": { + "S": "l3" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s06" + } + } + }, + { + "_id": "2:g0,1:8,2:p2,3:s08,", + "pk": "g0", + "base_pk": "p2", + "sk_n": { + "$numberDecimal": "8" + }, + "base_sk_s": "s08", + "item_data": { + "body": { + "S": "item 8" + }, + "exp.at": { + "N": "4102444808" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "8" + }, + "lsk": { + "S": "l1" + }, + "pk": { + "S": "p2" + }, + "sk": { + "S": "s08" + } + } + }, + { + "_id": "2:g1,1:1,2:p1,3:s01,", + "pk": "g1", + "base_pk": "p1", + "sk_n": { + "$numberDecimal": "1" + }, + "base_sk_s": "s01", + "item_data": { + "body": { + "S": "item 1" + }, + "exp.at": { + "N": "4102444801" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "1" + }, + "lsk": { + "S": "l8" + }, + "pk": { + "S": "p1" + }, + "sk": { + "S": "s01" + } + } + }, + { + "_id": "2:g1,1:3,2:p0,3:s03,", + "pk": "g1", + "base_pk": "p0", + "sk_n": { + "$numberDecimal": "3" + }, + "base_sk_s": "s03", + "item_data": { + "body": { + "S": "item 3" + }, + "exp.at": { + "N": "4102444803" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "3" + }, + "lsk": { + "S": "l6" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s03" + } + } + }, + { + "_id": "2:g1,1:5,2:p2,3:s05,", + "pk": "g1", + "base_pk": "p2", + "sk_n": { + "$numberDecimal": "5" + }, + "base_sk_s": "s05", + "item_data": { + "body": { + "S": "item 5" + }, + "exp.at": { + "N": "4102444805" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "5" + }, + "lsk": { + "S": "l4" + }, + "pk": { + "S": "p2" + }, + "sk": { + "S": "s05" + } + } + }, + { + "_id": "2:g1,1:7,2:p1,3:s07,", + "pk": "g1", + "base_pk": "p1", + "sk_n": { + "$numberDecimal": "7" + }, + "base_sk_s": "s07", + "item_data": { + "body": { + "S": "item 7" + }, + "exp.at": { + "N": "4102444807" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "7" + }, + "lsk": { + "S": "l2" + }, + "pk": { + "S": "p1" + }, + "sk": { + "S": "s07" + } + } + }, + { + "_id": "2:g1,1:9,2:p0,3:s09,", + "pk": "g1", + "base_pk": "p0", + "sk_n": { + "$numberDecimal": "9" + }, + "base_sk_s": "s09", + "item_data": { + "body": { + "S": "item 9" + }, + "exp.at": { + "N": "4102444809" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "9" + }, + "lsk": { + "S": "l0" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s09" + } + } + } + ] + }, + { + "name": "_ddb_86d32a95-5f27-4c2c-9e40-61c0b6acb9fb", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "pk": { + "$numberInt": "1" + }, + "sk_s": { + "$numberInt": "1" + } + }, + "name": "pk_1_sk_s_1", + "unique": true + }, + { + "key": { + "item_data.expires.N": { + "$numberInt": "1" + } + }, + "name": "idx_ttl_expires", + "sparse": true + } + ], + "documents": [ + { + "_id": "2:p0,3:s00,", + "pk": "p0", + "sk_s": "s00", + "item_data": { + "body": { + "S": "item 0" + }, + "expires": { + "N": "4102444800" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "0" + }, + "lsk": { + "S": "l9" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s00" + } + } + }, + { + "_id": "2:p0,3:s03,", + "pk": "p0", + "sk_s": "s03", + "item_data": { + "body": { + "S": "item 3" + }, + "expires": { + "N": "4102444803" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "3" + }, + "lsk": { + "S": "l6" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s03" + } + } + }, + { + "_id": "2:p0,3:s06,", + "pk": "p0", + "sk_s": "s06", + "item_data": { + "body": { + "S": "item 6" + }, + "expires": { + "N": "4102444806" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "6" + }, + "lsk": { + "S": "l3" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s06" + } + } + }, + { + "_id": "2:p0,3:s09,", + "pk": "p0", + "sk_s": "s09", + "item_data": { + "body": { + "S": "item 9" + }, + "expires": { + "N": "4102444809" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "9" + }, + "lsk": { + "S": "l0" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s09" + } + } + }, + { + "_id": "2:p1,3:s01,", + "pk": "p1", + "sk_s": "s01", + "item_data": { + "body": { + "S": "item 1" + }, + "expires": { + "N": "4102444801" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "1" + }, + "lsk": { + "S": "l8" + }, + "pk": { + "S": "p1" + }, + "sk": { + "S": "s01" + } + } + }, + { + "_id": "2:p1,3:s04,", + "pk": "p1", + "sk_s": "s04", + "item_data": { + "body": { + "S": "item 4" + }, + "expires": { + "N": "4102444804" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "4" + }, + "lsk": { + "S": "l5" + }, + "pk": { + "S": "p1" + }, + "sk": { + "S": "s04" + } + } + }, + { + "_id": "2:p1,3:s07,", + "pk": "p1", + "sk_s": "s07", + "item_data": { + "body": { + "S": "item 7" + }, + "expires": { + "N": "4102444807" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "7" + }, + "lsk": { + "S": "l2" + }, + "pk": { + "S": "p1" + }, + "sk": { + "S": "s07" + } + } + }, + { + "_id": "2:p2,3:s02,", + "pk": "p2", + "sk_s": "s02", + "item_data": { + "body": { + "S": "item 2" + }, + "expires": { + "N": "4102444802" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "2" + }, + "lsk": { + "S": "l7" + }, + "pk": { + "S": "p2" + }, + "sk": { + "S": "s02" + } + } + }, + { + "_id": "2:p2,3:s05,", + "pk": "p2", + "sk_s": "s05", + "item_data": { + "body": { + "S": "item 5" + }, + "expires": { + "N": "4102444805" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "5" + }, + "lsk": { + "S": "l4" + }, + "pk": { + "S": "p2" + }, + "sk": { + "S": "s05" + } + } + }, + { + "_id": "2:p2,3:s08,", + "pk": "p2", + "sk_s": "s08", + "item_data": { + "body": { + "S": "item 8" + }, + "expires": { + "N": "4102444808" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "8" + }, + "lsk": { + "S": "l1" + }, + "pk": { + "S": "p2" + }, + "sk": { + "S": "s08" + } + } + } + ] + }, + { + "name": "_ddb_be6507b1-cd5b-4663-abdb-d8b7c41c951f", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "pk": { + "$numberInt": "1" + }, + "sk_s": { + "$numberInt": "1" + } + }, + "name": "pk_1_sk_s_1", + "unique": true + }, + { + "key": { + "item_data.exp.at.N": { + "$numberInt": "1" + } + }, + "name": "idx_ttl_exp.at", + "sparse": true + } + ], + "documents": [ + { + "_id": "2:p0,3:s00,", + "pk": "p0", + "sk_s": "s00", + "item_data": { + "body": { + "S": "item 0" + }, + "exp.at": { + "N": "4102444800" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "0" + }, + "lsk": { + "S": "l9" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s00" + } + } + }, + { + "_id": "2:p0,3:s03,", + "pk": "p0", + "sk_s": "s03", + "item_data": { + "body": { + "S": "item 3" + }, + "exp.at": { + "N": "4102444803" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "3" + }, + "lsk": { + "S": "l6" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s03" + } + } + }, + { + "_id": "2:p0,3:s06,", + "pk": "p0", + "sk_s": "s06", + "item_data": { + "body": { + "S": "item 6" + }, + "exp.at": { + "N": "4102444806" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "6" + }, + "lsk": { + "S": "l3" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s06" + } + } + }, + { + "_id": "2:p0,3:s09,", + "pk": "p0", + "sk_s": "s09", + "item_data": { + "body": { + "S": "item 9" + }, + "exp.at": { + "N": "4102444809" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "9" + }, + "lsk": { + "S": "l0" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s09" + } + } + }, + { + "_id": "2:p1,3:s01,", + "pk": "p1", + "sk_s": "s01", + "item_data": { + "body": { + "S": "item 1" + }, + "exp.at": { + "N": "4102444801" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "1" + }, + "lsk": { + "S": "l8" + }, + "pk": { + "S": "p1" + }, + "sk": { + "S": "s01" + } + } + }, + { + "_id": "2:p1,3:s04,", + "pk": "p1", + "sk_s": "s04", + "item_data": { + "body": { + "S": "item 4" + }, + "exp.at": { + "N": "4102444804" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "4" + }, + "lsk": { + "S": "l5" + }, + "pk": { + "S": "p1" + }, + "sk": { + "S": "s04" + } + } + }, + { + "_id": "2:p1,3:s07,", + "pk": "p1", + "sk_s": "s07", + "item_data": { + "body": { + "S": "item 7" + }, + "exp.at": { + "N": "4102444807" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "7" + }, + "lsk": { + "S": "l2" + }, + "pk": { + "S": "p1" + }, + "sk": { + "S": "s07" + } + } + }, + { + "_id": "2:p2,3:s02,", + "pk": "p2", + "sk_s": "s02", + "item_data": { + "body": { + "S": "item 2" + }, + "exp.at": { + "N": "4102444802" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "2" + }, + "lsk": { + "S": "l7" + }, + "pk": { + "S": "p2" + }, + "sk": { + "S": "s02" + } + } + }, + { + "_id": "2:p2,3:s05,", + "pk": "p2", + "sk_s": "s05", + "item_data": { + "body": { + "S": "item 5" + }, + "exp.at": { + "N": "4102444805" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "5" + }, + "lsk": { + "S": "l4" + }, + "pk": { + "S": "p2" + }, + "sk": { + "S": "s05" + } + } + }, + { + "_id": "2:p2,3:s08,", + "pk": "p2", + "sk_s": "s08", + "item_data": { + "body": { + "S": "item 8" + }, + "exp.at": { + "N": "4102444808" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "8" + }, + "lsk": { + "S": "l1" + }, + "pk": { + "S": "p2" + }, + "sk": { + "S": "s08" + } + } + } + ] + }, + { + "name": "_ddb_ed7f4b33-6603-4360-9e00-50a4b4174133", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "pk": { + "$numberInt": "1" + }, + "sk_s": { + "$numberInt": "1" + }, + "base_pk": { + "$numberInt": "1" + }, + "base_sk_s": { + "$numberInt": "1" + } + }, + "name": "pk_1_sk_s_1_base_pk_1_base_sk_s_1" + } + ], + "documents": [ + { + "_id": "2:p0,2:l0,2:p0,3:s09,", + "pk": "p0", + "base_pk": "p0", + "sk_s": "l0", + "base_sk_s": "s09", + "item_data": { + "lsk": { + "S": "l0" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s09" + } + } + }, + { + "_id": "2:p0,2:l3,2:p0,3:s06,", + "pk": "p0", + "base_pk": "p0", + "sk_s": "l3", + "base_sk_s": "s06", + "item_data": { + "lsk": { + "S": "l3" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s06" + } + } + }, + { + "_id": "2:p0,2:l6,2:p0,3:s03,", + "pk": "p0", + "base_pk": "p0", + "sk_s": "l6", + "base_sk_s": "s03", + "item_data": { + "lsk": { + "S": "l6" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s03" + } + } + }, + { + "_id": "2:p0,2:l9,2:p0,3:s00,", + "pk": "p0", + "base_pk": "p0", + "sk_s": "l9", + "base_sk_s": "s00", + "item_data": { + "lsk": { + "S": "l9" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s00" + } + } + }, + { + "_id": "2:p1,2:l2,2:p1,3:s07,", + "pk": "p1", + "base_pk": "p1", + "sk_s": "l2", + "base_sk_s": "s07", + "item_data": { + "lsk": { + "S": "l2" + }, + "pk": { + "S": "p1" + }, + "sk": { + "S": "s07" + } + } + }, + { + "_id": "2:p1,2:l5,2:p1,3:s04,", + "pk": "p1", + "base_pk": "p1", + "sk_s": "l5", + "base_sk_s": "s04", + "item_data": { + "lsk": { + "S": "l5" + }, + "pk": { + "S": "p1" + }, + "sk": { + "S": "s04" + } + } + }, + { + "_id": "2:p1,2:l8,2:p1,3:s01,", + "pk": "p1", + "base_pk": "p1", + "sk_s": "l8", + "base_sk_s": "s01", + "item_data": { + "lsk": { + "S": "l8" + }, + "pk": { + "S": "p1" + }, + "sk": { + "S": "s01" + } + } + }, + { + "_id": "2:p2,2:l1,2:p2,3:s08,", + "pk": "p2", + "base_pk": "p2", + "sk_s": "l1", + "base_sk_s": "s08", + "item_data": { + "lsk": { + "S": "l1" + }, + "pk": { + "S": "p2" + }, + "sk": { + "S": "s08" + } + } + }, + { + "_id": "2:p2,2:l4,2:p2,3:s05,", + "pk": "p2", + "base_pk": "p2", + "sk_s": "l4", + "base_sk_s": "s05", + "item_data": { + "lsk": { + "S": "l4" + }, + "pk": { + "S": "p2" + }, + "sk": { + "S": "s05" + } + } + }, + { + "_id": "2:p2,2:l7,2:p2,3:s02,", + "pk": "p2", + "base_pk": "p2", + "sk_s": "l7", + "base_sk_s": "s02", + "item_data": { + "lsk": { + "S": "l7" + }, + "pk": { + "S": "p2" + }, + "sk": { + "S": "s02" + } + } + } + ] + }, + { + "name": "counters", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + } + ], + "documents": [ + { + "_id": "stream_seq:shardId-86d32a95-5f27-4c2c-9e40-61c0b6acb9fb-000000000001", + "value": { + "$numberLong": "7" + } + }, + { + "_id": "stream_seq:shardId-86d32a95-5f27-4c2c-9e40-61c0b6acb9fb-000000000003", + "value": { + "$numberLong": "3" + } + }, + { + "_id": "stream_seq:shardId-be6507b1-cd5b-4663-abdb-d8b7c41c951f-000000000001", + "value": { + "$numberLong": "7" + } + }, + { + "_id": "stream_seq:shardId-be6507b1-cd5b-4663-abdb-d8b7c41c951f-000000000003", + "value": { + "$numberLong": "3" + } + } + ] + }, + { + "name": "idempotency_tokens", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "created_at": { + "$numberInt": "1" + } + }, + "name": "created_at_1", + "expireAfterSeconds": { + "$numberInt": "540" + } + }, + { + "key": { + "account_id": { + "$numberInt": "1" + }, + "token": { + "$numberInt": "1" + } + }, + "name": "account_id_1_token_1", + "unique": true + } + ], + "documents": [] + }, + { + "name": "stream_records", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "created_at": { + "$numberInt": "1" + } + }, + "name": "created_at_1", + "expireAfterSeconds": { + "$numberInt": "86400" + } + }, + { + "key": { + "shard_id": { + "$numberInt": "1" + }, + "sequence_number": { + "$numberInt": "1" + } + }, + "name": "shard_id_1_sequence_number_1" + } + ], + "documents": [ + { + "_id": { + "$oid": "6ae3dca2833a09338997bbc6" + }, + "sequence_number": "000000000000000000001", + "shard_id": "shardId-86d32a95-5f27-4c2c-9e40-61c0b6acb9fb-000000000001", + "table_id": "86d32a95-5f27-4c2c-9e40-61c0b6acb9fb", + "event_name": "INSERT", + "record_data": { + "eventID": "ec12e427-a414-4144-98aa-51d008085b11", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "us-east-1", + "dynamodb": { + "ApproximateCreationDateTime": { + "$numberLong": "1793318050" + }, + "Keys": { + "pk": { + "S": "p0" + }, + "sk": { + "S": "s00" + } + }, + "NewImage": { + "body": { + "S": "item 0" + }, + "expires": { + "N": "4102444800" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "0" + }, + "lsk": { + "S": "l9" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s00" + } + }, + "SequenceNumber": "000000000000000000001", + "SizeBytes": { + "$numberLong": "45" + }, + "StreamViewType": "NEW_AND_OLD_IMAGES" + } + }, + "created_at": { + "$date": { + "$numberLong": "1793318050617" + } + } + }, + { + "_id": { + "$oid": "6ae3dca2833a09338997bbc7" + }, + "sequence_number": "000000000000000000001", + "shard_id": "shardId-86d32a95-5f27-4c2c-9e40-61c0b6acb9fb-000000000003", + "table_id": "86d32a95-5f27-4c2c-9e40-61c0b6acb9fb", + "event_name": "INSERT", + "record_data": { + "eventID": "31d394cc-3ec7-41bc-af56-cef355399242", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "us-east-1", + "dynamodb": { + "ApproximateCreationDateTime": { + "$numberLong": "1793318050" + }, + "Keys": { + "pk": { + "S": "p1" + }, + "sk": { + "S": "s01" + } + }, + "NewImage": { + "body": { + "S": "item 1" + }, + "expires": { + "N": "4102444801" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "1" + }, + "lsk": { + "S": "l8" + }, + "pk": { + "S": "p1" + }, + "sk": { + "S": "s01" + } + }, + "SequenceNumber": "000000000000000000001", + "SizeBytes": { + "$numberLong": "47" + }, + "StreamViewType": "NEW_AND_OLD_IMAGES" + } + }, + "created_at": { + "$date": { + "$numberLong": "1793318050624" + } + } + }, + { + "_id": { + "$oid": "6ae3dca2833a09338997bbc8" + }, + "sequence_number": "000000000000000000002", + "shard_id": "shardId-86d32a95-5f27-4c2c-9e40-61c0b6acb9fb-000000000001", + "table_id": "86d32a95-5f27-4c2c-9e40-61c0b6acb9fb", + "event_name": "INSERT", + "record_data": { + "eventID": "a2d6170a-5215-4ced-8df2-5c3ffd6ffb1e", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "us-east-1", + "dynamodb": { + "ApproximateCreationDateTime": { + "$numberLong": "1793318050" + }, + "Keys": { + "pk": { + "S": "p2" + }, + "sk": { + "S": "s02" + } + }, + "NewImage": { + "body": { + "S": "item 2" + }, + "expires": { + "N": "4102444802" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "2" + }, + "lsk": { + "S": "l7" + }, + "pk": { + "S": "p2" + }, + "sk": { + "S": "s02" + } + }, + "SequenceNumber": "000000000000000000002", + "SizeBytes": { + "$numberLong": "47" + }, + "StreamViewType": "NEW_AND_OLD_IMAGES" + } + }, + "created_at": { + "$date": { + "$numberLong": "1793318050629" + } + } + }, + { + "_id": { + "$oid": "6ae3dca2833a09338997bbc9" + }, + "sequence_number": "000000000000000000003", + "shard_id": "shardId-86d32a95-5f27-4c2c-9e40-61c0b6acb9fb-000000000001", + "table_id": "86d32a95-5f27-4c2c-9e40-61c0b6acb9fb", + "event_name": "INSERT", + "record_data": { + "eventID": "8f1a7f38-8f95-42fe-8df1-3bc0c3a3b47c", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "us-east-1", + "dynamodb": { + "ApproximateCreationDateTime": { + "$numberLong": "1793318050" + }, + "Keys": { + "pk": { + "S": "p0" + }, + "sk": { + "S": "s03" + } + }, + "NewImage": { + "body": { + "S": "item 3" + }, + "expires": { + "N": "4102444803" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "3" + }, + "lsk": { + "S": "l6" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s03" + } + }, + "SequenceNumber": "000000000000000000003", + "SizeBytes": { + "$numberLong": "47" + }, + "StreamViewType": "NEW_AND_OLD_IMAGES" + } + }, + "created_at": { + "$date": { + "$numberLong": "1793318050635" + } + } + }, + { + "_id": { + "$oid": "6ae3dca2833a09338997bbca" + }, + "sequence_number": "000000000000000000002", + "shard_id": "shardId-86d32a95-5f27-4c2c-9e40-61c0b6acb9fb-000000000003", + "table_id": "86d32a95-5f27-4c2c-9e40-61c0b6acb9fb", + "event_name": "INSERT", + "record_data": { + "eventID": "3e7b053a-e5df-451f-98c8-12a6984fe88b", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "us-east-1", + "dynamodb": { + "ApproximateCreationDateTime": { + "$numberLong": "1793318050" + }, + "Keys": { + "pk": { + "S": "p1" + }, + "sk": { + "S": "s04" + } + }, + "NewImage": { + "body": { + "S": "item 4" + }, + "expires": { + "N": "4102444804" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "4" + }, + "lsk": { + "S": "l5" + }, + "pk": { + "S": "p1" + }, + "sk": { + "S": "s04" + } + }, + "SequenceNumber": "000000000000000000002", + "SizeBytes": { + "$numberLong": "47" + }, + "StreamViewType": "NEW_AND_OLD_IMAGES" + } + }, + "created_at": { + "$date": { + "$numberLong": "1793318050641" + } + } + }, + { + "_id": { + "$oid": "6ae3dca2833a09338997bbcb" + }, + "sequence_number": "000000000000000000004", + "shard_id": "shardId-86d32a95-5f27-4c2c-9e40-61c0b6acb9fb-000000000001", + "table_id": "86d32a95-5f27-4c2c-9e40-61c0b6acb9fb", + "event_name": "INSERT", + "record_data": { + "eventID": "f57f29a6-85bb-45cd-880c-f9c871e75cc8", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "us-east-1", + "dynamodb": { + "ApproximateCreationDateTime": { + "$numberLong": "1793318050" + }, + "Keys": { + "pk": { + "S": "p2" + }, + "sk": { + "S": "s05" + } + }, + "NewImage": { + "body": { + "S": "item 5" + }, + "expires": { + "N": "4102444805" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "5" + }, + "lsk": { + "S": "l4" + }, + "pk": { + "S": "p2" + }, + "sk": { + "S": "s05" + } + }, + "SequenceNumber": "000000000000000000004", + "SizeBytes": { + "$numberLong": "47" + }, + "StreamViewType": "NEW_AND_OLD_IMAGES" + } + }, + "created_at": { + "$date": { + "$numberLong": "1793318050648" + } + } + }, + { + "_id": { + "$oid": "6ae3dca2833a09338997bbcc" + }, + "sequence_number": "000000000000000000005", + "shard_id": "shardId-86d32a95-5f27-4c2c-9e40-61c0b6acb9fb-000000000001", + "table_id": "86d32a95-5f27-4c2c-9e40-61c0b6acb9fb", + "event_name": "INSERT", + "record_data": { + "eventID": "ce0bbc81-4f39-4f0b-b9c8-c0681e358a16", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "us-east-1", + "dynamodb": { + "ApproximateCreationDateTime": { + "$numberLong": "1793318050" + }, + "Keys": { + "pk": { + "S": "p0" + }, + "sk": { + "S": "s06" + } + }, + "NewImage": { + "body": { + "S": "item 6" + }, + "expires": { + "N": "4102444806" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "6" + }, + "lsk": { + "S": "l3" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s06" + } + }, + "SequenceNumber": "000000000000000000005", + "SizeBytes": { + "$numberLong": "47" + }, + "StreamViewType": "NEW_AND_OLD_IMAGES" + } + }, + "created_at": { + "$date": { + "$numberLong": "1793318050653" + } + } + }, + { + "_id": { + "$oid": "6ae3dca2833a09338997bbcd" + }, + "sequence_number": "000000000000000000003", + "shard_id": "shardId-86d32a95-5f27-4c2c-9e40-61c0b6acb9fb-000000000003", + "table_id": "86d32a95-5f27-4c2c-9e40-61c0b6acb9fb", + "event_name": "INSERT", + "record_data": { + "eventID": "00200a41-ffb7-4b23-a081-82b56c70d150", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "us-east-1", + "dynamodb": { + "ApproximateCreationDateTime": { + "$numberLong": "1793318050" + }, + "Keys": { + "pk": { + "S": "p1" + }, + "sk": { + "S": "s07" + } + }, + "NewImage": { + "body": { + "S": "item 7" + }, + "expires": { + "N": "4102444807" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "7" + }, + "lsk": { + "S": "l2" + }, + "pk": { + "S": "p1" + }, + "sk": { + "S": "s07" + } + }, + "SequenceNumber": "000000000000000000003", + "SizeBytes": { + "$numberLong": "47" + }, + "StreamViewType": "NEW_AND_OLD_IMAGES" + } + }, + "created_at": { + "$date": { + "$numberLong": "1793318050659" + } + } + }, + { + "_id": { + "$oid": "6ae3dca2833a09338997bbce" + }, + "sequence_number": "000000000000000000006", + "shard_id": "shardId-86d32a95-5f27-4c2c-9e40-61c0b6acb9fb-000000000001", + "table_id": "86d32a95-5f27-4c2c-9e40-61c0b6acb9fb", + "event_name": "INSERT", + "record_data": { + "eventID": "64bab496-259f-4187-a786-e5bc28ec0a44", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "us-east-1", + "dynamodb": { + "ApproximateCreationDateTime": { + "$numberLong": "1793318050" + }, + "Keys": { + "pk": { + "S": "p2" + }, + "sk": { + "S": "s08" + } + }, + "NewImage": { + "body": { + "S": "item 8" + }, + "expires": { + "N": "4102444808" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "8" + }, + "lsk": { + "S": "l1" + }, + "pk": { + "S": "p2" + }, + "sk": { + "S": "s08" + } + }, + "SequenceNumber": "000000000000000000006", + "SizeBytes": { + "$numberLong": "47" + }, + "StreamViewType": "NEW_AND_OLD_IMAGES" + } + }, + "created_at": { + "$date": { + "$numberLong": "1793318050664" + } + } + }, + { + "_id": { + "$oid": "6ae3dca2833a09338997bbcf" + }, + "sequence_number": "000000000000000000007", + "shard_id": "shardId-86d32a95-5f27-4c2c-9e40-61c0b6acb9fb-000000000001", + "table_id": "86d32a95-5f27-4c2c-9e40-61c0b6acb9fb", + "event_name": "INSERT", + "record_data": { + "eventID": "0f77422e-8ece-42cd-a6a2-089ef2abbfa4", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "us-east-1", + "dynamodb": { + "ApproximateCreationDateTime": { + "$numberLong": "1793318050" + }, + "Keys": { + "pk": { + "S": "p0" + }, + "sk": { + "S": "s09" + } + }, + "NewImage": { + "body": { + "S": "item 9" + }, + "expires": { + "N": "4102444809" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "9" + }, + "lsk": { + "S": "l0" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s09" + } + }, + "SequenceNumber": "000000000000000000007", + "SizeBytes": { + "$numberLong": "47" + }, + "StreamViewType": "NEW_AND_OLD_IMAGES" + } + }, + "created_at": { + "$date": { + "$numberLong": "1793318050669" + } + } + }, + { + "_id": { + "$oid": "6ae3dca3833a09338997bbd4" + }, + "sequence_number": "000000000000000000001", + "shard_id": "shardId-be6507b1-cd5b-4663-abdb-d8b7c41c951f-000000000001", + "table_id": "be6507b1-cd5b-4663-abdb-d8b7c41c951f", + "event_name": "INSERT", + "record_data": { + "eventID": "853c9565-26af-4ed8-a06c-b3cef08d18f0", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "us-east-1", + "dynamodb": { + "ApproximateCreationDateTime": { + "$numberLong": "1793318051" + }, + "Keys": { + "pk": { + "S": "p0" + }, + "sk": { + "S": "s00" + } + }, + "NewImage": { + "body": { + "S": "item 0" + }, + "exp.at": { + "N": "4102444800" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "0" + }, + "lsk": { + "S": "l9" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s00" + } + }, + "SequenceNumber": "000000000000000000001", + "SizeBytes": { + "$numberLong": "44" + }, + "StreamViewType": "NEW_AND_OLD_IMAGES" + } + }, + "created_at": { + "$date": { + "$numberLong": "1793318051606" + } + } + }, + { + "_id": { + "$oid": "6ae3dca3833a09338997bbd5" + }, + "sequence_number": "000000000000000000001", + "shard_id": "shardId-be6507b1-cd5b-4663-abdb-d8b7c41c951f-000000000003", + "table_id": "be6507b1-cd5b-4663-abdb-d8b7c41c951f", + "event_name": "INSERT", + "record_data": { + "eventID": "6b6cfed6-4fb3-451d-a3f0-55de06959335", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "us-east-1", + "dynamodb": { + "ApproximateCreationDateTime": { + "$numberLong": "1793318051" + }, + "Keys": { + "pk": { + "S": "p1" + }, + "sk": { + "S": "s01" + } + }, + "NewImage": { + "body": { + "S": "item 1" + }, + "exp.at": { + "N": "4102444801" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "1" + }, + "lsk": { + "S": "l8" + }, + "pk": { + "S": "p1" + }, + "sk": { + "S": "s01" + } + }, + "SequenceNumber": "000000000000000000001", + "SizeBytes": { + "$numberLong": "46" + }, + "StreamViewType": "NEW_AND_OLD_IMAGES" + } + }, + "created_at": { + "$date": { + "$numberLong": "1793318051614" + } + } + }, + { + "_id": { + "$oid": "6ae3dca3833a09338997bbd6" + }, + "sequence_number": "000000000000000000002", + "shard_id": "shardId-be6507b1-cd5b-4663-abdb-d8b7c41c951f-000000000001", + "table_id": "be6507b1-cd5b-4663-abdb-d8b7c41c951f", + "event_name": "INSERT", + "record_data": { + "eventID": "98f9554d-ea84-493a-8f80-c3873a2e1638", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "us-east-1", + "dynamodb": { + "ApproximateCreationDateTime": { + "$numberLong": "1793318051" + }, + "Keys": { + "pk": { + "S": "p2" + }, + "sk": { + "S": "s02" + } + }, + "NewImage": { + "body": { + "S": "item 2" + }, + "exp.at": { + "N": "4102444802" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "2" + }, + "lsk": { + "S": "l7" + }, + "pk": { + "S": "p2" + }, + "sk": { + "S": "s02" + } + }, + "SequenceNumber": "000000000000000000002", + "SizeBytes": { + "$numberLong": "46" + }, + "StreamViewType": "NEW_AND_OLD_IMAGES" + } + }, + "created_at": { + "$date": { + "$numberLong": "1793318051621" + } + } + }, + { + "_id": { + "$oid": "6ae3dca3833a09338997bbd7" + }, + "sequence_number": "000000000000000000003", + "shard_id": "shardId-be6507b1-cd5b-4663-abdb-d8b7c41c951f-000000000001", + "table_id": "be6507b1-cd5b-4663-abdb-d8b7c41c951f", + "event_name": "INSERT", + "record_data": { + "eventID": "079ee850-96b2-4419-9765-4ad536ab5bc6", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "us-east-1", + "dynamodb": { + "ApproximateCreationDateTime": { + "$numberLong": "1793318051" + }, + "Keys": { + "pk": { + "S": "p0" + }, + "sk": { + "S": "s03" + } + }, + "NewImage": { + "body": { + "S": "item 3" + }, + "exp.at": { + "N": "4102444803" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "3" + }, + "lsk": { + "S": "l6" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s03" + } + }, + "SequenceNumber": "000000000000000000003", + "SizeBytes": { + "$numberLong": "46" + }, + "StreamViewType": "NEW_AND_OLD_IMAGES" + } + }, + "created_at": { + "$date": { + "$numberLong": "1793318051630" + } + } + }, + { + "_id": { + "$oid": "6ae3dca3833a09338997bbd8" + }, + "sequence_number": "000000000000000000002", + "shard_id": "shardId-be6507b1-cd5b-4663-abdb-d8b7c41c951f-000000000003", + "table_id": "be6507b1-cd5b-4663-abdb-d8b7c41c951f", + "event_name": "INSERT", + "record_data": { + "eventID": "07b3d7a0-ae43-4f1d-8471-a99be35ffad1", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "us-east-1", + "dynamodb": { + "ApproximateCreationDateTime": { + "$numberLong": "1793318051" + }, + "Keys": { + "pk": { + "S": "p1" + }, + "sk": { + "S": "s04" + } + }, + "NewImage": { + "body": { + "S": "item 4" + }, + "exp.at": { + "N": "4102444804" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "4" + }, + "lsk": { + "S": "l5" + }, + "pk": { + "S": "p1" + }, + "sk": { + "S": "s04" + } + }, + "SequenceNumber": "000000000000000000002", + "SizeBytes": { + "$numberLong": "46" + }, + "StreamViewType": "NEW_AND_OLD_IMAGES" + } + }, + "created_at": { + "$date": { + "$numberLong": "1793318051636" + } + } + }, + { + "_id": { + "$oid": "6ae3dca3833a09338997bbd9" + }, + "sequence_number": "000000000000000000004", + "shard_id": "shardId-be6507b1-cd5b-4663-abdb-d8b7c41c951f-000000000001", + "table_id": "be6507b1-cd5b-4663-abdb-d8b7c41c951f", + "event_name": "INSERT", + "record_data": { + "eventID": "07eb68e9-5f4d-4215-b0c0-8c9799eabe9b", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "us-east-1", + "dynamodb": { + "ApproximateCreationDateTime": { + "$numberLong": "1793318051" + }, + "Keys": { + "pk": { + "S": "p2" + }, + "sk": { + "S": "s05" + } + }, + "NewImage": { + "body": { + "S": "item 5" + }, + "exp.at": { + "N": "4102444805" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "5" + }, + "lsk": { + "S": "l4" + }, + "pk": { + "S": "p2" + }, + "sk": { + "S": "s05" + } + }, + "SequenceNumber": "000000000000000000004", + "SizeBytes": { + "$numberLong": "46" + }, + "StreamViewType": "NEW_AND_OLD_IMAGES" + } + }, + "created_at": { + "$date": { + "$numberLong": "1793318051645" + } + } + }, + { + "_id": { + "$oid": "6ae3dca3833a09338997bbda" + }, + "sequence_number": "000000000000000000005", + "shard_id": "shardId-be6507b1-cd5b-4663-abdb-d8b7c41c951f-000000000001", + "table_id": "be6507b1-cd5b-4663-abdb-d8b7c41c951f", + "event_name": "INSERT", + "record_data": { + "eventID": "b66e87e6-80d0-4a3b-a3b3-3908a1f48f6c", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "us-east-1", + "dynamodb": { + "ApproximateCreationDateTime": { + "$numberLong": "1793318051" + }, + "Keys": { + "pk": { + "S": "p0" + }, + "sk": { + "S": "s06" + } + }, + "NewImage": { + "body": { + "S": "item 6" + }, + "exp.at": { + "N": "4102444806" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "6" + }, + "lsk": { + "S": "l3" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s06" + } + }, + "SequenceNumber": "000000000000000000005", + "SizeBytes": { + "$numberLong": "46" + }, + "StreamViewType": "NEW_AND_OLD_IMAGES" + } + }, + "created_at": { + "$date": { + "$numberLong": "1793318051652" + } + } + }, + { + "_id": { + "$oid": "6ae3dca3833a09338997bbdb" + }, + "sequence_number": "000000000000000000003", + "shard_id": "shardId-be6507b1-cd5b-4663-abdb-d8b7c41c951f-000000000003", + "table_id": "be6507b1-cd5b-4663-abdb-d8b7c41c951f", + "event_name": "INSERT", + "record_data": { + "eventID": "215f1f12-4308-4670-8699-0775e89a586f", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "us-east-1", + "dynamodb": { + "ApproximateCreationDateTime": { + "$numberLong": "1793318051" + }, + "Keys": { + "pk": { + "S": "p1" + }, + "sk": { + "S": "s07" + } + }, + "NewImage": { + "body": { + "S": "item 7" + }, + "exp.at": { + "N": "4102444807" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "7" + }, + "lsk": { + "S": "l2" + }, + "pk": { + "S": "p1" + }, + "sk": { + "S": "s07" + } + }, + "SequenceNumber": "000000000000000000003", + "SizeBytes": { + "$numberLong": "46" + }, + "StreamViewType": "NEW_AND_OLD_IMAGES" + } + }, + "created_at": { + "$date": { + "$numberLong": "1793318051659" + } + } + }, + { + "_id": { + "$oid": "6ae3dca3833a09338997bbdc" + }, + "sequence_number": "000000000000000000006", + "shard_id": "shardId-be6507b1-cd5b-4663-abdb-d8b7c41c951f-000000000001", + "table_id": "be6507b1-cd5b-4663-abdb-d8b7c41c951f", + "event_name": "INSERT", + "record_data": { + "eventID": "4cf9191e-f8b0-4fc0-a2a0-b7db9478431e", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "us-east-1", + "dynamodb": { + "ApproximateCreationDateTime": { + "$numberLong": "1793318051" + }, + "Keys": { + "pk": { + "S": "p2" + }, + "sk": { + "S": "s08" + } + }, + "NewImage": { + "body": { + "S": "item 8" + }, + "exp.at": { + "N": "4102444808" + }, + "gpk": { + "S": "g0" + }, + "gsk": { + "N": "8" + }, + "lsk": { + "S": "l1" + }, + "pk": { + "S": "p2" + }, + "sk": { + "S": "s08" + } + }, + "SequenceNumber": "000000000000000000006", + "SizeBytes": { + "$numberLong": "46" + }, + "StreamViewType": "NEW_AND_OLD_IMAGES" + } + }, + "created_at": { + "$date": { + "$numberLong": "1793318051667" + } + } + }, + { + "_id": { + "$oid": "6ae3dca3833a09338997bbdd" + }, + "sequence_number": "000000000000000000007", + "shard_id": "shardId-be6507b1-cd5b-4663-abdb-d8b7c41c951f-000000000001", + "table_id": "be6507b1-cd5b-4663-abdb-d8b7c41c951f", + "event_name": "INSERT", + "record_data": { + "eventID": "6cfec9d6-4d4d-433c-a4ae-27b43450f064", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "us-east-1", + "dynamodb": { + "ApproximateCreationDateTime": { + "$numberLong": "1793318051" + }, + "Keys": { + "pk": { + "S": "p0" + }, + "sk": { + "S": "s09" + } + }, + "NewImage": { + "body": { + "S": "item 9" + }, + "exp.at": { + "N": "4102444809" + }, + "gpk": { + "S": "g1" + }, + "gsk": { + "N": "9" + }, + "lsk": { + "S": "l0" + }, + "pk": { + "S": "p0" + }, + "sk": { + "S": "s09" + } + }, + "SequenceNumber": "000000000000000000007", + "SizeBytes": { + "$numberLong": "46" + }, + "StreamViewType": "NEW_AND_OLD_IMAGES" + } + }, + "created_at": { + "$date": { + "$numberLong": "1793318051674" + } + } + } + ] + }, + { + "name": "stream_shards", + "options": {}, + "indexes": [ + { + "key": { + "_id": { + "$numberInt": "1" + } + }, + "name": "_id_" + }, + { + "key": { + "shard_id": { + "$numberInt": "1" + } + }, + "name": "shard_id_1", + "unique": true + } + ], + "documents": [ + { + "_id": { + "$oid": "6ae3dca2833a09338997bbc2" + }, + "shard_id": "shardId-86d32a95-5f27-4c2c-9e40-61c0b6acb9fb-000000000000", + "table_id": "86d32a95-5f27-4c2c-9e40-61c0b6acb9fb", + "starting_sequence_number": "000000000000000000000", + "created_at": { + "$date": { + "$numberLong": "1793318050033" + } + } + }, + { + "_id": { + "$oid": "6ae3dca2833a09338997bbc3" + }, + "shard_id": "shardId-86d32a95-5f27-4c2c-9e40-61c0b6acb9fb-000000000001", + "table_id": "86d32a95-5f27-4c2c-9e40-61c0b6acb9fb", + "starting_sequence_number": "000000000000000000000", + "created_at": { + "$date": { + "$numberLong": "1793318050034" + } + } + }, + { + "_id": { + "$oid": "6ae3dca2833a09338997bbc4" + }, + "shard_id": "shardId-86d32a95-5f27-4c2c-9e40-61c0b6acb9fb-000000000002", + "table_id": "86d32a95-5f27-4c2c-9e40-61c0b6acb9fb", + "starting_sequence_number": "000000000000000000000", + "created_at": { + "$date": { + "$numberLong": "1793318050037" + } + } + }, + { + "_id": { + "$oid": "6ae3dca2833a09338997bbc5" + }, + "shard_id": "shardId-86d32a95-5f27-4c2c-9e40-61c0b6acb9fb-000000000003", + "table_id": "86d32a95-5f27-4c2c-9e40-61c0b6acb9fb", + "starting_sequence_number": "000000000000000000000", + "created_at": { + "$date": { + "$numberLong": "1793318050037" + } + } + }, + { + "_id": { + "$oid": "6ae3dca3833a09338997bbd0" + }, + "shard_id": "shardId-be6507b1-cd5b-4663-abdb-d8b7c41c951f-000000000000", + "table_id": "be6507b1-cd5b-4663-abdb-d8b7c41c951f", + "starting_sequence_number": "000000000000000000000", + "created_at": { + "$date": { + "$numberLong": "1793318051039" + } + } + }, + { + "_id": { + "$oid": "6ae3dca3833a09338997bbd1" + }, + "shard_id": "shardId-be6507b1-cd5b-4663-abdb-d8b7c41c951f-000000000001", + "table_id": "be6507b1-cd5b-4663-abdb-d8b7c41c951f", + "starting_sequence_number": "000000000000000000000", + "created_at": { + "$date": { + "$numberLong": "1793318051041" + } + } + }, + { + "_id": { + "$oid": "6ae3dca3833a09338997bbd2" + }, + "shard_id": "shardId-be6507b1-cd5b-4663-abdb-d8b7c41c951f-000000000002", + "table_id": "be6507b1-cd5b-4663-abdb-d8b7c41c951f", + "starting_sequence_number": "000000000000000000000", + "created_at": { + "$date": { + "$numberLong": "1793318051043" + } + } + }, + { + "_id": { + "$oid": "6ae3dca3833a09338997bbd3" + }, + "shard_id": "shardId-be6507b1-cd5b-4663-abdb-d8b7c41c951f-000000000003", + "table_id": "be6507b1-cd5b-4663-abdb-d8b7c41c951f", + "starting_sequence_number": "000000000000000000000", + "created_at": { + "$date": { + "$numberLong": "1793318051045" + } + } + } + ] + } + ] + } + } +} diff --git a/crates/storage-mongodb/tests/fixtures/released/dump.js b/crates/storage-mongodb/tests/fixtures/released/dump.js new file mode 100644 index 000000000..a6e4d5259 --- /dev/null +++ b/crates/storage-mongodb/tests/fixtures/released/dump.js @@ -0,0 +1,37 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 +// +// Print both ExtendDB databases as one fixture, in canonical Extended JSON: +// +// mongosh --quiet --file dump.js +// +// For each database: every collection with its type and options, its +// indexes as listIndexes returns them (key order kept, `v` and `ns` +// dropped), and every document. Keys keep their stored order, which +// compound `_id` lookups depend on. `load.js` loads the file back. +const out = { databases: {} }; +for (const name of ["extenddb_catalog", "extenddb_data"]) { + const d = db.getSiblingDB(name); + const collections = []; + for (const spec of d.getCollectionInfos({}, { nameOnly: false })) { + if (spec.type !== "collection") { + continue; + } + const coll = d.getCollection(spec.name); + const indexes = coll.getIndexes().map((i) => { + const copy = Object.assign({}, i); + delete copy.v; + delete copy.ns; + return copy; + }); + collections.push({ + name: spec.name, + options: spec.options, + indexes: indexes, + documents: coll.find({}).sort({ _id: 1 }).toArray(), + }); + } + collections.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)); + out.databases[name] = { collections: collections }; +} +print(EJSON.stringify(out, null, 1, { relaxed: false })); diff --git a/crates/storage-mongodb/tests/fixtures/released/load.js b/crates/storage-mongodb/tests/fixtures/released/load.js new file mode 100644 index 000000000..a6548e29c --- /dev/null +++ b/crates/storage-mongodb/tests/fixtures/released/load.js @@ -0,0 +1,28 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 +// +// Load a fixture `dump.js` printed into empty databases: +// +// mongosh --quiet --eval 'const FIXTURE = ""' --file load.js +// +// Each collection is created with its options, its indexes are created as +// listed (the `_id_` index exists with the collection), and its documents +// are inserted in order. The databases must not exist. +const fixture = EJSON.parse(fs.readFileSync(FIXTURE, "utf8"), { relaxed: false }); +for (const [name, database] of Object.entries(fixture.databases)) { + const d = db.getSiblingDB(name); + if (d.getCollectionNames().length !== 0) { + throw new Error(`database ${name} is not empty`); + } + for (const c of database.collections) { + d.createCollection(c.name, c.options); + const indexes = c.indexes.filter((i) => i.name !== "_id_"); + if (indexes.length > 0) { + d.runCommand({ createIndexes: c.name, indexes: indexes }); + } + if (c.documents.length > 0) { + d.getCollection(c.name).insertMany(c.documents, { ordered: true }); + } + } +} +print("loaded"); diff --git a/crates/storage-mongodb/tests/fixtures/released/make-fixture.sh b/crates/storage-mongodb/tests/fixtures/released/make-fixture.sh new file mode 100755 index 000000000..08df0c53a --- /dev/null +++ b/crates/storage-mongodb/tests/fixtures/released/make-fixture.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 +# +# Make the fixture of one released MongoDB catalog: +# +# make-fixture.sh [dotted-ttl] +# +# Starts mongo:7 as a one-node replica set in a container named , +# with its port and the API port published on 127.0.0.1 only, runs `init` +# with the image extenddb/extenddb-mongodb:, starts its server, +# provisions account 123456789012 with the IAM user cu-tester, two inline +# policies (one with a dynamodb:ResourceTag condition) and an access key +# through `extenddb manage`, creates the table seed.py describes, stops the +# server and writes both databases to .json beside this script +# with dump.js. The containers are removed at the end. +# +# With `dotted-ttl` it also creates the table cu_dotted with TTL on the +# attribute `exp.at` and then adds the index releases before v0.1.12 created +# for it, `idx_ttl_exp.at` on `item_data.exp.at.N` (sparse; +# `v0.1.11:crates/storage-mongodb/src/metadata_engine.rs:293-301`), which +# v0.1.12 and later do not create; the file is then dotted-ttl.json. +set -euo pipefail +here="$(cd "$(dirname "$0")" && pwd)" +version=$1; m=$2; mport=$3; aport=$4; variant=${5:-} +server="$m-server" +work="$(mktemp -d)" +cleanup() { + docker rm -f "$server" "$m" >/dev/null 2>&1 || true + rm -rf "$work" +} +trap cleanup EXIT + +docker run -d --name "$m" -p "127.0.0.1:$mport:$mport" -p "127.0.0.1:$aport:18443" \ + -v "$here:/fixtures:ro" mongo:7 mongod --replSet rs0 --bind_ip_all --port "$mport" >/dev/null +for _ in $(seq 1 60); do + docker exec "$m" mongosh --quiet --port "$mport" --eval 'db.runCommand({ping: 1}).ok' 2>/dev/null \ + | grep -qx 1 && break + sleep 1 +done +docker exec "$m" mongosh --quiet --port "$mport" \ + --eval "rs.initiate({_id: 'rs0', members: [{_id: 0, host: 'localhost:$mport'}]})" >/dev/null +for _ in $(seq 1 60); do + docker exec "$m" mongosh --quiet --port "$mport" --eval 'rs.status().myState' 2>/dev/null \ + | grep -qx 1 && break + sleep 1 +done + +printf '[storage]\nbackend = "mongodb"\n\n[storage.mongodb]\nconnection_string = "mongodb://localhost:%s/?replicaSet=rs0"\n' \ + "$mport" > "$work/extenddb.toml" +password=FixtureAdminPassw0rd1 +docker run --rm --network "container:$m" --user "$(id -u):$(id -g)" -v "$work:/var/lib/extenddb" \ + -e EXTENDDB_ADMIN_USER=admin -e EXTENDDB_ADMIN_PASSWORD=$password \ + "extenddb/extenddb-mongodb:$version" init --backend mongodb \ + --config /var/lib/extenddb/extenddb.toml --overwrite --bind-addr 0.0.0.0 \ + --tls-san localhost --tls-san 127.0.0.1 > "$work/init.log" 2>&1 +docker run -d --name "$server" --network "container:$m" --user "$(id -u):$(id -g)" \ + -v "$work:/var/lib/extenddb" "extenddb/extenddb-mongodb:$version" >/dev/null +for _ in $(seq 1 120); do + curl -sk "https://127.0.0.1:$aport/health" | grep -q healthy && break + sleep 0.5 +done +mg() { + docker exec -e EXTENDDB_PASSWORD=$password "$server" extenddb manage --user admin \ + --config /var/lib/extenddb/extenddb.toml "$@" +} +mg create-account --account-id 123456789012 --account-name cu-seed >/dev/null +mg create-user --account-id 123456789012 --user-name cu-tester >/dev/null +mg put-user-policy --account-id 123456789012 --user-name cu-tester --policy-name dynamodb-full \ + --policy-document '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"dynamodb:*","Resource":"*"}]}' \ + >/dev/null +mg put-user-policy --account-id 123456789012 --user-name cu-tester --policy-name deny-prod \ + --policy-document '{"Version":"2012-10-17","Statement":[{"Effect":"Deny","Action":"dynamodb:DeleteTable","Resource":"*","Condition":{"StringEquals":{"dynamodb:ResourceTag/env":"prod"}}}]}' \ + >/dev/null +mg create-access-key --account-id 123456789012 --user-name cu-tester > "$work/access-key.json" +key=$(python3 -c "import json,sys; print(json.load(open(sys.argv[1]))['access_key_id'])" "$work/access-key.json") +secret=$(python3 -c "import json,sys; print(json.load(open(sys.argv[1]))['secret_access_key'])" "$work/access-key.json") +python3 "$here/seed.py" "https://127.0.0.1:$aport" "$key" "$secret" +out="$here/$version.json" +if [ "$variant" = dotted-ttl ]; then + python3 "$here/seed.py" "https://127.0.0.1:$aport" "$key" "$secret" cu_dotted exp.at + docker exec "$m" mongosh --quiet --port "$mport" --eval ' + const d = db.getSiblingDB("extenddb_catalog"); + const t = d.tables.findOne({ "_id.table_name": "cu_dotted" }); + db.getSiblingDB("extenddb_data").getCollection("_ddb_" + t.table_id) + .createIndex({ "item_data.exp.at.N": 1 }, { name: "idx_ttl_exp.at", sparse: true });' >/dev/null + out="$here/dotted-ttl.json" +fi +docker stop "$server" >/dev/null +docker exec "$m" mongosh --quiet --port "$mport" --file /fixtures/dump.js > "$out" +echo "wrote $out" diff --git a/crates/storage-mongodb/tests/fixtures/released/seed.py b/crates/storage-mongodb/tests/fixtures/released/seed.py new file mode 100755 index 000000000..536419503 --- /dev/null +++ b/crates/storage-mongodb/tests/fixtures/released/seed.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 +"""Create the table the released-catalog fixtures and the fresh comparison +carry, through the DynamoDB API: + + seed.py [table [TTL attribute]] + +The table (default `cu_orders`): hash key `pk`, range key `sk`, the GSI +`by_gpk` (`gpk`, `gsk` number), the LSI `by_lsk` (`pk`, `lsk`), a +NEW_AND_OLD_IMAGES stream, TTL on the attribute (default `expires`), and ten +items. Exits 1 on the first step that fails. +""" +import sys +import time + +import boto3 +import urllib3 +from botocore.config import Config + + +def main(endpoint: str, key: str, secret: str, table: str, ttl: str) -> int: + urllib3.disable_warnings() + ddb = boto3.client( + "dynamodb", + endpoint_url=endpoint, + region_name="us-east-1", + aws_access_key_id=key, + aws_secret_access_key=secret, + verify=False, + config=Config(retries={"max_attempts": 3}), + ) + ddb.create_table( + TableName=table, + AttributeDefinitions=[ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "sk", "AttributeType": "S"}, + {"AttributeName": "gpk", "AttributeType": "S"}, + {"AttributeName": "gsk", "AttributeType": "N"}, + {"AttributeName": "lsk", "AttributeType": "S"}, + ], + KeySchema=[ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"}, + ], + GlobalSecondaryIndexes=[ + { + "IndexName": "by_gpk", + "KeySchema": [ + {"AttributeName": "gpk", "KeyType": "HASH"}, + {"AttributeName": "gsk", "KeyType": "RANGE"}, + ], + "Projection": {"ProjectionType": "ALL"}, + } + ], + LocalSecondaryIndexes=[ + { + "IndexName": "by_lsk", + "KeySchema": [ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "lsk", "KeyType": "RANGE"}, + ], + "Projection": {"ProjectionType": "KEYS_ONLY"}, + } + ], + BillingMode="PAY_PER_REQUEST", + StreamSpecification={"StreamEnabled": True, "StreamViewType": "NEW_AND_OLD_IMAGES"}, + ) + for _ in range(120): + status = ddb.describe_table(TableName=table)["Table"]["TableStatus"] + if status == "ACTIVE": + break + time.sleep(0.5) + else: + print(f"{table} did not become ACTIVE", file=sys.stderr) + return 1 + ddb.update_time_to_live( + TableName=table, + TimeToLiveSpecification={"Enabled": True, "AttributeName": ttl}, + ) + for i in range(10): + ddb.put_item( + TableName=table, + Item={ + "pk": {"S": f"p{i % 3}"}, + "sk": {"S": f"s{i:02d}"}, + "gpk": {"S": f"g{i % 2}"}, + "gsk": {"N": str(i)}, + "lsk": {"S": f"l{9 - i}"}, + ttl: {"N": str(4102444800 + i)}, + "body": {"S": f"item {i}"}, + }, + ) + print(f"seeded {table}") + return 0 + + +if __name__ == "__main__": + if len(sys.argv) not in (4, 5, 6): + sys.exit(__doc__) + table = sys.argv[4] if len(sys.argv) > 4 else "cu_orders" + ttl = sys.argv[5] if len(sys.argv) > 5 else "expires" + sys.exit(main(*sys.argv[1:4], table, ttl)) diff --git a/crates/storage-mongodb/tests/migrations_fresh.rs b/crates/storage-mongodb/tests/migrations_fresh.rs new file mode 100644 index 000000000..88dccb774 --- /dev/null +++ b/crates/storage-mongodb/tests/migrations_fresh.rs @@ -0,0 +1,313 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Fresh installs on MongoDB: what is a catalog, what migration 1 creates, +//! reruns, `--reconverge`, `--forget`, and the statuses the ledger and the +//! settings decide. + +mod migrations_support; + +use std::time::Duration; + +use bson::{Document, doc}; +use extenddb_storage::migrations::{ + CatalogStatus, MigrationBackend, MigrationError, MigrationId, Outcome, Runner, exit, +}; +use extenddb_storage_mongodb::migrations::{ + Definitions, MigrationLock, MongoMigrations, Timing, describe_store, lock, +}; +use migrations_support::{Deployment, server}; + +const CATALOG_EXPECTED: &str = include_str!("../migrations/catalog/0001_baseline.expected.json"); +const DATA_EXPECTED: &str = include_str!("../migrations/data/0001_baseline.expected.json"); + +async fn definitions(d: &Deployment) -> (Definitions, Definitions) { + let migrations = MongoMigrations::open(&d.target, None).await.expect("open"); + let stores = migrations.stores(); + let catalog = describe_store(stores, &d.catalog()).await.expect("catalog"); + let data = describe_store(stores, &d.data()).await.expect("data"); + migrations.close().await; + (catalog, data) +} + +#[tokio::test(flavor = "multi_thread")] +async fn stores_that_are_not_catalogs_are_uninitialized_and_migrate_writes_nothing_to_them() { + let Some(uri) = server() else { return }; + // An empty store. + let empty = Deployment::new(&uri, "empty").await; + assert_eq!(empty.status().await, CatalogStatus::Uninitialized); + let refused = lock( + &empty.target, + false, + Duration::from_secs(2), + Timing::PRODUCTION, + ) + .await + .err() + .expect("refused"); + assert_eq!( + refused, + MigrationError::Status(CatalogStatus::Uninitialized) + ); + assert_eq!(refused.exit_code(), exit::ERROR); + assert!(Deployment::collections(&empty.catalog()).await.is_empty()); + + // A store holding only the migration lock, left by a holder that died. + let locked = Deployment::new(&uri, "lockonly").await; + MigrationLock::acquire(&locked.target, Duration::from_secs(5), Timing::PRODUCTION) + .await + .expect("lock") + .abandon(); + assert_eq!( + Deployment::collections(&locked.catalog()).await, + ["settings"] + ); + assert_eq!(locked.status().await, CatalogStatus::Uninitialized); + + // Another program's database. + let other = Deployment::new(&uri, "other").await; + other + .catalog() + .collection::("orders") + .insert_one(doc! { "order": 1 }) + .await + .expect("insert"); + assert_eq!(other.status().await, CatalogStatus::Uninitialized); + let refused = lock( + &other.target, + false, + Duration::from_secs(2), + Timing::PRODUCTION, + ) + .await + .err() + .expect("refused"); + assert_eq!(refused.exit_code(), exit::ERROR); + assert_eq!(Deployment::collections(&other.catalog()).await, ["orders"]); + for d in [empty, locked, other] { + d.drop_databases().await; + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn a_fresh_store_gets_every_migration_and_the_expected_definitions() { + let Some(uri) = server() else { return }; + let d = Deployment::new(&uri, "fresh").await; + let report = d.create().await; + assert_eq!((report.from, report.to), (None, Some(4))); + let applied: Vec<(u32, String)> = report + .applied + .iter() + .map(|s| (s.id, s.store_name.clone())) + .collect(); + assert_eq!( + applied, + [ + (1, d.target.catalog.clone()), + (1, d.target.data.clone()), + (2, d.target.catalog.clone()), + (3, d.target.catalog.clone()), + (4, d.target.catalog.clone()), + ] + ); + assert_eq!(d.status().await, CatalogStatus::Current { version: 4 }); + let rows = d.catalog_ledger().await; + let outcomes: Vec<(u32, Outcome)> = rows.iter().map(|r| (r.id, r.outcome)).collect(); + assert_eq!( + outcomes, + [ + (1, Outcome::Applied), + (2, Outcome::Applied), + (3, Outcome::NotApplicable), + (4, Outcome::NotApplicable), + ] + ); + assert!( + rows.iter() + .all(|r| r.applied_by == "test" && !r.applied_at.is_empty()) + ); + let data_rows = Deployment::documents(&d.data(), "extenddb_migrations").await; + assert_eq!(data_rows.len(), 1); + assert_eq!(data_rows[0].get_i64("_id"), Ok(1)); + assert_eq!(d.setting("catalog_version").await.as_deref(), Some("4")); + assert_eq!(d.setting("resource_tag_migration").await, None); + let history = Deployment::documents(&d.catalog(), "schema_history").await; + assert_eq!(history.len(), 1); + assert_eq!(history[0].get_str("_id"), Ok("001_initial")); + + let (catalog, data) = definitions(&d).await; + if std::env::var("EXTENDDB_BLESS_EXPECTED").as_deref() == Ok("1") { + let dir = env!("CARGO_MANIFEST_DIR"); + for (path, defs) in [("catalog", &catalog), ("data", &data)] { + let text = serde_json::to_string_pretty(defs).expect("json") + "\n"; + std::fs::write( + format!("{dir}/migrations/{path}/0001_baseline.expected.json"), + text, + ) + .expect("write"); + } + } + let expected_catalog: Definitions = serde_json::from_str(CATALOG_EXPECTED).expect("catalog"); + let expected_data: Definitions = serde_json::from_str(DATA_EXPECTED).expect("data"); + assert_eq!(catalog, expected_catalog); + assert_eq!(data, expected_data); + d.drop_databases().await; +} + +#[tokio::test(flavor = "multi_thread")] +async fn a_second_run_applies_nothing_and_reconverge_keeps_the_rows() { + let Some(uri) = server() else { return }; + let d = Deployment::new(&uri, "rerun").await; + d.create().await; + let before = d.catalog_ledger().await; + let again = d.migrate().await.expect("migrate"); + assert!(again.applied.is_empty()); + assert_eq!((again.from, again.to), (Some(4), Some(4))); + + let reconverged = d.reconverge().await.expect("reconverge"); + assert_eq!(reconverged.applied.len(), 5); + assert_eq!( + d.catalog_ledger().await, + before, + "reconverge leaves the rows as they are" + ); + assert_eq!(d.status().await, CatalogStatus::Current { version: 4 }); + d.drop_databases().await; +} + +#[tokio::test(flavor = "multi_thread")] +async fn forget_clears_an_unreleased_id_and_migrate_applies_it_again() { + let Some(uri) = server() else { return }; + let d = Deployment::new(&uri, "forget").await; + d.create().await; + let held = MigrationLock::acquire(&d.target, Duration::from_secs(5), Timing::PRODUCTION) + .await + .expect("lock"); + let migrations = MongoMigrations::open(&d.target, Some(&held)) + .await + .expect("open"); + Runner::new(&migrations, "test") + .forget(1) + .await + .expect("forget 1"); + migrations.close().await; + held.release().await; + // Migration 1 is gone from both ledgers, so the version is 0. + assert!( + Deployment::documents(&d.data(), "extenddb_migrations") + .await + .is_empty() + ); + assert_eq!(d.setting("catalog_version").await.as_deref(), Some("0")); + let CatalogStatus::Behind { version, pending } = d.status().await else { + panic!("not behind"); + }; + assert_eq!(version, Some(0)); + assert_eq!(pending.iter().map(|p| p.id).collect::>(), [1, 1]); + let report = d.migrate().await.expect("migrate"); + assert_eq!(report.applied.len(), 2); + assert_eq!(d.status().await, CatalogStatus::Current { version: 4 }); + d.drop_databases().await; +} + +#[tokio::test(flavor = "multi_thread")] +async fn a_changed_checksum_a_newer_catalog_and_an_older_release_are_reported() { + let Some(uri) = server() else { return }; + let d = Deployment::new(&uri, "statuses").await; + d.create().await; + let ledger = d.catalog().collection::("extenddb_migrations"); + let settings = d.catalog().collection::("settings"); + + // A later binary's breaking migration, then an additive one. + let row = |id: i64, kind: &str| { + doc! { "_id": id, "name": "later", "kind": kind, "checksum": "c", "outcome": "applied", + "notes": "", "applied_at": bson::DateTime::now(), "applied_by": "9.9.9" } + }; + ledger.insert_one(row(5, "breaking")).await.expect("insert"); + let status = d.status().await; + assert_eq!(status.label(), "AheadIncompatible"); + assert_eq!(status.exit_code(), exit::AHEAD_INCOMPATIBLE); + ledger + .delete_one(doc! { "_id": 5_i64 }) + .await + .expect("delete"); + ledger.insert_one(row(5, "additive")).await.expect("insert"); + assert_eq!(d.status().await.label(), "AheadCompatible"); + assert!(d.migrate().await.expect("migrate").applied.is_empty()); + ledger + .delete_one(doc! { "_id": 5_i64 }) + .await + .expect("delete"); + + // A released migrate --yes writes the marker again, and a released + // binary's dotted version. + settings + .insert_one(doc! { "_id": "resource_tag_migration", "value": "applied" }) + .await + .expect("marker"); + assert_eq!(d.status().await.label(), "WrittenByOlderRelease"); + let refused = d.migrate().await.expect_err("refused"); + assert_eq!(refused.exit_code(), exit::ATTENTION); + d.reconverge().await.expect("reconverge"); + assert_eq!(d.setting("resource_tag_migration").await, None); + assert_eq!(d.status().await, CatalogStatus::Current { version: 4 }); + settings + .update_one( + doc! { "_id": "catalog_version" }, + doc! { "$set": { "value": "0.0.2" } }, + ) + .await + .expect("version"); + assert_eq!(d.status().await.label(), "WrittenByOlderRelease"); + d.reconverge().await.expect("reconverge"); + assert_eq!(d.setting("catalog_version").await.as_deref(), Some("4")); + + // A recorded checksum this binary's body does not have. + ledger + .update_one( + doc! { "_id": 1_i64 }, + doc! { "$set": { "checksum": "edited" } }, + ) + .await + .expect("checksum"); + let status = d.status().await; + assert_eq!(status.label(), "Changed"); + assert_eq!(status.exit_code(), exit::ATTENTION); + assert!(status.to_string().contains("edited"), "{status}"); + d.drop_databases().await; +} + +#[tokio::test(flavor = "multi_thread")] +async fn a_ledger_document_this_binary_cannot_read_stops_the_status() { + let Some(uri) = server() else { return }; + let d = Deployment::new(&uri, "badrow").await; + d.create().await; + let ledger = d.data().collection::("extenddb_migrations"); + ledger + .update_one(doc! { "_id": 1_i64 }, doc! { "$unset": { "kind": "" } }) + .await + .expect("edit"); + let migrations = MongoMigrations::open(&d.target, None).await.expect("open"); + let error = Runner::new(&migrations, "") + .status() + .await + .expect_err("an error"); + migrations.close().await; + assert!(error.to_string().contains("has no kind"), "{error}"); + ledger + .update_one( + doc! { "_id": 1_i64 }, + doc! { "$set": { "kind": "sideways" } }, + ) + .await + .expect("edit"); + let migrations = MongoMigrations::open(&d.target, None).await.expect("open"); + let error = Runner::new(&migrations, "") + .status() + .await + .expect_err("an error"); + migrations.close().await; + assert!(error.to_string().contains("\"sideways\""), "{error}"); + assert_eq!(MigrationId::LATEST.number(), 4); + d.drop_databases().await; +} diff --git a/crates/storage-mongodb/tests/migrations_guards.rs b/crates/storage-mongodb/tests/migrations_guards.rs new file mode 100644 index 000000000..2b460e741 --- /dev/null +++ b/crates/storage-mongodb/tests/migrations_guards.rs @@ -0,0 +1,294 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! What stops a MongoDB run and what a rerun finishes: the post-check, a +//! failed write, indexes a deployment lacks, and a run stopped at every +//! request it sends. + +mod migrations_support; + +use std::time::Duration; + +use bson::{Document, doc}; +use extenddb_storage::migrations::{CatalogStatus, MigrationBackend, Runner}; +use extenddb_storage_mongodb::migrations::{ + Definitions, MigrationLock, MongoMigrations, Timing, describe_store, lock, +}; +use migrations_support::{Deployment, fixture, server}; + +const CATALOG_EXPECTED: &str = include_str!("../migrations/catalog/0001_baseline.expected.json"); +const DATA_EXPECTED: &str = include_str!("../migrations/data/0001_baseline.expected.json"); + +async fn index_names(db: &mongodb::Database, collection: &str) -> Vec { + let mut names = db + .collection::(collection) + .list_index_names() + .await + .expect("indexes"); + names.sort(); + names +} + +/// Run every migration with the runner stopped before request `requests`. +/// Returns whether the run finished, or the error. +async fn run_stopped_after(d: &Deployment, requests: usize) -> Result<(), String> { + let creating = d.status().await == CatalogStatus::Uninitialized; + let (held, target) = lock( + &d.target, + creating, + Duration::from_secs(10), + Timing::PRODUCTION, + ) + .await + .expect("lock"); + let migrations = MongoMigrations::open(&target, Some(&held)) + .await + .expect("open") + .stop_after(requests); + let finished = Runner::new(&migrations, "test") + .migrate() + .await + .map(|_| ()) + .map_err(|e| e.to_string()); + migrations.close().await; + held.release().await; + finished +} + +/// Both databases match a fresh install's definitions. +async fn assert_fresh_definitions(d: &Deployment) { + let migrations = MongoMigrations::open(&d.target, None).await.expect("open"); + let catalog = describe_store(migrations.stores(), &d.catalog()) + .await + .expect("catalog"); + let data = describe_store(migrations.stores(), &d.data()) + .await + .expect("data"); + migrations.close().await; + let expected_catalog: Definitions = serde_json::from_str(CATALOG_EXPECTED).expect("json"); + let expected_data: Definitions = serde_json::from_str(DATA_EXPECTED).expect("json"); + for (name, want) in &expected_catalog { + assert_eq!(catalog.get(name), Some(want), "{name}"); + } + for (name, want) in &expected_data { + assert_eq!(data.get(name), Some(want), "{name}"); + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn a_store_whose_objects_differ_records_nothing_and_names_the_object() { + let Some(uri) = server() else { return }; + let d = Deployment::new(&uri, "differ").await; + d.catalog() + .run_command(doc! { "create": "tags", "capped": true, "size": 4096 }) + .await + .expect("capped"); + let (held, target) = lock(&d.target, true, Duration::from_secs(5), Timing::PRODUCTION) + .await + .expect("lock"); + let migrations = MongoMigrations::open(&target, Some(&held)) + .await + .expect("open"); + let error = Runner::new(&migrations, "test") + .migrate() + .await + .expect_err("refused"); + migrations.close().await; + held.release().await; + assert!(error.to_string().contains("the collection tags"), "{error}"); + assert!( + Deployment::documents(&d.catalog(), "extenddb_migrations") + .await + .is_empty() + ); + assert_eq!(d.status().await, CatalogStatus::Uninitialized); + d.drop_databases().await; + + // An index of the same key with other options fails to create, by the + // server's own conflict code, and nothing is recorded. + let d = Deployment::new(&uri, "conflict").await; + d.catalog() + .collection::("accounts") + .create_index( + mongodb::IndexModel::builder() + .keys(doc! { "account_name": 1 }) + .build(), + ) + .await + .expect("index"); + let (held, target) = lock(&d.target, true, Duration::from_secs(5), Timing::PRODUCTION) + .await + .expect("lock"); + let migrations = MongoMigrations::open(&target, Some(&held)) + .await + .expect("open"); + let error = Runner::new(&migrations, "test") + .migrate() + .await + .expect_err("refused"); + migrations.close().await; + held.release().await; + assert!(error.to_string().contains("code 86"), "{error}"); + assert!(error.to_string().contains("account_name"), "{error}"); + assert_eq!(d.status().await, CatalogStatus::Uninitialized); + d.drop_databases().await; +} + +#[tokio::test(flavor = "multi_thread")] +async fn a_failed_write_stops_the_run_instead_of_being_discarded() { + let Some(uri) = server() else { return }; + let d = Deployment::new(&uri, "refusedwrite").await; + // Released bootstraps discarded this error and recorded 001_initial + // anyway (MongoDB item 13). + d.catalog() + .run_command(doc! { + "create": "schema_history", + "validator": { "_id": { "$ne": "001_initial" } }, + }) + .await + .expect("validator"); + let (held, target) = lock(&d.target, true, Duration::from_secs(5), Timing::PRODUCTION) + .await + .expect("lock"); + let migrations = MongoMigrations::open(&target, Some(&held)) + .await + .expect("open"); + let error = Runner::new(&migrations, "test") + .migrate() + .await + .expect_err("refused"); + migrations.close().await; + held.release().await; + assert!(error.to_string().contains("code 121"), "{error}"); + assert!( + Deployment::documents(&d.catalog(), "extenddb_migrations") + .await + .is_empty() + ); + assert_eq!(d.setting("catalog_version").await, None); + d.drop_databases().await; +} + +#[tokio::test(flavor = "multi_thread")] +async fn migrate_creates_the_indexes_a_deployment_lacks() { + let Some(uri) = server() else { return }; + let d = Deployment::new(&uri, "missingidx").await; + fixture::load(&d, &fixture::read("0.1.14")).await; + for (db, collection, index) in [ + (d.data(), "idempotency_tokens", "account_id_1_token_1"), + (d.data(), "stream_shards", "shard_id_1"), + (d.catalog(), "tables", "table_id_1"), + ] { + db.collection::(collection) + .drop_index(index) + .await + .expect("drop index"); + } + d.migrate().await.expect("migrate"); + assert!( + index_names(&d.data(), "idempotency_tokens") + .await + .contains(&"account_id_1_token_1".to_owned()) + ); + assert!( + index_names(&d.data(), "stream_shards") + .await + .contains(&"shard_id_1".to_owned()) + ); + assert!( + index_names(&d.catalog(), "tables") + .await + .contains(&"table_id_1".to_owned()) + ); + assert_eq!(d.status().await, CatalogStatus::Current { version: 4 }); + assert_fresh_definitions(&d).await; + d.drop_databases().await; +} + +/// The runner stopped before each request in turn, as a migrator killed at +/// that point stops, then run again: every rerun finishes with the +/// definitions of a fresh install and no marker, and on a released catalog +/// with the policy rewritten. Returns how many runs stopped. +async fn stop_at_every_request(uri: &str, state: Option<&str>) -> usize { + let mut stopped_runs = 0; + for requests in 0.. { + let d = Deployment::new(uri, "stopped").await; + if let Some(state) = state { + fixture::load(&d, &fixture::read(state)).await; + } + let finished = run_stopped_after(&d, requests).await.is_ok(); + if !finished { + stopped_runs += 1; + if let Err(e) = run_stopped_after(&d, usize::MAX).await { + panic!("{state:?}: the rerun after {requests}: {e}"); + } + } + assert_eq!( + d.status().await, + CatalogStatus::Current { version: 4 }, + "{requests}" + ); + assert_fresh_definitions(&d).await; + assert_eq!(d.setting("resource_tag_migration").await, None); + if state.is_some() { + let deny = d + .catalog() + .collection::("iam_policies") + .find_one(doc! { "policy_name": "deny-prod" }) + .await + .expect("policy") + .expect("deny-prod"); + let text = deny + .get_document("policy_document") + .expect("document") + .to_string(); + assert!(text.contains("aws:ResourceTag/env"), "{requests}: {text}"); + assert!(!text.contains("dynamodb:ResourceTag"), "{requests}: {text}"); + } + d.drop_databases().await; + if finished { + break; + } + } + stopped_runs +} + +#[tokio::test(flavor = "multi_thread")] +async fn a_fresh_run_stopped_at_any_request_finishes_on_the_next_run() { + let Some(uri) = server() else { return }; + assert!(stop_at_every_request(&uri, None).await > 10); +} + +#[tokio::test(flavor = "multi_thread")] +async fn an_upgrade_stopped_at_any_request_finishes_on_the_next_run() { + let Some(uri) = server() else { return }; + assert!(stop_at_every_request(&uri, Some("0.1.12")).await > 10); +} + +#[tokio::test(flavor = "multi_thread")] +async fn a_lock_left_by_a_killed_holder_is_taken_once_it_expires() { + let Some(uri) = server() else { return }; + let d = Deployment::new(&uri, "expired").await; + let timing = Timing { + lease: Duration::from_secs(2), + renew_every: Duration::from_millis(500), + margin: Duration::from_millis(800), + idle_limit: Duration::from_secs(30), + retry_every: Duration::from_millis(100), + ..Timing::PRODUCTION + }; + MigrationLock::acquire(&d.target, Duration::from_secs(5), timing) + .await + .expect("lock") + .abandon(); + let refused = MigrationLock::acquire(&d.target, Duration::from_millis(500), timing) + .await + .err() + .expect("held"); + assert_eq!(refused.exit_code(), 6); + let again = MigrationLock::acquire(&d.target, Duration::from_secs(5), timing) + .await + .expect("taken after the lease"); + again.release().await; + d.drop_databases().await; +} diff --git a/crates/storage-mongodb/tests/migrations_holders.rs b/crates/storage-mongodb/tests/migrations_holders.rs new file mode 100644 index 000000000..2d3fe5a81 --- /dev/null +++ b/crates/storage-mongodb/tests/migrations_holders.rs @@ -0,0 +1,354 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! What a migrator and a server do while they hold the migration lock or a +//! serving registration: migration 2's store calls keep the lock while they +//! complete and stop at its deadline, a server that renews its registration +//! stays registered, a server starts beside a lock whose lease ended, and a +//! dropped lock is released. Also the collision `migrate --yes` reads before +//! its first write. See `migrations_support` for the server these tests need. + +mod migrations_support; + +use std::time::{Duration, Instant}; + +use bson::{Document, doc}; +use extenddb_storage::management_store::OpError; +use extenddb_storage::migrations::{ + CatalogStatus, HasManagementStore, HeldLock, MigrationBackend, MigrationError, +}; +use extenddb_storage_mongodb::migrations::{ + MigrationLock, MongoMigrations, ServingRegistration, Timing, lock, migrate_holding, + register_and_check, status_holding, +}; +use migrations_support::{Deployment, fixture, server}; + +/// The id of the deployment's first account: `account_id`, or `_id` for the +/// default account a released `init` wrote without one. +async fn first_account(d: &Deployment) -> String { + let account = d + .catalog() + .collection::("accounts") + .find_one(doc! {}) + .await + .expect("read accounts") + .expect("an account"); + account + .get_str("account_id") + .or_else(|_| account.get_str("_id")) + .expect("an account id") + .to_owned() +} + +/// Add IAM users `0` to `` to `account`, each +/// with one inline policy holding `document`. +async fn add_users(d: &Deployment, account: &str, prefix: &str, count: usize, document: &Document) { + let now = bson::DateTime::now(); + let mut users = Vec::new(); + let mut policies = Vec::new(); + for i in 0..count { + let name = format!("{prefix}{i}"); + users.push(doc! { + "account_id": account, + "user_name": &name, + "user_arn": format!("arn:aws:iam::{account}:user/{name}"), + "created_at": now, + }); + policies.push(doc! { + "account_id": account, + "principal_type": "user", + "principal_name": &name, + "policy_name": "p", + "policy_document": document.clone(), + "created_at": now, + }); + } + let catalog = d.catalog(); + catalog + .collection::("iam_users") + .insert_many(users) + .await + .expect("insert users"); + catalog + .collection::("iam_policies") + .insert_many(policies) + .await + .expect("insert policies"); +} + +fn allow_get_item() -> Document { + doc! { + "Version": "2012-10-17", + "Statement": [{ "Effect": "Allow", "Action": "dynamodb:GetItem", "Resource": "*" }], + } +} + +/// Migration 2 sends two requests per user through the leased management +/// store, for its `detect` and again for its run. Each one that completes is +/// progress, so a scan that lasts longer than the idle limit keeps the lock +/// and the migration is applied. +#[tokio::test(flavor = "multi_thread")] +async fn migration_2_keeps_the_lock_through_a_scan_longer_than_the_idle_limit() { + let Some(uri) = server() else { return }; + let d = Deployment::new(&uri, "scan").await; + fixture::load(&d, &fixture::read("0.1.14")).await; + let account = first_account(&d).await; + add_users(&d, &account, "scan", 1000, &allow_get_item()).await; + let timing = Timing { + lease: Duration::from_secs(20), + renew_every: Duration::from_millis(100), + margin: Duration::from_secs(5), + idle_limit: Duration::from_millis(300), + ..Timing::PRODUCTION + }; + let (held, target) = lock(&d.target, false, Duration::from_secs(10), timing) + .await + .expect("lock"); + let started = Instant::now(); + let applied = migrate_holding(&target, "test", &held).await; + let took = started.elapsed(); + held.release().await; + applied.expect("every migration applies"); + // A shorter run would apply without the progress the scan records. + assert!(took > timing.idle_limit * 4, "the run took {took:?}"); + assert_eq!(d.status().await, CatalogStatus::Current { version: 4 }); + assert!(d.catalog_ledger().await.iter().any(|row| row.id == 2)); + d.drop_databases().await; +} + +/// `migrate --yes` reads the status under the lock before it runs, and on a +/// catalog behind migration 2 the status runs migration 2's `detect`. Read +/// as the holder, its calls keep the lock through a scan longer than the +/// idle limit, and the run that follows applies. +#[tokio::test(flavor = "multi_thread")] +async fn the_status_read_under_the_lock_keeps_it_through_a_long_scan() { + let Some(uri) = server() else { return }; + let d = Deployment::new(&uri, "lockedstatus").await; + fixture::load(&d, &fixture::read("0.1.14")).await; + let account = first_account(&d).await; + add_users(&d, &account, "status", 1000, &allow_get_item()).await; + let timing = Timing { + lease: Duration::from_secs(20), + renew_every: Duration::from_millis(100), + margin: Duration::from_secs(5), + idle_limit: Duration::from_millis(300), + ..Timing::PRODUCTION + }; + let (held, target) = lock(&d.target, false, Duration::from_secs(10), timing) + .await + .expect("lock"); + let started = Instant::now(); + let status = status_holding(&target, &held).await; + let took = started.elapsed(); + let applied = migrate_holding(&target, "test", &held).await; + held.release().await; + status.expect("the status"); + // A shorter read would keep the lock without the progress it records. + assert!( + took > timing.idle_limit * 2, + "the status read took {took:?}" + ); + applied.expect("every migration applies"); + assert_eq!(d.status().await, CatalogStatus::Current { version: 4 }); + d.drop_databases().await; +} + +/// Past the lock's deadline the leased store sends nothing: a policy write +/// migration 2 would send is refused before it reaches the database. +#[tokio::test(flavor = "multi_thread")] +async fn migration_2_writes_nothing_after_the_lock_deadline() { + let Some(uri) = server() else { return }; + let d = Deployment::new(&uri, "storedeadline").await; + fixture::load(&d, &fixture::read("0.1.14")).await; + let account = first_account(&d).await; + add_users(&d, &account, "late", 1, &allow_get_item()).await; + // No renewal before the deadline, 1.2 s after the claim. + let timing = Timing { + lease: Duration::from_secs(2), + renew_every: Duration::from_secs(60), + margin: Duration::from_millis(800), + ..Timing::PRODUCTION + }; + let held = MigrationLock::acquire(&d.target, Duration::from_secs(5), timing) + .await + .expect("lock"); + let migrations = MongoMigrations::open(&d.target, Some(&held)) + .await + .expect("open"); + let store = migrations.stores().management_store(); + let policy = serde_json::json!({ + "Version": "2012-10-17", + "Statement": [{ "Effect": "Allow", "Action": "dynamodb:Query", "Resource": "*" }], + }); + store + .put_policy(&account, "user", "late0", "before", &policy) + .await + .expect("before the deadline"); + tokio::time::sleep(Duration::from_millis(1300)).await; + let error = store + .put_policy(&account, "user", "late0", "after", &policy) + .await + .expect_err("past the deadline"); + let OpError::Internal(message) = &error else { + panic!("{error:?}"); + }; + assert!(message.contains("last renewed"), "{message}"); + migrations.close().await; + held.release().await; + let mut names: Vec = Deployment::documents(&d.catalog(), "iam_policies") + .await + .iter() + .filter(|p| p.get_str("principal_name") == Ok("late0")) + .map(|p| p.get_str("policy_name").unwrap_or_default().to_owned()) + .collect(); + names.sort(); + assert_eq!(names, ["before", "p"]); + d.drop_databases().await; +} + +/// A server that renews its registration stays registered: after three of +/// its registration lifetimes `migrate --yes` still finds it and waits. +#[tokio::test(flavor = "multi_thread")] +async fn a_renewed_registration_outlives_three_registration_lifetimes() { + let Some(uri) = server() else { return }; + let d = Deployment::new(&uri, "renewed").await; + d.create().await; + let timing = Timing { + registration: Duration::from_secs(2), + registration_deadline: Duration::from_millis(1500), + renew_every: Duration::from_millis(200), + retry_every: Duration::from_millis(100), + ..Timing::PRODUCTION + }; + let serving = ServingRegistration::register(&d.target, "test", timing) + .await + .expect("register"); + tokio::time::sleep(timing.registration * 3 + Duration::from_millis(500)).await; + assert_eq!(*serving.lost().borrow(), None, "the registration was lost"); + let error = lock( + &d.target, + false, + Duration::from_millis(500), + Timing::PRODUCTION, + ) + .await + .err() + .expect("a server is registered"); + assert_eq!(error.exit_code(), 6, "{error}"); + assert!( + error + .to_string() + .contains(&format!("pid {}", std::process::id())), + "{error}" + ); + serving.release().await; + d.drop_databases().await; +} + +/// A lock whose lease ended, as a killed migrator leaves it, does not stop a +/// server from starting; a live one does. +#[tokio::test(flavor = "multi_thread")] +async fn a_server_starts_beside_a_lock_whose_lease_ended() { + let Some(uri) = server() else { return }; + let d = Deployment::new(&uri, "expiredlock").await; + d.create().await; + let timing = Timing { + lease: Duration::from_secs(2), + renew_every: Duration::from_millis(500), + margin: Duration::from_millis(800), + retry_every: Duration::from_millis(100), + ..Timing::PRODUCTION + }; + let claimed = Instant::now(); + MigrationLock::acquire(&d.target, Duration::from_secs(5), timing) + .await + .expect("lock") + .abandon(); + let refused = register_and_check(&d.target, "test", Timing::PRODUCTION) + .await + .err() + .expect("the lease has not ended"); + assert_eq!(refused.exit_code(), 6, "{refused}"); + tokio::time::sleep( + (timing.lease + Duration::from_millis(500)).saturating_sub(claimed.elapsed()), + ) + .await; + register_and_check(&d.target, "test", Timing::PRODUCTION) + .await + .expect("serves once the lease ended") + .release() + .await; + d.drop_databases().await; +} + +/// `migrate --yes`, `--forget` and `init` release the lock by dropping the +/// shared [`HeldLock`], which has no release call, so a claim right after +/// the drop takes the lock at once. A single-threaded runtime, as the +/// commands use. +#[tokio::test] +async fn a_dropped_lock_is_released_at_once() { + let Some(uri) = server() else { return }; + let d = Deployment::new(&uri, "dropped").await; + fixture::load(&d, &fixture::read("0.1.14")).await; + let (held, target) = lock( + &d.target, + false, + Duration::from_secs(10), + Timing::PRODUCTION, + ) + .await + .expect("lock"); + migrate_holding(&target, "test", &held) + .await + .expect("migrate"); + drop(HeldLock::new(target.connection_string.clone(), held)); + MigrationLock::acquire(&d.target, Duration::from_secs(1), Timing::PRODUCTION) + .await + .expect("released by the drop") + .release() + .await; + d.drop_databases().await; +} + +/// A policy migration 2 cannot rewrite is found before the first write: +/// `migrate --yes` stops with `NeedsAttention` and records nothing, +/// migration 1 included. +#[tokio::test(flavor = "multi_thread")] +async fn a_collision_is_found_before_the_first_write_and_nothing_is_recorded() { + let Some(uri) = server() else { return }; + let d = Deployment::new(&uri, "collision").await; + fixture::load(&d, &fixture::read("0.1.14")).await; + let account = first_account(&d).await; + let both_forms = doc! { + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Deny", + "Action": "dynamodb:*", + "Resource": "*", + "Condition": { "StringEquals": { + "aws:ResourceTag/env": "prod", + "dynamodb:ResourceTag/env": "dev", + } }, + }], + }; + add_users(&d, &account, "collider", 1, &both_forms).await; + let error = d.migrate().await.expect_err("needs attention"); + let MigrationError::Status(CatalogStatus::NeedsAttention { items }) = &error else { + panic!("{error}"); + }; + assert!( + items.iter().any(|item| item.contains("collider0")), + "{items:?}" + ); + assert!( + Deployment::documents(&d.catalog(), "extenddb_migrations") + .await + .is_empty() + ); + assert!( + Deployment::documents(&d.data(), "extenddb_migrations") + .await + .is_empty() + ); + d.drop_databases().await; +} diff --git a/crates/storage-mongodb/tests/migrations_locks.rs b/crates/storage-mongodb/tests/migrations_locks.rs new file mode 100644 index 000000000..7bcd81a38 --- /dev/null +++ b/crates/storage-mongodb/tests/migrations_locks.rs @@ -0,0 +1,274 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The MongoDB migration lock and serving registration: two migrators, the +//! lease and its deadline, a hung holder, and servers beside migrations. + +mod migrations_support; + +use std::time::{Duration, Instant}; + +use bson::{Document, doc}; +use extenddb_storage::migrations::{CatalogStatus, MigrationError, Runner}; +use extenddb_storage_mongodb::migrations::{ + MigrationLock, MongoMigrations, ServingRegistration, Timing, lock, migrate_holding, + register_and_check, +}; +use migrations_support::{Deployment, fixture, server}; + +/// A lease of 4 s renewed every 1 s, acted on for 2.4 s after a renewal, and +/// registrations of 6 s that a server gives up on 4 s after its last renewal. +/// A renewal has 1.4 s to complete before the holder stops, enough on a +/// loaded CI runner; at 2 s and 0.5 s it had 0.7 s and failed under load. +const SHORT: Timing = Timing { + lease: Duration::from_secs(4), + renew_every: Duration::from_secs(1), + margin: Duration::from_millis(1600), + idle_limit: Duration::from_secs(30), + registration: Duration::from_secs(6), + registration_deadline: Duration::from_secs(4), + retry_every: Duration::from_millis(100), +}; + +#[tokio::test(flavor = "multi_thread")] +async fn two_migrators_started_together_apply_once_and_the_other_finds_nothing() { + let Some(uri) = server() else { return }; + let d = Deployment::new(&uri, "two").await; + fixture::load(&d, &fixture::read("0.1.12")).await; + let run = |target| async move { + let (held, target) = lock(&target, false, Duration::from_secs(60), Timing::PRODUCTION) + .await + .expect("lock"); + let report = migrate_holding(&target, "test", &held) + .await + .expect("migrate"); + held.release().await; + report + }; + let (a, b) = tokio::join!(run(d.target.clone()), run(d.target.clone())); + let mut applied = [a.applied.len(), b.applied.len()]; + applied.sort_unstable(); + assert_eq!( + applied, + [0, 5], + "one applies every step and the other finds nothing" + ); + assert_eq!(d.status().await, CatalogStatus::Current { version: 4 }); + d.drop_databases().await; +} + +#[tokio::test(flavor = "multi_thread")] +async fn the_holder_keeps_the_lock_for_three_leases_and_a_killed_one_is_replaced_within_a_lease() { + let Some(uri) = server() else { return }; + let d = Deployment::new(&uri, "lease").await; + let holder = MigrationLock::acquire(&d.target, Duration::from_secs(5), SHORT) + .await + .expect("lock"); + let started = Instant::now(); + let refused = MigrationLock::acquire(&d.target, SHORT.lease * 3, SHORT) + .await + .err() + .expect("refused for three leases"); + assert!(matches!(refused, MigrationError::Locked(_)), "{refused}"); + assert!(started.elapsed() >= SHORT.lease * 3); + holder.check().expect("the holder still holds it"); + let killed = Instant::now(); + holder.abandon(); + let next = MigrationLock::acquire(&d.target, Duration::from_secs(10), SHORT) + .await + .expect("acquired"); + let waited = killed.elapsed(); + assert!( + waited <= SHORT.lease + SHORT.renew_every + Duration::from_millis(500), + "acquired {waited:?} after the holder was killed" + ); + next.release().await; + d.drop_databases().await; +} + +#[tokio::test(flavor = "multi_thread")] +async fn a_holder_past_its_deadline_sends_nothing_and_records_nothing() { + let Some(uri) = server() else { return }; + let d = Deployment::new(&uri, "deadline").await; + // No renewal before the deadline: 2.4 s after the claim. + let timing = Timing { + renew_every: Duration::from_secs(60), + ..SHORT + }; + let held = MigrationLock::acquire(&d.target, Duration::from_secs(5), timing) + .await + .expect("lock"); + tokio::time::sleep(Duration::from_millis(2600)).await; + let migrations = MongoMigrations::open(&d.target, Some(&held)) + .await + .expect("open"); + let error = Runner::new(&migrations, "test") + .migrate() + .await + .expect_err("stopped"); + migrations.close().await; + held.release().await; + assert!(error.to_string().contains("last renewed"), "{error}"); + assert_eq!(Deployment::collections(&d.catalog()).await, ["settings"]); + assert!(Deployment::collections(&d.data()).await.is_empty()); + d.drop_databases().await; +} + +#[tokio::test(flavor = "multi_thread")] +async fn a_holder_whose_lock_another_process_took_stops_at_its_next_renewal() { + let Some(uri) = server() else { return }; + let d = Deployment::new(&uri, "taken").await; + let holder = MigrationLock::acquire(&d.target, Duration::from_secs(5), SHORT) + .await + .expect("lock"); + // Another process's claim, as it writes the document. + let settings = d.catalog().collection::("settings"); + settings + .update_one( + doc! { "_id": "migration_lock" }, + doc! { "$set": { "owner": "another" } }, + ) + .await + .expect("take the lock"); + tokio::time::sleep(SHORT.renew_every * 2).await; + let error = holder.check().expect_err("lost at the next renewal"); + assert!( + error.to_string().contains("no longer names this process"), + "{error}" + ); + // The release deletes this process's document only. + holder.release().await; + assert!( + settings + .find_one(doc! { "_id": "migration_lock", "owner": "another" }) + .await + .expect("read") + .is_some() + ); + d.drop_databases().await; +} + +#[tokio::test(flavor = "multi_thread")] +async fn a_holder_that_makes_no_progress_stops_renewing_and_loses_the_lock() { + let Some(uri) = server() else { return }; + let d = Deployment::new(&uri, "hung").await; + let timing = Timing { + idle_limit: Duration::from_secs(1), + ..SHORT + }; + let hung = MigrationLock::acquire(&d.target, Duration::from_secs(5), timing) + .await + .expect("lock"); + // The renewal task finds the holder idle at its first renewal, 1 s after + // the claim, and stops renewing; the lease then ends 4 s after the claim. + tokio::time::sleep(Duration::from_millis(2500)).await; + let error = hung.check().expect_err("lost"); + assert!(error.to_string().contains("no progress"), "{error}"); + let next = MigrationLock::acquire(&d.target, Duration::from_secs(10), SHORT) + .await + .expect("taken once the hung holder's lease ends"); + next.release().await; + hung.release().await; + d.drop_databases().await; +} + +#[tokio::test(flavor = "multi_thread")] +async fn migrate_waits_for_a_running_server_and_a_server_refuses_while_a_migration_runs() { + let Some(uri) = server() else { return }; + let d = Deployment::new(&uri, "serving").await; + d.create().await; + + let serving = register_and_check(&d.target, "test", Timing::PRODUCTION) + .await + .expect("serve"); + let error = lock(&d.target, false, Duration::from_secs(2), Timing::PRODUCTION) + .await + .err() + .expect("refused"); + assert_eq!(error.exit_code(), 6); + let message = error.to_string(); + assert!( + message.contains(&format!("pid {}", std::process::id())), + "{message}" + ); + // The refusal released the lock it took. + let settings = d.catalog().collection::("settings"); + assert!( + settings + .find_one(doc! { "_id": "migration_lock" }) + .await + .unwrap() + .is_none() + ); + // The preview and verify read the status beside the server. + assert_eq!(d.status().await, CatalogStatus::Current { version: 4 }); + serving.release().await; + let (held, _) = lock(&d.target, false, Duration::from_secs(2), Timing::PRODUCTION) + .await + .expect("no server runs"); + + let refused = register_and_check(&d.target, "test", Timing::PRODUCTION) + .await + .err() + .expect("a migration holds the lock"); + assert_eq!(refused.exit_code(), 6); + let servers = Deployment::documents(&d.catalog(), "extenddb_servers").await; + assert!( + servers.is_empty(), + "a refused server deletes its registration: {servers:?}" + ); + held.release().await; + register_and_check(&d.target, "test", Timing::PRODUCTION) + .await + .expect("serve") + .release() + .await; + d.drop_databases().await; +} + +#[tokio::test(flavor = "multi_thread")] +async fn a_server_refuses_a_catalog_it_cannot_serve_and_deletes_its_registration() { + let Some(uri) = server() else { return }; + let d = Deployment::new(&uri, "refuse").await; + let error = register_and_check(&d.target, "test", Timing::PRODUCTION) + .await + .err() + .expect("uninitialized"); + assert_eq!(error, MigrationError::Status(CatalogStatus::Uninitialized)); + fixture::load(&d, &fixture::read("0.1.14")).await; + let error = register_and_check(&d.target, "test", Timing::PRODUCTION) + .await + .err() + .expect("behind"); + assert_eq!(error.exit_code(), 3); + assert!( + Deployment::documents(&d.catalog(), "extenddb_servers") + .await + .is_empty() + ); + d.drop_databases().await; +} + +#[tokio::test(flavor = "multi_thread")] +async fn a_registration_that_cannot_be_renewed_is_reported_lost() { + let Some(uri) = server() else { return }; + let d = Deployment::new(&uri, "lost").await; + d.create().await; + let registration = ServingRegistration::register(&d.target, "test", SHORT) + .await + .expect("register"); + let mut lost = registration.lost(); + d.catalog() + .collection::("extenddb_servers") + .delete_many(doc! {}) + .await + .expect("delete"); + tokio::time::timeout(Duration::from_secs(5), lost.changed()) + .await + .expect("reported within a renewal") + .expect("changed"); + let reason = lost.borrow().clone().expect("a reason"); + assert!(reason.contains("deleted"), "{reason}"); + registration.release().await; + d.drop_databases().await; +} diff --git a/crates/storage-mongodb/tests/migrations_released.rs b/crates/storage-mongodb/tests/migrations_released.rs new file mode 100644 index 000000000..46e105cce --- /dev/null +++ b/crates/storage-mongodb/tests/migrations_released.rs @@ -0,0 +1,313 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Released MongoDB catalogs upgraded by the runner: the fixtures of the +//! 0.1.12, 0.1.13 and 0.1.14 images (0.1.12 stands for v0.1.4 to v0.1.11, +//! whose bootstrapper is the same), and `dotted-ttl`, a 0.1.12 catalog with +//! the TTL index releases before v0.1.12 created for a dotted attribute. +//! Each is compared after the upgrade with the definitions migration 1 +//! creates, and its per-table objects with what they were before, and the +//! code bodies' output with committed golden files. + +mod migrations_support; + +use std::collections::BTreeMap; + +use bson::{Document, doc}; +use extenddb_storage::migrations::{CatalogStatus, MigrationBackend}; +use extenddb_storage_mongodb::migrations::{Definitions, MongoMigrations, describe_store}; +use migrations_support::{Deployment, fixture, server}; +use serde_json::Value; + +const CATALOG_EXPECTED: &str = include_str!("../migrations/catalog/0001_baseline.expected.json"); +const DATA_EXPECTED: &str = include_str!("../migrations/data/0001_baseline.expected.json"); +const STATES: [&str; 4] = ["0.1.12", "0.1.13", "0.1.14", "dotted-ttl"]; + +/// The per-table collections and their indexes, named after the catalog's +/// table and index names instead of their ids: `T_` and +/// `I_
_`. +async fn per_table(d: &Deployment) -> BTreeMap> { + let mut names = BTreeMap::new(); + for t in Deployment::documents(&d.catalog(), "tables").await { + let table = t + .get_document("_id") + .unwrap() + .get_str("table_name") + .unwrap() + .to_owned(); + names.insert( + t.get_str("table_id").unwrap().to_owned(), + format!("T_{table}"), + ); + } + for i in Deployment::documents(&d.catalog(), "indexes").await { + let id = i.get_document("_id").unwrap(); + let table = names + .get(id.get_str("table_id").unwrap()) + .cloned() + .unwrap_or_default(); + names.insert( + i.get_str("index_id").unwrap().to_owned(), + format!("I_{}_{}", &table[2..], id.get_str("index_name").unwrap()), + ); + } + let mut objects = BTreeMap::new(); + for collection in Deployment::collections(&d.data()).await { + let Some(id) = collection.strip_prefix("_ddb_") else { + continue; + }; + let name = names + .get(id) + .cloned() + .unwrap_or_else(|| format!("unknown {id}")); + let cursor = d + .data() + .collection::(&collection) + .list_indexes() + .await + .unwrap(); + let mut indexes = index_texts(cursor).await; + indexes.sort(); + objects.insert(name, indexes); + } + objects +} + +/// The index definitions of a cursor, as (name, key and options). +async fn index_texts(cursor: mongodb::Cursor) -> Vec<(String, String)> { + use futures::TryStreamExt; + let models: Vec = cursor.try_collect().await.unwrap(); + models + .iter() + .map(|m| { + let name = m + .options + .as_ref() + .and_then(|o| o.name.clone()) + .unwrap_or_default(); + let options = m.options.as_ref().map(|o| { + format!( + "unique={:?} sparse={:?} ttl={:?} partial={:?}", + o.unique, o.sparse, o.expire_after, o.partial_filter_expression + ) + }); + (name, format!("{} {}", m.keys, options.unwrap_or_default())) + }) + .collect() +} + +async fn definitions(d: &Deployment) -> (Definitions, Definitions) { + let migrations = MongoMigrations::open(&d.target, None).await.expect("open"); + let catalog = describe_store(migrations.stores(), &d.catalog()) + .await + .expect("catalog"); + let data = describe_store(migrations.stores(), &d.data()) + .await + .expect("data"); + migrations.close().await; + (catalog, data) +} + +async fn setting_keys(d: &Deployment) -> Vec { + Deployment::documents(&d.catalog(), "settings") + .await + .iter() + .map(|s| s.get_str("_id").unwrap().to_owned()) + .collect() +} + +/// Write `value` to `tests/fixtures/golden/.txt` when +/// `EXTENDDB_BLESS_EXPECTED` is `1`, and compare it with the committed file. +fn golden(name: &str, value: &Value) { + let path = format!( + "{}/tests/fixtures/golden/{name}.txt", + env!("CARGO_MANIFEST_DIR") + ); + if std::env::var("EXTENDDB_BLESS_EXPECTED").as_deref() == Ok("1") { + std::fs::write(&path, serde_json::to_string_pretty(value).unwrap() + "\n").unwrap(); + } + let committed: Value = serde_json::from_str( + &std::fs::read_to_string(&path).unwrap_or_else(|e| panic!("{path}: {e}")), + ) + .unwrap(); + assert_eq!(&committed, value, "{path}"); +} + +#[tokio::test(flavor = "multi_thread")] +async fn every_released_catalog_upgrades_to_the_definitions_of_a_fresh_install() { + let Some(uri) = server() else { return }; + let expected_catalog: Definitions = serde_json::from_str(CATALOG_EXPECTED).unwrap(); + let expected_data: Definitions = serde_json::from_str(DATA_EXPECTED).unwrap(); + let fresh = Deployment::new(&uri, "freshkeys").await; + fresh.create().await; + let mut fresh_keys = setting_keys(&fresh).await; + fresh.drop_databases().await; + // `init` writes the encryption key and the data connection rows beside + // the runner, so a fresh install made by `init` has these keys as well. + fresh_keys.extend( + [ + "data_connection_string", + "data_database_name", + "encryption_key", + ] + .map(str::to_owned), + ); + fresh_keys.sort(); + + for state in STATES { + let d = Deployment::new(&uri, "released").await; + fixture::load(&d, &fixture::read(state)).await; + let before = per_table(&d).await; + let CatalogStatus::Behind { version, pending } = d.status().await else { + panic!("{state} is not behind"); + }; + assert_eq!(version, None, "{state} has no ledger"); + assert_eq!(pending.len(), 5, "{state}"); + let report = d.migrate().await.expect("migrate"); + assert_eq!((report.from, report.to), (None, Some(4)), "{state}"); + assert_eq!( + d.status().await, + CatalogStatus::Current { version: 4 }, + "{state}" + ); + + let (catalog, data) = definitions(&d).await; + for (name, want) in &expected_catalog { + assert_eq!(catalog.get(name), Some(want), "{state}: {name}"); + } + for (name, want) in &expected_data { + assert_eq!(data.get(name), Some(want), "{state}: {name}"); + } + // Collections migration 1 does not create: released catalogs and a + // fresh install have the same set. `counters` is created on first + // use by streams in both. + let extra_catalog: Vec<&String> = catalog + .keys() + .filter(|n| !expected_catalog.contains_key(*n)) + .collect(); + assert!(extra_catalog.is_empty(), "{state}: {extra_catalog:?}"); + let extra_data: Vec<&String> = data + .keys() + .filter(|n| !expected_data.contains_key(*n)) + .collect(); + assert_eq!(extra_data, ["counters"], "{state}"); + + // No migration changes a per-table object on MongoDB. + assert_eq!(per_table(&d).await, before, "{state}"); + if state == "dotted-ttl" { + // The accepted difference of this state: the legacy index stays + // until TTL is disabled on the table (inventory section 3.2, + // MongoDB item 4), and v1.0 creates none for a dotted name. + let dotted = &before["T_cu_dotted"]; + assert!( + dotted.iter().any(|(n, _)| n == "idx_ttl_exp.at"), + "{dotted:?}" + ); + } + let mut keys = setting_keys(&d).await; + keys.sort(); + assert_eq!(keys, fresh_keys, "{state}"); + let history = Deployment::documents(&d.catalog(), "schema_history").await; + assert_eq!(history.len(), 1, "{state}"); + d.drop_databases().await; + } +} + +/// Migration 2's body on the oldest released catalog: every stored policy +/// after the upgrade, which the committed golden file pins. +#[tokio::test(flavor = "multi_thread")] +async fn migration_2_rewrites_the_released_policies_as_the_golden_file_records() { + let Some(uri) = server() else { return }; + let d = Deployment::new(&uri, "golden2").await; + fixture::load(&d, &fixture::read("0.1.12")).await; + // A policy in the account a released `init` wrote without + // `account_id`, which the store's own listing skips. + let default = d + .catalog() + .collection::("accounts") + .find_one(doc! { "account_name": "default" }) + .await + .unwrap() + .unwrap(); + let default_id = default.get_str("_id").unwrap().to_owned(); + d.catalog() + .collection::("iam_users") + .insert_one(doc! { + "account_id": &default_id, "user_name": "ops", + "user_arn": format!("arn:aws:iam::{default_id}:user/ops"), + "created_at": bson::DateTime::from_millis(1_793_235_168_000), + }) + .await + .unwrap(); + d.catalog() + .collection::("iam_policies") + .insert_one(doc! { + "account_id": &default_id, "principal_type": "user", "principal_name": "ops", + "policy_name": "deny-prod-default", + "policy_document": { "Version": "2012-10-17", "Statement": [{ + "Effect": "Deny", "Action": "dynamodb:*", "Resource": "*", + "Condition": { "StringEquals": { "dynamodb:ResourceTag/env": "prod" } }, + }] }, + "created_at": bson::DateTime::from_millis(1_793_235_168_000), + }) + .await + .unwrap(); + d.migrate().await.expect("migrate"); + let policies: Vec = Deployment::documents(&d.catalog(), "iam_policies") + .await + .iter() + .map(|p| { + let account = if p.get_str("account_id") == Ok(default_id.as_str()) { + "" + } else { + p.get_str("account_id").unwrap() + }; + serde_json::json!({ + "account": account, + "principal": format!("{} {}", p.get_str("principal_type").unwrap(), p.get_str("principal_name").unwrap()), + "policy": p.get_str("policy_name").unwrap(), + "document": serde_json::to_value(p.get_document("policy_document").unwrap()).unwrap(), + }) + }) + .collect(); + let rows = d.catalog_ledger().await; + let notes: Vec = rows[1] + .notes + .lines() + .map(|l| l.replace(&default_id, "")) + .collect(); + golden( + "0002_resource_tag_condition_keys", + &serde_json::json!({ "policies": policies, "notes": notes }), + ); + d.drop_databases().await; +} + +/// Migration 1's bodies on the oldest released catalog: both databases' +/// definitions, the settings keys and the legacy ledger, which the +/// committed golden file pins. +#[tokio::test(flavor = "multi_thread")] +async fn migration_1_upgrades_the_oldest_release_as_the_golden_file_records() { + let Some(uri) = server() else { return }; + let d = Deployment::new(&uri, "golden1").await; + fixture::load(&d, &fixture::read("0.1.12")).await; + d.migrate_through(extenddb_storage::migrations::MigrationId::Baseline) + .await + .expect("migration 1"); + let (catalog, data) = definitions(&d).await; + let history: Vec = Deployment::documents(&d.catalog(), "schema_history") + .await + .iter() + .map(|h| h.get_str("_id").unwrap().to_owned()) + .collect(); + golden( + "0001_baseline", + &serde_json::json!({ + "catalog": catalog, + "data": data, + "settings": setting_keys(&d).await, + "schema_history": history, + }), + ); + d.drop_databases().await; +} diff --git a/crates/storage-mongodb/tests/migrations_support/fixture.rs b/crates/storage-mongodb/tests/migrations_support/fixture.rs new file mode 100644 index 000000000..b931d444d --- /dev/null +++ b/crates/storage-mongodb/tests/migrations_support/fixture.rs @@ -0,0 +1,206 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Released catalogs, as `tests/fixtures/released/dump.js` prints them, and +//! their loading into a deployment's databases. +//! +//! The fixtures are canonical Extended JSON. They are read here with every +//! object's key order kept, because MongoDB compares embedded documents, +//! such as the compound `_id` of `tables` and `indexes`, field by field in +//! order, and an index key's order is the index. + +use std::collections::BTreeMap; + +use bson::{Bson, Document, doc}; +use serde::de::{Deserialize, Deserializer, MapAccess, SeqAccess, Visitor}; + +use super::Deployment; + +/// A JSON value with its objects' key order kept. +enum Json { + Null, + Bool(bool), + Number(serde_json::Number), + String(String), + Array(Vec), + Object(Vec<(String, Json)>), +} + +impl<'de> Deserialize<'de> for Json { + fn deserialize>(deserializer: D) -> Result { + struct Ordered; + impl<'de> Visitor<'de> for Ordered { + type Value = Json; + fn expecting(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + f.write_str("JSON") + } + fn visit_unit(self) -> Result { + Ok(Json::Null) + } + fn visit_bool(self, v: bool) -> Result { + Ok(Json::Bool(v)) + } + fn visit_i64(self, v: i64) -> Result { + Ok(Json::Number(v.into())) + } + fn visit_u64(self, v: u64) -> Result { + Ok(Json::Number(v.into())) + } + fn visit_f64(self, v: f64) -> Result { + Ok(serde_json::Number::from_f64(v).map_or(Json::Null, Json::Number)) + } + fn visit_str(self, v: &str) -> Result { + Ok(Json::String(v.to_owned())) + } + fn visit_string(self, v: String) -> Result { + Ok(Json::String(v)) + } + fn visit_seq>(self, mut seq: A) -> Result { + let mut items = Vec::new(); + while let Some(item) = seq.next_element()? { + items.push(item); + } + Ok(Json::Array(items)) + } + fn visit_map>(self, mut map: A) -> Result { + let mut fields = Vec::new(); + while let Some((k, v)) = map.next_entry()? { + fields.push((k, v)); + } + Ok(Json::Object(fields)) + } + } + deserializer.deserialize_any(Ordered) + } +} + +impl Json { + fn plain(&self) -> serde_json::Value { + match self { + Self::Null => serde_json::Value::Null, + Self::Bool(b) => (*b).into(), + Self::Number(n) => serde_json::Value::Number(n.clone()), + Self::String(s) => s.clone().into(), + Self::Array(items) => items.iter().map(Self::plain).collect(), + Self::Object(fields) => serde_json::Value::Object( + fields.iter().map(|(k, v)| (k.clone(), v.plain())).collect(), + ), + } + } + + /// The BSON value: an object whose first key starts with `$` is an + /// Extended JSON wrapper (`$numberInt`, `$date`, `$oid`, ...), whose own + /// key order does not matter; every other object keeps its order. + fn bson(&self) -> Bson { + match self { + Self::Object(fields) if fields.first().is_some_and(|(k, _)| k.starts_with('$')) => { + Bson::try_from(self.plain()).expect("an Extended JSON value") + } + Self::Object(fields) => { + Bson::Document(fields.iter().map(|(k, v)| (k.clone(), v.bson())).collect()) + } + Self::Array(items) => Bson::Array(items.iter().map(Self::bson).collect()), + Self::String(s) => Bson::String(s.clone()), + Self::Bool(b) => Bson::Boolean(*b), + Self::Null => Bson::Null, + Self::Number(n) => n + .as_i64() + .map_or_else(|| Bson::Double(n.as_f64().unwrap_or(0.0)), Bson::Int64), + } + } +} + +/// One collection of a fixture. +pub struct Collection { + pub name: String, + pub options: Document, + pub indexes: Vec, + pub documents: Vec, +} + +/// A released catalog: each database's collections. +pub struct Fixture { + pub databases: BTreeMap>, +} + +/// The fixture `tests/fixtures/released/.json`. +/// +/// # Panics +/// +/// The file is missing or is not a fixture. +pub fn read(name: &str) -> Fixture { + let path = format!( + "{}/tests/fixtures/released/{name}.json", + env!("CARGO_MANIFEST_DIR") + ); + let text = std::fs::read_to_string(&path).unwrap_or_else(|e| panic!("{path}: {e}")); + let root = match serde_json::from_str::(&text) + .expect("fixture JSON") + .bson() + { + Bson::Document(d) => d, + other => panic!("{path}: {other:?}"), + }; + let mut databases = BTreeMap::new(); + for (name, database) in root.get_document("databases").expect("databases") { + let collections = database + .as_document() + .and_then(|d| d.get_array("collections").ok()) + .expect("collections") + .iter() + .map(|c| { + let c = c.as_document().expect("collection"); + let docs = |field: &str| -> Vec { + c.get_array(field) + .expect(field) + .iter() + .map(|d| d.as_document().expect("document").clone()) + .collect() + }; + Collection { + name: c.get_str("name").expect("name").to_owned(), + options: c.get_document("options").cloned().unwrap_or_default(), + indexes: docs("indexes"), + documents: docs("documents"), + } + }) + .collect(); + databases.insert(name.clone(), collections); + } + Fixture { databases } +} + +/// Load `fixture` into `deployment`'s databases, which must be empty: +/// `extenddb_catalog` into its catalog database, `extenddb_data` into its +/// data database. +pub async fn load(deployment: &Deployment, fixture: &Fixture) { + for (name, collections) in &fixture.databases { + let db = match name.as_str() { + "extenddb_catalog" => deployment.catalog(), + "extenddb_data" => deployment.data(), + other => panic!("the fixture names the database {other}"), + }; + for c in collections { + let mut create = doc! { "create": &c.name }; + create.extend(c.options.clone()); + db.run_command(create).await.expect("create"); + let indexes: Vec = c + .indexes + .iter() + .filter(|i| i.get_str("name") != Ok("_id_")) + .map(|i| Bson::Document(i.clone())) + .collect(); + if !indexes.is_empty() { + db.run_command(doc! { "createIndexes": &c.name, "indexes": indexes }) + .await + .expect("createIndexes"); + } + if !c.documents.is_empty() { + db.collection::(&c.name) + .insert_many(c.documents.clone()) + .await + .expect("insert"); + } + } + } +} diff --git a/crates/storage-mongodb/tests/migrations_support/mod.rs b/crates/storage-mongodb/tests/migrations_support/mod.rs new file mode 100644 index 000000000..567687a41 --- /dev/null +++ b/crates/storage-mongodb/tests/migrations_support/mod.rs @@ -0,0 +1,186 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! What the MongoDB migration tests share: the server, a deployment in +//! databases of its own, and the runner's entry points. +//! +//! The tests run against the replica set `EXTENDDB_TEST_MONGODB_URI` names +//! and skip without it, unless `EXTENDDB_TEST_MONGODB_REQUIRED` is `1`, which +//! CI sets so a missing server fails the job. Each deployment uses a catalog +//! and a data database named after the test, and drops them at the end. + +#![allow(dead_code)] + +pub mod fixture; + +use std::sync::atomic::{AtomicU32, Ordering}; +use std::time::Duration; + +use bson::{Document, doc}; +use extenddb_storage::migrations::{ + CatalogStatus, LedgerRow, MigrationError, MigrationId, RunReport, Runner, +}; +use extenddb_storage_mongodb::migrations::{ + MigrationLock, MongoMigrations, MongoTarget, Timing, catalog_status, lock, migrate_holding, +}; +use futures::TryStreamExt; + +/// The server the tests run against, or `None` to skip. +/// +/// # Panics +/// +/// When `EXTENDDB_TEST_MONGODB_REQUIRED` is `1` and no server is named. +pub fn server() -> Option { + match std::env::var("EXTENDDB_TEST_MONGODB_URI") { + Ok(uri) if !uri.is_empty() => Some(uri), + _ => { + assert!( + std::env::var("EXTENDDB_TEST_MONGODB_REQUIRED").as_deref() != Ok("1"), + "EXTENDDB_TEST_MONGODB_REQUIRED is 1 and EXTENDDB_TEST_MONGODB_URI names no server" + ); + eprintln!("skipped: set EXTENDDB_TEST_MONGODB_URI to a MongoDB replica set"); + None + } + } +} + +static NEXT: AtomicU32 = AtomicU32::new(0); + +/// One deployment in databases of its own. +pub struct Deployment { + pub target: MongoTarget, + pub client: mongodb::Client, +} + +impl Deployment { + /// A deployment named after `tag`, whose databases do not exist yet. + pub async fn new(uri: &str, tag: &str) -> Self { + let n = NEXT.fetch_add(1, Ordering::SeqCst); + let base = format!("t{}_{n}_{tag}", std::process::id()); + let target = + MongoTarget::with_databases(uri, format!("{base}_catalog"), format!("{base}_data")); + let client = mongodb::Client::with_uri_str(uri).await.expect("client"); + let deployment = Self { target, client }; + deployment.drop_databases().await; + deployment + } + + pub fn catalog(&self) -> mongodb::Database { + self.client.database(&self.target.catalog) + } + + pub fn data(&self) -> mongodb::Database { + self.client.database(&self.target.data) + } + + pub async fn drop_databases(&self) { + self.catalog().drop().await.expect("drop catalog"); + self.data().drop().await.expect("drop data"); + } + + /// `init`'s run: take the lock on a store that is not a catalog and + /// apply every migration. + pub async fn create(&self) -> RunReport { + let (held, target) = lock( + &self.target, + true, + Duration::from_secs(10), + Timing::PRODUCTION, + ) + .await + .expect("lock"); + let report = migrate_holding(&target, "test", &held) + .await + .expect("create"); + held.release().await; + report + } + + /// `migrate --yes`. + pub async fn migrate(&self) -> Result { + let (held, target) = lock( + &self.target, + false, + Duration::from_secs(10), + Timing::PRODUCTION, + ) + .await?; + let result = migrate_holding(&target, "test", &held).await; + held.release().await; + result + } + + /// `migrate --yes --reconverge`. + pub async fn reconverge(&self) -> Result { + let (held, target) = lock( + &self.target, + false, + Duration::from_secs(10), + Timing::PRODUCTION, + ) + .await?; + let migrations = MongoMigrations::open(&target, Some(&held)).await?; + let result = Runner::new(&migrations, "test").reconverge().await; + migrations.close().await; + held.release().await; + result + } + + /// `migrate --yes` through migration `id` only. + pub async fn migrate_through(&self, id: MigrationId) -> Result { + let held = + MigrationLock::acquire(&self.target, Duration::from_secs(10), Timing::PRODUCTION) + .await?; + let migrations = MongoMigrations::open(&self.target, Some(&held)).await?; + let result = Runner::new(&migrations, "test").migrate_through(id).await; + migrations.close().await; + held.release().await; + result + } + + pub async fn status(&self) -> CatalogStatus { + catalog_status(&self.target).await.expect("status") + } + + /// The catalog ledger's rows, in id order. + pub async fn catalog_ledger(&self) -> Vec { + let migrations = MongoMigrations::open(&self.target, None) + .await + .expect("open"); + let rows = Runner::new(&migrations, "") + .catalog_ledger() + .await + .expect("ledger"); + migrations.close().await; + rows + } + + /// Every document of `collection` in `db`. + pub async fn documents(db: &mongodb::Database, collection: &str) -> Vec { + db.collection::(collection) + .find(doc! {}) + .sort(doc! { "_id": 1 }) + .await + .expect("find") + .try_collect() + .await + .expect("collect") + } + + /// The `value` of a catalog settings document, if it exists. + pub async fn setting(&self, key: &str) -> Option { + self.catalog() + .collection::("settings") + .find_one(doc! { "_id": key }) + .await + .expect("setting") + .map(|d| d.get_str("value").unwrap_or_default().to_owned()) + } + + /// The collection names of `db`, sorted. + pub async fn collections(db: &mongodb::Database) -> Vec { + let mut names = db.list_collection_names().await.expect("names"); + names.sort(); + names + } +} diff --git a/crates/storage-postgres/src/bootstrapper.rs b/crates/storage-postgres/src/bootstrapper.rs index 23461be07..d081b1f0c 100755 --- a/crates/storage-postgres/src/bootstrapper.rs +++ b/crates/storage-postgres/src/bootstrapper.rs @@ -16,7 +16,7 @@ use extenddb_storage::bootstrapper::{ }, }; use extenddb_storage::management_store::{OpError, OpResult}; -use extenddb_storage::migrations::{AdminRole, MigrationId, Target}; +use extenddb_storage::migrations::{AdminRole, Target}; use sqlx::PgPool; use sqlx::postgres::{PgConnectOptions, PgPoolOptions}; use tokio::sync::OnceCell; @@ -120,16 +120,6 @@ impl PostgresBootstrapper { } } -/// The error of a bootstrapper method the commands do not call on this -/// backend: its catalog migrates through -/// [`crate::migrations::PostgresCatalogMigrations`] and the shared runner. -fn through_the_runner(method: &str) -> OpError { - OpError::Internal(format!( - "postgres migrates through its CatalogMigrations; the bootstrapper method {method} is \ - not used" - )) -} - #[async_trait] impl Bootstrapper for PostgresBootstrapper { async fn ensure_app_user(&self) -> OpResult<()> { @@ -207,18 +197,6 @@ impl Bootstrapper for PostgresBootstrapper { .await } - async fn run_catalog_migrations(&self) -> OpResult<()> { - Err(through_the_runner("run_catalog_migrations")) - } - - async fn run_data_migrations(&self) -> OpResult<()> { - Err(through_the_runner("run_data_migrations")) - } - - async fn pending_data_migrations(&self) -> OpResult> { - Err(through_the_runner("pending_data_migrations")) - } - async fn record_data_connection(&self) -> OpResult<()> { let pool = self.app_pool(&self.config.catalog_db).await?; let data_conn = self.app_connection_url(&self.config.data_db); @@ -369,10 +347,6 @@ impl Bootstrapper for PostgresBootstrapper { }) } - async fn is_catalog_initialized(&self) -> OpResult { - Err(through_the_runner("is_catalog_initialized")) - } - async fn list_table_names(&self) -> OpResult> { let Ok(pool) = self.app_pool(&self.config.catalog_db).await else { return Ok(Vec::new()); @@ -420,14 +394,6 @@ impl Bootstrapper for PostgresBootstrapper { Ok(()) } - async fn read_catalog_version(&self) -> OpResult> { - Err(through_the_runner("read_catalog_version")) - } - - fn expected_catalog_version(&self) -> String { - MigrationId::LATEST.number().to_string() - } - fn catalog_database_name(&self) -> String { self.config.catalog_db.clone() } diff --git a/crates/storage-postgres/src/lib.rs b/crates/storage-postgres/src/lib.rs index dfda9d9eb..a24be840f 100755 --- a/crates/storage-postgres/src/lib.rs +++ b/crates/storage-postgres/src/lib.rs @@ -168,7 +168,7 @@ pub fn backend() -> extenddb_storage::Backend { }) }, server_components: server_components_factory, - catalog_migrations: Some(&migrations::PostgresCatalogMigrations), + catalog_migrations: &migrations::PostgresCatalogMigrations, } } diff --git a/crates/storage-postgres/src/operations.rs b/crates/storage-postgres/src/operations.rs index 65171799f..a00ed25d2 100644 --- a/crates/storage-postgres/src/operations.rs +++ b/crates/storage-postgres/src/operations.rs @@ -58,14 +58,6 @@ impl OperationsEngine for PostgresOperationsEngine { Ok(()) } - /// The highest migration this binary carries: the catalog is versioned - /// by its ledger, and its `catalog_version` row holds the same integer. - fn catalog_version(&self) -> String { - extenddb_storage::migrations::MigrationId::LATEST - .number() - .to_string() - } - fn setting_default(&self, key: &str) -> Option { use extenddb_core::settings_keys::INDEX_PROPAGATION_DELAY_MS; match key { diff --git a/crates/storage-sqlite/src/bootstrapper.rs b/crates/storage-sqlite/src/bootstrapper.rs index d4c65b16c..5374db1db 100644 --- a/crates/storage-sqlite/src/bootstrapper.rs +++ b/crates/storage-sqlite/src/bootstrapper.rs @@ -179,18 +179,6 @@ impl Bootstrapper for SqliteBootstrapper { Ok(()) // Catalog and data share one file. } - async fn run_catalog_migrations(&self) -> OpResult<()> { - Err(through_the_runner("run_catalog_migrations")) - } - - async fn run_data_migrations(&self) -> OpResult<()> { - Err(through_the_runner("run_data_migrations")) - } - - async fn pending_data_migrations(&self) -> OpResult> { - Err(through_the_runner("pending_data_migrations")) - } - async fn record_data_connection(&self) -> OpResult<()> { let pool = self.pool().await?; sqlx::query( @@ -306,10 +294,6 @@ impl Bootstrapper for SqliteBootstrapper { }) } - async fn is_catalog_initialized(&self) -> OpResult { - Err(through_the_runner("is_catalog_initialized")) - } - async fn list_table_names(&self) -> OpResult> { let Ok(pool) = self.pool().await else { return Ok(Vec::new()); @@ -353,16 +337,6 @@ impl Bootstrapper for SqliteBootstrapper { Ok(()) } - async fn read_catalog_version(&self) -> OpResult> { - Err(through_the_runner("read_catalog_version")) - } - - fn expected_catalog_version(&self) -> String { - extenddb_storage::migrations::MigrationId::LATEST - .number() - .to_string() - } - fn catalog_database_name(&self) -> String { self.path.clone() } @@ -385,16 +359,6 @@ impl Bootstrapper for SqliteBootstrapper { } } -/// The error of a bootstrapper method the commands do not call on this -/// backend: its catalog migrates through -/// [`crate::migrations::SqliteCatalogMigrations`] and the shared runner. -fn through_the_runner(method: &str) -> OpError { - OpError::Internal(format!( - "sqlite migrates through its CatalogMigrations; the bootstrapper method {method} is \ - not used" - )) -} - #[cfg(all(test, unix))] mod lock_tests { use super::SqliteBootstrapper; diff --git a/crates/storage-sqlite/src/lib.rs b/crates/storage-sqlite/src/lib.rs index 7e9b278cb..567343611 100644 --- a/crates/storage-sqlite/src/lib.rs +++ b/crates/storage-sqlite/src/lib.rs @@ -100,7 +100,7 @@ pub fn backend() -> extenddb_storage::Backend { management_store: sqlite_management_store_factory, diagnostics_store: sqlite_diagnostics_store_factory, server_components: sqlite_server_components_factory, - catalog_migrations: Some(&migrations::SqliteCatalogMigrations), + catalog_migrations: &migrations::SqliteCatalogMigrations, } } diff --git a/crates/storage-sqlite/src/operations.rs b/crates/storage-sqlite/src/operations.rs index 13551cb0e..4e87578e1 100644 --- a/crates/storage-sqlite/src/operations.rs +++ b/crates/storage-sqlite/src/operations.rs @@ -50,14 +50,6 @@ impl OperationsEngine for SqliteOperationsEngine { Ok(()) } - /// The highest migration this binary carries: the catalog is versioned - /// by its ledger, and its `catalog_version` row holds the same integer. - fn catalog_version(&self) -> String { - extenddb_storage::migrations::MigrationId::LATEST - .number() - .to_string() - } - fn setting_default(&self, key: &str) -> Option { use extenddb_core::settings_keys::INDEX_PROPAGATION_DELAY_MS; match key { diff --git a/crates/storage/src/backend.rs b/crates/storage/src/backend.rs index 0c7007788..9e1474423 100644 --- a/crates/storage/src/backend.rs +++ b/crates/storage/src/backend.rs @@ -58,10 +58,8 @@ pub struct Backend { /// Creates the assembled server components for `serve`. pub server_components: ServerComponentsFactory, /// The backend's catalog migrations on the shared runner, which `init`, - /// `migrate`, `verify` and `serve` use; `None` for a backend that still - /// migrates through its bootstrapper and checks its catalog in its own - /// server factory. - pub catalog_migrations: Option<&'static dyn crate::migrations::CatalogMigrations>, + /// `migrate`, `verify` and `serve` use. + pub catalog_migrations: &'static dyn crate::migrations::CatalogMigrations, } static BACKEND: OnceLock = OnceLock::new(); diff --git a/crates/storage/src/bootstrapper.rs b/crates/storage/src/bootstrapper.rs index 354a85eb7..500803ec5 100755 --- a/crates/storage/src/bootstrapper.rs +++ b/crates/storage/src/bootstrapper.rs @@ -41,19 +41,6 @@ pub struct AdminBootstrapResult { pub from_env: bool, } -/// Outcome of [`Bootstrapper::repair_lost_sort_key_definitions`]. -#[derive(Debug, Default, Clone)] -pub struct KeyDefinitionRepair { - /// Tables whose sort key definitions were restored, as - /// `table_name: attribute (TYPE)` lines. When the caller passed `apply = - /// false` these are the repairs that WOULD be made. - pub repaired: Vec, - /// Tables that are damaged but could not be repaired automatically, because - /// no physical sort key column was found to read the type back from. These - /// need a human: guessing a type would resolve the key to the wrong column. - pub needs_attention: Vec, -} - /// High-level bootstrap operations for storage backends. /// /// Covers the init, destroy, and migrate command paths. Implementations @@ -73,68 +60,6 @@ pub trait Bootstrapper: Send + Sync { /// Create the data database. Returns error if it already exists. async fn create_data_db(&self) -> OpResult<()>; - /// Run catalog schema migrations (creates tables, indexes, etc.). - async fn run_catalog_migrations(&self) -> OpResult<()>; - - /// Run data schema migrations (stream tables, sequences, etc.). - async fn run_data_migrations(&self) -> OpResult<()>; - - /// Restore sort key attribute definitions that issue #259 deleted from table - /// metadata, reading each type back from the physical sort key column. - /// - /// Before the fix, `UpdateTable` replaced a table's stored attribute - /// definitions with the request's subset, so adding a GSI dropped the base - /// table's own key definitions and keyed reads silently returned another item - /// in the same partition. The write path no longer does that, but a table - /// damaged by an older build stays damaged, and it now fails its reads loudly - /// instead (see `TableKeyInfo::validate_sort_key_definitions`). This repairs - /// such tables so they serve correct reads again. - /// - /// With `apply` false this only detects and reports, writing nothing, so a - /// `migrate` run that has not been confirmed with `--yes` never mutates the - /// catalog. With `apply` true the repair is written. - /// - /// Idempotent: a table whose definitions are intact is left untouched, so - /// running it on a healthy deployment reports nothing and changes nothing. - /// - /// The default implementation repairs nothing, which is correct for a backend - /// that never wrote the definitions in the first place. - async fn repair_lost_sort_key_definitions(&self, apply: bool) -> OpResult { - let _ = apply; - Ok(KeyDefinitionRepair::default()) - } - - /// Acquire a cross-process lock so that concurrent `migrate` runs, such as - /// several replicas starting at once, don't race each other applying schema - /// changes. Blocks until the lock is available. - /// - /// The default is a no-op, which means a backend that does not override this - /// gets no serialization at all. Implement it if concurrent bootstrap is a - /// supported deployment for your backend. - /// - /// Implementations need not be re-entrant, and callers must not acquire - /// twice without releasing. Every acquire must be paired with - /// [`Bootstrapper::release_migration_lock`], and the backend must also - /// release the lock if the process dies without doing so. - async fn acquire_migration_lock(&self) -> OpResult<()> { - Ok(()) - } - - /// Release the lock taken by [`Bootstrapper::acquire_migration_lock`], and - /// do nothing if no lock is held. Defaults to a no-op. - async fn release_migration_lock(&self) -> OpResult<()> { - Ok(()) - } - - /// Filenames of data-database migrations that have not yet been applied. - /// - /// Excludes a pre-tracking baseline migration that already exists and will - /// be adopted (recorded without re-running) rather than applied. Data - /// migrations are tracked in the data database's own ledger, independent of - /// the catalog version, so `migrate` uses this to report accurately and to - /// decide whether confirmation is required — without running anything. - async fn pending_data_migrations(&self) -> OpResult>; - /// Record the data database connection string in the catalog. async fn record_data_connection(&self) -> OpResult<()>; @@ -152,9 +77,6 @@ pub trait Bootstrapper: Send + Sync { env_password: Option<&str>, ) -> OpResult; - /// Check if the catalog is already initialized (has schema). - async fn is_catalog_initialized(&self) -> OpResult; - /// List table names in the catalog (for destroy confirmation display). async fn list_table_names(&self) -> OpResult>; @@ -164,12 +86,6 @@ pub trait Bootstrapper: Send + Sync { /// Drop both catalog and data databases. Destructive and irreversible. async fn drop_databases(&self, data_db: &str) -> OpResult<()>; - /// Read the current catalog schema version. - async fn read_catalog_version(&self) -> OpResult>; - - /// Get the expected catalog version for this binary. - fn expected_catalog_version(&self) -> String; - /// Return the catalog database name for display. fn catalog_database_name(&self) -> String; @@ -187,6 +103,13 @@ pub trait Bootstrapper: Send + Sync { crate::migrations::Target::at(self.catalog_connection_url()) } + /// The migration lock `init` holds while it writes the encryption key, + /// the default account, the admin user and the data connection. A + /// backend whose lock has a lease checks it before each of those writes, + /// as its runner does before each migration request; the default keeps + /// nothing. + fn hold(&self, _lock: &crate::migrations::HeldLock) {} + /// Generate the backend-specific configuration subsection for extenddb.toml. /// Returns only the [`storage.backend_name`] section content, not the [storage] header. fn generate_backend_config_section(&self) -> String; @@ -609,7 +532,7 @@ pub mod helpers { /// new required method is added, or a defaulted one loses its default, this /// stops compiling, which is the same breakage an out-of-tree backend crate /// would hit. It also pins object safety by boxing as `dyn Bootstrapper`, and -/// checks that the defaulted migration-lock methods are usable no-ops. +/// checks that the defaulted `hold` is a usable no-op. #[cfg(test)] mod out_of_tree_compat_tests { use super::*; @@ -631,15 +554,6 @@ mod out_of_tree_compat_tests { async fn create_data_db(&self) -> OpResult<()> { Ok(()) } - async fn run_catalog_migrations(&self) -> OpResult<()> { - Ok(()) - } - async fn run_data_migrations(&self) -> OpResult<()> { - Ok(()) - } - async fn pending_data_migrations(&self) -> OpResult> { - Ok(Vec::new()) - } async fn record_data_connection(&self) -> OpResult<()> { Ok(()) } @@ -661,9 +575,6 @@ mod out_of_tree_compat_tests { from_env: false, }) } - async fn is_catalog_initialized(&self) -> OpResult { - Ok(true) - } async fn list_table_names(&self) -> OpResult> { Ok(Vec::new()) } @@ -673,12 +584,6 @@ mod out_of_tree_compat_tests { async fn drop_databases(&self, _data_db: &str) -> OpResult<()> { Ok(()) } - async fn read_catalog_version(&self) -> OpResult> { - Ok(None) - } - fn expected_catalog_version(&self) -> String { - "0.0.0".to_owned() - } fn catalog_database_name(&self) -> String { "minimal".to_owned() } @@ -693,14 +598,14 @@ mod out_of_tree_compat_tests { } } - /// A backend that does not implement the migration lock still works, and - /// the defaults are no-ops rather than errors. - #[tokio::test] - async fn migration_lock_defaults_to_a_usable_no_op() { + /// A backend that keeps no lease takes the default `hold`, which keeps + /// nothing and needs no lock of its own kind. + #[test] + fn hold_defaults_to_a_no_op() { let bootstrapper: Box = Box::new(MinimalBootstrapper); - assert!(bootstrapper.acquire_migration_lock().await.is_ok()); - assert!(bootstrapper.release_migration_lock().await.is_ok()); - // Releasing without acquiring must also be harmless. - assert!(bootstrapper.release_migration_lock().await.is_ok()); + bootstrapper.hold(&crate::migrations::HeldLock::new( + "minimal://".to_owned(), + (), + )); } } diff --git a/crates/storage/src/migrations/advice.rs b/crates/storage/src/migrations/advice.rs index 11de2ac9a..65edeb195 100644 --- a/crates/storage/src/migrations/advice.rs +++ b/crates/storage/src/migrations/advice.rs @@ -8,7 +8,6 @@ use super::catalog::{CatalogMigrations, Target}; use super::registry::{MigrationId, RELEASED_THROUGH}; use super::status::{CatalogStatus, MigrationError, Pending}; -use super::table::Kind; /// What a message needs beyond the status. #[derive(Debug, Clone, Copy)] @@ -203,26 +202,36 @@ fn capitalized(text: &str) -> String { /// `Pending (all breaking): 2 name (catalog), ...`, or each kind beside its /// migration when the kinds differ. fn pending_line(pending: &[Pending]) -> String { - let kinds: Vec = pending.iter().map(|p| p.kind).collect(); - let same = kinds.windows(2).all(|w| w[0] == w[1]); - let items: Vec = pending + // One item per id and kind of store: a body that runs in several stores + // of one kind (Cassandra's account keyspaces) is named once, with the + // number of stores. + let mut groups: Vec<(&Pending, usize)> = Vec::new(); + for p in pending { + match groups + .iter_mut() + .find(|(g, _)| g.id == p.id && g.store == p.store && g.kind == p.kind) + { + Some((_, stores)) => *stores += 1, + None => groups.push((p, 1)), + } + } + let same = groups.windows(2).all(|w| w[0].0.kind == w[1].0.kind); + let items: Vec = groups .iter() - .map(|p| { + .map(|(p, stores)| { + let store = match stores { + 1 => p.store.as_str().to_owned(), + n => format!("{} in {n} stores", p.store.as_str()), + }; if same { - format!("{} {} ({})", p.id, p.name, p.store.as_str()) + format!("{} {} ({store})", p.id, p.name) } else { - format!( - "{} {} ({}, {})", - p.id, - p.name, - p.store.as_str(), - p.kind.as_str() - ) + format!("{} {} ({store}, {})", p.id, p.name, p.kind.as_str()) } }) .collect(); - match kinds.first() { - Some(kind) if same => format!("Pending (all {}): {}.", kind.as_str(), items.join(", ")), + match groups.first() { + Some((p, _)) if same => format!("Pending (all {}): {}.", p.kind.as_str(), items.join(", ")), _ => format!("Pending: {}.", items.join(", ")), } } @@ -299,6 +308,55 @@ mod tests { assert!(message.contains("Stop every server that uses this catalog")); } + /// A body that runs in every account keyspace is named once per id with + /// the number of keyspaces, not once per keyspace. + #[test] + fn a_body_in_many_stores_of_one_kind_is_named_once() { + let baseline = MigrationId::ALL[0]; + let entry = |store, store_name: &str, kind| Pending { + id: baseline.number(), + name: baseline.name(), + store, + store_name: store_name.to_owned(), + kind, + }; + let mut list = vec![entry( + StoreKind::Catalog, + "extenddb_catalog", + Kind::Breaking, + )]; + for keyspace in ["acct_a", "acct_b", "acct_c"] { + list.push(entry(StoreKind::Account, keyspace, Kind::Breaking)); + } + let message = status_message( + &CatalogStatus::Behind { + version: None, + pending: list.clone(), + }, + &ADVICE, + ); + let line = format!( + "Pending (all breaking): {id} {name} (catalog), {id} {name} (account in 3 stores).", + id = baseline.number(), + name = baseline.name() + ); + assert!(message.contains(&line), "{message}"); + + list[1].kind = Kind::Additive; + let message = status_message( + &CatalogStatus::Behind { + version: None, + pending: list, + }, + &ADVICE, + ); + assert!(message.contains("(account, additive), "), "{message}"); + assert!( + message.contains("(account in 2 stores, breaking)."), + "{message}" + ); + } + #[test] fn mixed_kinds_are_named_per_migration() { let mut list = pending(Kind::Breaking); diff --git a/crates/storage/src/migrations/catalog.rs b/crates/storage/src/migrations/catalog.rs index 77788c5c1..3da93b762 100644 --- a/crates/storage/src/migrations/catalog.rs +++ b/crates/storage/src/migrations/catalog.rs @@ -4,11 +4,8 @@ //! The installed backend's catalog migrations, as `init`, `migrate`, //! `verify` and `serve` use them. //! -//! The app crate does not name a backend crate, so a backend that migrates -//! through the shared runner installs a [`CatalogMigrations`] in its -//! [`Backend`](crate::Backend). A backend without one still migrates through -//! its [`Bootstrapper`](crate::bootstrapper::Bootstrapper), and the commands -//! take that path for it. +//! The app crate does not name a backend crate, so every backend installs +//! its [`CatalogMigrations`] in its [`Backend`](crate::Backend). use std::any::Any; use std::time::Duration; @@ -180,6 +177,19 @@ pub trait CatalogMigrations: Send + Sync { /// beside a server. async fn status(&self, target: &Target) -> Result; + /// The catalog status, read under a lock this backend took, which + /// [`run_locked`] and [`forget_locked`] read before they act. The + /// status as [`CatalogMigrations::status`] reads it by default; a + /// backend whose lock ends when its holder stops making progress reads + /// it as the holder, so a long status read keeps the lock. + async fn status_under_lock( + &self, + _lock: &HeldLock, + target: &Target, + ) -> Result { + self.status(target).await + } + /// The catalog store's ledger rows, in id order, for `verify` to print /// their notes. Read-only; empty for a store with no ledger. async fn catalog_ledger(&self, target: &Target) -> Result, MigrationError>; @@ -201,6 +211,14 @@ pub trait CatalogMigrations: Send + Sync { /// took ([`Runner::forget`](super::Runner::forget)). async fn forget(&self, lock: &HeldLock, id: u32) -> Result<(), MigrationError>; + /// Release a lock this backend took. The default drops it, which frees + /// a lock its connection or file holds; a backend whose lock is a row + /// (Cassandra) deletes the row here, so the next migration or server + /// need not wait for its TTL. + async fn release(&self, lock: HeldLock) { + drop(lock); + } + /// Notes about the deployment that its status does not show, for the /// preview and `verify` to print and `migrate --yes` to check before it /// takes the lock ([`before_lock`]). On PostgreSQL: tables and sequences @@ -287,7 +305,7 @@ pub async fn run_locked( release: &str, kind: RunKind, ) -> Result { - let status = migrations.status(target).await?; + let status = migrations.status_under_lock(lock, target).await?; if let Some(refusal) = refusal(&status, kind) { return Err(refusal); } @@ -308,7 +326,7 @@ pub async fn forget_locked( lock: &HeldLock, id: u32, ) -> Result<(), MigrationError> { - let status = migrations.status(target).await?; + let status = migrations.status_under_lock(lock, target).await?; if status == CatalogStatus::Uninitialized { return Err(MigrationError::Status(status)); } @@ -339,21 +357,17 @@ fn refusal(status: &CatalogStatus, kind: RunKind) -> Option { } /// The installed backend's catalog migrations, or `None` when no backend is -/// installed or the installed one still migrates through its bootstrapper. +/// installed. #[must_use] pub fn installed() -> Option<&'static dyn CatalogMigrations> { - crate::backend::try_backend().and_then(|backend| backend.catalog_migrations) + crate::backend::try_backend().map(|backend| backend.catalog_migrations) } /// What `extenddb version`, `verify` and the `serve` banner call the catalog -/// this binary needs: `migration `, the highest id it carries, on a -/// backend on the runner; else the backend's catalog version string. +/// this binary needs: `migration `, the highest id it carries. #[must_use] pub fn binary_catalog_label() -> String { - if installed().is_some() { - return format!("migration {}", MigrationId::LATEST.number()); - } - crate::operations::catalog_version().unwrap_or_else(|_| "unknown".to_owned()) + format!("migration {}", MigrationId::LATEST.number()) } #[cfg(test)] diff --git a/crates/storage/src/migrations/m0002_resource_tag_condition_keys.rs b/crates/storage/src/migrations/m0002_resource_tag_condition_keys.rs index 3800915fa..c82989d9e 100644 --- a/crates/storage/src/migrations/m0002_resource_tag_condition_keys.rs +++ b/crates/storage/src/migrations/m0002_resource_tag_condition_keys.rs @@ -277,8 +277,10 @@ impl CodeBody for ResourceTagC } /// A pass that could not read or write a policy must not be recorded: the -/// next run finishes it. The error carries the first failure's database -/// error code, so the runner runs the body again after a lock timeout. +/// next run finishes it. The error carries the first failure's text, such +/// as a lapsed migration lock or a node that did not answer, and its +/// database error code when it has one, so the runner runs the body again +/// after a lock timeout. fn refuse_lookup_errors(report: &MigrationReport) -> Result<(), MigrationError> { if report.errors == 0 { return Ok(()); @@ -292,7 +294,14 @@ fn refuse_lookup_errors(report: &MigrationReport) -> Result<(), MigrationError> message: format!("{failed}, the first with {message}"), sqlstate: sqlstate.clone(), }, - _ => MigrationError::Store(failed), + Some( + OpError::Validation(message) + | OpError::AlreadyExists(message) + | OpError::NotFound(message) + | OpError::HasDependents(message) + | OpError::Internal(message), + ) => MigrationError::Store(format!("{failed}, the first with {message}")), + None => MigrationError::Store(failed), }) } @@ -304,3 +313,32 @@ fn collision_line(n: &NeedsAttention) -> String { n.account_id, n.principal, n.policy_name ) } + +#[cfg(test)] +mod tests { + use super::{MigrationError, MigrationReport, OpError, refuse_lookup_errors}; + + /// The first failure's own text reaches the operator whatever its kind: + /// a store under a lease that lapsed, or a request a node did not + /// answer, returns an internal error, and its text says which. + #[test] + fn a_failed_pass_names_its_first_error() { + let report = MigrationReport { + errors: 2, + first_error: Some(OpError::Internal( + "the migration lock in ks_catalog.settings was not renewed in time".to_owned(), + )), + ..MigrationReport::default() + }; + let refused = refuse_lookup_errors(&report); + let Err(MigrationError::Store(message)) = refused else { + panic!("{refused:?}"); + }; + assert_eq!( + message, + "resource-tag rewrite: 2 policy lookup(s) or write(s) failed, the first with the \ + migration lock in ks_catalog.settings was not renewed in time" + ); + assert!(refuse_lookup_errors(&MigrationReport::default()).is_ok()); + } +} diff --git a/crates/storage/src/operations.rs b/crates/storage/src/operations.rs index c8497f934..e48ba2cfe 100644 --- a/crates/storage/src/operations.rs +++ b/crates/storage/src/operations.rs @@ -28,9 +28,6 @@ pub trait OperationsEngine: Send + Sync { /// parameterized queries are not possible (e.g., CREATE DATABASE, DROP DATABASE). fn validate_identifier(&self, name: &str, label: &str) -> Result<(), StorageError>; - /// Get the catalog schema version for this backend. - fn catalog_version(&self) -> String; - /// Check if a configuration key contains sensitive data that should be redacted. fn is_sensitive_key(&self, key: &str) -> bool; @@ -68,11 +65,6 @@ pub fn get_operations_engine() -> Result<&'static dyn OperationsEngine, StorageE // Convenience functions that delegate to the operations engine -/// Get the catalog version for a backend. -pub fn catalog_version() -> Result { - get_operations_engine().map(OperationsEngine::catalog_version) -} - /// The installed backend's default for the runtime setting `key` /// ([`OperationsEngine::setting_default`]); `None` when no backend is /// installed or its engine does not read the key. From e0e10681f6f823b92cf54eccef4585e64b8c41ed Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Fri, 30 Oct 2026 19:34:25 +0000 Subject: [PATCH 02/45] ci: check the Cassandra and MongoDB catalogs, migrations and locks docs/schema/cassandra.txt and docs/schema/mongodb.txt hold the schema a fresh install creates. The catalog check jobs dump a fresh install's schema (devtools/schema-dump on Cassandra, devtools/check-mongodb-schema on MongoDB) and fail when it differs from the committed file. Cassandra: the storage job runs tests/migrations.rs and tests/migrations_stopped.rs. New jobs run the migration tests on a two-node cluster, check the permissions init grants under CassandraAuthorizer, and run the catalog checks: the schema, the lock and the serving registration, and a catalog the last 0.1.x release wrote. MongoDB: devtools/run-mongodb-tests runs the catalog migration tests against its replica set, and the catalog job checks the schema, the lock, a user with readWrite on the two databases and no other role, the released catalogs and the last 0.1.x image. The new test steps set EXTENDDB_TEST_CASSANDRA=required or EXTENDDB_TEST_MONGODB_REQUIRED=1, so a test without its server fails rather than skips. --- .github/workflows/integration-cassandra.yml | 254 ++++++++++- .github/workflows/integration-mongodb.yml | 44 +- devtools/catalog_checks.py | 44 +- devtools/check-cassandra-authorizer | 261 ++++++++++++ devtools/check-cassandra-catalog-lock | 326 ++++++++++++++ devtools/check-cassandra-older-binary | 446 ++++++++++++++++++++ devtools/check-mongodb-auth | 139 ++++++ devtools/check-mongodb-catalog-lock | 185 ++++++++ devtools/check-mongodb-older-binary | 184 ++++++++ devtools/check-mongodb-released | 206 +++++++++ devtools/check-mongodb-schema | 80 ++++ devtools/run-mongodb-tests | 10 + devtools/schema-dump | 98 ++++- docs/schema/cassandra.txt | 290 +++++++++++++ docs/schema/mongodb.txt | 65 +++ 15 files changed, 2620 insertions(+), 12 deletions(-) create mode 100755 devtools/check-cassandra-authorizer create mode 100755 devtools/check-cassandra-catalog-lock create mode 100755 devtools/check-cassandra-older-binary create mode 100755 devtools/check-mongodb-auth create mode 100755 devtools/check-mongodb-catalog-lock create mode 100755 devtools/check-mongodb-older-binary create mode 100755 devtools/check-mongodb-released create mode 100755 devtools/check-mongodb-schema create mode 100644 docs/schema/cassandra.txt create mode 100644 docs/schema/mongodb.txt diff --git a/.github/workflows/integration-cassandra.yml b/.github/workflows/integration-cassandra.yml index 703372dd0..ff98233ec 100644 --- a/.github/workflows/integration-cassandra.yml +++ b/.github/workflows/integration-cassandra.yml @@ -26,16 +26,19 @@ concurrency: # talk to a live node at 127.0.0.1:9042 with default cassandra/cassandra # credentials. This workflow enables them to run in CI. # -# A single-node Cassandra installation is enough. the tests create their own +# A single-node Cassandra installation is enough for every job except +# cassandra-two-nodes, which starts two nodes. The tests create their own # keyspaces with replication factor (RF) of 1 (one). # # The cassandra:4.1 image does not recognize CASSANDRA_AUTHENTICATOR as an -# environment variable for configuring authentication, so each job patches -# cassandra.yaml inside the running container to enable password auth, and -# restarts it before running any tests. The health-check on the service container -# uses the default AllowAllAuthenticator auth (no auth challenge) before the -# patch step runs. After the restart a second wait loop confirms the node is -# back up with PasswordAuthenticator active. +# environment variable for configuring authentication, so each job on a +# service container patches cassandra.yaml inside the running container to +# enable password auth (cassandra-authorizer also enables +# CassandraAuthorizer), and restarts it before running any tests. The +# health-check on the service container uses the default AllowAllAuthenticator +# auth (no auth challenge) before the patch step runs. After the restart a +# second wait loop confirms the node is back up with PasswordAuthenticator +# active. cassandra-two-nodes keeps the default AllowAllAuthenticator. jobs: cassandra-storage: @@ -84,6 +87,20 @@ jobs: echo "Cassandra did not become ready" >&2; exit 1 - name: Unit tests run: cargo test -p extenddb-storage-cassandra --lib + - name: Catalog migration integration tests + # Migration 1's definitions, the status of keyspaces that are not + # catalogs, the lock's lease, the serving registration and + # CreateAccount's account migrations (tests/migrations.rs). + run: cargo test -p extenddb-storage-cassandra --test migrations + env: + EXTENDDB_TEST_CASSANDRA: required + - name: Catalog migration crash and load tests + # Runs stopped before a write and run again, migration 2's rewrite + # of a released catalog's policies, and its status read and ledger + # row on catalogs with many policies (tests/migrations_stopped.rs). + run: cargo test -p extenddb-storage-cassandra --test migrations_stopped + env: + EXTENDDB_TEST_CASSANDRA: required - name: TTL integration tests continue-on-error: true # TTL integration tests are known-flaky under CI load; tracked in #338 run: cargo test -p extenddb-storage-cassandra --test ttl_integration -- --test-threads=1 @@ -211,16 +228,235 @@ jobs: - name: Check production Cassandra feature graph run: cargo check --release --no-default-features --features cassandra + # The current schema, serve and migrate on one catalog, and the v0.1.14 + # release against catalogs this binary created and migrated. + cassandra-catalog-checks: + runs-on: extenddb_ubuntu-2404_4-core + timeout-minutes: 60 + services: + cassandra: + image: cassandra:4.1 + ports: + - 127.0.0.1:9042:9042 + env: + CASSANDRA_CLUSTER_NAME: extenddb-ci + HEAP_NEWSIZE: 128M + MAX_HEAP_SIZE: 1024M + options: >- + --health-cmd "cqlsh -e 'SELECT release_version FROM system.local'" + --health-interval 15s + --health-timeout 10s + --health-retries 20 + --health-start-period 60s + steps: + - uses: actions/checkout@v6 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: true + - name: Configure Cassandra authentication + run: | + docker exec ${{ job.services.cassandra.id }} sed -i \ + 's/^authenticator: .*/authenticator: PasswordAuthenticator/' \ + /etc/cassandra/cassandra.yaml + docker restart ${{ job.services.cassandra.id }} + for i in $(seq 1 90); do + if docker exec ${{ job.services.cassandra.id }} \ + cqlsh -u cassandra -p cassandra \ + -e 'SELECT release_version FROM system.local' \ + >/dev/null 2>&1; then + echo "Cassandra ready with PasswordAuthenticator after ${i}s" + exit 0 + fi + sleep 2 + done + echo "Cassandra did not become ready" >&2; exit 1 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Install Python dependencies + run: pip install -r requirements.txt + - name: Build release (cassandra backend) + run: cargo build --release --no-default-features --features cassandra + + # A fresh install's schema against the committed file. On a difference + # the job fails with the diff and uploads the regenerated file to commit. + - name: Check the current schema + env: + EXTENDDB_CASSANDRA_PASSWORD: cassandra + run: | + ./target/release/extenddb init --config schema-check.toml \ + --cassandra-contact-points 127.0.0.1:9042 --cassandra-user cassandra \ + --keyspace-prefix schema_check --extenddb-user schema_check_app \ + --bind-addr 127.0.0.1 + devtools/schema-dump cassandra ${{ job.services.cassandra.id }} schema_check \ + > schema-cassandra.txt + if ! diff -u docs/schema/cassandra.txt schema-cassandra.txt; then + echo "docs/schema/cassandra.txt differs from a fresh install: commit the schema-cassandra artifact as docs/schema/cassandra.txt" + exit 1 + fi + + - name: Upload the regenerated schema + if: failure() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: schema-cassandra + path: schema-cassandra.txt + if-no-files-found: ignore + + - name: Check serve and migrate on one catalog + run: >- + devtools/check-cassandra-catalog-lock ./target/release/extenddb + ${{ job.services.cassandra.id }} 127.0.0.1:9042 18454 + + # No released image carries the Cassandra backend, so the job builds + # the last 0.1.x release from its tag. + - name: Build the v0.1.14 release (cassandra backend) + run: | + git fetch --depth 1 origin tag v0.1.14 + git worktree add "$RUNNER_TEMP/extenddb-v0.1.14" v0.1.14 + cargo build --release --no-default-features --features cassandra \ + --manifest-path "$RUNNER_TEMP/extenddb-v0.1.14/Cargo.toml" \ + --target-dir "$RUNNER_TEMP/target-v0.1.14" + + - name: Check the last 0.1.x release against migrated catalogs + run: >- + devtools/check-cassandra-older-binary ./target/release/extenddb + ${{ runner.temp }}/target-v0.1.14/release/extenddb + ${{ job.services.cassandra.id }} 127.0.0.1:9042 18455 + + # Two nodes, so a statement that follows a CREATE or ALTER can reach the + # node that did not coordinate it (#637), and two migrators reach the + # catalog through different coordinators. + cassandra-two-nodes: + runs-on: extenddb_ubuntu-2404_4-core + timeout-minutes: 45 + steps: + - uses: actions/checkout@v6 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: true + - name: Start a two-node cluster + run: | + docker network create cassandra-two-nodes + for node in 1 2; do + seeds=() + if [ "$node" = 2 ]; then + seeds=(-e CASSANDRA_SEEDS=cassandra-1) + fi + docker run -d --name "cassandra-$node" --network cassandra-two-nodes \ + -p "127.0.0.1:$((9041 + node)):9042" "${seeds[@]}" \ + -e CASSANDRA_CLUSTER_NAME=extenddb-ci \ + -e HEAP_NEWSIZE=128M -e MAX_HEAP_SIZE=1024M cassandra:4.1 + for i in $(seq 1 120); do + if docker exec "cassandra-$node" \ + cqlsh -e 'SELECT release_version FROM system.local' >/dev/null 2>&1; then + echo "cassandra-$node ready after ${i} tries" + break + fi + sleep 2 + done + done + for i in $(seq 1 60); do + if [ "$(docker exec cassandra-1 nodetool status | grep -c '^UN')" = 2 ]; then + docker exec cassandra-1 nodetool status + exit 0 + fi + sleep 2 + done + docker exec cassandra-1 nodetool status + echo "the second node did not join" >&2; exit 1 + - name: Catalog migration integration tests on two nodes + run: cargo test -p extenddb-storage-cassandra --test migrations + env: + EXTENDDB_TEST_CASSANDRA: required + EXTENDDB_TEST_CASSANDRA_CONTACT_POINT: 127.0.0.1:9042 + EXTENDDB_TEST_CASSANDRA_SECOND_CONTACT_POINT: 127.0.0.1:9043 + # The containers carry fixed names and publish 9042 and 9043, so a + # runner that is not ephemeral needs them gone before its next run. + - name: Remove the two nodes + if: always() + run: | + docker rm -f -v cassandra-1 cassandra-2 || true + docker network rm cassandra-two-nodes || true + + # init, migrate, serve and CreateAccount with the application role init + # creates, on a node that enforces permissions. + cassandra-authorizer: + runs-on: extenddb_ubuntu-2404_4-core + timeout-minutes: 45 + services: + cassandra: + image: cassandra:4.1 + ports: + - 127.0.0.1:9042:9042 + env: + CASSANDRA_CLUSTER_NAME: extenddb-ci + HEAP_NEWSIZE: 128M + MAX_HEAP_SIZE: 1024M + options: >- + --health-cmd "cqlsh -e 'SELECT release_version FROM system.local'" + --health-interval 15s + --health-timeout 10s + --health-retries 20 + --health-start-period 60s + steps: + - uses: actions/checkout@v6 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: true + - name: Configure Cassandra authentication and CassandraAuthorizer + run: | + docker exec ${{ job.services.cassandra.id }} sed -i \ + -e 's/^authenticator: .*/authenticator: PasswordAuthenticator/' \ + -e 's/^authorizer: .*/authorizer: CassandraAuthorizer/' \ + /etc/cassandra/cassandra.yaml + docker restart ${{ job.services.cassandra.id }} + for i in $(seq 1 90); do + if docker exec ${{ job.services.cassandra.id }} \ + cqlsh -u cassandra -p cassandra \ + -e 'SELECT release_version FROM system.local' \ + >/dev/null 2>&1; then + echo "Cassandra ready with CassandraAuthorizer after ${i}s" + exit 0 + fi + sleep 2 + done + echo "Cassandra did not become ready" >&2; exit 1 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Install Python dependencies + run: pip install -r requirements.txt + - name: Build release (cassandra backend) + run: cargo build --release --no-default-features --features cassandra + - name: Check init, migrate, serve and CreateAccount under CassandraAuthorizer + run: >- + devtools/check-cassandra-authorizer ./target/release/extenddb + ${{ job.services.cassandra.id }} 127.0.0.1:9042 18456 + cassandra-integration: runs-on: extenddb_ubuntu-2404_4-core timeout-minutes: 75 - needs: [cassandra-storage, cassandra-pytest, cassandra-rust, cassandra-production-build] + needs: + - cassandra-storage + - cassandra-pytest + - cassandra-rust + - cassandra-production-build + - cassandra-catalog-checks + - cassandra-two-nodes + - cassandra-authorizer if: always() steps: - run: | if [ "${{ needs.cassandra-storage.result }}" != "success" ] || \ [ "${{ needs.cassandra-pytest.result }}" != "success" ] || \ [ "${{ needs.cassandra-rust.result }}" != "success" ] || \ - [ "${{ needs.cassandra-production-build.result }}" != "success" ]; then + [ "${{ needs.cassandra-production-build.result }}" != "success" ] || \ + [ "${{ needs.cassandra-catalog-checks.result }}" != "success" ] || \ + [ "${{ needs.cassandra-two-nodes.result }}" != "success" ] || \ + [ "${{ needs.cassandra-authorizer.result }}" != "success" ]; then exit 1 fi diff --git a/.github/workflows/integration-mongodb.yml b/.github/workflows/integration-mongodb.yml index 03223f967..50f1f635e 100644 --- a/.github/workflows/integration-mongodb.yml +++ b/.github/workflows/integration-mongodb.yml @@ -119,6 +119,47 @@ jobs: - name: Run MongoDB rust integration suite run: devtools/run-mongodb-tests -- --rust --rust-integration + # The catalog migrations against released catalogs and released binaries, + # each check on a mongo:7 replica set of its own that the script starts + # with its ports on 127.0.0.1 and removes. The schema check uploads the + # regenerated file to commit when it differs. + mongodb-catalog: + runs-on: extenddb_ubuntu-2404_4-core + timeout-minutes: 40 + steps: + - uses: actions/checkout@v6 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: true + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Install Python dependencies + run: pip install -r requirements.txt + - name: Build release (mongodb backend) + run: cargo build --release --no-default-features --features mongodb-test-hooks + - name: Check the current schema + run: devtools/check-mongodb-schema ./target/release/extenddb schema-mongodb.txt 27071 + - name: Upload the regenerated schema + if: failure() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: schema-mongodb + path: schema-mongodb.txt + if-no-files-found: ignore + - name: Check serve and migrate on one catalog + run: devtools/check-mongodb-catalog-lock ./target/release/extenddb 27041 18460 + - name: Check the commands and the server as a readWrite user + run: devtools/check-mongodb-auth ./target/release/extenddb 27081 18490 + - name: Check the released catalogs against a fresh install + run: devtools/check-mongodb-released ./target/release/extenddb 27061 18480 + - name: Check the last 0.1.x release against migrated catalogs + run: | + docker pull extenddb/extenddb-mongodb:0.1.14 + devtools/check-mongodb-older-binary ./target/release/extenddb \ + extenddb/extenddb-mongodb:0.1.14 27051 18470 + mongodb-production-build: runs-on: extenddb_ubuntu-2404_4-core timeout-minutes: 40 @@ -134,12 +175,13 @@ jobs: mongodb-integration: runs-on: extenddb_ubuntu-2404_4-core timeout-minutes: 40 - needs: [mongodb-pytest, mongodb-rust, mongodb-production-build, mongodb-snapshot] + needs: [mongodb-pytest, mongodb-rust, mongodb-catalog, mongodb-production-build, mongodb-snapshot] if: always() steps: - run: | if [ "${{ needs.mongodb-pytest.result }}" != "success" ] || \ [ "${{ needs.mongodb-rust.result }}" != "success" ] || \ + [ "${{ needs.mongodb-catalog.result }}" != "success" ] || \ [ "${{ needs.mongodb-production-build.result }}" != "success" ] || \ [ "${{ needs.mongodb-snapshot.result }}" != "success" ]; then exit 1 diff --git a/devtools/catalog_checks.py b/devtools/catalog_checks.py index a60f29970..ddfa8d94b 100644 --- a/devtools/catalog_checks.py +++ b/devtools/catalog_checks.py @@ -5,7 +5,10 @@ server. devtools/check-sqlite-catalog-lock, check-sqlite-older-binary, -check-postgres-catalog-lock and check-postgres-older-binary import it. Each +check-postgres-catalog-lock, check-postgres-older-binary, +check-cassandra-catalog-lock, check-cassandra-older-binary, +check-cassandra-authorizer and the check-mongodb-* scripts import it; the +MongoDB ones start their replica sets with `start_mongo`. Each check prints one line, `ok` or `FAIL` and what was checked, with the command's output indented under a FAIL; `finish` prints the count of failed checks or `all checks passed` and returns the script's exit code. @@ -107,3 +110,42 @@ def finish() -> int: return 1 print("all checks passed") return 0 + + +def docker(*args: str, timeout: float = 120) -> subprocess.CompletedProcess: + """Run `docker` with `args`, capturing its output.""" + return subprocess.run(["docker", *args], capture_output=True, text=True, check=False, timeout=timeout) + + +def mongosh(container: str, port: int, script: str, timeout: float = 60) -> tuple[int, str]: + """Run `script` with mongosh in a container `start_mongo` started.""" + done = docker("exec", container, "mongosh", "--quiet", "--port", str(port), "--eval", script, timeout=timeout) + return done.returncode, done.stdout + done.stderr + + +def start_mongo(container: str, port: int, mongod_args: tuple[str, ...] = (), mounts: tuple[str, ...] = ()) -> bool: + """Start mongo:7 as a one-node replica set named rs0 in `container`, + listening on `port` inside the container and published on 127.0.0.1 + only, so the same connection string works from the host and from a + container that shares its network. Waits for the member to be primary. + `mongod_args` are added to mongod's command line, `mounts` are `-v` + arguments; a set started with `--keyFile` initiates through the + localhost exception.""" + docker("rm", "-f", container) + volumes = [arg for mount in mounts for arg in ("-v", mount)] + started = docker( + "run", "-d", "--name", container, "-p", f"127.0.0.1:{port}:{port}", *volumes, + "mongo:7", "mongod", "--replSet", "rs0", "--bind_ip_all", "--port", str(port), *mongod_args, + ) + if started.returncode != 0: + return report(False, f"start {container}", started.stderr) + for _ in range(60): + if mongosh(container, port, "db.runCommand({ping: 1}).ok")[1].strip().endswith("1"): + break + time.sleep(1) + mongosh(container, port, f"rs.initiate({{_id: 'rs0', members: [{{_id: 0, host: 'localhost:{port}'}}]}})") + for _ in range(60): + if mongosh(container, port, "db.hello().isWritablePrimary")[1].strip().endswith("true"): + return True + time.sleep(1) + return report(False, f"{container} did not become primary", docker("logs", container).stdout[-2000:]) diff --git a/devtools/check-cassandra-authorizer b/devtools/check-cassandra-authorizer new file mode 100755 index 000000000..6fd32acae --- /dev/null +++ b/devtools/check-cassandra-authorizer @@ -0,0 +1,261 @@ +#!/usr/bin/env python3 +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 +"""init, migrate, serve and CreateAccount on a Cassandra node that runs +CassandraAuthorizer, as the application role init creates: + + devtools/check-cassandra-authorizer \\ + [port] + + and are as in devtools/check-cassandra-catalog-lock. +The node must run PasswordAuthenticator and CassandraAuthorizer; the script +reads the authorizer from system_views.settings and fails on any other, so +it never passes without testing the grants. + +- init, as the admin role, creates the application role and grants it + SELECT, MODIFY, CREATE and ALTER on the catalog keyspace, CREATE on ALL + KEYSPACES, and the same four with DROP on the default account's keyspace + (LIST ALL PERMISSIONS OF the role); +- as that role: verify reports Current, migrate --yes finds nothing to do, + and migrate --yes --reconverge runs every migration again; +- serve starts, and CreateAccount through the management API creates an + account whose keyspace records migration 1 in its ledger; CreateTable, + PutItem, GetItem and DeleteTable work in that account; +- with the server stopped, verify reports Current and migrate --yes + --reconverge runs again over both account keyspaces; +- with the new account's keyspace dropped, as a CreateAccount that stopped + before creating it leaves an accounts row, migrate --yes as the admin role + creates the keyspace and grants the application role its permissions on + it, and the server serves the account again as the application role. + +Everything the script creates (the role, its keyspaces, the temporary +directory) is removed at the end. The server binds 127.0.0.1 only. +""" +import os +import re +import secrets +import shutil +import subprocess +import sys +import tempfile +import time + +from catalog_checks import expect_exit, finish, healthy, report, run, start, stop + +HERE = os.path.dirname(os.path.realpath(__file__)) +ADMIN_USER = os.environ.get("CASSANDRA_ADMIN_USER", "cassandra") +ADMIN_PASSWORD = os.environ.get("CASSANDRA_ADMIN_PASSWORD", "cassandra") +ADMIN_LOGIN_PASSWORD = "AdminPassw0rd!x" +ACCOUNT_ID = "123456789012" # the account devtools/provision-test-credentials creates + + +def cqlsh(container: str, statements: str) -> tuple[int, str]: + """Run CQL statements in the node's cqlsh as the admin role.""" + done = subprocess.run( + ["docker", "exec", "-i", container, "cqlsh", "-u", ADMIN_USER, "-p", ADMIN_PASSWORD], + input=statements, + capture_output=True, + text=True, + check=False, + timeout=120, + ) + return done.returncode, done.stdout + done.stderr + + +def permissions(container: str, role: str) -> set[tuple[str, str]]: + """The role's (resource, permission) pairs, from LIST ALL PERMISSIONS.""" + code, output = cqlsh(container, f"LIST ALL PERMISSIONS OF '{role}';\n") + if code != 0: + report(False, f"list the permissions of {role}", output) + return set() + found = set() + for line in output.splitlines(): + cells = [cell.strip() for cell in line.split("|")] + if len(cells) == 4 and cells[3] and cells[3] != "permission": + found.add((cells[2], cells[3])) + return found + + +def item_operations(port: int, keys: dict, where: str) -> None: + """CreateTable, PutItem, GetItem and DeleteTable through the server with + the account's credentials, each reported.""" + try: + import boto3 + import urllib3 + + urllib3.disable_warnings() + client = boto3.client( + "dynamodb", + endpoint_url=f"https://127.0.0.1:{port}", + region_name="us-east-1", + aws_access_key_id=keys["AWS_ACCESS_KEY_ID"], + aws_secret_access_key=keys["AWS_SECRET_ACCESS_KEY"], + verify=False, + ) + client.create_table( + TableName="granted", + AttributeDefinitions=[{"AttributeName": "pk", "AttributeType": "S"}], + KeySchema=[{"AttributeName": "pk", "KeyType": "HASH"}], + BillingMode="PAY_PER_REQUEST", + ) + for _ in range(120): + if client.describe_table(TableName="granted")["Table"]["TableStatus"] == "ACTIVE": + break + time.sleep(0.5) + client.put_item(TableName="granted", Item={"pk": {"S": "a"}, "v": {"S": "b"}}) + item = client.get_item(TableName="granted", Key={"pk": {"S": "a"}}).get("Item") + report(item == {"pk": {"S": "a"}, "v": {"S": "b"}}, f"CreateTable, PutItem and GetItem {where}") + client.delete_table(TableName="granted") + report(True, f"DeleteTable {where}") + except Exception as e: # noqa: BLE001 - reported as a failed check + report(False, f"the item operations {where}", repr(e)) + + +def main(binary: str, container: str, contact_point: str, port: int) -> int: + binary = os.path.realpath(binary) + stem = f"eddb_authz_{secrets.token_hex(4)}" + app_password = f"App{secrets.token_hex(8)}" + work = tempfile.mkdtemp(prefix=f"{stem}.") + env = dict(os.environ, HOME=os.path.join(work, "home")) + config = os.path.join(work, "extenddb.toml") + server = None + try: + code, output = cqlsh( + container, "SELECT value FROM system_views.settings WHERE name = 'authorizer';\n" + ) + if not report( + code == 0 and "CassandraAuthorizer" in output, + "the node runs CassandraAuthorizer", + output, + ): + return finish() + code, output = run( + [ + binary, "init", "--config", config, + "--cassandra-contact-points", contact_point, + "--cassandra-user", ADMIN_USER, + "--keyspace-prefix", stem, "--extenddb-user", stem, + "--bind-addr", "127.0.0.1", + ], + dict( + env, + EXTENDDB_CASSANDRA_PASSWORD=ADMIN_PASSWORD, + EXTENDDB_APP_PASSWORD=app_password, + EXTENDDB_ADMIN_USER="admin", + EXTENDDB_ADMIN_PASSWORD=ADMIN_LOGIN_PASSWORD, + ), + timeout=240, + ) + if not report(code == 0, f"exit {code}: init under CassandraAuthorizer", output): + return finish() + code, output = cqlsh(container, "SELECT keyspace_name FROM system_schema.keyspaces;\n") + accounts = [w for w in output.split() if w.startswith(f"{stem}_account_")] + if not report(len(accounts) == 1, "init created the default account's keyspace", output): + return finish() + catalog, default_account = f"{stem}_catalog", accounts[0] + granted = permissions(container, stem) + wanted = {(f"", p) for p in ("SELECT", "MODIFY", "CREATE", "ALTER")} + wanted |= {(f"", p) for p in ("SELECT", "MODIFY", "CREATE", "ALTER", "DROP")} + wanted.add(("", "CREATE")) + report( + wanted <= granted and (f"", "DROP") not in granted, + "init granted the application role its permissions, and no DROP on the catalog", + "\n".join(f"{r} {p}" for r, p in sorted(granted)), + ) + + migrate = [binary, "migrate", "--config", config] + verify = [binary, "verify", "--config", config] + expect_exit(0, "verify as the application role", verify, env, contains="OK: Current") + expect_exit(0, "migrate --yes as the application role", migrate + ["--yes"], env, contains="Up to date") + expect_exit( + 0, + "migrate --yes --reconverge as the application role", + migrate + ["--yes", "--reconverge"], + env, + contains="Ran again 1 baseline", + ) + + log = os.path.join(work, "serve.log") + server = start( + [binary, "serve", "--config", config, "--foreground", "--port", str(port)], env, log + ) + if not report(healthy(f"https://127.0.0.1:{port}/health", server, 90), "serve as the application role"): + with open(log, encoding="utf-8") as f: + report(False, "the server log", f.read()[-3000:]) + return finish() + code, output = run( + [sys.executable, os.path.join(HERE, "provision-test-credentials")], + dict( + env, + EXTENDDB_TEST_ENDPOINT=f"https://127.0.0.1:{port}", + EXTENDDB_ADMIN_USER="admin", + EXTENDDB_ADMIN_PASSWORD=ADMIN_LOGIN_PASSWORD, + ), + ) + keys = dict(re.findall(r"export (AWS_[A-Z_]+)=(\S+)", output)) + if not report(code == 0 and "AWS_ACCESS_KEY_ID" in keys, "CreateAccount and its credentials", output): + return finish() + new_account = f"{stem}_account_{ACCOUNT_ID}" + code, output = cqlsh(container, f"SELECT id, outcome FROM {new_account}.extenddb_migrations;\n") + report( + code == 0 and re.search(r"^\s*1\s*\|\s*applied", output, re.M) is not None, + "the new account's keyspace records migration 1", + output, + ) + item_operations(port, keys, "in the new account") + stop(server) + server = None + expect_exit(0, "verify over both account keyspaces", verify, env, contains="OK: Current") + expect_exit( + 0, + "migrate --yes --reconverge over both account keyspaces", + migrate + ["--yes", "--reconverge"], + env, + contains="Ran again 1 baseline", + ) + + # An accounts row whose keyspace is missing, as a CreateAccount that + # stopped before creating it leaves one: migrate --yes as the admin + # role creates the keyspace, and the application role then holds its + # permissions on it and serves the account. + code, output = cqlsh(container, f"DROP KEYSPACE {new_account};\n") + report(code == 0, "drop the new account's keyspace", output) + expect_exit( + 0, + "migrate --yes as the admin role creates the missing account keyspace", + migrate + ["--yes", "--cassandra-user", ADMIN_USER], + dict(env, EXTENDDB_CASSANDRA_PASSWORD=ADMIN_PASSWORD), + ) + granted = permissions(container, stem) + wanted = {(f"", p) for p in ("SELECT", "MODIFY", "CREATE", "ALTER", "DROP")} + report( + wanted <= granted, + "the application role holds its permissions on the keyspace migrate created", + "\n".join(f"{r} {p}" for r, p in sorted(granted)), + ) + server = start( + [binary, "serve", "--config", config, "--foreground", "--port", str(port)], env, log + ) + if not report(healthy(f"https://127.0.0.1:{port}/health", server, 90), "serve again as the application role"): + with open(log, encoding="utf-8") as f: + report(False, "the server log", f.read()[-3000:]) + return finish() + item_operations(port, keys, "in the account whose keyspace migrate created") + stop(server) + server = None + finally: + if server is not None: + stop(server) + code, output = cqlsh(container, "SELECT keyspace_name FROM system_schema.keyspaces;\n") + statements = "".join( + f"DROP KEYSPACE IF EXISTS {word};\n" for word in output.split() if word.startswith(f"{stem}_") + ) + cqlsh(container, statements + f"DROP ROLE IF EXISTS '{stem}';\n") + shutil.rmtree(work, ignore_errors=True) + return finish() + + +if __name__ == "__main__": + if len(sys.argv) not in (4, 5): + sys.exit(__doc__) + sys.exit(main(sys.argv[1], sys.argv[2], sys.argv[3], int(sys.argv[4]) if len(sys.argv) == 5 else 18456)) diff --git a/devtools/check-cassandra-catalog-lock b/devtools/check-cassandra-catalog-lock new file mode 100755 index 000000000..c02ea4b27 --- /dev/null +++ b/devtools/check-cassandra-catalog-lock @@ -0,0 +1,326 @@ +#!/usr/bin/env python3 +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 +"""Serve and migrate on one Cassandra catalog, with a release binary: + + devtools/check-cassandra-catalog-lock \\ + [port] + + is the Cassandra node's Docker container, which the script reads +and writes with `docker exec ... cqlsh` as the admin role +($CASSANDRA_ADMIN_USER and $CASSANDRA_ADMIN_PASSWORD, default cassandra and +cassandra). is the node's host:port as the binary reaches it, +for example 127.0.0.1:9042. The script creates its own role and keyspaces +with `init` and removes them at the end. + +- a store that is not a catalog: for a keyspace prefix with no keyspace, a + keyspace whose settings table holds only the lock row, and another + program's keyspace, migrate --yes exits 1 naming `extenddb init`; for + another program's keyspace whose settings table has the catalog's key and + value columns and a row of its own, it exits 1 saying the keyspace is not + an ExtendDB catalog; each keyspace's tables and settings rows are left as + they were; +- verify, the migrate preview and migrate --list exit 0 on the fresh catalog; +- the preview and verify run beside a server and exit 0; +- migrate --yes waits for the server's registration and exits 6, naming + "extenddb serve", when it outlasts --lock-timeout; +- a stopped server removes its registration, so migrate --yes runs at once; +- while another session holds the migration lock (the lock row, written by + the script with a lightweight transaction as a migration would), serve and + a second migrate --yes exit 6, and the preview exits 0; +- once the lock is free, migrate --yes --reconverge, migrate --forget 3 and + the migrate --yes that applies 3 again run, and serve runs; +- a server that cannot renew its registration for 30 s, with the node + paused, stops with exit 1 and removes its PID file. + +init's HOME points into a temporary directory, so its TLS certificate is not +written to the caller's home. The server binds 127.0.0.1 only. +""" +import os +import secrets +import shutil +import subprocess +import sys +import tempfile +import time +import tomllib + +from catalog_checks import expect_exit, finish, healthy, report, run, start, stop + +ADMIN_USER = os.environ.get("CASSANDRA_ADMIN_USER", "cassandra") +ADMIN_PASSWORD = os.environ.get("CASSANDRA_ADMIN_PASSWORD", "cassandra") + + +def cqlsh(container: str, statements: str) -> tuple[int, str]: + """Run CQL statements in the node's cqlsh as the admin role.""" + done = subprocess.run( + ["docker", "exec", "-i", container, "cqlsh", "-u", ADMIN_USER, "-p", ADMIN_PASSWORD], + input=statements, + capture_output=True, + text=True, + check=False, + timeout=120, + ) + return done.returncode, done.stdout + done.stderr + + +def config_with_prefix(source: str, target: str, prefix: str) -> None: + """A copy of the config `init` wrote, with another keyspace prefix.""" + with open(source, encoding="utf-8") as f: + lines = f.read().splitlines() + with open(target, "w", encoding="utf-8") as f: + for line in lines: + if line.strip().startswith("keyspace_prefix"): + line = f'keyspace_prefix = "{prefix}"' + f.write(line + "\n") + os.chmod(target, 0o600) + + +def tables(container: str, keyspace: str) -> str: + code, output = cqlsh( + container, + f"SELECT table_name FROM system_schema.tables WHERE keyspace_name = '{keyspace}';\n", + ) + return output if code == 0 else f"cqlsh failed: {output}" + + +def settings_keys(container: str, keyspace: str) -> str: + """The keys of the keyspace's settings table, or "" without one.""" + if "settings" not in tables(container, keyspace).split(): + return "" + code, output = cqlsh(container, f"SELECT key FROM {keyspace}.settings;\n") + return output if code == 0 else f"cqlsh failed: {output}" + + +def not_a_catalog(binary: str, env: dict, container: str, config: str, stem: str, work: str) -> None: + """migrate --yes refuses every store that is not a catalog with exit 1: + naming `extenddb init` for one init would create or adopt, and saying it + is not a catalog, and what it holds, for another program's keyspace, + which init refuses too.""" + cases = { + "a prefix with no keyspace": ("", "extenddb init"), + # Under its own lock, migrate --yes finds this keyspace holding only + # the lock row it wrote. + "a keyspace whose settings table holds only the lock row": ( + "CREATE KEYSPACE {ks} WITH replication = {{'class': 'SimpleStrategy', " + "'replication_factor': 1}};\n" + "CREATE TABLE {ks}.settings (key text PRIMARY KEY, value text);\n", + "extenddb init", + ), + "another program's keyspace": ( + "CREATE KEYSPACE {ks} WITH replication = {{'class': 'SimpleStrategy', " + "'replication_factor': 1}};\n" + "CREATE TABLE {ks}.settings (name text PRIMARY KEY, val int);\n" + "CREATE TABLE {ks}.orders (id int PRIMARY KEY);\n", + "is not an ExtendDB catalog", + ), + "another program's keyspace with a settings table of the catalog's columns": ( + "CREATE KEYSPACE {ks} WITH replication = {{'class': 'SimpleStrategy', " + "'replication_factor': 1}};\n" + "CREATE TABLE {ks}.settings (key text PRIMARY KEY, value text);\n" + "INSERT INTO {ks}.settings (key, value) VALUES ('theme', 'dark');\n", + "is not an ExtendDB catalog", + ), + } + for i, (label, (setup, named)) in enumerate(cases.items()): + prefix = f"{stem}_x{i}" + keyspace = f"{prefix}_catalog" + if setup: + code, output = cqlsh(container, setup.format(ks=keyspace)) + if not report(code == 0, f"set up {label}", output): + continue + other = os.path.join(work, f"x{i}.toml") + config_with_prefix(config, other, prefix) + before = tables(container, keyspace) + settings_keys(container, keyspace) + expect_exit( + 1, + f"migrate --yes on {label}", + [binary, "migrate", "--config", other, "--yes", "--lock-timeout", "5"], + env, + contains=named, + ) + expect_exit(1, f"the migrate preview on {label}", [binary, "migrate", "--config", other], env, contains=named) + after = tables(container, keyspace) + settings_keys(container, keyspace) + report(after == before, f"{label} is left as it was") + # Another process's lock row in a keyspace that is not a catalog yet: the + # preview, which takes no lock, reports it uninitialized. + keyspace = f"{stem}_x1_catalog" + code, output = cqlsh( + container, + "CONSISTENCY QUORUM;\nSERIAL CONSISTENCY SERIAL;\n" + f"INSERT INTO {keyspace}.settings (key, value) VALUES ('migration_lock', 'another') " + "IF NOT EXISTS USING TTL 300;\n", + ) + if report(code == 0 and "True" in output, "another process holds the lock row", output): + expect_exit( + 1, + "the migrate preview on a keyspace holding only another process's lock row", + [binary, "migrate", "--config", os.path.join(work, "x1.toml")], + env, + contains="extenddb init", + ) + + +def main(binary: str, container: str, contact_point: str, port: int) -> int: + binary = os.path.realpath(binary) + stem = f"eddb_lock_{secrets.token_hex(4)}" + app_password = f"App{secrets.token_hex(8)}" + work = tempfile.mkdtemp(prefix=f"{stem}.") + env = dict(os.environ, HOME=os.path.join(work, "home")) + config = os.path.join(work, "extenddb.toml") + serve = [binary, "serve", "--config", config, "--foreground", "--write-pid-file", "--port", str(port)] + servers: list[subprocess.Popen] = [] + paused = False + try: + code, output = run( + [ + binary, "init", "--config", config, + "--cassandra-contact-points", contact_point, + "--cassandra-user", ADMIN_USER, + "--keyspace-prefix", stem, "--extenddb-user", stem, + "--bind-addr", "127.0.0.1", + ], + dict(env, EXTENDDB_CASSANDRA_PASSWORD=ADMIN_PASSWORD, EXTENDDB_APP_PASSWORD=app_password), + timeout=180, + ) + if not report(code == 0, f"exit {code}: init", output): + return finish() + with open(config, "rb") as f: + run_dir = os.path.expanduser( + tomllib.load(f).get("server", {}).get("run_dir", "~/.extenddb/run").replace( + "~", env["HOME"], 1 + ) + ) + pid_file = os.path.join(run_dir, f"extenddb-{port}.pid") + migrate = [binary, "migrate", "--config", config] + expect_exit(0, "verify on the fresh catalog", [binary, "verify", "--config", config], env, contains="OK: Current") + expect_exit(0, "the migrate preview on the fresh catalog", migrate, env, contains="Up to date") + expect_exit(0, "migrate --list", migrate + ["--list"], env, contains="not applicable") + not_a_catalog(binary, env, container, config, stem, work) + + log = os.path.join(work, "serve.log") + server = start(serve, env, log) + servers.append(server) + if not healthy(f"https://127.0.0.1:{port}/health", server): + stop(server) + with open(log, encoding="utf-8") as f: + report(False, "the server became healthy", f.read()) + return finish() + print("ok the server is healthy") + expect_exit(0, "migrate preview beside a server", migrate, env) + expect_exit(0, "verify beside a server", [binary, "verify", "--config", config], env) + expect_exit( + 6, + "migrate --yes while a server runs", + migrate + ["--yes", "--lock-timeout", "3"], + env, + contains="extenddb serve", + ) + stop(server) + started = time.monotonic() + expect_exit( + 0, + "migrate --yes once the server stopped, without waiting for its registration's TTL", + migrate + ["--yes", "--lock-timeout", "5"], + env, + ) + report(time.monotonic() - started < 30, "the stopped server's registration was removed") + + keyspace = f"{stem}_catalog" + code, output = cqlsh( + container, + "CONSISTENCY QUORUM;\nSERIAL CONSISTENCY SERIAL;\n" + f"INSERT INTO {keyspace}.settings (key, value) VALUES ('migration_lock', " + "'held-by-check') IF NOT EXISTS USING TTL 300;\n", + ) + report(code == 0 and "True" in output, "the script holds the migration lock", output) + expect_exit(6, "serve while a migration holds the lock", serve, env, contains="migration lock") + expect_exit( + 6, + "a second migrate --yes while a migration holds the lock", + migrate + ["--yes", "--lock-timeout", "2"], + env, + ) + expect_exit(0, "migrate preview while a migration holds the lock", migrate, env) + code, output = cqlsh( + container, + "CONSISTENCY QUORUM;\nSERIAL CONSISTENCY SERIAL;\n" + f"DELETE FROM {keyspace}.settings WHERE key = 'migration_lock' IF value = " + "'held-by-check';\n", + ) + report(code == 0 and "True" in output, "the script releases the lock", output) + + expect_exit( + 0, + "migrate --yes --reconverge once the lock is free, every body run again", + migrate + ["--yes", "--reconverge", "--lock-timeout", "5"], + env, + contains="Ran again 1 baseline", + ) + expect_exit(0, "migrate --forget 3", migrate + ["--forget", "3"], env) + expect_exit(3, "the preview lists migration 3 as pending", migrate, env, contains="3 sort_key_definitions") + expect_exit( + 0, + "migrate --yes applies migration 3 again", + migrate + ["--yes", "--lock-timeout", "5"], + env, + contains="Applied 3 sort_key_definitions", + ) + + server = start(serve, env, log) + servers.append(server) + if not report(healthy(f"https://127.0.0.1:{port}/health", server), "serve once the lock is free"): + return finish() + report(os.path.exists(pid_file), f"the server wrote its PID file {pid_file}") + # The node stops answering: the server cannot renew its registration + # and must stop before the row's 45 s TTL ends. + subprocess.run(["docker", "pause", container], check=True, capture_output=True) + paused = True + started = time.monotonic() + try: + code = server.wait(timeout=60) + except subprocess.TimeoutExpired: + code = None + waited = time.monotonic() - started + subprocess.run(["docker", "unpause", container], check=True, capture_output=True) + paused = False + with open(log, encoding="utf-8") as f: + report( + code == 1 and waited < 45, + f"a server that cannot renew its registration stops with exit {code} after " + f"{waited:.0f} s", + f.read()[-3000:], + ) + report(not os.path.exists(pid_file), "the stopped server removed its PID file") + finally: + if paused: + subprocess.run(["docker", "unpause", container], check=False) + for server in servers: + stop(server) + cleanup(container, stem) + shutil.rmtree(work, ignore_errors=True) + return finish() + + +def cleanup(container: str, stem: str) -> None: + """Drop every keyspace with the script's prefix, and its role.""" + for _ in range(30): + code, _ = cqlsh(container, "SELECT now() FROM system.local;\n") + if code == 0: + break + time.sleep(2) + code, output = cqlsh(container, "SELECT keyspace_name FROM system_schema.keyspaces;\n") + keyspaces = [ + word.strip() + for word in output.split() + if word.strip().startswith(f"{stem}_") + ] + statements = "".join(f"DROP KEYSPACE IF EXISTS {ks};\n" for ks in keyspaces) + statements += f"DROP ROLE IF EXISTS '{stem}';\n" + cqlsh(container, statements) + + +if __name__ == "__main__": + if len(sys.argv) not in (4, 5): + sys.exit(__doc__) + sys.exit(main(sys.argv[1], sys.argv[2], sys.argv[3], int(sys.argv[4]) if len(sys.argv) == 5 else 18454)) diff --git a/devtools/check-cassandra-older-binary b/devtools/check-cassandra-older-binary new file mode 100755 index 000000000..daacb98cf --- /dev/null +++ b/devtools/check-cassandra-older-binary @@ -0,0 +1,446 @@ +#!/usr/bin/env python3 +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 +"""A Cassandra catalog the last 0.1.x release created, upgraded by this +binary, and that release run against v1.0 catalogs: + + devtools/check-cassandra-older-binary <0.1.14 binary> \\ + [port] + +<0.1.14 binary> is `extenddb` built from the v0.1.14 tag with +`--no-default-features --features cassandra` (there is no released Cassandra +image). and are as in +devtools/check-cassandra-catalog-lock; the node uses PasswordAuthenticator +with the admin role $CASSANDRA_ADMIN_USER and $CASSANDRA_ADMIN_PASSWORD +(default cassandra and cassandra). + +- Released catalog: the 0.1.14 binary's `init` creates a deployment, and its + server is seeded with a table that has a sort key, a GSI, an LSI and a + stream, a table with TTL, items and a backup. This binary's preview exits 3 + and `migrate --yes` applies every migration. A fresh deployment from this + binary's `init`, seeded the same way through this binary's server, is then + compared with it: the columns of every table of the catalog keyspace and of + each account keyspace (per-table tables named by their table and index), + the settings keys and the ledgers. The only accepted difference is the + control_plane_delay_seconds row the 0.1.14 baseline seeded. +- Two migrators: on another 0.1.14 catalog, two `migrate --yes` started + together both exit 0, one applies the migrations and the other waits for + the lock and finds nothing to do. +- Older binary: on the fresh and the upgraded v1.0 catalog the 0.1.14 binary + runs `verify`, `migrate --yes` and `serve`. The script prints what each run + changed and checks what this binary then reports: after a run that wrote + the catalog, `migrate` and `verify` exit 5 naming WrittenByOlderRelease and + `migrate --yes --reconverge` restores Current with the schema the catalog + had before; the 0.1.14 `serve` refuses a v1.0 catalog. + +Everything the script creates (roles, keyspaces, the temporary directory) is +removed at the end. The servers bind 127.0.0.1 only. +""" +import csv +import io +import os +import re +import secrets +import shutil +import subprocess +import sys +import tempfile +import time + +from catalog_checks import expect_exit, finish, healthy, report, run, start, stop + +HERE = os.path.dirname(os.path.realpath(__file__)) +ADMIN_USER = os.environ.get("CASSANDRA_ADMIN_USER", "cassandra") +ADMIN_PASSWORD = os.environ.get("CASSANDRA_ADMIN_PASSWORD", "cassandra") +ADMIN_LOGIN_PASSWORD = "AdminPassw0rd!x" + +# What a 0.1.14 catalog upgraded by this binary may hold that a fresh one +# does not: the row the 0.1.14 baseline seeded, whose value cannot be told +# from one an operator set (SPEC A3). +ACCEPTED = {"+ setting control_plane_delay_seconds"} + + +def cqlsh(container: str, statements: str) -> tuple[int, str]: + done = subprocess.run( + ["docker", "exec", "-i", container, "cqlsh", "-u", ADMIN_USER, "-p", ADMIN_PASSWORD], + input=statements, + capture_output=True, + text=True, + check=False, + timeout=180, + ) + return done.returncode, done.stdout + done.stderr + + +def copy(container: str, table: str, columns: str) -> list[list[str]]: + """Every row of `table`, as `cqlsh COPY ... TO STDOUT` prints them.""" + code, output = cqlsh(container, f"COPY {table} ({columns}) TO STDOUT;\n") + if code != 0: + raise RuntimeError(f"COPY {table}: {output}") + width = len(columns.split(",")) + rows = [] + for row in csv.reader(io.StringIO(output)): + if len(row) == width and not row[0].startswith("Using "): + rows.append(row) + return rows + + +class Deployment: + def __init__(self, stem: str, binary: str, work: str, contact_point: str, old: bool) -> None: + self.stem = stem + self.binary = binary + self.old = old + self.config = os.path.join(work, f"{stem}.toml") + self.app_password = f"App{secrets.token_hex(8)}" + self.contact_point = contact_point + self.env = dict(os.environ, HOME=os.path.join(work, "home")) + + def init(self) -> tuple[int, str]: + # v0.1.14 reads the admin password from --cassandra-pass only. + password = ["--cassandra-pass", ADMIN_PASSWORD] if self.old else [] + code, output = run( + [ + self.binary, "init", "--config", self.config, + "--cassandra-contact-points", self.contact_point, + "--cassandra-user", ADMIN_USER, *password, + "--keyspace-prefix", self.stem, "--extenddb-user", self.stem, + "--bind-addr", "127.0.0.1", + ], + dict( + self.env, + EXTENDDB_CASSANDRA_PASSWORD=ADMIN_PASSWORD, + EXTENDDB_APP_PASSWORD=self.app_password, + EXTENDDB_ADMIN_USER="admin", + EXTENDDB_ADMIN_PASSWORD=ADMIN_LOGIN_PASSWORD, + ), + timeout=240, + ) + if code == 0 and self.old: + # v0.1.14 writes the role and password lines commented out; with + # PasswordAuthenticator the operator sets them to the role init + # created. + with open(self.config, encoding="utf-8") as f: + lines = f.read().splitlines() + with open(self.config, "w", encoding="utf-8") as f: + for line in lines: + if line.startswith("# username = "): + line = f'username = "{self.stem}"' + elif line.startswith("# password = "): + line = f'password = "{self.app_password}"' + f.write(line + "\n") + return code, output + + def snapshot(self, container: str) -> list[str]: + """The deployment's columns, settings keys and ledger rows, with the + keyspace prefix, account ids and table ids replaced by names.""" + prefix = f"{self.stem}_" + accounts = { + row[0]: row[1] + for row in copy(container, f"{self.stem}_catalog.accounts", "account_id, account_name") + } + names = {} + for account_id, table_name, table_id in copy( + container, f"{self.stem}_catalog.tables", "account_id, table_name, table_id" + ): + names[f"items_{table_id.replace('-', '_')}"] = f"items_<{table_name}>" + for row in copy(container, f"{self.stem}_catalog.indexes", "table_id, index_name, index_id"): + if row[0] == table_id: + names[f"index_{row[2].replace('-', '_')}"] = f"index_<{table_name}/{row[1]}>" + def keyspace(name: str) -> str: + rest = name[len(prefix):] + if rest == "catalog": + return "catalog" + account_id = rest.removeprefix("account_") + return f"account<{accounts.get(account_id, account_id)}>" + lines = [] + for ks, table, column, kind, position, order, kind_type in copy( + container, + "system_schema.columns", + "keyspace_name, table_name, column_name, kind, position, clustering_order, type", + ): + if ks.startswith(prefix): + lines.append( + f"column {keyspace(ks)}.{names.get(table, table)}.{column} {kind} {position} " + f"{order} {kind_type}" + ) + for (key,) in copy(container, f"{self.stem}_catalog.settings", "key"): + if key != "migration_lock": + lines.append(f"setting {key}") + for ks in sorted({line.split()[1].split(".")[0] for line in lines if line.startswith("column ")}): + physical = ( + f"{self.stem}_catalog" + if ks == "catalog" + else f"{self.stem}_account_" + + next((i for i, n in accounts.items() if f"account<{n}>" == ks), ks) + ) + # system_schema gives a partition key column position 0. + if any(line.startswith(f"column {ks}.extenddb_migrations.id ") for line in lines): + for row in copy(container, f"{physical}.extenddb_migrations", "id, name, outcome"): + lines.append(f"ledger {ks} {' '.join(row)}") + for row in copy(container, f"{physical}.schema_history", "version, description"): + lines.append(f"legacy {ks} {' '.join(row)}") + return sorted(lines) + + def schema(self, container: str) -> list[str]: + return [line for line in self.snapshot(container) if line.startswith("column ")] + + +def diff(before: list[str], after: list[str]) -> list[str]: + return [f"- {line}" for line in before if line not in after] + [ + f"+ {line}" for line in after if line not in before + ] + + +def seed(binary: str, deployment: Deployment, port: int, work: str) -> bool: + """Start `binary`'s server on the deployment and create the seeded + objects through it.""" + log = os.path.join(work, f"seed-{deployment.stem}.log") + server = start( + [binary, "serve", "--config", deployment.config, "--foreground", "--port", str(port)], + deployment.env, + log, + ) + try: + if not healthy(f"https://127.0.0.1:{port}/health", server, 90): + with open(log, encoding="utf-8") as f: + report(False, f"{deployment.stem}: the server became healthy", f.read()[-3000:]) + return False + code, output = run( + [sys.executable, os.path.join(HERE, "provision-test-credentials")], + dict( + deployment.env, + EXTENDDB_TEST_ENDPOINT=f"https://127.0.0.1:{port}", + EXTENDDB_ADMIN_USER="admin", + EXTENDDB_ADMIN_PASSWORD=ADMIN_LOGIN_PASSWORD, + ), + ) + keys = dict(re.findall(r"export (AWS_[A-Z_]+)=(\S+)", output)) + if not report(code == 0 and "AWS_ACCESS_KEY_ID" in keys, f"{deployment.stem}: credentials", output): + return False + import boto3 + import urllib3 + + urllib3.disable_warnings() + client = boto3.client( + "dynamodb", + endpoint_url=f"https://127.0.0.1:{port}", + region_name="us-east-1", + aws_access_key_id=keys["AWS_ACCESS_KEY_ID"], + aws_secret_access_key=keys["AWS_SECRET_ACCESS_KEY"], + verify=False, + ) + client.create_table( + TableName="seeded", + AttributeDefinitions=[ + {"AttributeName": n, "AttributeType": "S"} for n in ("pk", "sk", "g", "l") + ], + KeySchema=[ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"}, + ], + GlobalSecondaryIndexes=[ + { + "IndexName": "gsi1", + "KeySchema": [{"AttributeName": "g", "KeyType": "HASH"}], + "Projection": {"ProjectionType": "ALL"}, + } + ], + LocalSecondaryIndexes=[ + { + "IndexName": "lsi1", + "KeySchema": [ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "l", "KeyType": "RANGE"}, + ], + "Projection": {"ProjectionType": "ALL"}, + } + ], + BillingMode="PAY_PER_REQUEST", + StreamSpecification={"StreamEnabled": True, "StreamViewType": "NEW_AND_OLD_IMAGES"}, + ) + client.create_table( + TableName="expiring", + AttributeDefinitions=[{"AttributeName": "pk", "AttributeType": "S"}], + KeySchema=[{"AttributeName": "pk", "KeyType": "HASH"}], + BillingMode="PAY_PER_REQUEST", + ) + for table in ("seeded", "expiring"): + for _ in range(120): + if client.describe_table(TableName=table)["Table"]["TableStatus"] == "ACTIVE": + break + time.sleep(0.5) + client.update_time_to_live( + TableName="expiring", + TimeToLiveSpecification={"Enabled": True, "AttributeName": "expires"}, + ) + for i in range(5): + client.put_item( + TableName="seeded", + Item={"pk": {"S": f"p{i}"}, "sk": {"S": "s"}, "g": {"S": "g"}, "l": {"S": f"l{i}"}}, + ) + client.create_backup(TableName="seeded", BackupName="seeded-backup") + who = "the 0.1.14 binary" if deployment.old else "this binary" + print(f"ok {deployment.stem}: seeded through {who}'s server") + return True + except Exception as e: # noqa: BLE001 - reported as a failed check + report(False, f"{deployment.stem}: seeding", repr(e)) + return False + finally: + stop(server) + + +def older_binary_runs(label: str, d: Deployment, binary: str, old: str, container: str, port: int) -> None: + """The 0.1.14 binary's verify, migrate --yes and serve on a v1.0 catalog, + and what this binary then reports.""" + migrate = [binary, "migrate", "--config", d.config] + for command in (["verify"], ["migrate", "--yes"], ["serve", "--foreground", "--port", str(port)]): + what = " ".join(command[:2]) + before = d.snapshot(container) + code, output = run( + [old, command[0], "--config", d.config, *command[1:]], + dict(d.env, EXTENDDB_CASSANDRA_PASSWORD=ADMIN_PASSWORD), + timeout=60, + ) + after = d.snapshot(container) + changes = diff(before, after) + print(f" {label}: the 0.1.14 {what}: exit {code}; {len(changes)} change(s)") + for line in changes: + print(f" | {line}") + for line in output.strip().splitlines()[-3:]: + print(f" | > {line}") + if command[0] == "serve": + report(code not in (0, -1), f"exit {code}: {label}: the 0.1.14 serve refuses a v1.0 catalog", output) + if not changes: + expect_exit(0, f"{label}: after the 0.1.14 {what}, the catalog is Current", migrate, d.env) + continue + expect_exit( + 5, + f"{label}: after the 0.1.14 {what}, migrate reports WrittenByOlderRelease", + migrate, + d.env, + contains="a release before 1.0 wrote to this catalog", + ) + expect_exit(5, f"{label}: after the 0.1.14 {what}, verify exits 5", [binary, "verify", "--config", d.config], d.env) + expect_exit(0, f"{label}: migrate --yes --reconverge", migrate + ["--yes", "--reconverge"], d.env) + expect_exit(0, f"{label}: verify reports Current", [binary, "verify", "--config", d.config], d.env, contains="OK: Current") + report( + d.schema(container) == [line for line in before if line.startswith("column ")], + f"{label}: the schema is the one it had before the 0.1.14 {what}", + ) + + +def marker_written_by_older_migrate(label: str, d: Deployment, binary: str, container: str) -> None: + """The row a 0.1.14 `migrate --yes` writes on a catalog whose legacy + files are all recorded (its resource-tag step, read in its + crates/app/src/cmd_migrate.rs). That migrate has no Cassandra password + option, so on a node with PasswordAuthenticator, the only kind its `init` + can use, it cannot log in; the script writes the row as it would.""" + before = d.schema(container) + code, output = cqlsh( + container, + f"INSERT INTO {d.stem}_catalog.settings (key, value) VALUES " + "('resource_tag_migration', 'applied');\n", + ) + if not report(code == 0, f"{label}: write the row a 0.1.14 migrate --yes writes", output): + return + migrate = [binary, "migrate", "--config", d.config] + expect_exit( + 5, + f"{label}: with the 0.1.14 marker row, migrate reports WrittenByOlderRelease", + migrate, + d.env, + contains="a release before 1.0 wrote to this catalog", + ) + expect_exit(5, f"{label}: with the 0.1.14 marker row, migrate --yes refuses", migrate + ["--yes"], d.env) + expect_exit(0, f"{label}: migrate --yes --reconverge removes the row", migrate + ["--yes", "--reconverge"], d.env) + expect_exit(0, f"{label}: verify reports Current", [binary, "verify", "--config", d.config], d.env, contains="OK: Current") + report(d.schema(container) == before, f"{label}: the schema is the one it had before the row") + + +def main(binary: str, old: str, container: str, contact_point: str, port: int) -> int: + binary = os.path.realpath(binary) + old = os.path.realpath(old) + work = tempfile.mkdtemp(prefix="eddb-cass-older.") + tag = secrets.token_hex(3) + upgraded = Deployment(f"eddb_up_{tag}", old, work, contact_point, True) + fresh = Deployment(f"eddb_fr_{tag}", binary, work, contact_point, False) + raced = Deployment(f"eddb_rc_{tag}", old, work, contact_point, True) + try: + for d, who in ((upgraded, "0.1.14"), (fresh, "this binary's"), (raced, "0.1.14")): + code, output = d.init() + report(code == 0, f"exit {code}: {who} init of {d.stem}", output) + seeded = seed(old, upgraded, port, work) and seed(binary, fresh, port, work) + + migrate = [binary, "migrate", "--config", upgraded.config] + expect_exit(3, "the preview on the 0.1.14 catalog lists every migration", migrate, upgraded.env, contains="1 baseline") + expect_exit(0, "migrate --yes on the 0.1.14 catalog", migrate + ["--yes"], upgraded.env, contains="Applied 1 baseline") + expect_exit(0, "a second migrate --yes finds nothing to do", migrate + ["--yes"], upgraded.env, contains="Up to date") + if seeded: + fresh_lines = fresh.snapshot(container) + upgraded_lines = upgraded.snapshot(container) + missing = [ + f"{label}: {needle}" + for label, lines in (("fresh", fresh_lines), ("upgraded", upgraded_lines)) + for needle in ("items_", "index_", "index_", "ledger account<") + if not any(needle in line for line in lines) + ] + report( + not missing, + "the comparison covers the seeded table's per-table tables and every ledger", + "\n".join(missing), + ) + differences = diff(fresh_lines, upgraded_lines) + unexpected = [line for line in differences if line not in ACCEPTED] + report( + not unexpected, + f"the upgraded 0.1.14 catalog equals a fresh one seeded the same way " + f"({len(differences)} accepted difference(s))", + "\n".join(unexpected), + ) + + # Two migrators started together on a 0.1.14 catalog. + cmd = [binary, "migrate", "--config", raced.config, "--yes"] + logs = [os.path.join(work, f"race-{i}.log") for i in (1, 2)] + procs = [start(cmd, raced.env, log) for log in logs] + codes = [p.wait(timeout=180) for p in procs] + outputs = [] + for log in logs: + with open(log, encoding="utf-8") as f: + outputs.append(f.read()) + applied = sum("Applied 1 baseline" in o for o in outputs) + waited = sum("Up to date" in o for o in outputs) + report( + codes == [0, 0] and applied == 1 and waited == 1, + f"two migrate --yes started together: exits {codes}, {applied} applied, {waited} found " + "nothing to do", + "\n----\n".join(outputs), + ) + + for label, d in (("fresh", fresh), ("upgraded", upgraded)): + older_binary_runs(label, d, binary, old, container, port) + marker_written_by_older_migrate(label, d, binary, container) + finally: + for d in (upgraded, fresh, raced): + code, output = cqlsh(container, "SELECT keyspace_name FROM system_schema.keyspaces;\n") + statements = "".join( + f"DROP KEYSPACE IF EXISTS {word};\n" + for word in output.split() + if word.startswith(f"{d.stem}_") + ) + cqlsh(container, statements + f"DROP ROLE IF EXISTS '{d.stem}';\n") + shutil.rmtree(work, ignore_errors=True) + return finish() + + +if __name__ == "__main__": + if len(sys.argv) not in (5, 6): + sys.exit(__doc__) + sys.exit( + main( + sys.argv[1], + sys.argv[2], + sys.argv[3], + sys.argv[4], + int(sys.argv[5]) if len(sys.argv) == 6 else 18455, + ) + ) diff --git a/devtools/check-mongodb-auth b/devtools/check-mongodb-auth new file mode 100755 index 000000000..d3bec4f95 --- /dev/null +++ b/devtools/check-mongodb-auth @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 +"""The commands and the server as a MongoDB user with readWrite only: + + devtools/check-mongodb-auth [mongo port] [api port] + +Starts mongo:7 as a one-node replica set with access control (a key file) +in the container extenddb-check-auth-, published on 127.0.0.1 +only, and creates the user `extenddb` with the roles readWrite on +extenddb_catalog and readWrite on extenddb_data and nothing else. As that +user: `init`, the migrate preview and `verify` exit 0, the server starts, +an account, IAM user and access key are created through `extenddb manage`, +the table seed.py describes is created and written through the DynamoDB +API, and on a catalog behind the binary `migrate --yes` applies what is +missing. The container is removed at the end with the temporary +directory. +""" +import os +import shutil +import subprocess +import sys +import tempfile +import time + +from catalog_checks import docker, expect_exit, finish, healthy, report, run, start, stop + +HERE = os.path.dirname(os.path.realpath(__file__)) +SEED = os.path.join(HERE, "..", "crates", "storage-mongodb", "tests", "fixtures", "released", "seed.py") +ROOT_PASSWORD = "CheckRootPassw0rd1" +APP_PASSWORD = "CheckAppPassw0rd1" + + +def shell(container: str, port: int, script: str, root: bool = True) -> tuple[int, str]: + login = ["-u", "root", "-p", ROOT_PASSWORD, "--authenticationDatabase", "admin"] if root else [] + done = docker("exec", container, "mongosh", "--quiet", "--port", str(port), *login, "--eval", script) + return done.returncode, done.stdout + done.stderr + + +def start_with_auth(container: str, port: int) -> bool: + docker("rm", "-f", container) + keyfile = "head -c 756 /dev/urandom | base64 -w0 > /tmp/key && chmod 400 /tmp/key && chown mongodb /tmp/key" + started = docker( + "run", "-d", "--name", container, "-p", f"127.0.0.1:{port}:{port}", "--entrypoint", "bash", "mongo:7", + "-c", f"{keyfile} && exec docker-entrypoint.sh mongod --replSet rs0 --bind_ip_all --port {port} " + "--keyFile /tmp/key", + ) + if not report(started.returncode == 0, f"start {container} with access control", started.stderr): + return False + for _ in range(60): + if shell(container, port, "db.runCommand({ping: 1}).ok", root=False)[1].strip().endswith("1"): + break + time.sleep(1) + # The localhost exception allows the first user and the initiation. + shell(container, port, f"rs.initiate({{_id: 'rs0', members: [{{_id: 0, host: 'localhost:{port}'}}]}})", + root=False) + for _ in range(60): + if shell(container, port, "db.hello().isWritablePrimary", root=False)[1].strip().endswith("true"): + break + time.sleep(1) + code, out = shell(container, port, f"db.getSiblingDB('admin').createUser({{user: 'root', pwd: " + f"'{ROOT_PASSWORD}', roles: ['root']}})", root=False) + if not report(code == 0, "create the administrator", out): + return False + code, out = shell(container, port, f"db.getSiblingDB('admin').createUser({{user: 'extenddb', pwd: " + f"'{APP_PASSWORD}', roles: [{{role: 'readWrite', db: 'extenddb_catalog'}}, " + f"{{role: 'readWrite', db: 'extenddb_data'}}]}})") + return report(code == 0, "create the application user with readWrite on both databases", out) + + +def main(binary: str, port: int, api: int) -> int: + binary = os.path.realpath(binary) + container = f"extenddb-check-auth-{port}" + work = tempfile.mkdtemp() + server = None + try: + if not start_with_auth(container, port): + return finish() + os.chdir(work) + env = dict(os.environ, HOME=os.path.join(work, "home"), EXTENDDB__SERVER__RUN_DIR=os.path.join(work, "run"), + EXTENDDB_ADMIN_USER="admin", EXTENDDB_ADMIN_PASSWORD="CheckAdminPassw0rd1") + uri = (f"mongodb://extenddb:{APP_PASSWORD}@127.0.0.1:{port}/?replicaSet=rs0&directConnection=true" + "&authSource=admin") + with open("extenddb.toml", "w", encoding="utf-8") as f: + f.write(f'[storage]\nbackend = "mongodb"\n\n[storage.mongodb]\nconnection_string = "{uri}"\n') + config = os.path.join(work, "extenddb.toml") + expect_exit(0, "init as the application user", + [binary, "init", "--backend", "mongodb", "--config", config, "--overwrite"], env, timeout=120) + expect_exit(0, "migrate preview as the application user", [binary, "migrate", "--config", config], env) + expect_exit(0, "verify as the application user", [binary, "verify", "--config", config], env) + server = start([binary, "serve", "--config", config, "--foreground", "--port", str(api)], env, "serve.log") + if not healthy(f"https://127.0.0.1:{api}/health", server, 60): + with open("serve.log", encoding="utf-8") as f: + report(False, "the server starts as the application user", f.read()) + return finish() + report(True, "the server starts as the application user") + manage_env = dict(env, EXTENDDB_PASSWORD="CheckAdminPassw0rd1", EXTENDDB__SERVER__PORT=str(api)) + manage = [binary, "manage", "--user", "admin", "--config", config] + for step in (["create-account", "--account-id", "123456789012", "--account-name", "auth"], + ["create-user", "--account-id", "123456789012", "--user-name", "tester"], + ["put-user-policy", "--account-id", "123456789012", "--user-name", "tester", "--policy-name", + "all", "--policy-document", + '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"dynamodb:*","Resource":"*"}]}']): + expect_exit(0, f"manage {step[0]}", manage + step, manage_env) + code, out = run(manage + ["create-access-key", "--account-id", "123456789012", "--user-name", "tester"], + manage_env) + if report(code == 0, "manage create-access-key", out): + import json + created = json.loads(out[out.index("{"):]) + seed_env = {k: v for k, v in os.environ.items() if k != "REQUESTS_CA_BUNDLE"} + done = subprocess.run([sys.executable, SEED, f"https://127.0.0.1:{api}", created["access_key_id"], + created["secret_access_key"]], capture_output=True, text=True, env=seed_env, + check=False, timeout=180) + report(done.returncode == 0, "CreateTable with a GSI, an LSI, a stream and TTL, and ten PutItems", + done.stdout + done.stderr) + stop(server) + server = None + code, out = shell(container, port, "db.getSiblingDB('extenddb_catalog').extenddb_migrations.deleteOne({_id: 2})") + report(code == 0, "remove migration 2's catalog row as the administrator", out) + expect_exit(3, "migrate preview on a catalog behind the binary", [binary, "migrate", "--config", config], env) + expect_exit(0, "migrate --yes as the application user", [binary, "migrate", "--config", config, "--yes"], env) + expect_exit(0, "verify after migrate --yes", [binary, "verify", "--config", config], env) + finally: + if server is not None: + stop(server) + docker("rm", "-f", "-v", container) + os.chdir("/") + shutil.rmtree(work, ignore_errors=True) + return finish() + + +if __name__ == "__main__": + if len(sys.argv) not in (2, 3, 4): + sys.exit(__doc__) + sys.exit(main( + sys.argv[1], + int(sys.argv[2]) if len(sys.argv) > 2 else 27081, + int(sys.argv[3]) if len(sys.argv) > 3 else 18490, + )) diff --git a/devtools/check-mongodb-catalog-lock b/devtools/check-mongodb-catalog-lock new file mode 100755 index 000000000..c369a858f --- /dev/null +++ b/devtools/check-mongodb-catalog-lock @@ -0,0 +1,185 @@ +#!/usr/bin/env python3 +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 +"""Serve and migrate on one MongoDB catalog, with a release binary: + + devtools/check-mongodb-catalog-lock [mongo port] [api port] + +On a mongo:7 replica set this script starts in the container +extenddb-check-lock-, published on 127.0.0.1 only: + +- before init, migrate --yes and verify exit 1 naming extenddb init and + create no database; +- a second init exits 1 saying the catalog is already initialized; +- the migrate preview and verify run beside a server and exit 0, and + migrate --yes exits 6 naming the server's host and pid; +- a server stopped with SIGTERM deletes its registration, so migrate --yes + then runs at once; +- while a migration holds the lock, a server exits 6 and a second + migrate --yes exits 6, and the preview still runs; +- a server refuses a catalog behind this binary with 3, and the command it + prints, run exactly as printed, applies what is missing; +- a server killed with SIGKILL keeps migrate --yes waiting until its + registration expires, at most 45 s after its last renewal; +- a server whose database stops answering exits 1 within 30 s, because its + registration counts as lost 30 s after its last renewal, which it sent + before the database stopped answering, and removes its PID file. + +Everything runs in a new temporary directory, with HOME pointed into it so +init's TLS certificate is not written to the caller's home. The server +binds 127.0.0.1 only. The container is removed at the end. +""" +import os +import shlex +import shutil +import signal +import socket +import subprocess +import sys +import tempfile +import time + +from catalog_checks import docker, expect_exit, finish, healthy, mongosh, report, start, start_mongo, stop + + +def live_servers(container: str, port: int) -> int: + """The registrations whose expiry has not passed.""" + _, out = mongosh( + container, port, + "db.getSiblingDB('extenddb_catalog').extenddb_servers" + ".countDocuments({$expr: {$gt: ['$expires_at', '$$NOW']}})", + ) + return int(out.strip().splitlines()[-1]) + + +def main(binary: str, mongo_port: int, api_port: int) -> int: + binary = os.path.realpath(binary) + container = f"extenddb-check-lock-{mongo_port}" + work = tempfile.mkdtemp() + servers = [] + try: + if not start_mongo(container, mongo_port): + return finish() + os.chdir(work) + env = dict( + os.environ, + HOME=os.path.join(work, "home"), + EXTENDDB__SERVER__RUN_DIR=os.path.join(work, "run"), + EXTENDDB_ADMIN_PASSWORD="CheckAdminPassw0rd1", + ) + uri = f"mongodb://127.0.0.1:{mongo_port}/?replicaSet=rs0&directConnection=true" + with open("extenddb.toml", "w", encoding="utf-8") as f: + f.write(f'[storage]\nbackend = "mongodb"\n\n[storage.mongodb]\nconnection_string = "{uri}"\n') + migrate = [binary, "migrate", "--config", "extenddb.toml"] + verify = [binary, "verify", "--config", "extenddb.toml"] + serve = [binary, "serve", "--config", "extenddb.toml", "--foreground", "--port", str(api_port)] + url = f"https://127.0.0.1:{api_port}/health" + + output = expect_exit(1, "migrate --yes before init", migrate + ["--yes", "--lock-timeout", "1"], env) + report("extenddb init" in output, "the refusal names extenddb init", output) + expect_exit(1, "verify before init", verify, env, contains="extenddb init") + _, names = mongosh(container, mongo_port, "db.getMongo().getDBNames().join(',')") + report("extenddb" not in names, "nothing was created before init", names) + + expect_exit(0, "init", [binary, "init", "--backend", "mongodb", "--config", "extenddb.toml", + "--overwrite"], env, timeout=120) + expect_exit(1, "a second init", [binary, "init", "--backend", "mongodb", "--config", + "extenddb.toml", "--overwrite"], env, contains="already initialized") + + server = start(serve, env, "serve.log") + servers.append(server) + if not healthy(url, server, 60): + stop(server) + with open("serve.log", encoding="utf-8") as f: + report(False, "the server did not become healthy", f.read()) + return finish() + report(live_servers(container, mongo_port) == 1, "the server registered") + expect_exit(0, "migrate preview beside a server", migrate, env) + expect_exit(0, "verify beside a server", verify, env) + expect_exit(6, "migrate --yes while a server runs", migrate + ["--yes", "--lock-timeout", "2"], env, + contains=f"pid {server.pid} on {socket.gethostname()}") + stop(server) + report(server.returncode == 0, f"the server stopped on SIGTERM with exit {server.returncode}") + report(live_servers(container, mongo_port) == 0, "the stopped server deleted its registration") + expect_exit(0, "migrate --yes once the server stopped", migrate + ["--yes", "--lock-timeout", "2"], env) + + # Another migration's lock, as its claim writes it. + mongosh(container, mongo_port, "db.getSiblingDB('extenddb_catalog').settings.insertOne(" + "{_id: 'migration_lock', owner: 'check', expires_at: new Date(Date.now() + 120000)})") + expect_exit(6, "serve while a migration holds the lock", serve, env, timeout=60) + expect_exit(6, "a second migrate --yes while a migration holds the lock", + migrate + ["--yes", "--lock-timeout", "1"], env) + expect_exit(0, "migrate preview while a migration holds the lock", migrate, env) + report(live_servers(container, mongo_port) == 0, "the refused server deleted its registration") + mongosh(container, mongo_port, "db.getSiblingDB('extenddb_catalog').settings.deleteOne({_id: 'migration_lock'})") + + # A catalog behind this binary: the server refuses with 3 and prints + # the command that applies what is missing. + mongosh(container, mongo_port, "db.getSiblingDB('extenddb_catalog').extenddb_migrations.deleteOne({_id: 4})") + expect_exit(3, "verify on a catalog behind this binary", verify, env) + output = expect_exit(3, "serve on a catalog behind this binary", serve, env, timeout=60) + advised = next((line for line in output.splitlines() + if line.startswith(" ") and " migrate " in f" {line} "), "") + words = shlex.split(advised) + if report(bool(words) and words[0] == "extenddb", f"serve prints the command to run: {advised!r}", output): + expect_exit(0, "the printed command, run as printed", [binary, *words[1:]], env) + expect_exit(0, "verify after the printed command", verify, env) + + # A server killed without deleting its registration blocks + # migrate --yes until the registration expires, 45 s after its last + # renewal. + server = start(serve, env, "serve-killed.log") + servers.append(server) + if healthy(url, server, 60): + server.kill() + server.wait() + killed = time.monotonic() + expect_exit(0, "migrate --yes after a killed server's registration expires", + migrate + ["--yes", "--lock-timeout", "70"], env, timeout=90) + waited = time.monotonic() - killed + report(waited <= 50, f"migrate --yes ran {waited:.0f} s after the server was killed") + else: + report(False, "the server did not start for the kill check") + + # A server whose database stops answering exits 1 and removes its + # PID file. + server = start(serve + ["--write-pid-file"], env, "serve-paused.log") + servers.append(server) + pid_file = os.path.join(work, "run", f"extenddb-{api_port}.pid") + if healthy(url, server, 60) and report(os.path.exists(pid_file), "the server wrote its PID file"): + docker("pause", container) + paused = time.monotonic() + try: + server.wait(timeout=90) + except subprocess.TimeoutExpired: + pass + took = time.monotonic() - paused + docker("unpause", container) + with open("serve-paused.log", encoding="utf-8") as f: + log = f.read() + report(server.returncode == 1 and took <= 31, + f"the server exited {server.returncode} {took:.0f} s after its database stopped answering", log) + report("registration" in log, "its last line names the lost registration", log) + report(not os.path.exists(pid_file), "the server removed its PID file") + else: + report(False, "the server did not start for the pause check") + finally: + for server in servers: + if server.poll() is None: + server.send_signal(signal.SIGTERM) + stop(server) + docker("unpause", container) + docker("rm", "-f", "-v", container) + os.chdir("/") + shutil.rmtree(work, ignore_errors=True) + return finish() + + +if __name__ == "__main__": + if len(sys.argv) not in (2, 3, 4): + sys.exit(__doc__) + sys.exit(main( + sys.argv[1], + int(sys.argv[2]) if len(sys.argv) > 2 else 27041, + int(sys.argv[3]) if len(sys.argv) > 3 else 18460, + )) diff --git a/devtools/check-mongodb-older-binary b/devtools/check-mongodb-older-binary new file mode 100755 index 000000000..f9e77c0a2 --- /dev/null +++ b/devtools/check-mongodb-older-binary @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 +"""Catalogs this binary created and migrated, against the last 0.1.x image: + + devtools/check-mongodb-older-binary \\ + [image] [mongo port] [api port] + +Two catalogs, each in a mongo:7 replica set of its own: a fresh one from +`init`, and the released 0.1.14 fixture upgraded with `migrate --yes`. On +each, the image (by default extenddb/extenddb-mongodb:0.1.14) runs `serve`, +which must refuse, then `verify`, `migrate --yes` and `serve` again, and the +script prints what each run changed, as a diff of the catalog's settings, +ledgers and indexes before and after. Then this binary must report the catalog as +written by an older release: `verify` and the migrate preview exit 5 and +`serve` refuses with 5; `migrate --yes --reconverge` must restore Current +with the definitions the catalog had before the image ran, and delete the +marker the image wrote. + +The containers publish only on 127.0.0.1, the image runs as the caller's +uid, and everything is removed at the end with the temporary directory. +""" +import difflib +import os +import shutil +import subprocess +import sys +import tempfile + +from catalog_checks import docker, expect_exit, finish, mongosh, report, run, start_mongo + +HERE = os.path.dirname(os.path.realpath(__file__)) +FIXTURES = os.path.join(HERE, "..", "crates", "storage-mongodb", "tests", "fixtures", "released") + +DUMP = """ +const out = []; +for (const name of ["extenddb_catalog", "extenddb_data"]) { + const d = db.getSiblingDB(name); + for (const c of d.getCollectionNames().sort()) { + if (c.startsWith("_ddb_")) continue; + for (const i of d.getCollection(c).getIndexes()) { + const o = Object.assign({}, i); delete o.v; delete o.ns; + out.push(`index ${name}.${c} ${EJSON.stringify(o)}`); + } + } + for (const c of ["settings", "schema_history", "extenddb_migrations"]) { + for (const doc of d.getCollection(c).find({}).sort({_id: 1}).toArray()) { + const o = Object.assign({}, doc); + if (o._id === "encryption_key") o.value = ""; + delete o.applied_at; + out.push(`row ${name}.${c} ${EJSON.stringify(o)}`); + } + } +} +print(out.join("\\n")); +""" + + +def dump(container: str, port: int) -> list[str]: + code, out = mongosh(container, port, DUMP) + if code != 0: + report(False, "dump the catalog", out) + return out.splitlines() + + +def definitions(lines: list[str]) -> list[str]: + return [line for line in lines if line.startswith("index ")] + + +def image_run(image: str, container: str, work: str, env: dict, *args: str, timeout: float = 60) -> tuple[int, str]: + """Run the image with `args` in `container`'s network, with the work + directory at the same path. A run still going after `timeout` is a + server that started; it is removed, and its code is -1.""" + name = f"{container}-image" + cmd = ["docker", "run", "--rm", "--name", name, "--network", f"container:{container}", + "--user", f"{os.getuid()}:{os.getgid()}", "-v", f"{work}:{work}", "-w", work, + "-e", f"HOME={env['HOME']}", "-e", f"EXTENDDB__SERVER__RUN_DIR={env['EXTENDDB__SERVER__RUN_DIR']}", + image, *args] + try: + done = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout, check=False) + return done.returncode, done.stdout + done.stderr + except subprocess.TimeoutExpired as e: + docker("rm", "-f", name) + return -1, f"still running after {timeout:g} s\n{e.stdout or ''}{e.stderr or ''}" + + +def changes(before: list[str], after: list[str]) -> list[str]: + return [line for line in difflib.unified_diff(before, after, lineterm="", n=0) + if line[:1] in "+-" and not line.startswith(("+++", "---"))] + + +def check(label: str, binary: str, image: str, port: int, api: int, fixture: str | None) -> None: + container = f"extenddb-check-older-{port}" + work = tempfile.mkdtemp() + try: + if not start_mongo(container, port, mounts=(f"{FIXTURES}:/fixtures:ro",)): + return + env = dict(os.environ, HOME=os.path.join(work, "home"), + EXTENDDB__SERVER__RUN_DIR=os.path.join(work, "run"), + EXTENDDB_ADMIN_PASSWORD="CheckAdminPassw0rd1") + os.makedirs(env["EXTENDDB__SERVER__RUN_DIR"]) + # The servers' TLS certificate, where the default config looks for it. + tls = os.path.join(env["HOME"], ".extenddb", "tls") + os.makedirs(tls) + run(["openssl", "req", "-x509", "-newkey", "rsa:2048", "-nodes", "-days", "1", "-subj", "/CN=localhost", + "-keyout", os.path.join(tls, "key.pem"), "-out", os.path.join(tls, "cert.pem")]) + config = os.path.join(work, "extenddb.toml") + uri = f"mongodb://localhost:{port}/?replicaSet=rs0&directConnection=true" + with open(config, "w", encoding="utf-8") as f: + f.write(f'[storage]\nbackend = "mongodb"\n\n[storage.mongodb]\nconnection_string = "{uri}"\n') + if fixture is None: + expect_exit(0, f"{label}: init", [binary, "init", "--backend", "mongodb", "--config", config, + "--overwrite"], env, timeout=120) + else: + code, out = mongosh(container, port, f'const FIXTURE = "/fixtures/{fixture}.json"; load("/fixtures/load.js")') + report(code == 0 and "loaded" in out, f"{label}: load the {fixture} fixture", out) + expect_exit(0, f"{label}: migrate --yes", [binary, "migrate", "--config", config, "--yes"], env) + expect_exit(0, f"{label}: verify before the image runs", [binary, "verify", "--config", config], env) + before = dump(container, port) + + serve = ["serve", "--config", config, "--foreground", "--port", str(api)] + code, out = image_run(image, container, work, env, *serve, timeout=40) + report(code > 0 and "resource-tag" in out, + f"{label}: the image's serve refuses the catalog for its marker (exit {code})", out) + report(changes(before, dump(container, port)) == [], f"{label}: the refused serve changed nothing") + + code, out = image_run(image, container, work, env, "verify", "--config", config) + print(f" {label}: the image's verify exits {code}: {out.strip().splitlines()[-1] if out.strip() else ''}") + code, out = image_run(image, container, work, env, "migrate", "--config", config, "--yes") + report(code == 0, f"{label}: the image's migrate --yes exits {code}", out) + after = dump(container, port) + changed = changes(before, after) + print(f" {label}: the image's migrate --yes changed:") + for line in changed: + print(f" | {line}") + report(any("resource_tag_migration" in line for line in changed), + f"{label}: the image's migrate --yes wrote the resource_tag_migration row", "\n".join(changed)) + # With the marker back, the image's server starts: measured, as the + # upgrade manual's table of released binaries records. + code, out = image_run(image, container, work, env, *serve, timeout=40) + print(f" {label}: the image's serve after its migrate --yes: " + f"{'still serving after 40 s, stopped' if code == -1 else f'exit {code}'}") + for line in changes(after, dump(container, port)): + print(f" | {line}") + + for what, cmd in [("verify", [binary, "verify", "--config", config]), + ("migrate preview", [binary, "migrate", "--config", config])]: + expect_exit(5, f"{label}: {what} after the image wrote to the catalog", cmd, env, + contains="release before 1.0") + expect_exit(5, f"{label}: serve after the image wrote to the catalog", + [binary, *serve], env, contains="--reconverge", timeout=60) + expect_exit(0, f"{label}: migrate --yes --reconverge", + [binary, "migrate", "--config", config, "--yes", "--reconverge"], env) + expect_exit(0, f"{label}: verify after --reconverge", [binary, "verify", "--config", config], env) + restored = dump(container, port) + report(definitions(restored) == definitions(before), + f"{label}: --reconverge restored the definitions the catalog had", + "\n".join(changes(definitions(before), definitions(restored)))) + report(not any("resource_tag_migration" in line for line in restored), + f"{label}: --reconverge deleted the marker") + finally: + docker("rm", "-f", "-v", container) + shutil.rmtree(work, ignore_errors=True) + + +def main(binary: str, image: str, port: int, api: int) -> int: + binary = os.path.realpath(binary) + code, out = run(["docker", "image", "inspect", image]) + if not report(code == 0, f"the image {image} is present (pull it first)", out[-500:]): + return finish() + check("fresh", binary, image, port, api, None) + check("upgraded 0.1.14", binary, image, port + 1, api + 1, "0.1.14") + return finish() + + +if __name__ == "__main__": + if len(sys.argv) not in (2, 3, 4, 5): + sys.exit(__doc__) + sys.exit(main( + sys.argv[1], + sys.argv[2] if len(sys.argv) > 2 else "extenddb/extenddb-mongodb:0.1.14", + int(sys.argv[3]) if len(sys.argv) > 3 else 27051, + int(sys.argv[4]) if len(sys.argv) > 4 else 18470, + )) diff --git a/devtools/check-mongodb-released b/devtools/check-mongodb-released new file mode 100755 index 000000000..5fdbed6c8 --- /dev/null +++ b/devtools/check-mongodb-released @@ -0,0 +1,206 @@ +#!/usr/bin/env python3 +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 +"""Released MongoDB catalogs upgraded by a release binary, compared with a +fresh install seeded the same way: + + devtools/check-mongodb-released [mongo port] [api port] + +The fresh install is `init`, then this binary's server, then the account, +IAM user, policies and access key the fixtures carry, and the tables +`cu_orders` and `cu_dotted` from +crates/storage-mongodb/tests/fixtures/released/seed.py. Each fixture in that +directory is loaded into a mongo:7 replica set of its own and upgraded: +the migrate preview exits 3, `migrate --yes` 0, the preview and `verify` 0; +then the server answers DescribeTable, a GSI query, a Scan and a PutItem +with a new access key. Its catalog collections and indexes, its per-table +collections and indexes (named after the tables and indexes the catalog +records), and its settings keys must equal the fresh install's, apart from +the lines accepted-differences.json lists for that state with a reason. + +Containers are named extenddb-check-released-, publish only on +127.0.0.1 and are removed at the end with the temporary directory. +""" +import json +import os +import shutil +import sys +import tempfile + +from catalog_checks import docker, expect_exit, finish, healthy, mongosh, report, run, start, start_mongo, stop + +HERE = os.path.dirname(os.path.realpath(__file__)) +FIXTURES = os.path.realpath(os.path.join(HERE, "..", "crates", "storage-mongodb", "tests", "fixtures", "released")) +STATES = ["0.1.12", "0.1.13", "0.1.14", "dotted-ttl"] +ADMIN_PASSWORD = "FixtureAdminPassw0rd1" + +NORMALIZE = """ +const out = []; +const cat = db.getSiblingDB("extenddb_catalog"); +const names = {}; +for (const t of cat.tables.find().toArray()) names[t.table_id] = "T_" + t._id.table_name; +for (const i of cat.indexes.find().toArray()) { + names[i.index_id] = "I_" + names[i._id.table_id].slice(2) + "_" + i._id.index_name; +} +for (const dbName of ["extenddb_catalog", "extenddb_data"]) { + const d = db.getSiblingDB(dbName); + for (const c of d.getCollectionNames().sort()) { + const shown = c.startsWith("_ddb_") ? (names[c.slice(5)] || "unknown") : c; + const spec = d.getCollectionInfos({name: c})[0]; + out.push(`collection ${dbName}.${shown} ${EJSON.stringify(spec.options)}`); + for (const i of d.getCollection(c).getIndexes()) { + const o = Object.assign({}, i); delete o.v; delete o.ns; + out.push(`index ${dbName}.${shown} ${EJSON.stringify(o)}`); + } + } +} +for (const s of cat.settings.find().sort({_id: 1}).toArray()) out.push(`setting ${s._id}`); +print(out.sort().join("\\n")); +""" + + +class Deployment: + """A mongo:7 replica set with a config, a HOME and a run directory.""" + + def __init__(self, binary: str, port: int, api: int, work: str, tls: str | None = None) -> None: + self.binary, self.port, self.api = binary, port, api + self.container = f"extenddb-check-released-{port}" + self.dir = os.path.join(work, str(port)) + os.makedirs(os.path.join(self.dir, "run")) + self.env = dict(os.environ, HOME=os.path.join(self.dir, "home"), + EXTENDDB__SERVER__RUN_DIR=os.path.join(self.dir, "run"), + EXTENDDB_ADMIN_USER="admin", EXTENDDB_ADMIN_PASSWORD=ADMIN_PASSWORD) + self.config = os.path.join(self.dir, "extenddb.toml") + uri = f"mongodb://127.0.0.1:{port}/?replicaSet=rs0&directConnection=true" + with open(self.config, "w", encoding="utf-8") as f: + f.write(f'[storage]\nbackend = "mongodb"\n\n[storage.mongodb]\nconnection_string = "{uri}"\n') + # A loaded fixture's server uses the certificate the fresh install's + # `init` made, where the default config looks for it. + if tls is not None: + shutil.copytree(tls, os.path.join(self.env["HOME"], ".extenddb", "tls")) + + def cmd(self, *args: str) -> list[str]: + return [self.binary, *args, "--config", self.config] + + def manage(self, *args: str) -> tuple[int, str]: + env = dict(self.env, EXTENDDB_PASSWORD=ADMIN_PASSWORD, EXTENDDB__SERVER__PORT=str(self.api)) + return run([self.binary, "manage", "--user", "admin", "--config", self.config, *args], env) + + def serve(self): + server = start(self.cmd("serve") + ["--foreground", "--port", str(self.api)], self.env, + os.path.join(self.dir, "serve.log")) + return server, healthy(f"https://127.0.0.1:{self.api}/health", server, 60) + + def access_key(self, label: str, account: bool) -> tuple[str, str] | None: + steps = [("create-account", "--account-id", "123456789012", "--account-name", "cu-seed"), + ("create-user", "--account-id", "123456789012", "--user-name", "cu-tester"), + ("put-user-policy", "--account-id", "123456789012", "--user-name", "cu-tester", + "--policy-name", "dynamodb-full", "--policy-document", + '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"dynamodb:*","Resource":"*"}]}')] + for step in steps if account else []: + code, out = self.manage(*step) + if not report(code == 0, f"{label}: manage {step[0]}", out): + return None + code, out = self.manage("create-access-key", "--account-id", "123456789012", "--user-name", "cu-tester") + if not report(code == 0, f"{label}: manage create-access-key", out): + return None + created = json.loads(out[out.index("{"):]) + return created["access_key_id"], created["secret_access_key"] + + def normalized(self) -> list[str]: + code, out = mongosh(self.container, self.port, NORMALIZE) + report(code == 0, f"read {self.container}", out) + return out.splitlines() + + +def smoke(label: str, endpoint: str, key: str, secret: str) -> None: + import boto3 + import urllib3 + urllib3.disable_warnings() + ddb = boto3.client("dynamodb", endpoint_url=endpoint, region_name="us-east-1", aws_access_key_id=key, + aws_secret_access_key=secret, verify=False) + try: + table = ddb.describe_table(TableName="cu_orders")["Table"] + query = ddb.query(TableName="cu_orders", IndexName="by_gpk", KeyConditionExpression="gpk = :g", + ExpressionAttributeValues={":g": {"S": "g0"}}) + scan = ddb.scan(TableName="cu_orders") + ddb.put_item(TableName="cu_orders", Item={"pk": {"S": "p9"}, "sk": {"S": "after"}, "gpk": {"S": "g1"}, + "gsk": {"N": "99"}, "lsk": {"S": "l"}}) + report(table["TableStatus"] == "ACTIVE" and query["Count"] == 5 and scan["Count"] == 10, + f"{label}: DescribeTable, a GSI query ({query['Count']} items), a Scan ({scan['Count']} items) " + "and a PutItem after the upgrade") + except Exception as e: # noqa: BLE001 - every failure is the check's result + report(False, f"{label}: the upgraded server answers the DynamoDB API", repr(e)) + + +def main(binary: str, port: int, api: int) -> int: + binary = os.path.realpath(binary) + with open(os.path.join(FIXTURES, "accepted-differences.json"), encoding="utf-8") as f: + accepted = json.load(f) + work = tempfile.mkdtemp() + deployments = [] + try: + fresh = Deployment(binary, port, api, work) + deployments.append(fresh) + if not start_mongo(fresh.container, port): + return finish() + expect_exit(0, "fresh: init", [*fresh.cmd("init"), "--backend", "mongodb", "--overwrite"], fresh.env, + timeout=120) + server, up = fresh.serve() + keys = fresh.access_key("fresh", account=True) if report(up, "fresh: the server is healthy") else None + if keys is not None: + for table, ttl in [("cu_orders", "expires"), ("cu_dotted", "exp.at")]: + code, out = run([sys.executable, os.path.join(FIXTURES, "seed.py"), f"https://127.0.0.1:{api}", + *keys, table, ttl], dict(os.environ)) + report(code == 0, f"fresh: seed {table}", out) + stop(server) + reference = fresh.normalized() + + for n, state in enumerate(STATES, start=1): + d = Deployment(binary, port + n, api + n, work, os.path.join(fresh.env["HOME"], ".extenddb", "tls")) + deployments.append(d) + if not start_mongo(d.container, d.port, mounts=(f"{FIXTURES}:/fixtures:ro",)): + continue + code, out = mongosh(d.container, d.port, f'const FIXTURE = "/fixtures/{state}.json"; load("/fixtures/load.js")') + report(code == 0 and "loaded" in out, f"{state}: load the fixture", out) + expect_exit(3, f"{state}: migrate preview", d.cmd("migrate"), d.env) + expect_exit(0, f"{state}: migrate --yes", d.cmd("migrate") + ["--yes"], d.env) + expect_exit(0, f"{state}: migrate preview after --yes", d.cmd("migrate"), d.env) + expect_exit(0, f"{state}: verify", d.cmd("verify"), d.env) + upgraded = d.normalized() + server, up = d.serve() + if report(up, f"{state}: the upgraded server is healthy"): + keys = d.access_key(state, account=False) + if keys is not None: + smoke(state, f"https://127.0.0.1:{d.api}", *keys) + stop(server) + tables = {line.split()[1].split(".", 1)[1] for line in upgraded + if line.startswith(("collection extenddb_data.T_", "collection extenddb_data.I_"))} + wanted = [line for line in reference + if not line.startswith(("collection extenddb_data.T_", "collection extenddb_data.I_", + "index extenddb_data.T_", "index extenddb_data.I_")) + or line.split()[1].split(".", 1)[1] in tables] + allowed = {entry["line"] for entry in accepted.get(state, [])} + only_upgraded = sorted(set(upgraded) - set(wanted) - allowed) + only_fresh = sorted(set(wanted) - set(upgraded) - allowed) + report(not only_upgraded and not only_fresh, + f"{state}: the upgraded catalog equals a fresh install seeded the same way", + "\n".join([f"only upgraded: {line}" for line in only_upgraded] + + [f"only fresh: {line}" for line in only_fresh])) + unused = allowed - (set(upgraded) ^ set(wanted)) + report(not unused, f"{state}: every accepted difference occurs", "\n".join(sorted(unused))) + finally: + for d in deployments: + docker("rm", "-f", "-v", d.container) + shutil.rmtree(work, ignore_errors=True) + return finish() + + +if __name__ == "__main__": + if len(sys.argv) not in (2, 3, 4): + sys.exit(__doc__) + sys.exit(main( + sys.argv[1], + int(sys.argv[2]) if len(sys.argv) > 2 else 27061, + int(sys.argv[3]) if len(sys.argv) > 3 else 18480, + )) diff --git a/devtools/check-mongodb-schema b/devtools/check-mongodb-schema new file mode 100755 index 000000000..4bb5d21e0 --- /dev/null +++ b/devtools/check-mongodb-schema @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 +"""The schema of a fresh MongoDB install, against docs/schema/mongodb.txt: + + devtools/check-mongodb-schema [mongo port] + +Runs `init` on a mongo:7 replica set this script starts in the container +extenddb-check-schema- (published on 127.0.0.1 only, removed at +the end), then writes both databases' collections with their options and +indexes with their keys and options, one object per line in a fixed order, +to . A fresh install has no per-table collections. Exits 1 +with the diff when the file differs from docs/schema/mongodb.txt; the +output file is then the one to commit there. +""" +import difflib +import os +import shutil +import sys +import tempfile + +from catalog_checks import docker, expect_exit, finish, mongosh, report, start_mongo + +HERE = os.path.dirname(os.path.realpath(__file__)) +COMMITTED = os.path.join(HERE, "..", "docs", "schema", "mongodb.txt") + +SCHEMA = """ +const out = []; +for (const name of ["extenddb_catalog", "extenddb_data"]) { + const d = db.getSiblingDB(name); + for (const spec of d.getCollectionInfos().sort((a, b) => (a.name < b.name ? -1 : 1))) { + out.push(`collection ${name}.${spec.name} type ${spec.type} options ${EJSON.stringify(spec.options)}`); + if (spec.type !== "collection") continue; + const indexes = d.getCollection(spec.name).getIndexes().sort((a, b) => (a.name < b.name ? -1 : 1)); + for (const i of indexes) { + const o = Object.assign({}, i); delete o.v; delete o.ns; delete o.name; + const key = EJSON.stringify(o.key); delete o.key; + out.push(`index ${name}.${spec.name} ${i.name} key ${key} options ${EJSON.stringify(o)}`); + } + } +} +print(out.join("\\n")); +""" + + +def main(binary: str, output: str, port: int) -> int: + binary = os.path.realpath(binary) + container = f"extenddb-check-schema-{port}" + work = tempfile.mkdtemp() + try: + if not start_mongo(container, port): + return finish() + config = os.path.join(work, "extenddb.toml") + uri = f"mongodb://127.0.0.1:{port}/?replicaSet=rs0&directConnection=true" + with open(config, "w", encoding="utf-8") as f: + f.write(f'[storage]\nbackend = "mongodb"\n\n[storage.mongodb]\nconnection_string = "{uri}"\n') + env = dict(os.environ, HOME=os.path.join(work, "home"), EXTENDDB_ADMIN_PASSWORD="CheckAdminPassw0rd1") + expect_exit(0, "init", [binary, "init", "--backend", "mongodb", "--config", config, "--overwrite"], + env, timeout=120) + code, schema = mongosh(container, port, SCHEMA) + if not report(code == 0, "read the schema", schema): + return finish() + with open(output, "w", encoding="utf-8") as f: + f.write(schema.strip() + "\n") + with open(COMMITTED, encoding="utf-8") as f: + committed = f.read() + diff = list(difflib.unified_diff(committed.splitlines(), schema.strip().splitlines(), + "docs/schema/mongodb.txt", "fresh install", lineterm="")) + report(not diff, f"a fresh install matches docs/schema/mongodb.txt (else commit {output} there)", + "\n".join(diff)) + finally: + docker("rm", "-f", "-v", container) + shutil.rmtree(work, ignore_errors=True) + return finish() + + +if __name__ == "__main__": + if len(sys.argv) not in (3, 4): + sys.exit(__doc__) + sys.exit(main(sys.argv[1], sys.argv[2], int(sys.argv[3]) if len(sys.argv) > 3 else 27071)) diff --git a/devtools/run-mongodb-tests b/devtools/run-mongodb-tests index 2a287798a..6c69e6993 100755 --- a/devtools/run-mongodb-tests +++ b/devtools/run-mongodb-tests @@ -287,6 +287,16 @@ if [[ " ${RUN_TESTS_ARGS[*]} " == *" --rust"* ]]; then EXTENDDB_TEST_MONGODB_URI="mongodb://localhost:$MONGO_PORT/?replicaSet=rs0&directConnection=true" \ cargo test -p extenddb-storage-mongodb --lib -- --ignored records_live echo "" + # The catalog migration tests: each runs in databases of its own on this + # server, and EXTENDDB_TEST_MONGODB_REQUIRED turns a skip for a missing + # server into a failure. + echo "=== Running MongoDB catalog migration tests (migrations_*) ===" + echo "" + EXTENDDB_TEST_MONGODB_URI="mongodb://localhost:$MONGO_PORT/?replicaSet=rs0&directConnection=true" \ + EXTENDDB_TEST_MONGODB_REQUIRED=1 \ + cargo test -p extenddb-storage-mongodb --test migrations_fresh --test migrations_guards \ + --test migrations_holders --test migrations_locks --test migrations_released + echo "" fi echo "=== Running test suite via devtools/run-tests --backend mongodb ===" echo " passthrough args: ${RUN_TESTS_ARGS[*]}" diff --git a/devtools/schema-dump b/devtools/schema-dump index 2e027636c..f46dbf399 100755 --- a/devtools/schema-dump +++ b/devtools/schema-dump @@ -6,6 +6,7 @@ order, so a fresh install can be compared with docs/schema/.txt. devtools/schema-dump sqlite devtools/schema-dump postgres + devtools/schema-dump cassandra On PostgreSQL the dump lists tables, columns, indexes, constraints, sequences, functions, triggers, views and types; the per-table `_ddb_*` @@ -13,7 +14,13 @@ tables are left out, because no migration creates them, and so are the objects an extension owns (pgvector's functions and types), so a server with pgvector and one without print the same file. A migration that creates any other kind of object adds a query for it here. On SQLite every schema object -is listed; a fresh install has no per-table objects. +is listed; a fresh install has no per-table objects. On Cassandra the dump +reads system_schema through cqlsh in the node's Docker container, as the +admin role $CASSANDRA_ADMIN_USER with $CASSANDRA_ADMIN_PASSWORD (default +cassandra and cassandra), and lists the tables, columns and secondary +indexes of the catalog keyspace and of the one account keyspace a fresh +install has, init's default account, whose id is left out because it is +random. """ import sqlite3 import sys @@ -144,6 +151,89 @@ def postgres_schema(base: str, database: str) -> list[str]: return lines +def cassandra_rows(container: str, table: str, columns: str) -> list[list[str]]: + """Every row of a system_schema table, read with `cqlsh COPY ... TO + STDOUT` in the node's container as the admin role.""" + import csv + import io + import os + import subprocess + + done = subprocess.run( + [ + "docker", "exec", "-i", container, "cqlsh", + "-u", os.environ.get("CASSANDRA_ADMIN_USER", "cassandra"), + "-p", os.environ.get("CASSANDRA_ADMIN_PASSWORD", "cassandra"), + ], + input=f"COPY {table} ({columns}) TO STDOUT;\n", + capture_output=True, + text=True, + check=False, + timeout=120, + ) + if done.returncode != 0: + sys.exit(f"COPY {table}: {done.stdout}{done.stderr}") + width = len(columns.split(",")) + return [ + row + for row in csv.reader(io.StringIO(done.stdout)) + if len(row) == width and not row[0].startswith("Using ") + ] + + +# The order cqlsh's DESCRIBE gives a table's columns. +CASSANDRA_KINDS = {"partition_key": 0, "clustering": 1, "static": 2, "regular": 3} + + +def cassandra_schema(container: str, keyspace: str) -> list[str]: + tables = sorted( + name + for ks, name in cassandra_rows(container, "system_schema.tables", "keyspace_name, table_name") + if ks == keyspace + ) + columns = sorted( + ( + (table, CASSANDRA_KINDS.get(kind, 4), int(position), column, kind, order, kind_type) + for ks, table, column, kind, position, order, kind_type in cassandra_rows( + container, + "system_schema.columns", + "keyspace_name, table_name, column_name, kind, position, clustering_order, type", + ) + if ks == keyspace + ), + ) + indexes = sorted( + (table, index, kind, options) + for ks, table, index, kind, options in cassandra_rows( + container, "system_schema.indexes", "keyspace_name, table_name, index_name, kind, options" + ) + if ks == keyspace + ) + lines = [f"table {name}" for name in tables] + for table, _, position, column, kind, order, kind_type in columns: + detail = f"{kind_type} {kind}" + if kind in ("partition_key", "clustering"): + detail += f" {position}" + if kind == "clustering": + detail += f" {order}" + lines.append(f"column {table}.{column}: {detail}") + lines += [f"index {table}.{index}: {kind} {options}" for table, index, kind, options in indexes] + return lines + + +def cassandra_account_keyspace(container: str, prefix: str) -> str: + """The keyspace of the one account a fresh install has (init's default + account), whose id is random.""" + found = sorted( + ks + for (ks,) in cassandra_rows(container, "system_schema.keyspaces", "keyspace_name") + if ks.startswith(f"{prefix}_account_") + ) + if len(found) != 1: + sys.exit(f"expected one account keyspace with the prefix {prefix}, found {found}") + return found[0] + + def main(argv: list[str]) -> None: if len(argv) == 2 and argv[0] == "sqlite": print("\n".join(sqlite_schema(argv[1]))) @@ -152,6 +242,12 @@ def main(argv: list[str]) -> None: for store, database in (("catalog", catalog), ("data", data)): print(f"# {store} database") print("\n".join(postgres_schema(base, database))) + elif len(argv) == 3 and argv[0] == "cassandra": + container, prefix = argv[1:] + account = cassandra_account_keyspace(container, prefix) + for store, keyspace in (("catalog", f"{prefix}_catalog"), ("account", account)): + print(f"# {store} keyspace") + print("\n".join(cassandra_schema(container, keyspace))) else: sys.exit(__doc__) diff --git a/docs/schema/cassandra.txt b/docs/schema/cassandra.txt new file mode 100644 index 000000000..e4f20e073 --- /dev/null +++ b/docs/schema/cassandra.txt @@ -0,0 +1,290 @@ +# catalog keyspace +table access_keys +table accounts +table admin_users +table backups_by_account +table backups_by_arn +table backups_by_table +table continuous_backups +table extenddb_migrations +table extenddb_servers +table iam_group_members +table iam_groups +table iam_permissions_boundaries +table iam_policies +table iam_role_tags +table iam_roles +table iam_sessions +table iam_user_tags +table iam_users +table idempotency_tokens_by_account +table indexes +table login_attempts +table metrics +table schema_history +table settings +table tables +table tags +column access_keys.access_key_id: text partition_key 0 +column access_keys.account_id: text regular +column access_keys.created_at: timestamp regular +column access_keys.is_active: boolean regular +column access_keys.secret_key_encrypted: blob regular +column access_keys.user_name: text regular +column accounts.account_id: text partition_key 0 +column accounts.account_name: text regular +column accounts.created_at: timestamp regular +column admin_users.admin_name: text partition_key 0 +column admin_users.created_at: timestamp regular +column admin_users.password_hash: text regular +column backups_by_account.account_id: text partition_key 0 +column backups_by_account.created_at: timestamp clustering 0 desc +column backups_by_account.backup_arn: text clustering 1 asc +column backups_by_account.backup_name: text regular +column backups_by_account.backup_size_bytes: bigint regular +column backups_by_account.table_arn: text regular +column backups_by_account.table_id: text regular +column backups_by_account.table_name: text regular +column backups_by_arn.account_id: text partition_key 0 +column backups_by_arn.backup_arn: text partition_key 1 +column backups_by_arn.active_restores: map regular +column backups_by_arn.attribute_definitions: text regular +column backups_by_arn.backup_name: text regular +column backups_by_arn.backup_size_bytes: bigint regular +column backups_by_arn.backup_status: text regular +column backups_by_arn.backup_type: text regular +column backups_by_arn.billing_mode: text regular +column backups_by_arn.created_at: timestamp regular +column backups_by_arn.definition: text regular +column backups_by_arn.item_count: bigint regular +column backups_by_arn.key_schema: text regular +column backups_by_arn.provisioned_throughput: text regular +column backups_by_arn.stream_specification: text regular +column backups_by_arn.table_arn: text regular +column backups_by_arn.table_created_at: timestamp regular +column backups_by_arn.table_id: text regular +column backups_by_arn.table_name: text regular +column backups_by_table.account_id: text partition_key 0 +column backups_by_table.table_name: text partition_key 1 +column backups_by_table.created_at: timestamp clustering 0 desc +column backups_by_table.backup_arn: text clustering 1 asc +column backups_by_table.backup_name: text regular +column backups_by_table.backup_size_bytes: bigint regular +column backups_by_table.table_arn: text regular +column backups_by_table.table_id: text regular +column continuous_backups.account_id: text partition_key 0 +column continuous_backups.table_name: text clustering 0 asc +column continuous_backups.pitr_enabled: boolean regular +column extenddb_migrations.id: int partition_key 0 +column extenddb_migrations.applied_at: timestamp regular +column extenddb_migrations.applied_by: text regular +column extenddb_migrations.checksum: text regular +column extenddb_migrations.kind: text regular +column extenddb_migrations.name: text regular +column extenddb_migrations.notes: text regular +column extenddb_migrations.outcome: text regular +column extenddb_servers.server_id: text partition_key 0 +column extenddb_servers.host: text regular +column extenddb_servers.pid: int regular +column extenddb_servers.release: text regular +column extenddb_servers.started_at: text regular +column iam_group_members.account_id: text partition_key 0 +column iam_group_members.group_name: text partition_key 1 +column iam_group_members.user_name: text clustering 0 asc +column iam_groups.account_id: text partition_key 0 +column iam_groups.group_name: text clustering 0 asc +column iam_groups.created_at: timestamp regular +column iam_groups.group_arn: text regular +column iam_permissions_boundaries.account_id: text partition_key 0 +column iam_permissions_boundaries.principal_type: text clustering 0 asc +column iam_permissions_boundaries.principal_name: text clustering 1 asc +column iam_permissions_boundaries.policy_document: text regular +column iam_policies.account_id: text partition_key 0 +column iam_policies.principal_type: text clustering 0 asc +column iam_policies.principal_name: text clustering 1 asc +column iam_policies.policy_name: text clustering 2 asc +column iam_policies.created_at: timestamp regular +column iam_policies.policy_document: text regular +column iam_role_tags.account_id: text partition_key 0 +column iam_role_tags.role_name: text partition_key 1 +column iam_role_tags.tag_key: text clustering 0 asc +column iam_role_tags.tag_value: text regular +column iam_roles.account_id: text partition_key 0 +column iam_roles.role_name: text clustering 0 asc +column iam_roles.created_at: timestamp regular +column iam_roles.permissions_boundary_arn: text regular +column iam_roles.role_arn: text regular +column iam_roles.trust_policy: text regular +column iam_sessions.session_token: text partition_key 0 +column iam_sessions.access_key_id: text regular +column iam_sessions.account_id: text regular +column iam_sessions.created_at: timestamp regular +column iam_sessions.expires_at: timestamp regular +column iam_sessions.role_name: text regular +column iam_sessions.secret_key_encrypted: blob regular +column iam_sessions.session_name: text regular +column iam_sessions.session_policy: text regular +column iam_sessions.session_tags: text regular +column iam_user_tags.account_id: text partition_key 0 +column iam_user_tags.user_name: text partition_key 1 +column iam_user_tags.tag_key: text clustering 0 asc +column iam_user_tags.tag_value: text regular +column iam_users.account_id: text partition_key 0 +column iam_users.user_name: text clustering 0 asc +column iam_users.created_at: timestamp regular +column iam_users.password_hash: text regular +column iam_users.user_arn: text regular +column idempotency_tokens_by_account.account_id: text partition_key 0 +column idempotency_tokens_by_account.token: text clustering 0 asc +column idempotency_tokens_by_account.created_at: timestamp regular +column idempotency_tokens_by_account.fingerprint: text regular +column indexes.table_id: text partition_key 0 +column indexes.index_name: text clustering 0 asc +column indexes.backfill_heartbeat: timestamp regular +column indexes.index_id: text regular +column indexes.index_status: text regular +column indexes.index_type: text regular +column indexes.key_schema: text regular +column indexes.projection: text regular +column indexes.propagation_delay_ms: int regular +column indexes.provisioned_throughput: text regular +column login_attempts.principal: text partition_key 0 +column login_attempts.attempted_at: timestamp clustering 0 desc +column login_attempts.source_ip: text regular +column login_attempts.success: boolean regular +column metrics.bucket: timestamp partition_key 0 +column metrics.metric: text partition_key 1 +column metrics.table_name: text partition_key 2 +column metrics.index_name: text partition_key 3 +column metrics.operation: text partition_key 4 +column metrics.count: bigint regular +column metrics.max: double regular +column metrics.min: double regular +column metrics.sum: double regular +column schema_history.version: int partition_key 0 +column schema_history.applied_at: timestamp regular +column schema_history.description: text regular +column settings.key: text partition_key 0 +column settings.value: text regular +column tables.account_id: text partition_key 0 +column tables.table_name: text clustering 0 asc +column tables.attribute_definitions: text regular +column tables.billing_mode: text regular +column tables.created_at: timestamp regular +column tables.deletion_protection_enabled: boolean regular +column tables.item_count: bigint regular +column tables.key_schema: text regular +column tables.on_demand_throughput: text regular +column tables.provisioned_throughput: text regular +column tables.restore_date_time: timestamp regular +column tables.restore_heartbeat: timestamp regular +column tables.restore_owner: uuid regular +column tables.restore_source_backup_arn: text regular +column tables.restore_source_table_arn: text regular +column tables.sse_specification: text regular +column tables.status_transition_at: timestamp regular +column tables.stream_label: text regular +column tables.stream_specification: text regular +column tables.table_arn: text regular +column tables.table_class: text regular +column tables.table_id: text regular +column tables.table_size_bytes: bigint regular +column tables.table_status: text regular +column tables.ttl_attribute: text regular +column tables.ttl_backfill_cursor: text regular +column tables.ttl_cleanup_generation: uuid regular +column tables.ttl_generation: uuid regular +column tables.ttl_index_ready: boolean regular +column tables.ttl_sweep_owner: uuid regular +column tags.resource_arn: text partition_key 0 +column tags.tag_key: text clustering 0 asc +column tags.tag_value: text regular +index tables.tables_table_id_idx: COMPOSITES {'target': 'table_id'} +# account keyspace +table backup_items +table extenddb_migrations +table gsi_pending +table index_propagation_holds +table schema_history +table stream_records +table stream_shards +table transaction_ledger +table ttl_expiration_buckets +table ttl_expirations +table ttl_reconcile_pending +table ttl_repair_inflight +table ttl_repair_inflight_partitions +column backup_items.backup_arn: text partition_key 0 +column backup_items.bucket: int partition_key 1 +column backup_items.item_index: bigint clustering 0 asc +column backup_items.item_data: text regular +column extenddb_migrations.id: int partition_key 0 +column extenddb_migrations.applied_at: timestamp regular +column extenddb_migrations.applied_by: text regular +column extenddb_migrations.checksum: text regular +column extenddb_migrations.kind: text regular +column extenddb_migrations.name: text regular +column extenddb_migrations.notes: text regular +column extenddb_migrations.outcome: text regular +column gsi_pending.worker_partition: int partition_key 0 +column gsi_pending.ready_at: timestamp clustering 0 asc +column gsi_pending.id: timeuuid clustering 1 asc +column gsi_pending.last_ready_at: timestamp static +column gsi_pending.index_context: text regular +column gsi_pending.new_item: text regular +column gsi_pending.old_item: text regular +column gsi_pending.table_id: text regular +column index_propagation_holds.table_id: text partition_key 0 +column index_propagation_holds.index_id: text clustering 0 asc +column schema_history.version: int partition_key 0 +column schema_history.applied_at: timestamp regular +column schema_history.description: text regular +column stream_records.shard_id: text partition_key 0 +column stream_records.sequence_number: text clustering 0 asc +column stream_records.created_at: timestamp regular +column stream_records.event_name: text regular +column stream_records.record_data: text regular +column stream_records.table_id: text regular +column stream_shards.shard_id: text partition_key 0 +column stream_shards.created_at: timestamp regular +column stream_shards.ending_sequence_number: text regular +column stream_shards.parent_shard_id: text regular +column stream_shards.starting_sequence_number: text regular +column stream_shards.table_id: text regular +column transaction_ledger.txn_id: uuid partition_key 0 +column transaction_ledger.client_token: text regular +column transaction_ledger.items_blob: text regular +column transaction_ledger.request_fingerprint: text regular +column transaction_ledger.started_at: bigint regular +column transaction_ledger.state: text regular +column ttl_expiration_buckets.table_id: text partition_key 0 +column ttl_expiration_buckets.generation: uuid clustering 0 asc +column ttl_expiration_buckets.bucket: bigint clustering 1 asc +column ttl_expiration_buckets.shard: int clustering 2 asc +column ttl_expirations.table_id: text partition_key 0 +column ttl_expirations.generation: uuid partition_key 1 +column ttl_expirations.bucket: bigint partition_key 2 +column ttl_expirations.shard: int partition_key 3 +column ttl_expirations.expires_at: bigint clustering 0 asc +column ttl_expirations.key_hash: text clustering 1 asc +column ttl_expirations.key_data: text clustering 2 asc +column ttl_expirations.state: text regular +column ttl_expirations.work_data: text regular +column ttl_expirations.work_id: uuid regular +column ttl_reconcile_pending.worker_partition: int partition_key 0 +column ttl_reconcile_pending.id: timeuuid clustering 0 asc +column ttl_reconcile_pending.account_id: text regular +column ttl_reconcile_pending.key_data: text regular +column ttl_reconcile_pending.table_id: text regular +column ttl_reconcile_pending.table_name: text regular +column ttl_repair_inflight.worker_partition: int partition_key 0 +column ttl_repair_inflight.repair_id: uuid clustering 0 asc +column ttl_repair_inflight.account_id: text regular +column ttl_repair_inflight.created_at: timestamp regular +column ttl_repair_inflight.generation: uuid regular +column ttl_repair_inflight.key_data: text regular +column ttl_repair_inflight.table_id: text regular +column ttl_repair_inflight.table_name: text regular +column ttl_repair_inflight_partitions.worker_partition: int partition_key 0 +index stream_shards.stream_shards_table_id_idx: COMPOSITES {'target': 'table_id'} +index transaction_ledger.idx_txn_started_at: COMPOSITES {'target': 'started_at'} diff --git a/docs/schema/mongodb.txt b/docs/schema/mongodb.txt new file mode 100644 index 000000000..44474e255 --- /dev/null +++ b/docs/schema/mongodb.txt @@ -0,0 +1,65 @@ +collection extenddb_catalog.access_keys type collection options {} +index extenddb_catalog.access_keys _id_ key {"_id":1} options {} +index extenddb_catalog.access_keys account_id_1_user_name_1 key {"account_id":1,"user_name":1} options {} +collection extenddb_catalog.accounts type collection options {} +index extenddb_catalog.accounts _id_ key {"_id":1} options {} +index extenddb_catalog.accounts account_name_1 key {"account_name":1} options {"unique":true} +collection extenddb_catalog.admin_users type collection options {} +index extenddb_catalog.admin_users _id_ key {"_id":1} options {} +collection extenddb_catalog.backups type collection options {} +index extenddb_catalog.backups _id_ key {"_id":1} options {} +index extenddb_catalog.backups account_id_1_table_name_1 key {"account_id":1,"table_name":1} options {} +collection extenddb_catalog.continuous_backups type collection options {} +index extenddb_catalog.continuous_backups _id_ key {"_id":1} options {} +collection extenddb_catalog.extenddb_migrations type collection options {} +index extenddb_catalog.extenddb_migrations _id_ key {"_id":1} options {} +collection extenddb_catalog.extenddb_servers type collection options {} +index extenddb_catalog.extenddb_servers _id_ key {"_id":1} options {} +index extenddb_catalog.extenddb_servers expires_at_1 key {"expires_at":1} options {"expireAfterSeconds":0} +collection extenddb_catalog.iam_groups type collection options {} +index extenddb_catalog.iam_groups _id_ key {"_id":1} options {} +index extenddb_catalog.iam_groups group_arn_1 key {"group_arn":1} options {"unique":true} +collection extenddb_catalog.iam_permissions_boundaries type collection options {} +index extenddb_catalog.iam_permissions_boundaries _id_ key {"_id":1} options {} +collection extenddb_catalog.iam_policies type collection options {} +index extenddb_catalog.iam_policies _id_ key {"_id":1} options {} +collection extenddb_catalog.iam_roles type collection options {} +index extenddb_catalog.iam_roles _id_ key {"_id":1} options {} +index extenddb_catalog.iam_roles role_arn_1 key {"role_arn":1} options {"unique":true} +collection extenddb_catalog.iam_sessions type collection options {} +index extenddb_catalog.iam_sessions _id_ key {"_id":1} options {} +index extenddb_catalog.iam_sessions access_key_id_1 key {"access_key_id":1} options {"unique":true} +index extenddb_catalog.iam_sessions expires_at_1 key {"expires_at":1} options {"expireAfterSeconds":0} +collection extenddb_catalog.iam_users type collection options {} +index extenddb_catalog.iam_users _id_ key {"_id":1} options {} +index extenddb_catalog.iam_users user_arn_1 key {"user_arn":1} options {"unique":true} +collection extenddb_catalog.indexes type collection options {} +index extenddb_catalog.indexes _id_ key {"_id":1} options {} +collection extenddb_catalog.login_attempts type collection options {} +index extenddb_catalog.login_attempts _id_ key {"_id":1} options {} +index extenddb_catalog.login_attempts principal_1_attempted_at_1 key {"principal":1,"attempted_at":1} options {} +collection extenddb_catalog.metrics type collection options {} +index extenddb_catalog.metrics _id.bucket_1 key {"_id.bucket":1} options {} +index extenddb_catalog.metrics _id_ key {"_id":1} options {} +collection extenddb_catalog.schema_history type collection options {} +index extenddb_catalog.schema_history _id_ key {"_id":1} options {} +collection extenddb_catalog.settings type collection options {} +index extenddb_catalog.settings _id_ key {"_id":1} options {} +collection extenddb_catalog.tables type collection options {} +index extenddb_catalog.tables _id_ key {"_id":1} options {} +index extenddb_catalog.tables table_id_1 key {"table_id":1} options {"unique":true} +collection extenddb_catalog.tags type collection options {} +index extenddb_catalog.tags _id_ key {"_id":1} options {} +collection extenddb_data.extenddb_migrations type collection options {} +index extenddb_data.extenddb_migrations _id_ key {"_id":1} options {} +collection extenddb_data.idempotency_tokens type collection options {} +index extenddb_data.idempotency_tokens _id_ key {"_id":1} options {} +index extenddb_data.idempotency_tokens account_id_1_token_1 key {"account_id":1,"token":1} options {"unique":true} +index extenddb_data.idempotency_tokens created_at_1 key {"created_at":1} options {"expireAfterSeconds":540} +collection extenddb_data.stream_records type collection options {} +index extenddb_data.stream_records _id_ key {"_id":1} options {} +index extenddb_data.stream_records created_at_1 key {"created_at":1} options {"expireAfterSeconds":86400} +index extenddb_data.stream_records shard_id_1_sequence_number_1 key {"shard_id":1,"sequence_number":1} options {} +collection extenddb_data.stream_shards type collection options {} +index extenddb_data.stream_shards _id_ key {"_id":1} options {} +index extenddb_data.stream_shards shard_id_1 key {"shard_id":1} options {"unique":true} From f2efa11b8cc475f8fa3aa3caf806666e4a000e4d Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Fri, 30 Oct 2026 19:34:26 +0000 Subject: [PATCH 03/45] docs: Cassandra and MongoDB in ADR-0023, the manuals and CONTRIBUTING.md ADR-0023 describes every backend on the runner and gives the Cassandra and MongoDB lock and registration values. The upgrade manual gives this release's Cassandra and MongoDB procedures, the GRANT statements a Cassandra deployment created before 1.0 needs under CassandraAuthorizer, and how a stopped Cassandra run is finished. docs/cassandra-setup.md says what init grants. The design docs, the architecture, admin and storage extension manuals and CONTRIBUTING.md's procedure for adding a migration cover all four backends, and docs/build-docs.py says extenddb version prints the catalog migration on every backend. --- CONTRIBUTING.md | 75 +++++++++++++++---- docs/adr/0023-catalog-migrations.md | 21 ++++-- docs/build-docs.py | 2 +- docs/cassandra-setup.md | 43 +++++++---- docs/design/04-component-storage.md | 14 ++-- docs/design/13-storage-mongodb.md | 52 ++++++------- docs/manuals/01-architecture-guide.md | 5 +- docs/manuals/05-admin-guide.md | 2 +- docs/manuals/07-upgrade-manual.md | 46 ++++++++++-- docs/manuals/12-extending-extenddb-storage.md | 2 +- 10 files changed, 179 insertions(+), 83 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 202d6441b..dc800aeba 100755 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -106,10 +106,7 @@ records the migrations it applied in a ledger table, `extenddb_migrations`, and `extenddb migrate --yes` applies the missing ones in order. [ADR-0023](docs/adr/0023-catalog-migrations.md) records the design. -SQLite and PostgreSQL migrate through this framework. Cassandra and MongoDB -still migrate through their bootstrappers: a Cassandra change is a numbered -CQL file listed in `crates/storage-cassandra/src/migrations.rs`, and a MongoDB -change is code in `crates/storage-mongodb/src/bootstrapper.rs`. +SQLite, PostgreSQL, Cassandra and MongoDB migrate through this framework. ### Procedure @@ -121,14 +118,17 @@ change is code in `crates/storage-mongodb/src/bootstrapper.rs`. is never reused, reordered or removed. 2. Add an arm for the id to each backend's table, `migration()` in - `crates/storage-sqlite/src/migrations/mod.rs` and in - `crates/storage-postgres/src/migrations/mod.rs`. Both are exhaustive + `crates/storage-sqlite/src/migrations/mod.rs`, + `crates/storage-postgres/src/migrations/mod.rs`, + `crates/storage-cassandra/src/migrations/mod.rs` and + `crates/storage-mongodb/src/migrations/mod.rs`. All four are exhaustive `match`es with no wildcard arm, so the workspace does not compile until every table has the arm. The arm is one of: - `Migration::Applies { kind, convergent, bodies }`, with one body for each - kind of store the migration changes: `StoreKind::Catalog`, and on - PostgreSQL also `StoreKind::Data`. The runner applies the catalog body + kind of store the migration changes: `StoreKind::Catalog`, on + PostgreSQL and MongoDB also `StoreKind::Data`, and on Cassandra also + `StoreKind::Account` (one body for every account keyspace). The runner applies the catalog body first, and it writes a catalog ledger row for every id, including an id whose only body runs in the data database. - `Migration::NotApplicable("reason")`, when the change has no effect on @@ -196,6 +196,40 @@ change is code in `crates/storage-mongodb/src/bootstrapper.rs`. `EXTENDDB_BLESS_EXPECTED=1` the test writes the golden file instead of comparing; read the file before you commit it. + On MongoDB every body is code; the backend has no file bodies. A body + sends every request through `MongoStores::request` + (`crates/storage-mongodb/src/migrations/backend.rs`), which checks first + that this process still holds the migration lock and waits at most 10 s + for the answer, and every request must be safe to send again: the body + is not in a transaction, and a migrator stopped part way runs it again. + The runner writes the ledger row, and in the catalog the + `catalog_version` row, in one transaction after the body. Accept an + object that already exists only by the server's error code, such as + NamespaceExists (48) from `create`, never by discarding a result. A body + that reaches the catalog through `ManagementStore` gets one that checks + the lock before each call. MongoDB's golden tests are in + `crates/storage-mongodb/tests/migrations_released.rs`; they need a + replica set, named by `EXTENDDB_TEST_MONGODB_URI`. + + On Cassandra a file body is CQL, `NNNN_.cql` in + `crates/storage-cassandra/migrations/catalog/` for the catalog keyspace or + `crates/storage-cassandra/migrations/account/` for the account keyspaces, + embedded with `file_body!` like the others. Cassandra has no transactions: + the runner sends the statements one by one, checks before each that this + process still holds the migration lock, and writes the ledger row at + `QUORUM` after the body, so every statement must be safe to run again + (`CREATE ... IF NOT EXISTS`, `ALTER TABLE ... ADD IF NOT EXISTS ...`). A + statement is `CREATE TABLE`, `CREATE INDEX`, `ALTER TABLE`, `INSERT INTO` + or `DELETE FROM`, naming its table without a keyspace: the runner writes + the keyspace in front of the table (`crates/storage-cassandra/src/migrations/cql.rs`) + and refuses a file with any other statement before it runs anything. After + each `CREATE` and `ALTER` the runner waits until every live node reports the + same schema version. The file is split on the semicolon character, so no + comment may hold one. The golden tests are in + `crates/storage-cassandra/src/migrations/tests.rs`; the + `0001_baseline.expected.json` beside each body holds the definitions the + runner compares the keyspace with the first time migration 1 is applied. + On PostgreSQL, a body that runs `CREATE INDEX CONCURRENTLY` builds every index through `IndexHelper` in `crates/storage-postgres/src/migrations/index_helper.rs`. `ensure` drops @@ -225,7 +259,10 @@ change is code in `crates/storage-mongodb/src/bootstrapper.rs`. the new binary. The job does not exist yet; the first PR that declares an additive migration adds it, and deletes the tests that refuse `Kind::Additive` until then (`no_migration_is_additive_before_the_old_binary_job` - in the SQLite crate, and its PostgreSQL counterpart in `crates/bin`). + in the SQLite crate and in `crates/storage-cassandra/src/migrations/tests.rs`, + its PostgreSQL counterpart in `crates/bin`, and + `no_migration_is_declared_additive_before_the_old_binary_job_exists` in + the MongoDB crate). Set `convergent: true` when every body is safe to run again on a store at any later migration. `extenddb migrate --yes --reconverge`, which repairs a @@ -234,17 +271,21 @@ change is code in `crates/storage-mongodb/src/bootstrapper.rs`. 5. Handle per-table objects. CreateTable and CreateAccount create objects at request time with the binary's own DDL: the `_ddb_` tables and their - indexes on PostgreSQL, the index tables on SQLite, and on Cassandra the + indexes on PostgreSQL, the index tables on SQLite, the `_ddb_` + collections and their indexes on MongoDB, and on Cassandra the `items_
` tables and each account's keyspace. Their shape depends on the release that created them. A migration that changes them is breaking. It lists the objects from the catalog rows, never by name in the data - store, where PostgreSQL's vector tables share the `_ddb_` prefix. It skips + store, where PostgreSQL's vector tables and MongoDB's index collections + share the `_ddb_` prefix. It skips a catalog row whose table is `DELETING` or whose physical table is missing, and adds a note for it. The same PR changes the DDL that creates - new objects, in `crates/storage-sqlite/src/data/ddl.rs` and - `crates/storage-postgres/src/data/ddl.rs`. Migration 4, - `m0004_index_table_base_key.rs` in each backend's `src/migrations/`, does - this. + new objects, in `crates/storage-sqlite/src/data/ddl.rs`, + `crates/storage-postgres/src/data/ddl.rs`, + `crates/storage-cassandra/src/data/ddl.rs` and + `crates/storage-mongodb/src/table_engine.rs`. Migration 4, + `m0004_index_table_base_key.rs` in the SQLite and PostgreSQL crates' + `src/migrations/`, does this. 6. Commit the current schema. Each backend's integration job regenerates `docs/schema/.txt` from a fresh install. When the result differs @@ -256,6 +297,9 @@ change is code in `crates/storage-mongodb/src/bootstrapper.rs`. ` for a deployment `init` created on that server. The dump lists the kinds of object `devtools/schema-dump`'s docstring names; a migration that creates another kind adds a query for it there. + On MongoDB, `devtools/check-mongodb-schema ` + runs `init` on a replica set of its own, writes the schema to the output + file and compares it with `docs/schema/mongodb.txt`. 7. Leave the checksum literals to the release. Each backend's `RELEASED` table pins the checksum, kind and convergent flag of every released body, @@ -272,6 +316,7 @@ change is code in `crates/storage-mongodb/src/bootstrapper.rs`. ```bash cargo test -p extenddb-storage-sqlite --lib the_released_bodies -- --nocapture cargo test -p extenddb-storage-postgres --lib the_released_bodies -- --nocapture + cargo test -p extenddb-storage-mongodb --lib the_released_bodies -- --nocapture ``` A released body does not change. If one has to, for example because a diff --git a/docs/adr/0023-catalog-migrations.md b/docs/adr/0023-catalog-migrations.md index 8c0f278c0..41e13b6af 100644 --- a/docs/adr/0023-catalog-migrations.md +++ b/docs/adr/0023-catalog-migrations.md @@ -213,8 +213,11 @@ that each backend implements, as follows. | Cassandra | A row in the catalog's `settings` table, written only by lightweight transactions at `SERIAL`, with a 60 s TTL renewed every 15 s | A row per server in `extenddb_servers`, renewed every 15 s with a 45 s TTL | | MongoDB | A document in the `settings` collection, claimed by a conditional update with majority write concern and renewed by its owner | A document per server in `extenddb_servers` that expires 45 s after its last renewal | - The Cassandra and MongoDB rows are the design those backends implement - when they move onto the runner; their intervals and TTLs may change then. + The Cassandra row's values are in + `crates/storage-cassandra/src/migrations/locks.rs` (the lock) and + `crates/storage-cassandra/src/migrations/registrations.rs` (the + registration); the MongoDB row's are in + `crates/storage-mongodb/src/migrations/locks.rs`. On PostgreSQL there are two layouts in which the serving key does not match the servers that run, and the upgrade manual's step that stops @@ -385,12 +388,14 @@ that each backend implements, as follows. fails the test. - On Cassandra a migration needs every replica of the catalog keyspace reachable, because the lock is taken at `SERIAL` and running servers are - read at `ALL`. When Cassandra moves onto the runner, - the runner is to wait for schema agreement after every DDL statement, reading `schema_version` from `system.local` on each node. -- SQLite and PostgreSQL run on the runner first. Cassandra and MongoDB keep - their bootstrapper migrations and the marker check in `serve.rs` until they - move onto it, and each backend's previous mechanism is deleted when it - moves. + read at `ALL`. The runner waits for schema agreement after every DDL + statement, before every ledger row and before it releases the lock, reading + `schema_version` from `system.local` on each live node + (`crates/storage-cassandra/src/migrations/agreement.rs`). +- Every backend runs on the runner. Each backend's previous mechanism was + deleted when it moved: the PostgreSQL and SQLite migration helpers first, + then the Cassandra and MongoDB bootstrapper migrations, with the marker + check in `serve.rs`. - The shared numbering and the ledger in every store are read by every later release, so reversing this decision means migrating every deployment's ledgers. A released body changes only through a `rebless` entry. diff --git a/docs/build-docs.py b/docs/build-docs.py index 4b76c6b99..1a43bae74 100755 --- a/docs/build-docs.py +++ b/docs/build-docs.py @@ -133,7 +133,7 @@ def get_extenddb_version() -> str: def get_catalog_version() -> str: """The catalog the manuals describe: `migration `, the last id of the `migration_ids!` list in crates/storage/src/migrations/registry.rs, as - `extenddb version` prints it on PostgreSQL and SQLite.""" + `extenddb version` prints it.""" registry = ROOT / "crates" / "storage" / "src" / "migrations" / "registry.rs" text = registry.read_text() ids = [int(n) for n in re.findall(r"^\s*[A-Z][A-Za-z0-9]* = (\d+), \"", text, re.MULTILINE)] diff --git a/docs/cassandra-setup.md b/docs/cassandra-setup.md index 5c507190a..5074e764a 100644 --- a/docs/cassandra-setup.md +++ b/docs/cassandra-setup.md @@ -110,21 +110,29 @@ The node must run `PasswordAuthenticator` (step 3). `init` always creates its application role with a password, so it fails against an unauthenticated (`AllowAllAuthenticator`) node — there is no credential-free path. -> **Authorizer.** Run the node with `AllowAllAuthorizer` (the default). `init` -> creates the application role but grants it no permissions, so under -> `CassandraAuthorizer` the server later fails to start with -> `User extenddb has no SELECT permission ...`. Until `init` issues the grants, -> only `AllowAllAuthorizer` works. +> **Authorizer.** The node may run `AllowAllAuthorizer` (the default) or +> `CassandraAuthorizer`. `init` reads the node's authorizer from +> `system_views.settings`; under any authorizer other than +> `AllowAllAuthorizer` it grants the application role SELECT, MODIFY, CREATE +> and ALTER on the catalog keyspace, CREATE ON ALL KEYSPACES (for +> CreateAccount), and SELECT, MODIFY, CREATE, ALTER and DROP on the default +> account's keyspace (DeleteTable drops the account's tables and DeleteAccount +> drops the keyspace). A keyspace that CreateAccount creates as the application +> role needs no grant: a role holds every permission on a keyspace it created. +> Under `AllowAllAuthorizer`, which grants everything and supports no GRANT, +> `init` grants nothing. init creates: - The catalog keyspace (`extenddb_catalog` by default) +- The default account's keyspace (`extenddb_account_` by default) - The application role ExtendDB uses at serve time (named `extenddb` by - default; override with `--extenddb-user` / `--extenddb-pass`) + default; override with `--extenddb-user` / `--extenddb-pass`), with the + grants above when the node's authorizer is not `AllowAllAuthorizer` - Admin user credentials (printed to stdout once — save them) - A self-signed TLS certificate at `~/.extenddb/tls/cert.pem` - The config file `extenddb.toml` -Per-account keyspaces are created later, when accounts are added. +Further account keyspaces are created when accounts are added. > **Save the admin credentials.** They are printed once and cannot be retrieved > later. @@ -157,8 +165,10 @@ Fields: `extenddb`). - `replication_factor` — keyspace replication factor. `init` always writes `1`, regardless of the cluster. For a multi-node cluster, edit this to 3 or more - **after** `init`; keyspaces created later (per account) use the config value. - Existing keyspaces are not altered retroactively. + **after** `init`; keyspaces created later (per account) use the config value, + except an account keyspace that `migrate --yes` creates for an account whose + CreateAccount stopped before its keyspace: that one takes the catalog + keyspace's replication. Existing keyspaces are not altered retroactively. - `datacenter` — the local datacenter name for `NetworkTopologyStrategy`. Currently, `init` always writes `datacenter1` and has no flag to change it, and the keyspaces it creates reference `datacenter1`, so the cluster's local datacenter must be @@ -231,7 +241,10 @@ section covers only what is specific to Cassandra. Set `replication_factor` to 3 or more for a multi-node cluster. `init` always writes `1`, so edit the config after `init`; keyspaces created afterward (per -account) use the value in the config. The `datacenter` must be `datacenter1`: +account) use the value in the config, except an account keyspace that +`migrate --yes` creates for an account whose CreateAccount stopped before its +keyspace, which takes the catalog keyspace's replication. The `datacenter` must +be `datacenter1`: `init` hardcodes that name into the keyspace definitions and provides no flag to change it, so your cluster's local datacenter has to be named `datacenter1` until that is configurable. ExtendDB creates its keyspaces with @@ -256,10 +269,12 @@ a password. Give ExtendDB a dedicated application role rather than the `cassandra` superuser, and supply its credentials through `username` and `password`; both are required. -Use `AllowAllAuthorizer` for now. `init` creates the application role but grants -it no permissions, so `CassandraAuthorizer` makes `serve` fail with -`User has no SELECT permission ...`. Until `init` issues the grants a -role needs, authorization must be left open at the Cassandra layer. +`CassandraAuthorizer` works: when the node's authorizer is not +`AllowAllAuthorizer`, `init` grants the application role SELECT, MODIFY, CREATE +and ALTER on the catalog keyspace, CREATE ON ALL KEYSPACES, and those four and +DROP on the default account's keyspace; a keyspace the role creates itself +(CreateAccount) needs no grant. Under `AllowAllAuthorizer` `init` grants +nothing, since that authorizer grants everything. ### Connection encryption diff --git a/docs/design/04-component-storage.md b/docs/design/04-component-storage.md index dcfda3a3d..6aad21e84 100755 --- a/docs/design/04-component-storage.md +++ b/docs/design/04-component-storage.md @@ -567,13 +567,17 @@ The ops run in table and key order. A ConditionCheck or Delete that finds no row ### 5.6 Migrations -Migrations are embedded in the binary at compile time via `include_str!` and applied in order by the -`catalog::run_migrations` helper. Each migration is tracked in the `schema_history` table. - -Migration files are numbered sequentially: +The catalog migrations are the numbered list every backend shares +([ADR-0023](../adr/0023-catalog-migrations.md)). PostgreSQL's bodies are SQL files embedded in the +binary at compile time via `include_str!`. The shared runner applies them in id order and records +each body in the `extenddb_migrations` ledger of the database it changed, and every id in the +catalog's ledger. The +`schema_history` table of a 0.1.x catalog stays in place and is not read. + +Migration files are named by id: ``` migrations/ -└── 001_initial_schema.sql +└── 0001_baseline.sql ``` ## 6. GSI Consistency Model diff --git a/docs/design/13-storage-mongodb.md b/docs/design/13-storage-mongodb.md index 5498e6cc0..378805bbb 100644 --- a/docs/design/13-storage-mongodb.md +++ b/docs/design/13-storage-mongodb.md @@ -46,7 +46,8 @@ creation. ## 3. Catalog Database Collections -Collections created by `run_catalog_migrations` in `bootstrapper.rs`. Two +Collections created by migration 1's catalog body, +`crates/storage-mongodb/src/migrations/m0001_baseline.rs`. Two additional collections (`iam_group_members`, `backup_items` — the latter is no longer used) are auto-created on first insert and not in the migration list. @@ -108,8 +109,10 @@ field is unset when the index flips to `ACTIVE`. ```json { "_id": "", "value": "..." } ``` -Bootstrapped keys include `catalog_version`, `encryption_key` (base64-encoded -256-bit AES-GCM key), `data_database_name`, and `data_connection_string`. +Keys include `catalog_version` (the catalog's migration id, which the +migration runner writes with each catalog ledger row), `encryption_key` +(base64-encoded 256-bit AES-GCM key), `data_database_name`, and +`data_connection_string`. ### 3.6 `admin_users` ```json @@ -832,12 +835,14 @@ in `lib.rs`). Reject `$` (operator injection), `.` (field-path traversal), null bytes, and non-ASCII. Runs before any query construction. -### 5.10 Catalog version check +### 5.10 Catalog status check -`read_catalog_version` returns the `settings.catalog_version` value; -`expected_catalog_version` returns the compiled-in `0.0.2`. The bin -layer compares them on `extenddb serve` startup — same pattern as the -PostgreSQL backend. +The catalog and the data database each hold a migration ledger, the +`extenddb_migrations` collection. `extenddb serve` reads the catalog's +status from both ledgers through the shared migration runner before it +opens the engine, and refuses a catalog with pending migrations or with a +breaking migration this binary does not know, as on every backend. +[ADR-0023](../adr/0023-catalog-migrations.md) records the design. ## 6. Crate Structure @@ -941,27 +946,16 @@ layer redacts the password from it for display via the ## 9. Bootstrapper Flow -**`extenddb init`:** - -1. Materialize both databases with sentinel collections (MongoDB creates - databases lazily on first write, so we explicitly create - `schema_history` in the catalog db and `idempotency_tokens` in the - data db). -2. `run_catalog_migrations` — create the 17 catalog collections and - their indexes; seed `settings.catalog_version = 0.0.2`; record - the migration in `schema_history`. -3. `create_data_db` — create `idempotency_tokens` (TTL + unique index), - `stream_shards` (unique on shard_id), `stream_records` (TTL + compound - query index). -4. `bootstrap_encryption_key` — generate a random 256-bit key, - base64-encode, insert into `settings` (idempotent — E11000 races are - silently ignored). -5. `bootstrap_default_account` — insert `default` account if none - exists. -6. `bootstrap_admin_user` — bcrypt-hash the password, insert into - `admin_users`. -7. `record_data_connection` — record `data_database_name` and - `data_connection_string` in `settings`. +**`extenddb init`:** takes the migration lock, whose claim creates the +catalog database, and checks under it that the catalog is not +initialized. It applies every migration through the shared runner: +migration 1 creates the catalog and data collections and their indexes, +and the runner records each body in its database's ledger once the body +completes. Still under the lock, it writes +the encryption key, the default account, the admin user and the data +connection rows, checking the lock's lease before each request as the +runner does, and then releases the lock. +[ADR-0023](../adr/0023-catalog-migrations.md) records the design. **`extenddb destroy`:** drop both databases. diff --git a/docs/manuals/01-architecture-guide.md b/docs/manuals/01-architecture-guide.md index 4978c3668..1d1c9016a 100755 --- a/docs/manuals/01-architecture-guide.md +++ b/docs/manuals/01-architecture-guide.md @@ -205,9 +205,8 @@ extenddb uses a dual-database architecture: - **Catalog database** (e.g., `extenddb_catalog`): Stores table metadata, account/user/group/role/policy definitions, access keys, settings, stream metadata, and metrics. Shared across all accounts. - **Data database** (e.g., `extenddb_catalog_data`): Stores user items, GSI/LSI data, and stream records. Each table gets its own PostgreSQL table. -On PostgreSQL and SQLite the catalog is versioned by its migration ledger, the `extenddb_migrations` table: the catalog version is the highest migration recorded in every store, migration 4 for this release, and `serve` reads the catalog's status at startup and refuses a catalog that is behind it or carries a breaking migration it does not know. The Cassandra catalog version is 0.0.4, stored in the `settings` table under the key `catalog_version`; Cassandra has no startup check (see the Upgrade Manual). -The MongoDB backend tracks its own catalog version, 0.0.2. -A mismatch between the compiled-in version and the stored one prevents a PostgreSQL or SQLite server from starting; run `extenddb migrate` to upgrade. +On PostgreSQL, SQLite, Cassandra and MongoDB the catalog is versioned by its migration ledger, the `extenddb_migrations` table (on MongoDB a collection, in the catalog database and in the data database): the catalog version is the highest migration recorded in every store, migration 4 for this release, and `serve` reads the catalog's status at startup and refuses a catalog that is behind it or carries a breaking migration it does not know. On Cassandra every account keyspace has a ledger beside the catalog keyspace's. +A mismatch between the binary's migrations and the catalog's ledger prevents a server from starting; run `extenddb migrate` to upgrade. ## Pluggable Architecture diff --git a/docs/manuals/05-admin-guide.md b/docs/manuals/05-admin-guide.md index 1c2589181..31fe07abd 100755 --- a/docs/manuals/05-admin-guide.md +++ b/docs/manuals/05-admin-guide.md @@ -925,7 +925,7 @@ Check that PostgreSQL is running and the connection string in `extenddb.toml` is **The catalog has pending migrations, or is ahead of the binary:** -On PostgreSQL and SQLite, `serve` reads the catalog's migration status before it opens the engine and refuses a catalog it cannot serve, printing the status and the command that resolves it. A catalog with pending migrations exits 3, for example `the catalog is at migration 3, and ExtendDB needs migration 4`, followed by the pending migrations and `extenddb migrate --config extenddb.toml --yes`. A catalog that a newer release migrated with a breaking migration exits 4 and names that release's migrations; upgrade the binary rather than the catalog. The [Upgrade Manual](07-upgrade-manual.md#the-catalog-status) lists every status with its exit code and the stop / migrate / start sequence. On Cassandra and MongoDB the catalog checks are those of the 0.1.x releases; their sections of the Upgrade Manual describe them. +On PostgreSQL, SQLite, Cassandra and MongoDB, `serve` reads the catalog's migration status before it opens the engine and refuses a catalog it cannot serve, printing the status and the command that resolves it. A catalog with pending migrations exits 3, for example `the catalog is at migration 3, and ExtendDB needs migration 4`, followed by the pending migrations and `extenddb migrate --config extenddb.toml --yes`. A catalog that a newer release migrated with a breaking migration exits 4 and names that release's migrations; upgrade the binary rather than the catalog. The [Upgrade Manual](07-upgrade-manual.md#the-catalog-status) lists every status with its exit code and the stop / migrate / start sequence. ### Authentication Errors diff --git a/docs/manuals/07-upgrade-manual.md b/docs/manuals/07-upgrade-manual.md index f238d5d08..3b9f73691 100755 --- a/docs/manuals/07-upgrade-manual.md +++ b/docs/manuals/07-upgrade-manual.md @@ -4,7 +4,7 @@ ## Current Status -This release versions the PostgreSQL and SQLite catalogs by a migration ledger and carries catalog migrations 1 to 4, which `extenddb migrate --list` prints. **Every PostgreSQL and SQLite deployment created by a 0.1.x release runs `extenddb migrate --yes` once** before a server of this release starts on it: the server refuses a catalog with pending migrations. Migration 1 brings the catalog of every 0.1.x release to the v1.0 schema. Cassandra stays at catalog 0.0.4 and MongoDB at 0.0.2, and their sections below apply unchanged. +This release versions the PostgreSQL, SQLite, Cassandra and MongoDB catalogs by a migration ledger and carries catalog migrations 1 to 4, which `extenddb migrate --list` prints. **Every deployment created by a 0.1.x release runs `extenddb migrate --yes` once** before a server of this release starts on it: the server refuses a catalog with pending migrations. Migration 1 brings the catalog of every 0.1.x release to the v1.0 schema. A backup that a 0.1.x release took can restore without its global and local secondary indexes: a SQLite backup taken by 0.1.14 records only the table's key schema and attribute definitions, and its restore creates the table with its items and no index. Take a new backup of each table after upgrading to v1.0, and keep the 0.1.x backups for their items only. @@ -14,9 +14,9 @@ The [general upgrade procedure](#general-upgrade-procedure) gives the sequence. ### The Migration System -Every backend shares one numbered list of catalog migrations, in `crates/storage/src/migrations/registry.rs`. For each id a backend declares the body it runs in each of its stores, a SQL file under `crates/storage-/migrations/` or code, or why the id does not apply to it. `extenddb migrate --yes` applies the pending bodies in id order, catalog store first, and commits each body with a row in that store's ledger, the `extenddb_migrations` table: the id, name, kind, checksum, notes, time, and the release that applied it. On PostgreSQL the catalog database and the data database each have a ledger; on SQLite the database file has one. +Every backend shares one numbered list of catalog migrations, in `crates/storage/src/migrations/registry.rs`. For each id a backend declares the body it runs in each of its stores, a SQL file under `crates/storage-/migrations/` or code, or why the id does not apply to it. `extenddb migrate --yes` applies the pending bodies in id order, catalog store first, and commits each body with a row in that store's ledger, the `extenddb_migrations` table: the id, name, kind, checksum, notes, time, and the release that applied it. On PostgreSQL the catalog database and the data database each have a ledger; on SQLite the database file has one. On MongoDB the bodies are code, and the catalog database and the data database each have a ledger collection: there a body's requests are each safe to send again, and its ledger row and, in the catalog, the `catalog_version` row commit in one transaction after the body, so a migrator stopped part way runs the body again. On Cassandra the bodies are CQL files and code, and the catalog keyspace and each account keyspace have a ledger table. Cassandra has no transactions: every statement of a body is safe to send again and a body's ledger row is written after it, so a run that stopped or failed part way, for example with `no answer within 20 s` from a loaded node, finishes when `migrate --yes` runs again. -The catalog's version is the highest id such that it and every lower id are recorded in every store its bodies touch. The `catalog_version` row in `settings` holds the same integer, written with each catalog row, for tools that read it. The `schema_history` tables of 0.1.x PostgreSQL catalogs stay in place and are not read. +The catalog's version is the highest id such that it and every lower id are recorded in every store its bodies touch. The `catalog_version` row in `settings` holds the same integer, written with each catalog row, for tools that read it; on Cassandra a stopped run can leave it one ahead of the ledger, or one behind after migration 1's last account keyspace, until `migrate --yes` runs again. The `schema_history` tables of 0.1.x PostgreSQL catalogs stay in place and are not read. Running `extenddb migrate --yes` on an up-to-date catalog changes nothing. @@ -41,7 +41,7 @@ A migration is additive when the previous release serves the catalog correctly w ## Writing a New Migration -Writing a catalog migration is contributor work. The procedure, with a worked example and the tests to run on each backend, is in the [catalog migration section](../../CONTRIBUTING.md#adding-a-catalog-migration) of `CONTRIBUTING.md` in the repository root. `extenddb migrate --list` prints the migrations a PostgreSQL or SQLite binary carries, with each one's kind on its backend; a Cassandra or MongoDB binary refuses `--list`, because those backends still migrate through their bootstrappers. +Writing a catalog migration is contributor work. The procedure, with a worked example and the tests to run on each backend, is in the [catalog migration section](../../CONTRIBUTING.md#adding-a-catalog-migration) of `CONTRIBUTING.md` in the repository root. `extenddb migrate --list` prints the migrations the binary carries, with each one's kind on its backend. ## General Upgrade Procedure @@ -53,7 +53,7 @@ For releases that include catalog changes. The names below in angle brackets are extenddb stop --config extenddb.toml ``` -2. **Back up the databases** (PostgreSQL; for SQLite see [SQLite](#sqlite) below) +2. **Back up the databases** (PostgreSQL; for SQLite see [SQLite](#sqlite) below, for Cassandra see the [rollback procedure](#rollback-procedure)) ```bash pg_dump -h -U > catalog_backup_$(date +%Y%m%d).sql @@ -87,7 +87,39 @@ The PostgreSQL commands and `serve` work through PgBouncer in transaction mode a - Behind PgBouncer, the TCP keepalives the serving session sets (30 s) run on the database's connection to PgBouncer, which stays up when a serving host dies. The database keeps that host's key until PgBouncer closes the dead host's connection, which PgBouncer's own keepalive settings decide: by default the operating system's, two hours idle on Linux. Until then `migrate --yes` exits 6 naming the dead server. Set `tcp_keepalive = 1`, `tcp_keepidle = 10`, `tcp_keepintvl = 5` and `tcp_keepcnt = 4` in PgBouncer's `[pgbouncer]` section for the same 30 s, or end the session with `SELECT pg_terminate_backend()` on the pid `migrate` names. - After a failover of the catalog's server, the new primary holds no server's key, while a server that served through the old primary goes on serving through connections that reconnect to the new primary, until it counts its key as lost: at most 30 s after its last renewal. A `migrate --yes` in those 30 s does not see that server. Wait 30 s after a failover before you run `migrate --yes`, or stop the servers first. -Without `--yes`, `migrate` reads the [catalog status](#the-catalog-status), prints each pending migration, and changes nothing; it takes no lock, so it runs beside a server. With `--yes` it takes the migration lock, waiting up to `--lock-timeout` seconds (60 by default) for running servers and other migrations, reads the status again under the lock, and applies what is pending. It refuses a store that is not a catalog with exit status 1 and the advice to run `extenddb init`. The exit statuses are those in the status table; a script under `set -e` can run the preview as `extenddb migrate ... || [ $? -eq 3 ]`. +On MongoDB, `migrate` connects with the connection string in the config file, which the server uses too. A user with the `readWrite` role on `extenddb_catalog` and on `extenddb_data` and no other role can run `init`, `migrate --yes`, `verify` and `serve`. `migrate --yes` takes the migration lock, a document in the catalog's `settings` collection, and then waits for the registration every running server keeps in the catalog's `extenddb_servers` collection, and after `--lock-timeout` exits 6 naming each server's host and pid. A server deletes its registration when it stops; a server that was killed leaves it until it expires, at most 45 s after its last renewal. A server that cannot renew its registration for 30 s, for example because the database stopped answering, exits 1 and removes its PID file. The lock and the registrations expire by the clock of the replica set's primary, so keep the clocks of its members in sync with NTP. + +On Cassandra, `migrate` connects as the role the config file names, which the server connects as too. `--cassandra-user ` connects as another role, with its password in `EXTENDDB_CASSANDRA_PASSWORD` or `--cassandra-pass`; a wrong password is reported as a connection timeout. Migration 1 adopts a 0.1.x catalog: it keeps every setting except the `resource_tag_migration` row, which it deletes, and records itself in a ledger in the catalog keyspace and in every account keyspace. On a cluster that runs `CassandraAuthorizer`, `init` grants the application role SELECT, MODIFY, CREATE and ALTER on the catalog keyspace, those four and DROP on the default account's keyspace, and CREATE ON ALL KEYSPACES, which CreateAccount needs to create a keyspace per account and which also lets the role create tables in every keyspace on the cluster; a run of `migrate --yes --cassandra-user ` that creates a missing account keyspace grants the application role the five permissions on it. A 0.1.x `init` granted the application role nothing, so a deployment created before 1.0 on such a cluster needs the statements below before `migrate`. `migrate --yes` takes the migration lock, a row in the catalog's `settings` table written by lightweight transactions, and then waits for the registration every running server of this release keeps in the catalog's `extenddb_servers` table, and after `--lock-timeout` exits 6 naming each server. A server deletes its registration when it stops; a server that was killed leaves it until it expires, at most 45 s after its last renewal. A server that cannot renew its registration for 30 s exits 1. A migration needs every replica of the catalog keyspace reachable: the lock is written at `SERIAL`, the registrations are read at `ALL`, and after each schema statement the runner waits until every live node reports the same schema version. `serve` refuses a catalog with pending migrations, exiting 3 under `--foreground` with the pending migrations and the `extenddb migrate --config --yes` command to run. A 0.1.x server refuses a migrated catalog. + +On a deployment created before 1.0 on a cluster that runs `CassandraAuthorizer`, this release's `migrate`, `verify` and `serve`, connected as the application role, fail with `Unauthorized ... has no SELECT permission on
_catalog.accounts>` until the role holds its permissions. With every server stopped, run the statements below in `cqlsh` as a superuser; the application role cannot grant them to itself. `` and `` are `username` and `keyspace_prefix` under `[storage.cassandra]` in the config file. Each account has its own keyspace, so list the accounts first: + +``` +SELECT account_id FROM _catalog.accounts; +``` + +Grant on the catalog keyspace and on all keyspaces: + +``` +GRANT SELECT ON KEYSPACE _catalog TO ''; +GRANT MODIFY ON KEYSPACE _catalog TO ''; +GRANT CREATE ON KEYSPACE _catalog TO ''; +GRANT ALTER ON KEYSPACE _catalog TO ''; +GRANT CREATE ON ALL KEYSPACES TO ''; +``` + +Then, for each account id whose keyspace `_account_` exists: + +``` +GRANT SELECT ON KEYSPACE _account_ TO ''; +GRANT MODIFY ON KEYSPACE _account_ TO ''; +GRANT CREATE ON KEYSPACE _account_ TO ''; +GRANT ALTER ON KEYSPACE _account_ TO ''; +GRANT DROP ON KEYSPACE _account_ TO ''; +``` + +A GRANT on an account keyspace that does not exist fails with `Resource ... doesn't exist`; skip that account, and `migrate --yes` creates its keyspace. A statement for a permission the role already holds prints `Role '' was already granted ...` and changes nothing, so a role granted some of these by hand runs the whole list. + +Without `--yes`, `migrate` reads the [catalog status](#the-catalog-status), prints each pending migration, and changes nothing; it takes no lock, so it runs beside a server. With `--yes` it takes the migration lock, waiting up to `--lock-timeout` seconds (60 by default) for running servers and other migrations, reads the status again under the lock, and applies what is pending. It refuses a store that is not a catalog with exit status 1 and the advice to run `extenddb init`; on Cassandra it refuses a keyspace that holds another program's tables with a message that names them. The exit statuses are those in the status table; a script under `set -e` can run the preview as `extenddb migrate ... || [ $? -eq 3 ]`. On PostgreSQL `migrate --yes` can also print: @@ -124,6 +156,8 @@ To discard data written after the upgrade as well, restore `` from its 3. Rebuild the previous version and start it +This manual has no tested rollback for Cassandra. Before migrating, with every server stopped, take a snapshot of the catalog keyspace and every account keyspace on every node with your usual Cassandra tooling; a rollback restores those snapshots, again with every server stopped, before the previous binary starts. + ## Version History The catalog versions below are those of the 0.1.x releases, and the commands, output and procedures in them are those of the 0.1.x binaries. This release upgrades the catalog of any 0.1.x release with the [general upgrade procedure](#general-upgrade-procedure): its migration 1 applies the changes each version below made. diff --git a/docs/manuals/12-extending-extenddb-storage.md b/docs/manuals/12-extending-extenddb-storage.md index 75f701d54..de77b2b25 100755 --- a/docs/manuals/12-extending-extenddb-storage.md +++ b/docs/manuals/12-extending-extenddb-storage.md @@ -405,7 +405,7 @@ The PostgreSQL implementation makes backend-specific choices. These are implemen 5. **Dual-database architecture** — catalog and data are separate PostgreSQL databases. Your backend might use a single database with keyspace separation, or a different topology entirely. -6. **SQL migrations** — migration files are raw SQL. Your backend needs its own schema initialization mechanism, exposed through `Bootstrapper`. +6. **SQL migrations**: migration files are raw SQL. Your backend applies the numbered catalog migrations through the shared runner: it implements `CatalogMigrations` (`crates/storage/src/migrations/catalog.rs`), installs it in its `Backend`, and declares a body or a reason for every id. Its `Bootstrapper` keeps database creation and the rows `init` writes after the migrations. [ADR-0023](../adr/0023-catalog-migrations.md) records the design and the [catalog migration section](../../CONTRIBUTING.md#adding-a-catalog-migration) of `CONTRIBUTING.md` the procedure. ## Summary of Traits and Implementations From a1db4bcdabd4571e2d541fc09e366a6eeeeba4e2 Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Fri, 30 Oct 2026 20:57:47 +0000 Subject: [PATCH 04/45] test(cassandra): fix two migration tests that failed in CI The fixture in tests/migrations_stopped.rs created its users with create_user, 32 at a time. create_user inserts the user row with a lightweight transaction, and every fixture user is in the same account's partition, so the 32 transactions contend for that partition. On the cassandra-storage runner one of them failed, and the_status_read_under_the_lock_keeps_it_through_a_long_scan stopped in its fixture with user: Internal("Database error"). On a node limited to one CPU the old fixture failed in both runs. The fixture now inserts the user row with the columns create_user writes, without the transaction, and adds the self-service policy create_user adds, so each user holds the same rows as before. a_bodys_request_to_a_node_that_stops_answering_fails_at_the_request_timeout sends its requests through a proxy that stops passing bytes. On the two-node runner the driver also read the second node from system.peers and connected to it directly, and the second node answered the request sent after the proxy went silent, 7 ms later. The test now uses a session that leaves every node but the proxy out of its query plans. On a local two-node cluster the old session's plan held both nodes and the second node answered the first request after the proxy went silent; the new session's plan holds the proxy only, and both of its requests waited for the 10 s request timeout. --- .../tests/migrations/lock.rs | 20 ++------ .../tests/migrations_stopped.rs | 46 ++++++++++++++++--- .../tests/migrations_support/mod.rs | 41 +++++++++++++++++ 3 files changed, 86 insertions(+), 21 deletions(-) diff --git a/crates/storage-cassandra/tests/migrations/lock.rs b/crates/storage-cassandra/tests/migrations/lock.rs index 56c65643d..fb79b0946 100644 --- a/crates/storage-cassandra/tests/migrations/lock.rs +++ b/crates/storage-cassandra/tests/migrations/lock.rs @@ -198,30 +198,20 @@ async fn a_server_and_a_migration_refuse_each_other() { /// timeout, through the store the body is handed, rather than when the lease /// cuts the body 35 to 65 s later (SPEC.md A6: every request under the lock /// is bounded by 10 s). The node is the test node behind a proxy that stops -/// passing bytes once the session is up. +/// passing bytes once the session is up, and the session sends every +/// request through the proxy, on one node or two. #[tokio::test] async fn a_bodys_request_to_a_node_that_stops_answering_fails_at_the_request_timeout() { use extenddb_storage::migrations::{HasManagementStore, MigrationBackend}; - use extenddb_storage_cassandra::migrations::{CassandraMigrations, Deployment}; + use extenddb_storage_cassandra::migrations::CassandraMigrations; if helpers::skip_without_cassandra() { return; } let proxy = migrations_support::SilencingProxy::in_front_of(helpers::test_contact_point()).await; - let config = helpers::test_config(); - let deployment = Deployment { - contact_points: vec![proxy.address.clone()], - keyspace_prefix: "p2m_silent".to_owned(), - username: config.username.clone(), - password: config.password.clone(), - app_role: config.username.clone(), - }; - let session = deployment - .connect() - .await - .expect("connect through the proxy"); - let migrations = CassandraMigrations::for_status(session, &deployment.keyspace_prefix); + let session = proxy.session().await; + let migrations = CassandraMigrations::for_status(session, "p2m_silent"); let store = migrations.stores().management_store(); // Through the proxy, a request is answered: the keyspace does not exist. let answered = store.list_policies("000000000000", "user", "alice").await; diff --git a/crates/storage-cassandra/tests/migrations_stopped.rs b/crates/storage-cassandra/tests/migrations_stopped.rs index 3157ba432..fdc467900 100644 --- a/crates/storage-cassandra/tests/migrations_stopped.rs +++ b/crates/storage-cassandra/tests/migrations_stopped.rs @@ -121,17 +121,51 @@ async fn released_catalog(session: &Arc, prefix: &str) { .expect("session"); } -/// Add `count` IAM users to [`ACCOUNT`], each with one inline policy holding -/// a `dynamodb:ResourceTag` key, 32 users at a time. -async fn add_tagged_users(store: &CassandraCatalogStore, count: usize) { +/// Add `count` IAM users to [`ACCOUNT`] of the catalog of `prefix`, 32 at a +/// time, each with the self-service policy `create_user` gives a user and +/// one inline policy holding a `dynamodb:ResourceTag` key. The user row is a +/// plain insert of the columns `create_user` writes: `create_user` inserts it +/// with a lightweight transaction, and 32 of those at once on the account's +/// partition contend with each other, so on a loaded node some of them fail. +async fn add_tagged_users(store: &CassandraCatalogStore, prefix: &str, count: usize) { use futures::StreamExt; let tagged: serde_json::Value = serde_json::from_str(TAGGED).expect("json"); + let insert = format!( + "INSERT INTO {prefix}_catalog.iam_users (account_id, user_name, user_arn, created_at) \ + VALUES (?, ?, ?, toTimestamp(now()))" + ); futures::stream::iter(0..count) .map(|i| { let tagged = &tagged; + let insert = &insert; async move { let name = format!("u{i:05}"); - store.create_user(ACCOUNT, &name, None).await.expect("user"); + let arn = format!("arn:aws:iam::{ACCOUNT}:user/{name}"); + store + .session() + .query_with_values( + insert, + cdrs_tokio::query_values!(ACCOUNT, name.as_str(), arn.as_str()), + ) + .await + .unwrap_or_else(|e| panic!("user {name}: {e}")); + let self_service = serde_json::json!({ + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Action": [ + "iam:CreateAccessKey", + "iam:DeleteAccessKey", + "iam:ListAccessKeys", + "iam:ChangePassword" + ], + "Resource": arn + }] + }); + store + .put_policy(ACCOUNT, "user", &name, "SelfServicePolicy", &self_service) + .await + .expect("self-service policy"); store .put_policy(ACCOUNT, "user", &name, "tagged", tagged) .await @@ -430,7 +464,7 @@ async fn migration_2_records_its_row_when_it_rewrote_many_policies() { let (prefix, _drops) = new_prefix("m2many"); released_catalog(&session, &prefix).await; let store = CassandraCatalogStore::catalog_only(session.clone(), prefix.clone()); - add_tagged_users(&store, USERS).await; + add_tagged_users(&store, &prefix, USERS).await; let (done, _) = run_stopped_after(&session, &prefix, usize::MAX, false, MigrationId::LATEST).await; done.expect("migrate records migration 2 over many policies"); @@ -480,7 +514,7 @@ async fn the_status_read_under_the_lock_keeps_it_through_a_long_scan() { let (prefix, _drops) = new_prefix("m2status"); released_catalog(&session, &prefix).await; let store = CassandraCatalogStore::catalog_only(session.clone(), prefix.clone()); - add_tagged_users(&store, USERS).await; + add_tagged_users(&store, &prefix, USERS).await; let times = LeaseTimes { ttl: Duration::from_secs(6), renew_every: Duration::from_millis(300), diff --git a/crates/storage-cassandra/tests/migrations_support/mod.rs b/crates/storage-cassandra/tests/migrations_support/mod.rs index ba6ae08a7..5ef870e2f 100644 --- a/crates/storage-cassandra/tests/migrations_support/mod.rs +++ b/crates/storage-cassandra/tests/migrations_support/mod.rs @@ -145,6 +145,47 @@ impl SilencingProxy { pub fn go_silent(&self) { self.silent.store(true, std::sync::atomic::Ordering::SeqCst); } + + /// A session to the test node that sends every request through the + /// proxy. The driver reads the other nodes of a cluster from + /// `system.peers` and connects to them directly, so on two nodes the + /// second one would answer while the proxy is silent; this session + /// leaves every node but the proxy out of its query plans. + pub async fn session(&self) -> Arc { + use cdrs_tokio::authenticators::StaticPasswordAuthenticatorProvider; + use cdrs_tokio::cluster::session::{SessionBuilder, TcpSessionBuilder}; + use cdrs_tokio::cluster::topology::NodeDistance; + use cdrs_tokio::cluster::{NodeInfo, NodeTcpConfigBuilder}; + use cdrs_tokio::load_balancing::RoundRobinLoadBalancingStrategy; + use cdrs_tokio::load_balancing::node_distance_evaluator::NodeDistanceEvaluator; + + struct OnlyThrough(std::net::SocketAddr); + + impl NodeDistanceEvaluator for OnlyThrough { + fn compute_distance(&self, node: &NodeInfo) -> Option { + (node.broadcast_rpc_address == self.0).then_some(NodeDistance::Local) + } + } + + let config = helpers::test_config(); + let proxy = self.address.parse().expect("the proxy's address"); + let nodes = NodeTcpConfigBuilder::new() + .with_contact_point(self.address.clone().into()) + .with_authenticator_provider(Arc::new(StaticPasswordAuthenticatorProvider::new( + config.username.unwrap_or_default(), + config.password.unwrap_or_default(), + ))) + .build() + .await + .expect("the proxy's node config"); + Arc::new( + TcpSessionBuilder::new(RoundRobinLoadBalancingStrategy::new(), nodes) + .with_node_distance_evaluator(Box::new(OnlyThrough(proxy))) + .build() + .await + .expect("connect through the proxy"), + ) + } } impl Drop for SilencingProxy { From f8b1a434336563a6adf40a38035246627764beeb Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Fri, 30 Oct 2026 22:55:40 +0000 Subject: [PATCH 05/45] test(cassandra): send each migrator's requests to its own node in the two-node test two_migrators_started_together_apply_a_legacy_catalog_once starts two migrators through sessions that reach the cluster at different nodes. The driver reads the other nodes from system.peers_v2 and sends requests to every node in turn, so each migrator sent its requests to both nodes, and the two migrators' requests did not always go through different coordinators. The test now gives each migrator a session whose query plans hold its own node only. On a local two-node cluster the plan of each such session held one node, a different one for each, while a session built as before held both. The schema agreement wait still asks every node for its version, on a connection to each, and it agreed through both sessions. pinned_session in tests/migrations_support builds that session. Like CassandraEngine::create_session, it logs in with the test credentials when both are set and gives up on the connect after 10 s, so a node that cannot be reached fails the test after 10 s instead of leaving it waiting until the job's timeout. The proxy test's session now comes from it. The proxy test now checks that the request sent before the proxy goes silent is answered by the node, which reports that the keyspace does not exist. Before, any error passed, so a proxy that passed nothing would not have been caught. With the proxy silenced before that request, the test now fails. The fixture in tests/migrations_stopped.rs binds password_hash to null, as create_user does, so each user row holds the same cells as before. Corrections to the previous commit's message, which says system.peers: on Cassandra 4.0 and later the driver reads system.peers_v2. The CI log does not say which of create_user's two statements failed, because create_user reports any driver error as Database error; the lightweight transaction is the statement the new fixture no longer sends. --- crates/storage-cassandra/tests/migrations.rs | 35 +++---- .../tests/migrations/lock.rs | 10 +- .../tests/migrations_stopped.rs | 12 ++- .../tests/migrations_support/mod.rs | 94 +++++++++++-------- 4 files changed, 88 insertions(+), 63 deletions(-) diff --git a/crates/storage-cassandra/tests/migrations.rs b/crates/storage-cassandra/tests/migrations.rs index 34317430b..f986d474a 100644 --- a/crates/storage-cassandra/tests/migrations.rs +++ b/crates/storage-cassandra/tests/migrations.rs @@ -27,12 +27,11 @@ use extenddb_storage::migrations::{ }; use extenddb_storage_cassandra::CassandraCatalogStore; use extenddb_storage_cassandra::cassandra_util::CassandraSession; -use extenddb_storage_cassandra::engine::CassandraEngine; use extenddb_storage_cassandra::migrations::{ CassandraCatalogMigrations, catalog_status, migrate_account, migrate_catalog_keyspace, postcheck, run_baseline_unrecorded, }; -use migrations_support::{TIMEOUT, cql, keyspace, new_prefix, session}; +use migrations_support::{TIMEOUT, cql, keyspace, new_prefix, pinned_session, session}; /// A catalog keyspace at migration 4 and one account keyspace. async fn deployment(session: &Arc, prefix: &str) -> String { @@ -66,20 +65,16 @@ fn target_of(prefix: &str) -> Target { )) } -/// A session through the node `EXTENDDB_TEST_CASSANDRA_SECOND_CONTACT_POINT` -/// names, the second node of a two-node cluster, so two migrators reach the -/// catalog through different coordinators; through the test node when it is -/// not set. -async fn second_session() -> Arc { - let mut config = helpers::test_config(); - if let Ok(second) = std::env::var("EXTENDDB_TEST_CASSANDRA_SECOND_CONTACT_POINT") { - config.contact_points = vec![second]; - } - Arc::new( - CassandraEngine::create_session(&config) - .await - .expect("connect to the second node"), - ) +/// Two sessions for two migrators, each sending all its requests to one +/// node: the test node, and the node `EXTENDDB_TEST_CASSANDRA_SECOND_CONTACT_POINT` +/// names, the second node of a two-node cluster, so the two migrators reach +/// the catalog through different coordinators. Both go to the test node +/// when it is not set. +async fn migrator_sessions() -> (Arc, Arc) { + let first = helpers::test_contact_point(); + let second = std::env::var("EXTENDDB_TEST_CASSANDRA_SECOND_CONTACT_POINT") + .unwrap_or_else(|_| first.clone()); + (pinned_session(&first).await, pinned_session(&second).await) } #[tokio::test] @@ -88,7 +83,7 @@ async fn two_migrators_started_together_apply_a_legacy_catalog_once() { return; } let session = session().await; - let second = second_session().await; + let (first_node, second_node) = migrator_sessions().await; let (prefix, _drops) = new_prefix("race"); let catalog = format!("{prefix}_catalog"); keyspace(&session, &catalog).await; @@ -101,8 +96,8 @@ async fn two_migrators_started_together_apply_a_legacy_catalog_once() { ) .await; let (first, other) = tokio::join!( - migrate_catalog_keyspace(session.clone(), &prefix, "test", TIMEOUT), - migrate_catalog_keyspace(second.clone(), &prefix, "test", TIMEOUT), + migrate_catalog_keyspace(first_node.clone(), &prefix, "test", TIMEOUT), + migrate_catalog_keyspace(second_node.clone(), &prefix, "test", TIMEOUT), ); let applied = [ first.expect("the first migrator").applied.len(), @@ -113,7 +108,7 @@ async fn two_migrators_started_together_apply_a_legacy_catalog_once() { "one migrator applies and the other finds nothing to do: {applied:?}" ); assert_eq!( - catalog_status(second.clone(), &prefix) + catalog_status(second_node.clone(), &prefix) .await .expect("status"), CatalogStatus::Current { version: 4 } diff --git a/crates/storage-cassandra/tests/migrations/lock.rs b/crates/storage-cassandra/tests/migrations/lock.rs index fb79b0946..caca77a24 100644 --- a/crates/storage-cassandra/tests/migrations/lock.rs +++ b/crates/storage-cassandra/tests/migrations/lock.rs @@ -213,9 +213,13 @@ async fn a_bodys_request_to_a_node_that_stops_answering_fails_at_the_request_tim let session = proxy.session().await; let migrations = CassandraMigrations::for_status(session, "p2m_silent"); let store = migrations.stores().management_store(); - // Through the proxy, a request is answered: the keyspace does not exist. - let answered = store.list_policies("000000000000", "user", "alice").await; - assert!(answered.is_err(), "{answered:?}"); + // Through the proxy, a request is answered: the node reports that the + // keyspace does not exist. + let answered = format!( + "{:?}", + store.list_policies("000000000000", "user", "alice").await + ); + assert!(answered.contains("does not exist"), "{answered}"); proxy.go_silent(); let started = Instant::now(); let answer = tokio::time::timeout( diff --git a/crates/storage-cassandra/tests/migrations_stopped.rs b/crates/storage-cassandra/tests/migrations_stopped.rs index fdc467900..12add12fb 100644 --- a/crates/storage-cassandra/tests/migrations_stopped.rs +++ b/crates/storage-cassandra/tests/migrations_stopped.rs @@ -131,8 +131,9 @@ async fn add_tagged_users(store: &CassandraCatalogStore, prefix: &str, count: us use futures::StreamExt; let tagged: serde_json::Value = serde_json::from_str(TAGGED).expect("json"); let insert = format!( - "INSERT INTO {prefix}_catalog.iam_users (account_id, user_name, user_arn, created_at) \ - VALUES (?, ?, ?, toTimestamp(now()))" + "INSERT INTO {prefix}_catalog.iam_users \ + (account_id, user_name, user_arn, password_hash, created_at) \ + VALUES (?, ?, ?, ?, toTimestamp(now()))" ); futures::stream::iter(0..count) .map(|i| { @@ -145,7 +146,12 @@ async fn add_tagged_users(store: &CassandraCatalogStore, prefix: &str, count: us .session() .query_with_values( insert, - cdrs_tokio::query_values!(ACCOUNT, name.as_str(), arn.as_str()), + cdrs_tokio::query_values!( + ACCOUNT, + name.as_str(), + arn.as_str(), + None::<&str> + ), ) .await .unwrap_or_else(|e| panic!("user {name}: {e}")); diff --git a/crates/storage-cassandra/tests/migrations_support/mod.rs b/crates/storage-cassandra/tests/migrations_support/mod.rs index 5ef870e2f..94e1961c0 100644 --- a/crates/storage-cassandra/tests/migrations_support/mod.rs +++ b/crates/storage-cassandra/tests/migrations_support/mod.rs @@ -146,46 +146,66 @@ impl SilencingProxy { self.silent.store(true, std::sync::atomic::Ordering::SeqCst); } - /// A session to the test node that sends every request through the - /// proxy. The driver reads the other nodes of a cluster from - /// `system.peers` and connects to them directly, so on two nodes the - /// second one would answer while the proxy is silent; this session - /// leaves every node but the proxy out of its query plans. + /// A session to the test node whose requests all go through the proxy. + /// On two nodes a session from `session()` would also send them straight + /// to the second node, which answers while the proxy is silent. pub async fn session(&self) -> Arc { - use cdrs_tokio::authenticators::StaticPasswordAuthenticatorProvider; - use cdrs_tokio::cluster::session::{SessionBuilder, TcpSessionBuilder}; - use cdrs_tokio::cluster::topology::NodeDistance; - use cdrs_tokio::cluster::{NodeInfo, NodeTcpConfigBuilder}; - use cdrs_tokio::load_balancing::RoundRobinLoadBalancingStrategy; - use cdrs_tokio::load_balancing::node_distance_evaluator::NodeDistanceEvaluator; - - struct OnlyThrough(std::net::SocketAddr); - - impl NodeDistanceEvaluator for OnlyThrough { - fn compute_distance(&self, node: &NodeInfo) -> Option { - (node.broadcast_rpc_address == self.0).then_some(NodeDistance::Local) - } - } + pinned_session(&self.address).await + } +} - let config = helpers::test_config(); - let proxy = self.address.parse().expect("the proxy's address"); - let nodes = NodeTcpConfigBuilder::new() - .with_contact_point(self.address.clone().into()) - .with_authenticator_provider(Arc::new(StaticPasswordAuthenticatorProvider::new( - config.username.unwrap_or_default(), - config.password.unwrap_or_default(), - ))) - .build() - .await - .expect("the proxy's node config"); - Arc::new( - TcpSessionBuilder::new(RoundRobinLoadBalancingStrategy::new(), nodes) - .with_node_distance_evaluator(Box::new(OnlyThrough(proxy))) - .build() - .await - .expect("connect through the proxy"), - ) +/// A session whose requests all go to the node reached at `contact_point`. +/// The driver reads the other nodes of a cluster from `system.peers_v2` +/// (`system.peers` before Cassandra 4.0) and sends requests to every node in +/// turn; this session leaves every node but that one out of its query plans. +/// The schema agreement wait still asks every node for its version, on a +/// connection to each. Like `CassandraEngine::create_session`, it logs in +/// with the test credentials when both are set and gives up on the connect +/// after 10 s. +// tests/migrations_stopped.rs compiles this module too and uses no pinned +// session. +#[allow(dead_code)] +pub async fn pinned_session(contact_point: &str) -> Arc { + use cdrs_tokio::authenticators::{ + NoneAuthenticatorProvider, StaticPasswordAuthenticatorProvider, + }; + use cdrs_tokio::cluster::session::{SessionBuilder, TcpSessionBuilder}; + use cdrs_tokio::cluster::topology::NodeDistance; + use cdrs_tokio::cluster::{NodeInfo, NodeTcpConfigBuilder}; + use cdrs_tokio::load_balancing::RoundRobinLoadBalancingStrategy; + use cdrs_tokio::load_balancing::node_distance_evaluator::NodeDistanceEvaluator; + + struct OnlyThrough(std::net::SocketAddr); + + impl NodeDistanceEvaluator for OnlyThrough { + fn compute_distance(&self, node: &NodeInfo) -> Option { + (node.broadcast_rpc_address == self.0).then_some(NodeDistance::Local) + } } + + let config = helpers::test_config(); + let address = contact_point + .parse() + .unwrap_or_else(|e| panic!("contact point {contact_point}: {e}")); + let nodes = NodeTcpConfigBuilder::new().with_contact_point(contact_point.to_owned().into()); + let nodes = if let (Some(username), Some(password)) = (&config.username, &config.password) { + nodes.with_authenticator_provider(Arc::new(StaticPasswordAuthenticatorProvider::new( + username, password, + ))) + } else { + nodes.with_authenticator_provider(Arc::new(NoneAuthenticatorProvider)) + }; + let nodes = nodes.build().await.expect("the node config"); + let session = tokio::time::timeout( + Duration::from_secs(10), + TcpSessionBuilder::new(RoundRobinLoadBalancingStrategy::new(), nodes) + .with_node_distance_evaluator(Box::new(OnlyThrough(address))) + .build(), + ) + .await + .unwrap_or_else(|_| panic!("no session to {contact_point} within 10 s")) + .unwrap_or_else(|e| panic!("connect to {contact_point}: {e}")); + Arc::new(session) } impl Drop for SilencingProxy { From b8584d5905a4e2a3555bb8f4f00cc5205e940ca2 Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Sat, 31 Oct 2026 00:00:30 +0000 Subject: [PATCH 06/45] test(cassandra): check that the proxy answered, and accept host names The proxy test's first request must now be answered through the proxy. The error a node returns names the server the driver read it from, and the check requires the proxy's address there as well as "does not exist". A session that also sent requests straight to the second node passed the old check whenever its next request, the one after the proxy went silent, fell to the proxy. On a local two-node cluster, a session built as before the previous commit failed the new check in three runs out of three, answered by the second node, and the pinned session passed it in three out of three. pinned_session resolves its contact point first, so a host name works as it does for CassandraEngine::create_session, and the session is given the address it resolved to. The doc comments on the pinned sessions now say what they hold: query plans with one node. The schema agreement wait still reads every node. A correction to the previous commit's message: create_user's account_exists read reports the driver's text, and only its two writes, the lightweight transaction and the self-service policy insert, report Database error. The CI log ruled out the read and could not tell the two writes apart. --- crates/storage-cassandra/tests/migrations.rs | 6 ++-- .../tests/migrations/lock.rs | 8 +++-- .../tests/migrations_support/mod.rs | 33 +++++++++++-------- 3 files changed, 28 insertions(+), 19 deletions(-) diff --git a/crates/storage-cassandra/tests/migrations.rs b/crates/storage-cassandra/tests/migrations.rs index f986d474a..81c60f89b 100644 --- a/crates/storage-cassandra/tests/migrations.rs +++ b/crates/storage-cassandra/tests/migrations.rs @@ -65,11 +65,11 @@ fn target_of(prefix: &str) -> Target { )) } -/// Two sessions for two migrators, each sending all its requests to one +/// Two sessions for two migrators, each with query plans that hold one /// node: the test node, and the node `EXTENDDB_TEST_CASSANDRA_SECOND_CONTACT_POINT` /// names, the second node of a two-node cluster, so the two migrators reach -/// the catalog through different coordinators. Both go to the test node -/// when it is not set. +/// the catalog through different coordinators. Both hold the test node when +/// it is not set. async fn migrator_sessions() -> (Arc, Arc) { let first = helpers::test_contact_point(); let second = std::env::var("EXTENDDB_TEST_CASSANDRA_SECOND_CONTACT_POINT") diff --git a/crates/storage-cassandra/tests/migrations/lock.rs b/crates/storage-cassandra/tests/migrations/lock.rs index caca77a24..c7f8a3c8a 100644 --- a/crates/storage-cassandra/tests/migrations/lock.rs +++ b/crates/storage-cassandra/tests/migrations/lock.rs @@ -214,12 +214,16 @@ async fn a_bodys_request_to_a_node_that_stops_answering_fails_at_the_request_tim let migrations = CassandraMigrations::for_status(session, "p2m_silent"); let store = migrations.stores().management_store(); // Through the proxy, a request is answered: the node reports that the - // keyspace does not exist. + // keyspace does not exist, and the driver names the proxy as the server + // that answered. let answered = format!( "{:?}", store.list_policies("000000000000", "user", "alice").await ); - assert!(answered.contains("does not exist"), "{answered}"); + assert!( + answered.contains("does not exist") && answered.contains(&proxy.address), + "{answered}" + ); proxy.go_silent(); let started = Instant::now(); let answer = tokio::time::timeout( diff --git a/crates/storage-cassandra/tests/migrations_support/mod.rs b/crates/storage-cassandra/tests/migrations_support/mod.rs index 94e1961c0..11cd3485d 100644 --- a/crates/storage-cassandra/tests/migrations_support/mod.rs +++ b/crates/storage-cassandra/tests/migrations_support/mod.rs @@ -146,22 +146,22 @@ impl SilencingProxy { self.silent.store(true, std::sync::atomic::Ordering::SeqCst); } - /// A session to the test node whose requests all go through the proxy. - /// On two nodes a session from `session()` would also send them straight + /// A session to the test node whose query plans hold only the proxy. On + /// two nodes a session from `session()` would also send requests straight /// to the second node, which answers while the proxy is silent. pub async fn session(&self) -> Arc { pinned_session(&self.address).await } } -/// A session whose requests all go to the node reached at `contact_point`. -/// The driver reads the other nodes of a cluster from `system.peers_v2` -/// (`system.peers` before Cassandra 4.0) and sends requests to every node in -/// turn; this session leaves every node but that one out of its query plans. -/// The schema agreement wait still asks every node for its version, on a -/// connection to each. Like `CassandraEngine::create_session`, it logs in -/// with the test credentials when both are set and gives up on the connect -/// after 10 s. +/// A session whose query plans hold only the node reached at +/// `contact_point`, a host name or address with a port. The driver reads +/// the other nodes of a cluster from `system.peers_v2` (`system.peers` +/// before Cassandra 4.0) and sends requests to every node in turn; this +/// session leaves every node but that one out of its query plans. The schema +/// agreement wait still asks every node for its version, on a connection to +/// each. Like `CassandraEngine::create_session`, it logs in with the test +/// credentials when both are set and gives up on the connect after 10 s. // tests/migrations_stopped.rs compiles this module too and uses no pinned // session. #[allow(dead_code)] @@ -184,10 +184,15 @@ pub async fn pinned_session(contact_point: &str) -> Arc { } let config = helpers::test_config(); - let address = contact_point - .parse() - .unwrap_or_else(|e| panic!("contact point {contact_point}: {e}")); - let nodes = NodeTcpConfigBuilder::new().with_contact_point(contact_point.to_owned().into()); + // The node reached at the contact point carries the address the driver + // connected to, so the contact point is resolved once and the session is + // given that address. + let address = tokio::net::lookup_host(contact_point) + .await + .ok() + .and_then(|mut found| found.next()) + .unwrap_or_else(|| panic!("contact point {contact_point} does not resolve")); + let nodes = NodeTcpConfigBuilder::new().with_contact_point(address.to_string().into()); let nodes = if let (Some(username), Some(password)) = (&config.username, &config.password) { nodes.with_authenticator_provider(Arc::new(StaticPasswordAuthenticatorProvider::new( username, password, From fe3b4f8c8b3eed3995e43fe596d690bb74776445 Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Sun, 1 Nov 2026 12:49:16 +0000 Subject: [PATCH 07/45] fix(app): list a body for many stores once in the migrate preview The preview printed one line per store that a migration was pending in, without the store's name, so on Cassandra a deployment with 4 account keyspaces after --forget 1 printed "1 baseline (account, breaking)" four times, and one with a thousand accounts would print a thousand. The preview now groups the pending lines by id, kind of store and kind, as the advice that verify and serve print already does: "1 baseline (account in 4 stores, breaking)". The grouping moves to a function the two share. --- crates/app/src/cmd_migrate_runner.rs | 35 ++++++++++++++-------- crates/app/src/cmd_migrate_runner_tests.rs | 28 +++++++++++++++++ crates/storage/src/migrations/advice.rs | 34 ++++++++++++++------- 3 files changed, 75 insertions(+), 22 deletions(-) diff --git a/crates/app/src/cmd_migrate_runner.rs b/crates/app/src/cmd_migrate_runner.rs index 7c3271b6b..fb24fead4 100644 --- a/crates/app/src/cmd_migrate_runner.rs +++ b/crates/app/src/cmd_migrate_runner.rs @@ -21,8 +21,8 @@ use std::time::Duration; use extenddb_config as config; use extenddb_storage::migrations::advice::{self, Advice, AdviceFor}; use extenddb_storage::migrations::{ - AdminRole, CatalogMigrations, CatalogStatus, Listed, MigrationError, MigrationId, RunKind, - RunReport, Target, before_lock, exit, forget_locked, run_locked, + AdminRole, CatalogMigrations, CatalogStatus, Listed, MigrationError, MigrationId, Pending, + RunKind, RunReport, Target, before_lock, exit, forget_locked, run_locked, }; use crate::cmd_migrate::MigrateArgs; @@ -231,16 +231,8 @@ async fn preview( ), } println!("Pending:"); - for p in pending { - let description = - MigrationId::from_number(p.id).map_or("", MigrationId::description); - println!( - " {} {} ({}, {}): {description}", - p.id, - p.name, - p.store.as_str(), - p.kind.as_str() - ); + for line in pending_lines(pending) { + println!("{line}"); } println!(); println!( @@ -257,6 +249,25 @@ async fn preview( Ok(status.exit_code()) } +/// The preview's lines for `pending`, one per id, kind of store and kind: +/// a body that runs in several stores of one kind, as in each of +/// Cassandra's account keyspaces, is one line with the number of stores. +fn pending_lines(pending: &[Pending]) -> Vec { + advice::grouped(pending) + .into_iter() + .map(|(p, stores)| { + let description = MigrationId::from_number(p.id).map_or("", MigrationId::description); + format!( + " {} {} ({}, {}): {description}", + p.id, + p.name, + advice::stores_label(p, stores), + p.kind.as_str() + ) + }) + .collect() +} + /// Take the lock, run, and print what the run did. async fn apply( migrations: &dyn CatalogMigrations, diff --git a/crates/app/src/cmd_migrate_runner_tests.rs b/crates/app/src/cmd_migrate_runner_tests.rs index e8eb6d107..6503b5330 100644 --- a/crates/app/src/cmd_migrate_runner_tests.rs +++ b/crates/app/src/cmd_migrate_runner_tests.rs @@ -482,3 +482,31 @@ fn the_password_variable_is_the_backends_own() { role("pg_admin", None) ); } + +/// The preview lists a body that runs in several stores of one kind once, +/// with the number of stores, as the advice does: a deployment with many +/// account keyspaces gets one line per migration, not one per keyspace. +#[test] +fn the_preview_lists_a_body_for_many_stores_once() { + let pending_in = |store: StoreKind, store_name: &str| Pending { + id: 1, + name: "baseline", + store, + store_name: store_name.to_owned(), + kind: Kind::Breaking, + }; + let mut pending = vec![pending_in(StoreKind::Catalog, "acme_catalog")]; + for n in 0..4 { + pending.push(pending_in(StoreKind::Account, &format!("acme_account_{n}"))); + } + let lines = super::pending_lines(&pending); + assert_eq!(lines.len(), 2, "{lines:?}"); + assert!( + lines[0].starts_with(" 1 baseline (catalog, breaking): "), + "{lines:?}" + ); + assert!( + lines[1].starts_with(" 1 baseline (account in 4 stores, breaking): "), + "{lines:?}" + ); +} diff --git a/crates/storage/src/migrations/advice.rs b/crates/storage/src/migrations/advice.rs index 65edeb195..a5bd39287 100644 --- a/crates/storage/src/migrations/advice.rs +++ b/crates/storage/src/migrations/advice.rs @@ -199,12 +199,12 @@ fn capitalized(text: &str) -> String { }) } -/// `Pending (all breaking): 2 name (catalog), ...`, or each kind beside its -/// migration when the kinds differ. -fn pending_line(pending: &[Pending]) -> String { - // One item per id and kind of store: a body that runs in several stores - // of one kind (Cassandra's account keyspaces) is named once, with the - // number of stores. +/// `pending` with one entry per id, kind of store and kind, in their first +/// order, and the number of stores each entry stands for: a body that runs +/// in several stores of one kind (Cassandra's account keyspaces) is listed +/// once. +#[must_use] +pub fn grouped(pending: &[Pending]) -> Vec<(&Pending, usize)> { let mut groups: Vec<(&Pending, usize)> = Vec::new(); for p in pending { match groups @@ -215,14 +215,28 @@ fn pending_line(pending: &[Pending]) -> String { None => groups.push((p, 1)), } } + groups +} + +/// The kind of store of a [`grouped`] entry, with the number of stores +/// when it stands for more than one: `account in 4 stores`. +#[must_use] +pub fn stores_label(pending: &Pending, stores: usize) -> String { + match stores { + 1 => pending.store.as_str().to_owned(), + n => format!("{} in {n} stores", pending.store.as_str()), + } +} + +/// `Pending (all breaking): 2 name (catalog), ...`, or each kind beside its +/// migration when the kinds differ. +fn pending_line(pending: &[Pending]) -> String { + let groups = grouped(pending); let same = groups.windows(2).all(|w| w[0].0.kind == w[1].0.kind); let items: Vec = groups .iter() .map(|(p, stores)| { - let store = match stores { - 1 => p.store.as_str().to_owned(), - n => format!("{} in {n} stores", p.store.as_str()), - }; + let store = stores_label(p, *stores); if same { format!("{} {} ({store})", p.id, p.name) } else { From d123441ac16127e0bc66b015531adc501407a3fe Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Sun, 1 Nov 2026 12:49:40 +0000 Subject: [PATCH 08/45] test(devtools): require the marker refusal from the 0.1.14 Cassandra server check-cassandra-older-binary passed when the 0.1.14 server exited non-zero for any reason, such as a port in use or a config it could not parse. It now also requires the server's refusal of the catalog for its missing marker ("this catalog has not had the resource-tag condition-key migration applied"), as the MongoDB check does. --- devtools/check-cassandra-older-binary | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/devtools/check-cassandra-older-binary b/devtools/check-cassandra-older-binary index daacb98cf..6ea584401 100755 --- a/devtools/check-cassandra-older-binary +++ b/devtools/check-cassandra-older-binary @@ -309,7 +309,13 @@ def older_binary_runs(label: str, d: Deployment, binary: str, old: str, containe for line in output.strip().splitlines()[-3:]: print(f" | > {line}") if command[0] == "serve": - report(code not in (0, -1), f"exit {code}: {label}: the 0.1.14 serve refuses a v1.0 catalog", output) + # For the marker that migration 1 removes, not for any other + # failure (a port in use, a config it cannot parse). + report( + code not in (0, -1) and "resource-tag condition-key migration" in output, + f"exit {code}: {label}: the 0.1.14 serve refuses a v1.0 catalog for its marker", + output, + ) if not changes: expect_exit(0, f"{label}: after the 0.1.14 {what}, the catalog is Current", migrate, d.env) continue From 5513001b1f22588f49b72e08dd8c3595d73b278e Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Sun, 1 Nov 2026 13:11:42 +0000 Subject: [PATCH 09/45] fix(storage-cassandra): refuse a keyspace whose tables only share catalog names init and the migration lock took a table for the catalog's by its name alone, and the names are common ones (metrics, tags, tables, accounts, settings, schema_history). A keyspace holding another program's metrics (id int PRIMARY KEY, reading double) and tags (name text PRIMARY KEY, color text), with a row each, received the catalog's other tables, the lock row and migration 1 before the post-check refused it, and the application role init created was left behind. A table of a catalog table's name now belongs to another program when it has a column migration 1 does not give it, or defines otherwise (kind, position, clustering order or type), and so does a row in any such table other than settings and the schema_history rows migration 1 inserts. A column that migration 1 adds with ALTER TABLE ... ADD may be missing, so a keyspace an init left before migration 1 recorded anything is still taken as a catalog to finish. init runs the check before it creates the application role, so a refused init creates nothing. --- crates/storage-cassandra/src/bootstrapper.rs | 49 ++++--- .../storage-cassandra/src/migrations/mod.rs | 127 ++++++++++++++---- .../src/migrations/postcheck.rs | 15 +++ crates/storage-cassandra/tests/migrations.rs | 109 ++++++++++++++- 4 files changed, 254 insertions(+), 46 deletions(-) diff --git a/crates/storage-cassandra/src/bootstrapper.rs b/crates/storage-cassandra/src/bootstrapper.rs index b4e98dfd8..3113ee5f2 100644 --- a/crates/storage-cassandra/src/bootstrapper.rs +++ b/crates/storage-cassandra/src/bootstrapper.rs @@ -289,9 +289,42 @@ impl CassandraBootstrapper { } } +impl CassandraBootstrapper { + /// Refuse a catalog keyspace that exists, is not a catalog, and holds + /// what another program put there ([`foreign_contents`]), naming it. + /// `init` calls it before it writes anything, the application role + /// included, and again before it creates the keyspace. + /// + /// [`foreign_contents`]: crate::migrations::foreign_contents + async fn refuse_a_foreign_keyspace(&self) -> OpResult<()> { + let keyspace = self.engine.catalog_keyspace(); + let session = self.engine.session_arc(); + let status = crate::migrations::catalog_status(session.clone(), &self.keyspace_prefix) + .await + .map_err(|e| OpError::Internal(e.to_string()))?; + if status != extenddb_storage::migrations::CatalogStatus::Uninitialized { + return Ok(()); + } + let foreign = crate::migrations::foreign_contents(&session, &keyspace) + .await + .map_err(|e| OpError::Internal(e.to_string()))?; + if foreign.is_empty() { + return Ok(()); + } + Err(OpError::AlreadyExists(format!( + "Keyspace '{keyspace}' exists and is not an ExtendDB catalog: it holds {}. Choose \ + another --keyspace-prefix.", + foreign.join(", ") + ))) + } +} + #[async_trait] impl Bootstrapper for CassandraBootstrapper { async fn ensure_app_user(&self) -> OpResult<()> { + // A keyspace of another program is refused before the role exists, + // so a refused `init` creates nothing. + self.refuse_a_foreign_keyspace().await?; let user = &self.config.app_user; let password = &self.config.app_password; @@ -401,21 +434,7 @@ impl Bootstrapper for CassandraBootstrapper { let keyspace = self.engine.catalog_keyspace(); println!("--- Creating catalog keyspace '{keyspace}'..."); let session = self.engine.session_arc(); - let status = crate::migrations::catalog_status(session.clone(), &self.keyspace_prefix) - .await - .map_err(|e| OpError::Internal(e.to_string()))?; - if status == extenddb_storage::migrations::CatalogStatus::Uninitialized { - let foreign = crate::migrations::foreign_contents(&session, &keyspace) - .await - .map_err(|e| OpError::Internal(e.to_string()))?; - if !foreign.is_empty() { - return Err(OpError::AlreadyExists(format!( - "Keyspace '{keyspace}' exists and is not an ExtendDB catalog: it holds {}. \ - Choose another --keyspace-prefix.", - foreign.join(", ") - ))); - } - } + self.refuse_a_foreign_keyspace().await?; self.engine .create_keyspace(&keyspace) .await diff --git a/crates/storage-cassandra/src/migrations/mod.rs b/crates/storage-cassandra/src/migrations/mod.rs index d886ee364..dcfdddc4d 100644 --- a/crates/storage-cassandra/src/migrations/mod.rs +++ b/crates/storage-cassandra/src/migrations/mod.rs @@ -54,8 +54,8 @@ pub use registrations::ServingRegistration; use crate::cassandra_util::CassandraSession; -/// Migration 1's catalog body, for the tests that read it. -#[cfg(test)] +/// Migration 1's catalog body: the tests read it, and +/// [`foreign_contents`] reads the `schema_history` rows it inserts. pub(crate) const CATALOG_BASELINE: &str = file_body!("migrations/catalog", "0001_baseline.cql"); /// Migration 1's account body, for the tests that read it. @@ -394,24 +394,32 @@ pub async fn create_lock_table( } /// What an uninitialized `keyspace` holds that makes it another program's: -/// the tables migration 1 does not create, a `settings` table without the -/// catalog's `key` and `value` columns, and `settings` rows other than the -/// migration lock's. Empty when the keyspace does not exist, and for -/// one an `init` created and stopped in before migration 1 recorded -/// anything, which holds only a catalog's tables (migration 1 converges -/// them) and at most the lock row. Read before anything is written to the -/// keyspace: `init` and the migration lock refuse it when this is not -/// empty, so another program's `settings` table never receives the lock -/// row. Call it only for a keyspace whose status is `Uninitialized`, since -/// a catalog's `settings` table holds many rows. +/// the tables migration 1 does not create; a table of a catalog table's +/// name with a column migration 1 does not give it, or defines otherwise; +/// a `settings` table without the catalog's `key` and `value` columns; +/// `settings` rows other than the migration lock's; `schema_history` rows +/// other than the ones migration 1 inserts; and a row in any other table. +/// Empty when the keyspace does not exist, and for one an `init` created +/// and stopped in before migration 1 recorded anything: that holds only +/// tables migration 1 creates, with the columns it gives them (a column a +/// later `ALTER TABLE ... ADD` adds may be missing, and migration 1 adds +/// it), no rows but migration 1's `schema_history` rows, and at most the +/// lock row. A table name alone shows nothing: the catalog's names +/// (`metrics`, `tags`, `tables`, `accounts`) are common ones. Read before +/// anything is written to the keyspace: `init` and the migration lock +/// refuse it when this is not empty, so another program's `settings` +/// table never receives the lock row. Call it only for a keyspace whose +/// status is `Uninitialized`, since a catalog's tables hold rows. /// /// # Errors /// -/// `system_schema` or the `settings` table cannot be read. +/// `system_schema` or a table of the keyspace cannot be read. pub async fn foreign_contents( session: &CassandraSession, keyspace: &str, ) -> Result, MigrationError> { + use cdrs_tokio::consistency::Consistency; + let expected: postcheck::Definitions = serde_json::from_str(CATALOG_EXPECTED) .map_err(|e| MigrationError::Store(format!("read the expected definitions: {e}")))?; let ours: std::collections::BTreeSet<&str> = expected @@ -425,30 +433,93 @@ pub async fn foreign_contents( .filter(|t| *t != "extenddb_migrations" && !ours.contains(t.as_str())) .map(|t| format!("the table {t}")) .collect(); - if !connect::has_columns(session, keyspace, "settings", &["key", "value"]).await? { - if tables.iter().any(|t| t == "settings") { + let actual = postcheck::describe(session, keyspace).await?; + for table in tables.iter().filter(|t| ours.contains(t.as_str())) { + let differing = postcheck::columns_not_in(&expected, &actual, table); + if table == "settings" + && !connect::has_columns(session, keyspace, "settings", &["key", "value"]).await? + { foreign .push("the table settings without the catalog's key and value columns".to_owned()); + continue; } - } else { - let found = connect::rows( - session, - &format!("SELECT key FROM {keyspace}.settings LIMIT 3"), - cdrs_tokio::query_values!(), - cdrs_tokio::consistency::Consistency::Quorum, - &format!("read the settings rows of {keyspace}"), - ) - .await?; - for row in &found { - let key: String = connect::column(row, "key", "read the settings rows")?; - if key != locks::LOCK_KEY { - foreign.push(format!("the settings row '{key}'")); + if !differing.is_empty() { + foreign.push(format!( + "the table {table}, whose column(s) {} migration 1 does not create", + differing.join(", ") + )); + continue; + } + let context = format!("read the rows of {keyspace}.{table}"); + match table.as_str() { + "settings" => { + let found = connect::rows( + session, + &format!("SELECT key FROM {keyspace}.settings LIMIT 3"), + cdrs_tokio::query_values!(), + Consistency::Quorum, + &context, + ) + .await?; + for row in &found { + let key: String = connect::column(row, "key", &context)?; + if key != locks::LOCK_KEY { + foreign.push(format!("the settings row '{key}'")); + } + } + } + "schema_history" => { + let found = connect::rows( + session, + &format!("SELECT version, description FROM {keyspace}.schema_history"), + cdrs_tokio::query_values!(), + Consistency::Quorum, + &context, + ) + .await?; + let inserted = baseline_history(); + for row in &found { + let version: i32 = connect::column(row, "version", &context)?; + let description: Option = + connect::nullable(row, "description", &context)?; + if !inserted.contains(&(version, description.as_deref().unwrap_or_default())) { + foreign.push(format!("the schema_history row of version {version}")); + } + } + } + _ => { + let found = connect::rows( + session, + &format!("SELECT * FROM {keyspace}.{table} LIMIT 1"), + cdrs_tokio::query_values!(), + Consistency::Quorum, + &context, + ) + .await?; + if !found.is_empty() { + foreign.push(format!("rows in the table {table}")); + } } } } Ok(foreign) } +/// The `schema_history` rows migration 1's catalog body inserts, as +/// `(version, description)`. +fn baseline_history() -> Vec<(i32, &'static str)> { + CATALOG_BASELINE + .lines() + .filter_map(|line| { + let values = line + .trim() + .strip_prefix("INSERT INTO schema_history (version, description) VALUES (")?; + let (version, rest) = values.split_once(", '")?; + Some((version.parse().ok()?, rest.strip_suffix("');")?)) + }) + .collect() +} + /// The authorizer the node runs, read from `system_views.settings` /// (Cassandra 4.0 and later). /// diff --git a/crates/storage-cassandra/src/migrations/postcheck.rs b/crates/storage-cassandra/src/migrations/postcheck.rs index f3dbfec99..96280b8a2 100644 --- a/crates/storage-cassandra/src/migrations/postcheck.rs +++ b/crates/storage-cassandra/src/migrations/postcheck.rs @@ -154,6 +154,21 @@ pub fn compare(expected: &Definitions, actual: &Definitions) -> Vec { differences } +/// The columns of `table` in `actual` that `expected` does not have, or +/// defines otherwise, by name. A column `expected` has and `actual` lacks +/// is not one of them. +#[must_use] +pub fn columns_not_in(expected: &Definitions, actual: &Definitions, table: &str) -> Vec { + actual + .columns + .iter() + .filter_map(|(key, found)| { + let (t, name) = key.split_once('.')?; + (t == table && expected.columns.get(key) != Some(found)).then(|| name.to_owned()) + }) + .collect() +} + /// Compare `keyspace` with `expected_json`; stop naming every difference. /// /// # Errors diff --git a/crates/storage-cassandra/tests/migrations.rs b/crates/storage-cassandra/tests/migrations.rs index 81c60f89b..05f1d4f78 100644 --- a/crates/storage-cassandra/tests/migrations.rs +++ b/crates/storage-cassandra/tests/migrations.rs @@ -28,8 +28,8 @@ use extenddb_storage::migrations::{ use extenddb_storage_cassandra::CassandraCatalogStore; use extenddb_storage_cassandra::cassandra_util::CassandraSession; use extenddb_storage_cassandra::migrations::{ - CassandraCatalogMigrations, catalog_status, migrate_account, migrate_catalog_keyspace, - postcheck, run_baseline_unrecorded, + CassandraCatalogMigrations, catalog_status, foreign_contents, migrate_account, + migrate_catalog_keyspace, postcheck, run_baseline_unrecorded, }; use migrations_support::{TIMEOUT, cql, keyspace, new_prefix, pinned_session, session}; @@ -263,7 +263,9 @@ async fn a_keyspace_that_is_not_a_catalog_is_uninitialized_and_migrate_refuses_i message.contains("is not an ExtendDB catalog") && message .contains("the table settings without the catalog's key and value columns") - && !message.contains("the table accounts"), + && message.contains( + "the table accounts, whose column(s) id migration 1 does not create" + ), "{message}" ), other => panic!("{:?}", other.err()), @@ -320,6 +322,107 @@ async fn a_keyspace_that_is_not_a_catalog_is_uninitialized_and_migrate_refuses_i assert_eq!(keys, ["theme"], "no lock row was written"); } +/// A table is not a catalog's for its name alone: another program's +/// `metrics` and `tags`, with columns migration 1 does not create and a +/// row each, are refused, and so is a `schema_history` with the catalog's +/// columns and a row migration 1 does not insert. A keyspace an `init` +/// left before migration 1 recorded anything, with migration 1's own +/// `schema_history` rows, is still a catalog to finish. +#[tokio::test] +async fn tables_with_catalog_names_and_other_columns_or_rows_are_refused() { + if helpers::skip_without_cassandra() { + return; + } + let session = session().await; + let migrations = CassandraCatalogMigrations; + let (named, _named_drops) = new_prefix("named"); + let catalog = format!("{named}_catalog"); + keyspace(&session, &catalog).await; + for statement in [ + format!("CREATE TABLE {catalog}.metrics (id int PRIMARY KEY, reading double)"), + format!("CREATE TABLE {catalog}.tags (name text PRIMARY KEY, color text)"), + format!("INSERT INTO {catalog}.metrics (id, reading) VALUES (1, 2.5)"), + format!("INSERT INTO {catalog}.tags (name, color) VALUES ('urgent', 'red')"), + ] { + cql(&session, &statement).await; + } + match migrations.lock(&target_of(&named), TIMEOUT).await { + Err(MigrationError::Refused(message)) => assert!( + message.contains("is not an ExtendDB catalog") + && message.contains("the table metrics, whose column(s) id, reading") + && message.contains("the table tags, whose column(s) color, name"), + "{message}" + ), + other => panic!("{:?}", other.err()), + } + assert_eq!( + helpers_tables(&session, &catalog).await, + ["metrics", "tags"], + "nothing was created" + ); + + let (history, _history_drops) = new_prefix("history"); + let catalog = format!("{history}_catalog"); + keyspace(&session, &catalog).await; + cql( + &session, + &format!( + "CREATE TABLE {catalog}.schema_history (version int PRIMARY KEY, description text, \ + applied_at timestamp)" + ), + ) + .await; + cql( + &session, + &format!( + "INSERT INTO {catalog}.schema_history (version, description) VALUES (1, \ + 'initial_schema')" + ), + ) + .await; + assert_eq!( + foreign_contents(&session, &catalog).await.expect("read"), + Vec::::new(), + "migration 1's own row" + ); + cql( + &session, + &format!( + "INSERT INTO {catalog}.schema_history (version, description) VALUES (9, 'theirs')" + ), + ) + .await; + match migrations.lock(&target_of(&history), TIMEOUT).await { + Err(MigrationError::Refused(message)) => assert!( + message.contains("the schema_history row of version 9") + && !message.contains("version 1"), + "{message}" + ), + other => panic!("{:?}", other.err()), + } +} + +/// The tables of `keyspace`, in name order. +async fn helpers_tables(session: &CassandraSession, keyspace: &str) -> Vec { + use cdrs_tokio::types::IntoRustByName; + let mut names: Vec = session + .query_with_values( + "SELECT table_name FROM system_schema.tables WHERE keyspace_name = ?", + cdrs_tokio::query_values!(keyspace), + ) + .await + .expect("read") + .response_body() + .expect("body") + .into_rows() + .unwrap_or_default() + .iter() + .map(|row| row.get_r_by_name("table_name").expect("table_name")) + .collect(); + names.sort(); + names +} + #[tokio::test] async fn a_ledger_that_cannot_be_read_stops_the_status() { if helpers::skip_without_cassandra() { From 9198970cbd2606c4530e3b2d1d9866ef27a9cbf0 Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Sun, 1 Nov 2026 14:25:07 +0000 Subject: [PATCH 10/45] fix(migrations): --reconverge skips migration 2 on Cassandra and MongoDB Migration 2 is now declared not convergent on Cassandra and MongoDB, as #666 declares it on SQLite and PostgreSQL. A dynamodb:ResourceTag key written through the API after the rewrite has no effect, as on the service, and running the rewrite again would give it one. A reconverge now leaves such a policy as it was written, also one that names both forms of a key, which the rewrite refuses, and says that it skipped migration 2. Tests on both backends: after a released catalog is migrated, policies with the dynamodb: form and one with both forms are written, the marker a 0.1.x release leaves is put back, and the reconverge leaves the text of every policy as it was and names migration 2 as skipped. Each test fails with migration 2 declared convergent. --- .../storage-cassandra/src/migrations/mod.rs | 5 +- .../tests/migrations_stopped.rs | 96 +++++++++++++++++++ crates/storage-mongodb/src/migrations/mod.rs | 5 +- .../storage-mongodb/src/migrations/tests.rs | 2 +- .../storage-mongodb/tests/migrations_fresh.rs | 14 ++- .../tests/migrations_released.rs | 91 ++++++++++++++++++ 6 files changed, 209 insertions(+), 4 deletions(-) diff --git a/crates/storage-cassandra/src/migrations/mod.rs b/crates/storage-cassandra/src/migrations/mod.rs index dcfdddc4d..6c5c764e3 100644 --- a/crates/storage-cassandra/src/migrations/mod.rs +++ b/crates/storage-cassandra/src/migrations/mod.rs @@ -98,7 +98,10 @@ pub fn migration(id: MigrationId) -> Migration { }, MigrationId::ResourceTagConditionKeys => Migration::Applies { kind: Kind::Breaking, - convergent: true, + // A `dynamodb:ResourceTag` key written through the API after the + // rewrite has no effect, as on the service; running the rewrite + // again would give it one. + convergent: false, bodies: vec![StoreBody::code( StoreKind::Catalog, &ResourceTagConditionKeys, diff --git a/crates/storage-cassandra/tests/migrations_stopped.rs b/crates/storage-cassandra/tests/migrations_stopped.rs index 12add12fb..9f4ccb1df 100644 --- a/crates/storage-cassandra/tests/migrations_stopped.rs +++ b/crates/storage-cassandra/tests/migrations_stopped.rs @@ -554,3 +554,99 @@ async fn the_status_read_under_the_lock_keeps_it_through_a_long_scan() { ); kept.expect("the status read kept the lock"); } + +/// Both forms on one key with different values, which migration 2 refuses. +const COLLIDING: &str = r#"{"Version":"2012-10-17","Statement":[{"Effect":"Deny","Action":"dynamodb:*","Resource":"*","Condition":{"StringEquals":{"dynamodb:ResourceTag/env":"prod","aws:ResourceTag/env":"dev"}}}]}"#; + +/// From 1.0 on a `dynamodb:ResourceTag` key written through the API has no +/// effect, as on the service. Migration 2 is not convergent, so a reconverge +/// leaves the policies written after it, in each place a policy is stored, +/// with the text they were written with, one that names both forms of a key +/// included, and names migration 2 as skipped. +#[tokio::test] +async fn a_reconverge_leaves_policies_written_after_migration_2_as_they_are() { + if helpers::skip_without_cassandra() { + return; + } + let session = session().await; + let (prefix, _drops) = new_prefix("m2rc"); + released_catalog(&session, &prefix).await; + let (done, _) = + run_stopped_after(&session, &prefix, usize::MAX, false, MigrationId::LATEST).await; + done.expect("migrate"); + let store = CassandraCatalogStore::catalog_only(session.clone(), prefix.clone()); + let tagged: serde_json::Value = serde_json::from_str(TAGGED).expect("json"); + let colliding: serde_json::Value = serde_json::from_str(COLLIDING).expect("json"); + for (principal_type, name, policy) in [ + ("user", "dev", "tagged-user"), + ("group", "devs", "tagged-group"), + ("role", "app", "tagged-role"), + ] { + store + .put_policy(ACCOUNT, principal_type, name, policy, &tagged) + .await + .expect("policy"); + } + store + .put_policy(ACCOUNT, "user", "dev", "both-forms", &colliding) + .await + .expect("policy"); + store + .set_user_boundary(ACCOUNT, "dev", &tagged) + .await + .expect("user boundary"); + store + .set_role_boundary(ACCOUNT, "app", &tagged) + .await + .expect("role boundary"); + let before = policies(&store).await; + assert_eq!( + before.matches("dynamodb:ResourceTag/env").count(), + 6, + "{before}" + ); + // What a release before 1.0 leaves when it writes to a migrated catalog. + cql( + &session, + &format!( + "INSERT INTO {prefix}_catalog.settings (key, value) VALUES \ + ('resource_tag_migration', 'applied')" + ), + ) + .await; + assert_eq!( + catalog_status(session.clone(), &prefix) + .await + .expect("status") + .label(), + "WrittenByOlderRelease" + ); + + let catalog = format!("{prefix}_catalog"); + let lock = MigrationLock::acquire(session.clone(), &catalog, LeaseTimes::DEFAULT, TIMEOUT) + .await + .expect("take the lock"); + let report = { + let migrations = CassandraMigrations::locked(&lock, &prefix); + Runner::new(&migrations, "test").reconverge().await + }; + lock.release().await; + let report = report.expect("reconverge"); + assert!(report.applied.iter().all(|step| step.id != 2), "{report:?}"); + assert!( + report + .notes + .iter() + .any(|n| n.starts_with("Skipped 2 resource_tag_condition_keys:")), + "{report:?}" + ); + assert_eq!( + catalog_status(session.clone(), &prefix) + .await + .expect("status"), + CatalogStatus::Current { + version: MigrationId::LATEST.number() + } + ); + assert_eq!(policies(&store).await, before); +} diff --git a/crates/storage-mongodb/src/migrations/mod.rs b/crates/storage-mongodb/src/migrations/mod.rs index bd765980f..1b8714bec 100644 --- a/crates/storage-mongodb/src/migrations/mod.rs +++ b/crates/storage-mongodb/src/migrations/mod.rs @@ -104,7 +104,10 @@ pub fn migration(id: MigrationId) -> Migration { }, MigrationId::ResourceTagConditionKeys => Migration::Applies { kind: Kind::Breaking, - convergent: true, + // A `dynamodb:ResourceTag` key written through the API after the + // rewrite has no effect, as on the service; running the rewrite + // again would give it one. + convergent: false, bodies: vec![StoreBody::code( StoreKind::Catalog, &ResourceTagConditionKeys, diff --git a/crates/storage-mongodb/src/migrations/tests.rs b/crates/storage-mongodb/src/migrations/tests.rs index 97dfbe2d1..651ad34fe 100644 --- a/crates/storage-mongodb/src/migrations/tests.rs +++ b/crates/storage-mongodb/src/migrations/tests.rs @@ -55,7 +55,7 @@ fn every_body_is_code_and_three_and_four_do_not_apply() { assert!(!reason.is_empty(), "{id}"); } assert!(migration(MigrationId::Baseline).convergent()); - assert!(migration(MigrationId::ResourceTagConditionKeys).convergent()); + assert!(!migration(MigrationId::ResourceTagConditionKeys).convergent()); } /// The lock, the registrations and the ledgers are read and written through diff --git a/crates/storage-mongodb/tests/migrations_fresh.rs b/crates/storage-mongodb/tests/migrations_fresh.rs index 88dccb774..95e5393fe 100644 --- a/crates/storage-mongodb/tests/migrations_fresh.rs +++ b/crates/storage-mongodb/tests/migrations_fresh.rs @@ -165,7 +165,19 @@ async fn a_second_run_applies_nothing_and_reconverge_keeps_the_rows() { assert_eq!((again.from, again.to), (Some(4), Some(4))); let reconverged = d.reconverge().await.expect("reconverge"); - assert_eq!(reconverged.applied.len(), 5); + // Every step but migration 2's, which is not convergent. + assert_eq!(reconverged.applied.len(), 4, "{reconverged:?}"); + assert!( + reconverged.applied.iter().all(|step| step.id != 2), + "{reconverged:?}" + ); + assert!( + reconverged + .notes + .iter() + .any(|n| n.starts_with("Skipped 2 resource_tag_condition_keys:")), + "{reconverged:?}" + ); assert_eq!( d.catalog_ledger().await, before, diff --git a/crates/storage-mongodb/tests/migrations_released.rs b/crates/storage-mongodb/tests/migrations_released.rs index 46e105cce..7577ae3ae 100644 --- a/crates/storage-mongodb/tests/migrations_released.rs +++ b/crates/storage-mongodb/tests/migrations_released.rs @@ -311,3 +311,94 @@ async fn migration_1_upgrades_the_oldest_release_as_the_golden_file_records() { ); d.drop_databases().await; } + +/// From 1.0 on a `dynamodb:ResourceTag` key written through the API has no +/// effect, as on the service. Migration 2 is not convergent, so a reconverge +/// leaves the policies written after it with the text they were written +/// with, one that names both forms of a key, which the rewrite refuses, +/// included, and names migration 2 as skipped. +#[tokio::test(flavor = "multi_thread")] +async fn a_reconverge_leaves_policies_written_after_migration_2_as_they_are() { + let Some(uri) = server() else { return }; + let d = Deployment::new(&uri, "reconverge2").await; + fixture::load(&d, &fixture::read("0.1.14")).await; + d.migrate().await.expect("migrate"); + let account = Deployment::documents(&d.catalog(), "accounts").await[0] + .get_str("_id") + .unwrap() + .to_owned(); + let tagged = doc! { "Version": "2012-10-17", "Statement": [{ + "Effect": "Deny", "Action": "dynamodb:*", "Resource": "*", + "Condition": { "StringEquals": { "dynamodb:ResourceTag/env": "prod" } }, + }] }; + let both_forms = doc! { "Version": "2012-10-17", "Statement": [{ + "Effect": "Deny", "Action": "dynamodb:*", "Resource": "*", + "Condition": { "StringEquals": { + "dynamodb:ResourceTag/env": "prod", "aws:ResourceTag/env": "dev", + } }, + }] }; + d.catalog() + .collection::("iam_users") + .insert_one(doc! { + "account_id": &account, "user_name": "after", + "user_arn": format!("arn:aws:iam::{account}:user/after"), + "created_at": bson::DateTime::from_millis(1_793_235_168_000), + }) + .await + .unwrap(); + for (name, document) in [("tagged-after", &tagged), ("both-forms-after", &both_forms)] { + d.catalog() + .collection::("iam_policies") + .insert_one(doc! { + "account_id": &account, "principal_type": "user", "principal_name": "after", + "policy_name": name, "policy_document": document, + "created_at": bson::DateTime::from_millis(1_793_235_168_000), + }) + .await + .unwrap(); + } + let documents = |policies: Vec| -> Vec { + policies + .iter() + .map(|p| { + format!( + "{} {}", + p.get_str("policy_name").unwrap(), + p.get_document("policy_document").unwrap() + ) + }) + .collect() + }; + let before = documents(Deployment::documents(&d.catalog(), "iam_policies").await); + assert_eq!( + before + .iter() + .filter(|p| p.contains("dynamodb:ResourceTag/env")) + .count(), + 2, + "{before:?}" + ); + // What a release before 1.0 leaves when it writes to a migrated catalog. + d.catalog() + .collection::("settings") + .insert_one(doc! { "_id": "resource_tag_migration", "value": "applied" }) + .await + .unwrap(); + assert_eq!(d.status().await.label(), "WrittenByOlderRelease"); + + let report = d.reconverge().await.expect("reconverge"); + assert!(report.applied.iter().all(|step| step.id != 2), "{report:?}"); + assert!( + report + .notes + .iter() + .any(|n| n.starts_with("Skipped 2 resource_tag_condition_keys:")), + "{report:?}" + ); + assert_eq!(d.status().await, CatalogStatus::Current { version: 4 }); + assert_eq!( + documents(Deployment::documents(&d.catalog(), "iam_policies").await), + before + ); + d.drop_databases().await; +} From 9889b533edbcdc19cb506679512f33777c7e9442 Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Sun, 1 Nov 2026 14:33:25 +0000 Subject: [PATCH 11/45] fix(storage): name a policy Cassandra or MongoDB cannot decode, and read the others Migration 2 reads each principal's policies and boundary through list_policy_documents and boundary_document, which #666 overrides on SQLite only. On Cassandra the trait's default went through list_policies, which fails the whole list when one stored document does not decode, so the status, verify and migrate --yes stopped with exit 1 and "Database error". On MongoDB list_policies and the boundary getters skip a document they cannot read as JSON, so migration 2 left such a policy as it was and reported nothing. Both stores now override the two methods and decode each document on its own, and their leased stores, which a migration reads through, pass the calls on. A document that does not decode is an item that needs attention, named with its account, principal, policy and error, as on SQLite. Tests on both backends: a principal with one undecodable policy, one tagged policy and a tagged boundary. The status under the lock is NeedsAttention and names the undecodable policy, and the rewrite rewrites the other two. Both tests fail without the overrides. --- .../src/management_store/mod.rs | 32 ++++++- .../src/management_store/policies.rs | 69 ++++++++++++- .../src/migrations/guarded.rs | 5 +- .../tests/migrations_stopped.rs | 69 +++++++++++++ .../storage-mongodb/src/management_store.rs | 90 ++++++++++++++++- .../storage-mongodb/src/migrations/guarded.rs | 5 +- .../tests/migrations_holders.rs | 96 +++++++++++++++++++ 7 files changed, 360 insertions(+), 6 deletions(-) diff --git a/crates/storage-cassandra/src/management_store/mod.rs b/crates/storage-cassandra/src/management_store/mod.rs index 28bf7bae1..896ebef37 100755 --- a/crates/storage-cassandra/src/management_store/mod.rs +++ b/crates/storage-cassandra/src/management_store/mod.rs @@ -7,7 +7,7 @@ //! file under the 500-line limit. use extenddb_storage::management_store::{ - AccessKeyCreated, AccountDetail, GroupDetail, OpResult, RoleDetail, UserDetail, + AccessKeyCreated, AccountDetail, GroupDetail, OpResult, RoleDetail, StoredDocument, UserDetail, }; use futures::future::BoxFuture; @@ -400,6 +400,21 @@ impl extenddb_storage::management_store::ManagementStore for CassandraCatalogSto }) } + fn list_policy_documents( + &self, + account_id: &str, + principal_type: &str, + principal_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_string(); + let principal_type = principal_type.to_string(); + let principal_name = principal_name.to_string(); + Box::pin(async move { + self.list_policy_documents_impl(&account_id, &principal_type, &principal_name) + .await + }) + } + // ── Permissions boundaries ───────────────────────────────────── fn set_user_boundary( @@ -473,6 +488,21 @@ impl extenddb_storage::management_store::ManagementStore for CassandraCatalogSto }) } + fn boundary_document( + &self, + account_id: &str, + principal_type: &str, + principal_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_string(); + let principal_type = principal_type.to_string(); + let principal_name = principal_name.to_string(); + Box::pin(async move { + self.boundary_document_impl(&account_id, &principal_type, &principal_name) + .await + }) + } + fn delete_role_boundary( &self, account_id: &str, diff --git a/crates/storage-cassandra/src/management_store/policies.rs b/crates/storage-cassandra/src/management_store/policies.rs index 5e452d2cd..dad65fed3 100644 --- a/crates/storage-cassandra/src/management_store/policies.rs +++ b/crates/storage-cassandra/src/management_store/policies.rs @@ -3,7 +3,7 @@ //! Policy and permissions-boundary operations for `CassandraCatalogStore`. -use extenddb_storage::management_store::{OpError, OpResult}; +use extenddb_storage::management_store::{OpError, OpResult, StoredDocument}; use crate::catalog_store::CassandraCatalogStore; @@ -131,6 +131,43 @@ impl CassandraCatalogStore { Ok(policies) } + /// A principal's policies for the resource-tag migration, each document + /// decoded on its own: one that does not decode comes back as its decode + /// error, and the others are still read. + pub(crate) async fn list_policy_documents_impl( + &self, + account_id: &str, + principal_type: &str, + principal_name: &str, + ) -> OpResult> { + let catalog_keyspace = self.catalog_keyspace(); + let query = format!( + "SELECT policy_name, policy_document FROM {catalog_keyspace}.iam_policies \ + WHERE account_id = ? AND principal_type = ? AND principal_name = ?" + ); + let rows = crate::cassandra_util::query_rows( + self.session(), + &query, + cdrs_tokio::query_values!(account_id, principal_type, principal_name), + "list_policy_documents", + ) + .await?; + let mut policies = crate::cassandra_util::map_rows( + rows, + |row| { + use crate::cassandra_util::get_column; + let doc_str: String = get_column(row, "policy_document", "list_policy_documents")?; + Ok::<_, OpError>(( + get_column::(row, "policy_name", "list_policy_documents")?, + serde_json::from_str(&doc_str).map_err(|e| e.to_string()), + )) + }, + "list_policy_documents", + )?; + policies.sort_by(|a, b| a.0.cmp(&b.0)); + Ok(policies) + } + // ── Permissions boundaries ───────────────────────────────────── pub(crate) async fn set_boundary_impl( @@ -198,6 +235,36 @@ impl CassandraCatalogStore { Ok(Some(doc)) } + /// A principal's permissions boundary for the resource-tag migration: + /// the decoded document, or its decode error. + pub(crate) async fn boundary_document_impl( + &self, + account_id: &str, + principal_type: &str, + principal_name: &str, + ) -> OpResult> { + let catalog_keyspace = self.catalog_keyspace(); + let query = format!( + "SELECT policy_document FROM {catalog_keyspace}.iam_permissions_boundaries \ + WHERE account_id = ? AND principal_type = ? AND principal_name = ?" + ); + let row = crate::cassandra_util::query_optional( + self.session(), + &query, + cdrs_tokio::query_values!(account_id, principal_type, principal_name), + "boundary_document", + ) + .await?; + let Some(row) = row else { + return Ok(None); + }; + let doc_str: String = + crate::cassandra_util::get_column(&row, "policy_document", "boundary_document")?; + Ok(Some( + serde_json::from_str(&doc_str).map_err(|e| e.to_string()), + )) + } + pub(crate) async fn delete_boundary_impl( &self, account_id: &str, diff --git a/crates/storage-cassandra/src/migrations/guarded.rs b/crates/storage-cassandra/src/migrations/guarded.rs index b1ac6f851..e83d67a83 100644 --- a/crates/storage-cassandra/src/migrations/guarded.rs +++ b/crates/storage-cassandra/src/migrations/guarded.rs @@ -20,7 +20,8 @@ use std::time::Duration; use extenddb_storage::management_store::{ AccessKeyCreated, AccountDetail, GroupDetail, GroupListEntry, ManagementStore, OpError, - OpResult, RoleDetail, RoleListEntry, SessionPolicyEntry, UserDetail, UserListEntry, + OpResult, RoleDetail, RoleListEntry, SessionPolicyEntry, StoredDocument, UserDetail, + UserListEntry, }; use futures::future::BoxFuture; @@ -135,11 +136,13 @@ impl ManagementStore for LeasedManagementStore { fn put_policy(&self, account_id: &str, principal_type: &str, principal_name: &str, policy_name: &str, document: &serde_json::Value) -> BoxFuture<'_, OpResult<()>>; fn delete_policy(&self, account_id: &str, principal_type: &str, principal_name: &str, policy_name: &str) -> BoxFuture<'_, OpResult<()>>; fn list_policies(&self, account_id: &str, principal_type: &str, principal_name: &str) -> BoxFuture<'_, OpResult>>; + fn list_policy_documents(&self, account_id: &str, principal_type: &str, principal_name: &str) -> BoxFuture<'_, OpResult>>; fn set_user_boundary(&self, account_id: &str, user_name: &str, document: &serde_json::Value) -> BoxFuture<'_, OpResult<()>>; fn get_user_boundary(&self, account_id: &str, user_name: &str) -> BoxFuture<'_, OpResult>>; fn delete_user_boundary(&self, account_id: &str, user_name: &str) -> BoxFuture<'_, OpResult<()>>; fn set_role_boundary(&self, account_id: &str, role_name: &str, document: &serde_json::Value) -> BoxFuture<'_, OpResult<()>>; fn get_role_boundary(&self, account_id: &str, role_name: &str) -> BoxFuture<'_, OpResult>>; + fn boundary_document(&self, account_id: &str, principal_type: &str, principal_name: &str) -> BoxFuture<'_, OpResult>>; fn delete_role_boundary(&self, account_id: &str, role_name: &str) -> BoxFuture<'_, OpResult<()>>; fn create_access_key(&self, account_id: &str, user_name: &str) -> BoxFuture<'_, OpResult>; fn delete_access_key(&self, account_id: &str, user_name: &str, key_id: &str) -> BoxFuture<'_, OpResult<()>>; diff --git a/crates/storage-cassandra/tests/migrations_stopped.rs b/crates/storage-cassandra/tests/migrations_stopped.rs index 9f4ccb1df..32694efd2 100644 --- a/crates/storage-cassandra/tests/migrations_stopped.rs +++ b/crates/storage-cassandra/tests/migrations_stopped.rs @@ -650,3 +650,72 @@ async fn a_reconverge_leaves_policies_written_after_migration_2_as_they_are() { ); assert_eq!(policies(&store).await, before); } + +/// A stored document that does not decode is named, with its account, +/// principal, policy and decode error, as an item a person must fix: the +/// status `migrate --yes` reads under the lock is NeedsAttention. The +/// principal's other policies and its boundary are still read, and the +/// rewrite rewrites them. +#[tokio::test] +async fn an_undecodable_policy_is_named_and_the_others_are_rewritten() { + if helpers::skip_without_cassandra() { + return; + } + let session = session().await; + let (prefix, _drops) = new_prefix("m2bad"); + released_catalog(&session, &prefix).await; + let store = CassandraCatalogStore::catalog_only(session.clone(), prefix.clone()); + let tagged: serde_json::Value = serde_json::from_str(TAGGED).expect("json"); + store + .create_user(ACCOUNT, "carol", None) + .await + .expect("user"); + for name in ["a_broken", "b_tagged"] { + store + .put_policy(ACCOUNT, "user", "carol", name, &tagged) + .await + .expect("policy"); + } + store + .set_user_boundary(ACCOUNT, "carol", &tagged) + .await + .expect("boundary"); + cql( + &session, + &format!( + "UPDATE {prefix}_catalog.iam_policies SET policy_document = '{{not json' WHERE \ + account_id = '{ACCOUNT}' AND principal_type = 'user' AND principal_name = 'carol' \ + AND policy_name = 'a_broken'" + ), + ) + .await; + + let catalog = format!("{prefix}_catalog"); + let lock = MigrationLock::acquire(session.clone(), &catalog, LeaseTimes::DEFAULT, TIMEOUT) + .await + .expect("take the lock"); + let status = status_holding(&lock, &prefix).await; + lock.release().await; + let Ok(CatalogStatus::NeedsAttention { items }) = &status else { + panic!("expected NeedsAttention: {status:?}"); + }; + assert!( + items.iter().any(|i| i.contains(&format!( + "account {ACCOUNT} user/carol policy a_broken has a stored document that cannot \ + be decoded" + )) && i.contains("line 1")), + "{items:?}" + ); + + let report = + extenddb_storage::policy_migration::migrate_resource_tag_condition_keys(&store, true).await; + assert_eq!(report.errors, 0, "{:?}", report.failures); + assert_eq!(report.unreadable.len(), 1, "{report:?}"); + let carol: Vec<&str> = report + .rewritten + .iter() + .filter(|r| r.principal == "user/carol") + .map(|r| r.policy_name.as_str()) + .collect(); + assert_eq!(carol, ["b_tagged", ""]); +} diff --git a/crates/storage-mongodb/src/management_store.rs b/crates/storage-mongodb/src/management_store.rs index 91e60e2b9..35db6b722 100644 --- a/crates/storage-mongodb/src/management_store.rs +++ b/crates/storage-mongodb/src/management_store.rs @@ -10,8 +10,8 @@ use mongodb::options::{FindOptions, UpdateOptions}; use extenddb_storage::management_store::{ AccessKeyCreated, AccountDetail, AdminEntry, GroupDetail, GroupListEntry, ManagementStore, - MetricsRow, OpError, OpResult, RoleDetail, RoleListEntry, SessionPolicyEntry, UserDetail, - UserListEntry, + MetricsRow, OpError, OpResult, RoleDetail, RoleListEntry, SessionPolicyEntry, StoredDocument, + UserDetail, UserListEntry, }; use crate::catalog_store::MongoCatalogStore; @@ -1451,6 +1451,54 @@ impl ManagementStore for MongoCatalogStore { }) } + fn list_policy_documents( + &self, + account_id: &str, + principal_type: &str, + principal_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let principal_type = principal_type.to_owned(); + let principal_name = principal_name.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("iam_policies"); + let opts = FindOptions::builder() + .sort(doc! { "policy_name": 1 }) + .build(); + let cursor = coll + .find(doc! { + "account_id": &account_id, + "principal_type": &principal_type, + "principal_name": &principal_name, + }) + .with_options(opts) + .await + .map_err(|e| { + tracing::error!("list_policy_documents: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let docs: Vec = cursor.try_collect().await.map_err(|e| { + tracing::error!("list_policy_documents cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(docs + .iter() + .map(|d| { + let name = d.get_str("policy_name").map_or_else( + |_| { + format!( + "", + d.get("_id").map(ToString::to_string).unwrap_or_default() + ) + }, + str::to_owned, + ); + (name, stored_document(d)) + }) + .collect()) + }) + } + fn set_user_boundary( &self, account_id: &str, @@ -1585,6 +1633,34 @@ impl ManagementStore for MongoCatalogStore { }) } + fn boundary_document( + &self, + account_id: &str, + principal_type: &str, + principal_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let principal_type = principal_type.to_owned(); + let principal_name = principal_name.to_owned(); + Box::pin(async move { + let coll = self + .catalog_db() + .collection::("iam_permissions_boundaries"); + let doc = coll + .find_one(doc! { + "account_id": &account_id, + "principal_type": &principal_type, + "principal_name": &principal_name, + }) + .await + .map_err(|e| { + tracing::error!("boundary_document: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(doc.as_ref().map(stored_document)) + }) + } + fn get_role_boundary( &self, account_id: &str, @@ -2486,3 +2562,13 @@ fn encrypt_secret(plaintext: &str, key_b64: &str, aad: &str) -> Result, result.extend_from_slice(&ciphertext); Ok(result) } + +/// A stored policy or boundary as the resource-tag migration reads it: its +/// `policy_document` as JSON, or why that cannot be read. `list_policies` +/// and the boundary getters skip such a document. +fn stored_document(d: &Document) -> StoredDocument { + match d.get("policy_document") { + None => Err("the stored policy has no policy_document field".to_owned()), + Some(b) => bson::from_bson::(b.clone()).map_err(|e| e.to_string()), + } +} diff --git a/crates/storage-mongodb/src/migrations/guarded.rs b/crates/storage-mongodb/src/migrations/guarded.rs index cef14f729..67cf341d7 100644 --- a/crates/storage-mongodb/src/migrations/guarded.rs +++ b/crates/storage-mongodb/src/migrations/guarded.rs @@ -21,7 +21,8 @@ use std::sync::Arc; use bson::{Document, doc}; use extenddb_storage::management_store::{ AccessKeyCreated, AccountDetail, GroupDetail, GroupListEntry, ManagementStore, OpError, - OpResult, RoleDetail, RoleListEntry, SessionPolicyEntry, UserDetail, UserListEntry, + OpResult, RoleDetail, RoleListEntry, SessionPolicyEntry, StoredDocument, UserDetail, + UserListEntry, }; use futures::future::BoxFuture; @@ -189,11 +190,13 @@ impl ManagementStore for LeasedManagementStore { fn put_policy(&self, account_id: &str, principal_type: &str, principal_name: &str, policy_name: &str, document: &serde_json::Value) -> BoxFuture<'_, OpResult<()>>; fn delete_policy(&self, account_id: &str, principal_type: &str, principal_name: &str, policy_name: &str) -> BoxFuture<'_, OpResult<()>>; fn list_policies(&self, account_id: &str, principal_type: &str, principal_name: &str) -> BoxFuture<'_, OpResult>>; + fn list_policy_documents(&self, account_id: &str, principal_type: &str, principal_name: &str) -> BoxFuture<'_, OpResult>>; fn set_user_boundary(&self, account_id: &str, user_name: &str, document: &serde_json::Value) -> BoxFuture<'_, OpResult<()>>; fn get_user_boundary(&self, account_id: &str, user_name: &str) -> BoxFuture<'_, OpResult>>; fn delete_user_boundary(&self, account_id: &str, user_name: &str) -> BoxFuture<'_, OpResult<()>>; fn set_role_boundary(&self, account_id: &str, role_name: &str, document: &serde_json::Value) -> BoxFuture<'_, OpResult<()>>; fn get_role_boundary(&self, account_id: &str, role_name: &str) -> BoxFuture<'_, OpResult>>; + fn boundary_document(&self, account_id: &str, principal_type: &str, principal_name: &str) -> BoxFuture<'_, OpResult>>; fn delete_role_boundary(&self, account_id: &str, role_name: &str) -> BoxFuture<'_, OpResult<()>>; fn create_access_key(&self, account_id: &str, user_name: &str) -> BoxFuture<'_, OpResult>; fn delete_access_key(&self, account_id: &str, user_name: &str, key_id: &str) -> BoxFuture<'_, OpResult<()>>; diff --git a/crates/storage-mongodb/tests/migrations_holders.rs b/crates/storage-mongodb/tests/migrations_holders.rs index 2d3fe5a81..8a7b90cdb 100644 --- a/crates/storage-mongodb/tests/migrations_holders.rs +++ b/crates/storage-mongodb/tests/migrations_holders.rs @@ -352,3 +352,99 @@ async fn a_collision_is_found_before_the_first_write_and_nothing_is_recorded() { ); d.drop_databases().await; } + +/// A stored document that cannot be read as JSON, here a policy whose +/// `policy_document` is binary data, is named, with its account, principal, +/// policy and the error, as an item a person must fix: the status +/// `migrate --yes` reads under the lock is NeedsAttention. The principal's +/// other policies and its boundary are still read, and the rewrite through +/// the store a migration uses rewrites them. +#[tokio::test(flavor = "multi_thread")] +async fn an_undecodable_policy_is_named_and_the_others_are_rewritten() { + let Some(uri) = server() else { return }; + let d = Deployment::new(&uri, "undecodable").await; + fixture::load(&d, &fixture::read("0.1.14")).await; + let account = first_account(&d).await; + let tagged = doc! { + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Deny", "Action": "dynamodb:*", "Resource": "*", + "Condition": { "StringEquals": { "dynamodb:ResourceTag/env": "prod" } }, + }], + }; + let now = bson::DateTime::now(); + let catalog = d.catalog(); + catalog + .collection::("iam_users") + .insert_one(doc! { + "account_id": &account, "user_name": "carol", + "user_arn": format!("arn:aws:iam::{account}:user/carol"), "created_at": now, + }) + .await + .expect("user"); + let binary = bson::Binary { + subtype: bson::spec::BinarySubtype::Generic, + bytes: b"{not json".to_vec(), + }; + for (name, document) in [ + ("a_broken", bson::Bson::Binary(binary)), + ("b_tagged", bson::Bson::Document(tagged.clone())), + ] { + catalog + .collection::("iam_policies") + .insert_one(doc! { + "account_id": &account, "principal_type": "user", "principal_name": "carol", + "policy_name": name, "policy_document": document, "created_at": now, + }) + .await + .expect("policy"); + } + catalog + .collection::("iam_permissions_boundaries") + .insert_one(doc! { + "account_id": &account, "principal_type": "user", "principal_name": "carol", + "policy_document": tagged, "created_at": now, + }) + .await + .expect("boundary"); + + let held = MigrationLock::acquire(&d.target, Duration::from_secs(10), Timing::PRODUCTION) + .await + .expect("lock"); + let status = status_holding(&d.target, &held).await; + let Ok(CatalogStatus::NeedsAttention { items }) = &status else { + panic!("expected NeedsAttention: {status:?}"); + }; + assert!( + items.iter().any(|i| i.contains(&format!( + "account {account} user/carol policy a_broken has a stored document that cannot be \ + decoded" + ))), + "{items:?}" + ); + let migrations = MongoMigrations::open(&d.target, Some(&held)) + .await + .expect("open"); + let report = extenddb_storage::policy_migration::migrate_resource_tag_condition_keys( + migrations.stores().management_store(), + true, + ) + .await; + migrations.close().await; + held.release().await; + assert_eq!(report.errors, 0, "{:?}", report.failures); + assert_eq!(report.unreadable.len(), 1, "{report:?}"); + assert!( + report.unreadable[0].error.contains("byte array"), + "{:?}", + report.unreadable + ); + let carol: Vec<&str> = report + .rewritten + .iter() + .filter(|r| r.principal == "user/carol") + .map(|r| r.policy_name.as_str()) + .collect(); + assert_eq!(carol, ["b_tagged", ""]); + d.drop_databases().await; +} From 1c3b2bf60e3faaf7ead63c338c1bbc62a311b9fa Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Sun, 1 Nov 2026 15:24:46 +0000 Subject: [PATCH 12/45] fix(server): register again on Cassandra and MongoDB, and release the registration last A Cassandra or MongoDB server whose registration could not be renewed for 30 s ended the process with std::process::exit(1), skipping the serve shutdown path, as a PostgreSQL server did before #666. Both backends now implement ServingRegistrar. On a loss the server answers new requests with ServiceUnavailable and registers again: a new row or document, then a read of the migration lock and of the catalog's status. It serves again when the status is the one it started with. A migration that holds the lock, a changed status, or 30 s without a registration stop it through the shutdown path with exit 1. The registration is now renewed every 5 s instead of every 15 s, and every second after a renewal failed, each renewal bounded by the interval. It counts as lost after 30 s without a successful renewal, so an outage shorter than that leaves it in place, and the server can register again during the 30 s after a loss. The registration was released while the other workers were told to stop, so a migration that waited for the servers could start while a worker still wrote to the catalog. ServingRegistrar gains release(), which serve calls after the workers have drained. PostgreSQL keeps the default, which does nothing, since its key ends with the session. A registration dropped without a release, as an attempt the server drops at its bound, deletes its row or document from a task of its own. Tests: on both backends, an attempt to register again replaces the row or document when the status is unchanged, and is refused, leaving none, while a migration holds the lock or once the status changed; a dropped registration's row or document is deleted. On Cassandra, a registration whose node stops answering is lost within the deadline plus one renewal's bound. The drain in serve releases the registration after the workers have stopped. Each of these fails with its change undone. --- crates/server/src/serve.rs | 79 ++++++- crates/storage-cassandra/src/lib.rs | 53 +---- .../storage-cassandra/src/migrations/mod.rs | 45 +++- .../src/migrations/registrar.rs | 151 +++++++++++++ .../src/migrations/registrations.rs | 204 +++++++++++++----- .../tests/migrations/lock.rs | 183 +++++++++++++++- crates/storage-mongodb/src/lib.rs | 58 ++--- .../storage-mongodb/src/migrations/locks.rs | 16 +- crates/storage-mongodb/src/migrations/mod.rs | 16 +- .../src/migrations/registrar.rs | 122 +++++++++++ .../src/migrations/registrations.rs | 51 +++-- .../tests/migrations_holders.rs | 5 +- .../storage-mongodb/tests/migrations_locks.rs | 93 ++++++++ crates/storage/src/hooks.rs | 11 +- docs/adr/0023-catalog-migrations.md | 4 +- docs/cassandra-setup.md | 8 + docs/manuals/07-upgrade-manual.md | 4 +- 17 files changed, 919 insertions(+), 184 deletions(-) create mode 100644 crates/storage-cassandra/src/migrations/registrar.rs create mode 100644 crates/storage-mongodb/src/migrations/registrar.rs diff --git a/crates/server/src/serve.rs b/crates/server/src/serve.rs index 61c45a013..68bd54a28 100644 --- a/crates/server/src/serve.rs +++ b/crates/server/src/serve.rs @@ -390,6 +390,8 @@ async fn serve_inner(params: ServeParams, port: u16) -> anyhow::Result<()> { let registrar = runtime_hooks .as_ref() .and_then(|hooks| hooks.serving_registrar()); + // Released after the drain, once no worker writes to the catalog. + let held = registrar.clone(); let admission = crate::registration::Admission::default(); // Dev mode: seed the credentials the SDK may sign with and verify against @@ -738,8 +740,9 @@ async fn serve_inner(params: ServeParams, port: u16) -> anyhow::Result<()> { let server_result = crate::start_server(listener, state, pid_file, tls_config).await; // The HTTP server has stopped accepting; drain the workers so in-flight - // cycles finish and the metrics flush worker writes its final bucket. - drain_workers(&shutdown, worker_handles).await; + // cycles finish and the metrics flush worker writes its final bucket, + // then give up the serving registration. + drain_then_release(&shutdown, worker_handles, held.as_deref()).await; server_result?; @@ -755,6 +758,30 @@ async fn serve_inner(params: ServeParams, port: u16) -> anyhow::Result<()> { Ok(()) } +/// Drain the workers, then release the serving registration: a migration +/// that waits for the servers starts once the registration is gone, so +/// releasing it first would let the migration run beside a worker that +/// still writes to the catalog. The release is bounded, like the drain. +async fn drain_then_release( + shutdown: &CancellationToken, + handles: Vec>, + registrar: Option<&dyn extenddb_storage::hooks::ServingRegistrar>, +) { + const RELEASE_TIMEOUT: Duration = Duration::from_secs(10); + + drain_workers(shutdown, handles).await; + if let Some(registrar) = registrar + && tokio::time::timeout(RELEASE_TIMEOUT, registrar.release()) + .await + .is_err() + { + tracing::warn!( + "The serving registration was not released within {}s; it expires by itself", + RELEASE_TIMEOUT.as_secs() + ); + } +} + /// Cancel the shutdown token and wait for every worker to return. /// /// Bounded by `DRAIN_TIMEOUT` so a worker stuck inside a backend call cannot @@ -810,3 +837,51 @@ pub fn log_to_syslog_raw(msg: &str) { pub fn log_to_syslog_raw(msg: &str) { eprintln!("{msg}"); } + +#[cfg(test)] +mod tests { + use std::sync::{Arc, Mutex}; + use std::time::Duration; + + use extenddb_storage::CancellationToken; + use extenddb_storage::hooks::{Reregistration, ServingRegistrar}; + + /// A registrar that records when it is released. + struct Recorder(Arc>>); + + #[async_trait::async_trait] + impl ServingRegistrar for Recorder { + async fn lost(&self) -> String { + std::future::pending().await + } + + async fn register_again(&self) -> Reregistration { + Reregistration::Unavailable(String::new()) + } + + async fn release(&self) { + self.0.lock().unwrap().push("released"); + } + } + + /// The registration is released after every worker has returned, so a + /// migration that waits for the servers cannot start while a worker + /// still writes to the catalog. + #[tokio::test] + async fn the_registration_is_released_after_the_workers_stop() { + let events = Arc::new(Mutex::new(Vec::new())); + let shutdown = CancellationToken::new(); + let worker = { + let (events, shutdown) = (events.clone(), shutdown.clone()); + tokio::spawn(async move { + shutdown.cancelled().await; + // A last write after the shutdown signal. + tokio::time::sleep(Duration::from_millis(200)).await; + events.lock().unwrap().push("worker stopped"); + }) + }; + let registrar = Recorder(events.clone()); + super::drain_then_release(&shutdown, vec![worker], Some(®istrar)).await; + assert_eq!(*events.lock().unwrap(), ["worker stopped", "released"]); + } +} diff --git a/crates/storage-cassandra/src/lib.rs b/crates/storage-cassandra/src/lib.rs index f7335d72e..40b36427b 100644 --- a/crates/storage-cassandra/src/lib.rs +++ b/crates/storage-cassandra/src/lib.rs @@ -48,7 +48,7 @@ pub use engine::{CassandraEngine, CassandraSession}; pub use metadata_engine::TtlBackfillCursor; use cdrs_tokio::types::IntoRustByName; -use extenddb_storage::hooks::{ServerRuntimeHooks, WorkerContext}; +use extenddb_storage::hooks::{ServerRuntimeHooks, ServingRegistrar, WorkerContext}; use extenddb_storage::server_components::{BackendError, ServerComponents}; use std::sync::Arc; @@ -61,8 +61,10 @@ struct CassandraRuntimeHooks { engine: Arc, control_plane_notify: Arc, gsi_worker_guard: std::sync::OnceLock, - /// The registration this server holds while it serves. - registration: Arc, + /// The registration this server holds while it serves, which the + /// server takes again when it is lost and releases after its workers + /// stop. + registration: Arc, } #[async_trait::async_trait] @@ -168,11 +170,6 @@ impl ServerRuntimeHooks for CassandraRuntimeHooks { .await; }); - let registration = tokio::spawn(hold_registration( - self.registration.clone(), - ctx.shutdown.clone(), - )); - vec![ control_plane, transaction_recovery, @@ -182,47 +179,17 @@ impl ServerRuntimeHooks for CassandraRuntimeHooks { ttl_repair, ttl_audit, restore_sweep, - registration, ] } fn backend_info(&self) -> Option { Some(format!("keyspace_prefix={}", self.engine.keyspace_prefix)) } -} -/// Hold the serving registration for the server's life: release it when -/// the server shuts down, so a migration need not wait for its TTL, and end -/// the process with exit status 1 once it could not be renewed for 30 s, -/// before its row expires and a migration could start under this server. -/// The exit runs the `atexit` handler that removes the PID file. -async fn hold_registration( - registration: Arc, - shutdown: extenddb_storage::hooks::CancellationToken, -) { - let mut lost = registration.lost(); - loop { - tokio::select! { - () = shutdown.cancelled() => { - registration.release().await; - return; - } - changed = lost.changed() => { - if changed.is_err() { - return; - } - let reason = lost.borrow().clone(); - if let Some(reason) = reason { - let message = format!( - "extenddb fatal: {reason}; the server stops, because a migration can \ - start on the catalog once the serving registration is gone" - ); - tracing::error!("{message}"); - eprintln!("{message}"); - std::process::exit(1); - } - } - } + /// The registration's row is lost when it cannot be renewed for 30 s; + /// the server then registers again or stops. + fn serving_registrar(&self) -> Option> { + Some(self.registration.clone() as Arc) } } @@ -369,7 +336,7 @@ fn server_components_factory( let engine = match CassandraEngine::new(cassandra_config, ®ion).await { Ok(engine) => engine, Err(e) => { - registration.release().await; + ServingRegistrar::release(registration.as_ref()).await; return Err(BackendError::ConnectionFailed { backend: "cassandra".to_string(), details: e.to_string(), diff --git a/crates/storage-cassandra/src/migrations/mod.rs b/crates/storage-cassandra/src/migrations/mod.rs index 6c5c764e3..aa272e30a 100644 --- a/crates/storage-cassandra/src/migrations/mod.rs +++ b/crates/storage-cassandra/src/migrations/mod.rs @@ -27,6 +27,7 @@ mod cql; mod guarded; mod locks; pub mod postcheck; +mod registrar; mod registrations; use std::sync::Arc; @@ -50,7 +51,8 @@ pub use connect::Deployment; pub(crate) use connect::{LOCKED_DDL_TIMEOUT, REQUEST_TIMEOUT}; pub(crate) use locks::LeaseHandle; pub use locks::{LeaseTimes, MigrationLock}; -pub use registrations::ServingRegistration; +pub use registrar::ServerRegistration; +pub use registrations::{RegistrationTimes, ServingRegistration}; use crate::cassandra_util::CassandraSession; @@ -627,7 +629,29 @@ pub async fn register_server( keyspace_prefix: &str, host: &str, release: &str, -) -> Result { +) -> Result { + register_server_timed( + session, + keyspace_prefix, + host, + release, + RegistrationTimes::DEFAULT, + ) + .await +} + +/// [`register_server`] with other registration times, for tests. +/// +/// # Errors +/// +/// As [`register_server`]. +pub async fn register_server_timed( + session: Arc, + keyspace_prefix: &str, + host: &str, + release: &str, + times: RegistrationTimes, +) -> Result { let catalog = format!("{keyspace_prefix}_catalog"); agreement::wait_for_agreement(&session, AGREEMENT_TIMEOUT).await?; if !connect::table_exists(&session, &catalog, "extenddb_servers").await? { @@ -642,7 +666,8 @@ pub async fn register_server( }); } let registration = - ServingRegistration::register(session.clone(), &catalog, host, release).await?; + ServingRegistration::register_timed(session.clone(), &catalog, host, release, times) + .await?; let checked = async { if let Some(holder) = locks::lock_holder(&session, &catalog).await? { return Err(MigrationError::Locked(format!( @@ -656,15 +681,23 @@ pub async fn register_server( "{status}; serving it, because a binary may serve migrations it does not \ know when every one of them is additive" ); - Ok(()) + Ok(status) } - CatalogStatus::Current { .. } => Ok(()), + status @ CatalogStatus::Current { .. } => Ok(status), status => Err(MigrationError::Status(status)), } } .await; match checked { - Ok(()) => Ok(registration), + Ok(status) => Ok(ServerRegistration::new( + session, + keyspace_prefix, + host, + release, + times, + status, + registration, + )), Err(e) => { registration.release().await; Err(e) diff --git a/crates/storage-cassandra/src/migrations/registrar.rs b/crates/storage-cassandra/src/migrations/registrar.rs new file mode 100644 index 000000000..4f92ac215 --- /dev/null +++ b/crates/storage-cassandra/src/migrations/registrar.rs @@ -0,0 +1,151 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The registration a server holds for its life, and the backend's side of +//! registering again once it is lost ([`ServingRegistrar`]). +//! +//! A registration is lost when no renewal of its row has succeeded for 30 s +//! (a node restart, an outage or a partition), before the row expires. The +//! server then refuses requests and asks for a new registration: a new row, +//! then a read of the migration lock, so a migration that holds it, or that +//! waits for the servers to stop while holding it, makes the attempt fail. +//! The catalog's status is read again under the new row, and the server +//! serves again only when it is the status the server started with; +//! otherwise a migration ran in between, and the server stops. + +use std::sync::{Arc, Mutex, PoisonError}; + +use async_trait::async_trait; +use extenddb_storage::hooks::{Reregistration, ServingRegistrar}; +use extenddb_storage::migrations::CatalogStatus; + +use super::registrations::{RegistrationTimes, ServingRegistration}; +use super::{catalog_status, locks}; +use crate::cassandra_util::CassandraSession; + +/// The serving registration of a running server, with what it needs to +/// take it again: the session, the catalog, the row's contents and the +/// status the server started with. +pub struct ServerRegistration { + session: Arc, + keyspace_prefix: String, + host: String, + release: String, + times: RegistrationTimes, + started: CatalogStatus, + current: Mutex>, +} + +impl ServerRegistration { + pub(super) fn new( + session: Arc, + keyspace_prefix: &str, + host: &str, + release: &str, + times: RegistrationTimes, + started: CatalogStatus, + registration: ServingRegistration, + ) -> Self { + Self { + session, + keyspace_prefix: keyspace_prefix.to_owned(), + host: host.to_owned(), + release: release.to_owned(), + times, + started, + current: Mutex::new(Arc::new(registration)), + } + } + + /// The status the server started with. + #[must_use] + pub fn started(&self) -> &CatalogStatus { + &self.started + } + + fn current(&self) -> Arc { + self.current + .lock() + .unwrap_or_else(PoisonError::into_inner) + .clone() + } + + fn catalog(&self) -> String { + format!("{}_catalog", self.keyspace_prefix) + } +} + +#[async_trait] +impl ServingRegistrar for ServerRegistration { + async fn lost(&self) -> String { + let mut lost = self.current().lost(); + loop { + let reason = lost.borrow_and_update().clone(); + if let Some(reason) = reason { + return reason; + } + if lost.changed().await.is_err() { + // Released without being lost: nothing more is reported on + // this registration. + std::future::pending::<()>().await; + } + } + } + + async fn register_again(&self) -> Reregistration { + let catalog = self.catalog(); + let registration = match ServingRegistration::register_timed( + self.session.clone(), + &catalog, + &self.host, + &self.release, + self.times, + ) + .await + { + Ok(registration) => registration, + Err(e) => return Reregistration::Unavailable(e.to_string()), + }; + match locks::lock_holder(&self.session, &catalog).await { + Ok(None) => {} + Ok(Some(holder)) => { + registration.release().await; + return Reregistration::Refused(format!( + "a migration holds the catalog's migration lock (holder {holder})" + )); + } + Err(e) => { + registration.release().await; + return Reregistration::Unavailable(format!( + "the migration lock could not be read: {e}" + )); + } + } + let status = match catalog_status(self.session.clone(), &self.keyspace_prefix).await { + Ok(status) => status, + Err(e) => { + registration.release().await; + return Reregistration::Unavailable(format!( + "the catalog's status could not be read: {e}" + )); + } + }; + if status != self.started { + registration.release().await; + return Reregistration::Refused(format!( + "the catalog's status changed while the server held no registration (when it \ + started: {}; now: {status}); start the server again to serve the catalog as it \ + is now", + self.started + )); + } + // Dropping the lost registration deletes its row, if it is still + // there, from a task of its own. + *self.current.lock().unwrap_or_else(PoisonError::into_inner) = Arc::new(registration); + Reregistration::Registered + } + + async fn release(&self) { + self.current().release().await; + } +} diff --git a/crates/storage-cassandra/src/migrations/registrations.rs b/crates/storage-cassandra/src/migrations/registrations.rs index 3d30a9879..7fe3dd9c1 100644 --- a/crates/storage-cassandra/src/migrations/registrations.rs +++ b/crates/storage-cassandra/src/migrations/registrations.rs @@ -4,11 +4,16 @@ //! The serving registration. //! //! A server registers in `extenddb_servers`: one row, written at `QUORUM` -//! with a TTL of 45 s and again every 15 s at `LOCAL_QUORUM`. A server that -//! cannot renew for 30 s stops (before its row expires), so `migrate --yes`, -//! which waits while a row exists, cannot run beside it. +//! with a TTL of 45 s and again every 5 s at `LOCAL_QUORUM`. A renewal that +//! fails is tried again every second. The registration counts as lost only +//! when no renewal has succeeded for 30 s, after a last attempt at that +//! deadline, so the server stops admitting requests at least 10 s before its +//! row expires and `migrate --yes`, which waits while a row exists, can see +//! it gone. The server then registers again with a new row +//! ([`super::registrar`]). use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; use cdrs_tokio::consistency::Consistency; @@ -22,9 +27,30 @@ use super::locks::MigrationLock; use crate::cassandra_util::CassandraSession; /// The times of a serving registration. -const REGISTRATION_TTL_SECS: i32 = 45; -const REGISTRATION_RENEW: Duration = Duration::from_secs(15); -const REGISTRATION_LOST_AFTER: Duration = Duration::from_secs(30); +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RegistrationTimes { + /// The row's TTL, in seconds. + pub ttl_secs: i32, + /// How often the row is written again. + pub renew_every: Duration, + /// How soon a renewal that failed is tried again. + pub retry_every: Duration, + /// How long after the last successful renewal the registration counts + /// as lost. Each renewal is bounded by `renew_every`, so the loss is + /// reported by `lost_after + renew_every`, which must stay under the TTL. + pub lost_after: Duration, +} + +impl RegistrationTimes { + /// A 45 s row renewed every 5 s, tried again every second after a + /// failure, and lost after 30 s without a renewal. + pub const DEFAULT: Self = Self { + ttl_secs: 45, + renew_every: Duration::from_secs(5), + retry_every: Duration::from_secs(1), + lost_after: Duration::from_secs(30), + }; +} /// One running server, as `migrate --yes` names it. #[derive(Debug, Clone, PartialEq, Eq)] @@ -148,6 +174,8 @@ pub struct ServingRegistration { server_id: String, lost: watch::Receiver>, renewal: JoinHandle<()>, + ttl_secs: i32, + released: AtomicBool, } impl ServingRegistration { @@ -162,13 +190,29 @@ impl ServingRegistration { keyspace: &str, host: &str, release: &str, + ) -> Result { + Self::register_timed(session, keyspace, host, release, RegistrationTimes::DEFAULT).await + } + + /// [`ServingRegistration::register`] with other times, for tests. + /// + /// # Errors + /// + /// As [`ServingRegistration::register`]. + pub async fn register_timed( + session: Arc, + keyspace: &str, + host: &str, + release: &str, + times: RegistrationTimes, ) -> Result { let server_id = uuid::Uuid::new_v4().to_string(); let started_at = chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true); let pid = i32::try_from(std::process::id()).unwrap_or(i32::MAX); let cql = format!( "INSERT INTO {keyspace}.extenddb_servers (server_id, host, pid, release, started_at) \ - VALUES (?, ?, ?, ?, ?) USING TTL {REGISTRATION_TTL_SECS}" + VALUES (?, ?, ?, ?, ?) USING TTL {}", + times.ttl_secs ); let values = || { cdrs_tokio::query_values!(server_id.as_str(), host, pid, release, started_at.as_str()) @@ -183,18 +227,26 @@ impl ServingRegistration { ) .await?; let (lost_tx, lost) = watch::channel(None); - let renewal = tokio::spawn(renew_registration(session.clone(), cql, values(), lost_tx)); + let renewal = tokio::spawn(renew_registration( + session.clone(), + cql, + values(), + times, + lost_tx, + )); Ok(Self { session, keyspace: keyspace.to_owned(), server_id, lost, renewal, + ttl_secs: times.ttl_secs, + released: AtomicBool::new(false), }) } - /// Changes to `Some(reason)` when the registration could not be renewed - /// for 30 s. + /// Changes to `Some(reason)` once no renewal has succeeded for + /// [`RegistrationTimes::lost_after`]. #[must_use] pub fn lost(&self) -> watch::Receiver> { self.lost.clone() @@ -203,74 +255,102 @@ impl ServingRegistration { /// Stop renewing and delete the row, so a migration need not wait for /// its TTL. pub async fn release(&self) { + self.released.store(true, Ordering::Release); self.renewal.abort(); - let cql = format!( - "DELETE FROM {}.extenddb_servers WHERE server_id = ?", - self.keyspace - ); - if let Err(e) = send( - &self.session, - &cql, - cdrs_tokio::query_values!(self.server_id.as_str()), - Consistency::Quorum, - None, - "remove this server from extenddb_servers", - ) - .await - { - tracing::warn!("{e}; the row expires within {REGISTRATION_TTL_SECS} s"); + if let Err(e) = delete_row(&self.session, &self.keyspace, &self.server_id).await { + tracing::warn!("{e}; the row expires within {} s", self.ttl_secs); } } } +/// A registration dropped without [`ServingRegistration::release`], as an +/// attempt to register again that the server drops at its bound, deletes +/// its row from a task of its own, so it leaves no registration behind. impl Drop for ServingRegistration { fn drop(&mut self) { self.renewal.abort(); + if self.released.load(Ordering::Acquire) { + return; + } + let Ok(runtime) = tokio::runtime::Handle::try_current() else { + return; + }; + let (session, keyspace, server_id) = ( + self.session.clone(), + self.keyspace.clone(), + self.server_id.clone(), + ); + runtime.spawn(async move { + if let Err(e) = delete_row(&session, &keyspace, &server_id).await { + tracing::warn!("{e}; the row expires by its TTL"); + } + }); } } +/// Delete a server's row at `QUORUM`. +async fn delete_row( + session: &CassandraSession, + keyspace: &str, + server_id: &str, +) -> Result<(), MigrationError> { + send( + session, + &format!("DELETE FROM {keyspace}.extenddb_servers WHERE server_id = ?"), + cdrs_tokio::query_values!(server_id), + Consistency::Quorum, + None, + "remove this server from extenddb_servers", + ) + .await + .map(drop) +} + +/// Write the row again every `renew_every`, each write bounded by +/// `renew_every`; after a failure try again every `retry_every`. Report the +/// registration lost once the attempt at `lost_after` after the last +/// successful renewal fails too. async fn renew_registration( session: Arc, cql: String, values: cdrs_tokio::query::QueryValues, + times: RegistrationTimes, lost: watch::Sender>, ) { let mut last_ok = Instant::now(); - let mut next = last_ok + REGISTRATION_RENEW; + let mut next = last_ok + times.renew_every; loop { - tokio::time::sleep_until(next.min(last_ok + REGISTRATION_LOST_AFTER)).await; - if last_ok.elapsed() >= REGISTRATION_LOST_AFTER { - let _ = lost.send(Some(format!( - "the serving registration was not renewed for {} s", - REGISTRATION_LOST_AFTER.as_secs() - ))); - return; - } - if Instant::now() < next { - continue; - } + let deadline = last_ok + times.lost_after; + tokio::time::sleep_until(next.min(deadline)).await; let sent = Instant::now(); // LOCAL_QUORUM, so a running server stays up while a remote // datacenter is unreachable. - match send( + let renewal = send( &session, &cql, values.clone(), Consistency::LocalQuorum, None, "renew the serving registration", - ) - .await - { - Ok(_) => { + ); + let why = match tokio::time::timeout(times.renew_every, renewal).await { + Ok(Ok(_)) => { last_ok = sent; - next = sent + REGISTRATION_RENEW; - } - Err(e) => { - tracing::warn!("{e}; retrying"); - next = Instant::now() + REGISTRATION_RENEW / 3; + next = sent + times.renew_every; + continue; } + Ok(Err(e)) => e.to_string(), + Err(_) => format!("no answer within {} s", times.renew_every.as_secs()), + }; + if sent >= deadline { + let _ = lost.send(Some(format!( + "the serving registration was not renewed for {} s (the last attempt: {why})", + sent.duration_since(last_ok).as_secs() + ))); + return; } + tracing::warn!("renewing the serving registration failed: {why}; retrying"); + next = Instant::now() + times.retry_every; } } @@ -278,20 +358,22 @@ async fn renew_registration( mod tests { use std::time::Duration; + /// The loss is reported by `lost_after + renew_every`, the last + /// attempt's bound, before the row's TTL ends. #[test] - fn a_server_stops_before_its_row_expires() { - assert!( - super::REGISTRATION_LOST_AFTER - < Duration::from_secs(super::REGISTRATION_TTL_SECS as u64) - ); - assert!(super::REGISTRATION_RENEW < super::REGISTRATION_LOST_AFTER); + fn a_server_stops_admitting_before_its_row_expires() { + let times = super::RegistrationTimes::DEFAULT; + let ttl = Duration::from_secs(u64::try_from(times.ttl_secs).expect("ttl")); + assert!(times.lost_after + times.renew_every < ttl); + assert!(times.retry_every < times.renew_every); + assert!(times.renew_every < times.lost_after); } /// The times ADR-0023 (decision 5) and the module documentation give the /// lock and the registration: a 60 s lock row renewed every 15 s, acted /// on until 25 s before it expires and no longer renewed after 30 s - /// without progress; a 45 s registration renewed every 15 s, whose server - /// stops after 30 s without a renewal. + /// without progress; a 45 s registration renewed every 5 s and every + /// second after a failure, lost after 30 s without a renewal. #[test] fn the_lease_and_registration_times_are_adr_0023s() { use crate::migrations::LeaseTimes; @@ -305,8 +387,14 @@ mod tests { stall: Duration::from_secs(30), } ); - assert_eq!(super::REGISTRATION_TTL_SECS, 45); - assert_eq!(super::REGISTRATION_RENEW, Duration::from_secs(15)); - assert_eq!(super::REGISTRATION_LOST_AFTER, Duration::from_secs(30)); + assert_eq!( + super::RegistrationTimes::DEFAULT, + super::RegistrationTimes { + ttl_secs: 45, + renew_every: Duration::from_secs(5), + retry_every: Duration::from_secs(1), + lost_after: Duration::from_secs(30), + } + ); } } diff --git a/crates/storage-cassandra/tests/migrations/lock.rs b/crates/storage-cassandra/tests/migrations/lock.rs index c7f8a3c8a..4387d7642 100644 --- a/crates/storage-cassandra/tests/migrations/lock.rs +++ b/crates/storage-cassandra/tests/migrations/lock.rs @@ -4,12 +4,15 @@ //! The migration lock's lease, the serving registration and the bound on //! a code body's requests under the lock. +use std::sync::Arc; use std::time::{Duration, Instant}; +use extenddb_storage::hooks::{Reregistration, ServingRegistrar}; use extenddb_storage::migrations::{CatalogMigrations, CatalogStatus, MigrationError}; +use extenddb_storage_cassandra::cassandra_util::CassandraSession; use extenddb_storage_cassandra::migrations::{ - CassandraCatalogMigrations, LeaseTimes, MigrationLock, ServingRegistration, register_server, - schema_statement_bounds, + CassandraCatalogMigrations, LeaseTimes, MigrationLock, RegistrationTimes, ServerRegistration, + ServingRegistration, register_server, schema_statement_bounds, }; use super::{deployment, target_of}; @@ -174,7 +177,7 @@ async fn a_server_and_a_migration_refuse_each_other() { ); migrations.release(lock).await; // The status read and the preview take no lock beside a server. - let registration: ServingRegistration = + let registration: ServerRegistration = register_server(session.clone(), &prefix, "127.0.0.1:3", "test") .await .expect("register once the lock is free"); @@ -295,3 +298,177 @@ async fn an_account_run_under_a_lapsed_lease_writes_nothing() { tables.len() ); } + +/// The ids of the servers registered in `catalog`. +async fn server_ids(session: &Arc, catalog: &str) -> Vec { + use cdrs_tokio::types::IntoRustByName; + session + .query(format!("SELECT server_id FROM {catalog}.extenddb_servers")) + .await + .expect("read") + .response_body() + .expect("body") + .into_rows() + .unwrap_or_default() + .iter() + .map(|row| { + let id: String = row.get_r_by_name("server_id").expect("id"); + id + }) + .collect() +} + +/// Wait up to 10 s for `catalog` to hold `count` server rows. +async fn wait_for_rows( + session: &Arc, + catalog: &str, + count: usize, +) -> Vec { + let started = Instant::now(); + loop { + let ids = server_ids(session, catalog).await; + if ids.len() == count || started.elapsed() > Duration::from_secs(10) { + return ids; + } + tokio::time::sleep(Duration::from_millis(200)).await; + } +} + +/// An attempt to register again writes a new row, reads the lock and the +/// status, and serves on when the status is the one the server started +/// with; the row it replaces is deleted. While a migration holds the lock, +/// or once the status changed, the attempt is refused and leaves no row. +#[tokio::test] +async fn a_server_registers_again_unless_a_migration_holds_the_catalog_or_its_status_changed() { + if helpers::skip_without_cassandra() { + return; + } + let session = session().await; + let (prefix, _drops) = new_prefix("again"); + deployment(&session, &prefix).await; + let catalog = format!("{prefix}_catalog"); + let registration = register_server(session.clone(), &prefix, "127.0.0.1:4", "test") + .await + .expect("register"); + let rows = wait_for_rows(&session, &catalog, 1).await; + let [first] = rows.as_slice() else { + panic!("one row: {rows:?}"); + }; + let first = first.clone(); + + assert_eq!( + registration.register_again().await, + Reregistration::Registered + ); + let ids = wait_for_rows(&session, &catalog, 1).await; + assert_eq!(ids.len(), 1, "{ids:?}"); + assert_ne!( + ids[0], first, + "the lost row is deleted and a new one written" + ); + + let lock = MigrationLock::acquire(session.clone(), &catalog, LeaseTimes::DEFAULT, TIMEOUT) + .await + .expect("take the lock"); + let refused = registration.register_again().await; + assert!( + matches!(&refused, Reregistration::Refused(why) if why.contains("migration lock")), + "{refused:?}" + ); + assert_eq!(wait_for_rows(&session, &catalog, 1).await, ids); + lock.release().await; + + cql( + &session, + &format!( + "INSERT INTO {catalog}.settings (key, value) VALUES ('resource_tag_migration', \ + 'applied')" + ), + ) + .await; + let refused = registration.register_again().await; + assert!( + matches!(&refused, Reregistration::Refused(why) if why.contains("status changed")), + "{refused:?}" + ); + assert_eq!(wait_for_rows(&session, &catalog, 1).await, ids); + registration.release().await; + assert_eq!( + wait_for_rows(&session, &catalog, 0).await, + Vec::::new() + ); +} + +/// A registration whose renewals get no answer is reported lost once no +/// renewal has succeeded for `lost_after`, after a last attempt at that +/// deadline, each attempt bounded by `renew_every`: before its row expires. +/// One dropped without a release deletes its row. +#[tokio::test] +async fn a_registration_that_gets_no_answer_is_lost_before_its_row_expires() { + if helpers::skip_without_cassandra() { + return; + } + let session = session().await; + let (prefix, _drops) = new_prefix("silent"); + deployment(&session, &prefix).await; + let catalog = format!("{prefix}_catalog"); + // A TTL long enough that no row expires during the test: a row that + // goes is deleted. + let times = RegistrationTimes { + ttl_secs: 60, + renew_every: Duration::from_millis(800), + retry_every: Duration::from_millis(200), + lost_after: Duration::from_secs(3), + }; + let proxy = migrations_support::SilencingProxy::in_front_of( + helpers::test_config().contact_points[0].clone(), + ) + .await; + let silent = proxy.session().await; + let registration = + ServingRegistration::register_timed(silent, &catalog, "127.0.0.1:5", "test", times) + .await + .expect("register"); + let mut lost = registration.lost(); + tokio::time::sleep(Duration::from_secs(1)).await; + let silenced = Instant::now(); + proxy.go_silent(); + tokio::time::timeout(Duration::from_secs(10), lost.changed()) + .await + .expect("reported") + .expect("changed"); + let after = silenced.elapsed(); + assert!( + after <= times.lost_after + times.renew_every + Duration::from_millis(500), + "lost {after:?} after the node went silent" + ); + let reason = lost.borrow().clone().expect("a reason"); + assert!(reason.contains("was not renewed for"), "{reason}"); + drop(registration); + + let before = server_ids(&session, &catalog).await; + let dropped = ServingRegistration::register_timed( + session.clone(), + &catalog, + "127.0.0.1:6", + "test", + times, + ) + .await + .expect("register"); + let after = server_ids(&session, &catalog).await; + let new: Vec<&String> = after.iter().filter(|id| !before.contains(id)).collect(); + let [id] = new.as_slice() else { + panic!("one new row: {before:?} then {after:?}"); + }; + let id = (*id).clone(); + drop(dropped); + let started = Instant::now(); + while server_ids(&session, &catalog).await.contains(&id) { + assert!( + started.elapsed() < Duration::from_secs(5), + "the dropped registration's row is still there" + ); + tokio::time::sleep(Duration::from_millis(200)).await; + } +} diff --git a/crates/storage-mongodb/src/lib.rs b/crates/storage-mongodb/src/lib.rs index 88e524fe2..c199de6f8 100644 --- a/crates/storage-mongodb/src/lib.rs +++ b/crates/storage-mongodb/src/lib.rs @@ -41,34 +41,20 @@ use extenddb_storage::error::StorageError; // Backend registration // ============================================================================ -use extenddb_storage::hooks::{ServerRuntimeHooks, WorkerContext}; +use extenddb_storage::hooks::{ServerRuntimeHooks, ServingRegistrar, WorkerContext}; use extenddb_storage::server_components::{BackendError, ServerComponents}; /// Backend-specific runtime hooks for `MongoDB`. struct MongoRuntimeHooks { engine: Arc, - /// The serving registration, released when the server shuts down. - registration: Arc>>, + /// The serving registration, which the server takes again when it is + /// lost and releases after its workers stop. + registration: Arc, } #[async_trait::async_trait] impl ServerRuntimeHooks for MongoRuntimeHooks { async fn spawn_workers(&self, ctx: &WorkerContext) -> Vec> { - let registration = self - .registration - .lock() - .unwrap_or_else(|e| e.into_inner()) - .take(); - let token = ctx.shutdown.clone(); - // The drain after the listener closes waits for this task, so the - // registration is deleted before the process ends and a migration - // can start at once. - let release_registration = tokio::spawn(async move { - token.cancelled().await; - if let Some(registration) = registration { - registration.release().await; - } - }); let storage_for_ttl = self.engine.clone(); let metrics = ctx.metrics.clone(); let ttl = tokio::spawn(async move { @@ -89,30 +75,17 @@ impl ServerRuntimeHooks for MongoRuntimeHooks { tracing::info!( "MongoDB backend: TTL, stream cleanup, GSI backfill, and control-plane workers spawned" ); - vec![ttl, stream, backfill, control_plane, release_registration] + vec![ttl, stream, backfill, control_plane] } fn backend_info(&self) -> Option { Some("mongodb".to_string()) } -} -/// End the process with exit status 1 once the serving registration is -/// lost: it has not been renewed for 30 s, so `migrate --yes` may no longer -/// see this server. The PID file is removed at exit. A registration released -/// at shutdown ends this task without exiting. -async fn stop_when_registration_lost(mut lost: tokio::sync::watch::Receiver>) { - while lost.changed().await.is_ok() { - let reason = lost.borrow().clone(); - if let Some(reason) = reason { - let message = format!( - "extenddb fatal: {reason}; the server stops, because a migration can start on \ - the catalog once the serving registration is gone" - ); - tracing::error!("{message}"); - eprintln!("{message}"); - std::process::exit(1); - } + /// The registration is lost when its document is deleted or cannot be + /// renewed for 30 s; the server then registers again or stops. + fn serving_registrar(&self) -> Option> { + Some(self.registration.clone() as Arc) } } @@ -160,8 +133,7 @@ fn server_components_factory( ) .await .map_err(BackendError::Catalog)?; - tokio::spawn(stop_when_registration_lost(registration.lost())); - let registration = Arc::new(std::sync::Mutex::new(Some(registration))); + let registration = Arc::new(registration); let built = build_components( &connection_string, ®ion, @@ -172,13 +144,7 @@ fn server_components_factory( ) .await; if built.is_err() { - let taken = registration - .lock() - .unwrap_or_else(|e| e.into_inner()) - .take(); - if let Some(registration) = taken { - registration.release().await; - } + ServingRegistrar::release(registration.as_ref()).await; } built }) @@ -192,7 +158,7 @@ async fn build_components( max_connections: Option, max_catalog_connections: Option, tx_read_concern: mongodb::options::ReadConcern, - registration: Arc>>, + registration: Arc, ) -> Result { // Create MongoEngine let engine = MongoEngine::new(connection_string, region, max_connections, tx_read_concern) diff --git a/crates/storage-mongodb/src/migrations/locks.rs b/crates/storage-mongodb/src/migrations/locks.rs index 92b4be65e..7d92c8f6e 100644 --- a/crates/storage-mongodb/src/migrations/locks.rs +++ b/crates/storage-mongodb/src/migrations/locks.rs @@ -57,8 +57,15 @@ pub struct Timing { pub idle_limit: Duration, /// A registration's `expires_at` after its server's last renewal. pub registration: Duration, - /// A server that has not renewed its registration for this long stops. + /// A server that has not renewed its registration for this long counts + /// it as lost. pub registration_deadline: Duration, + /// How often a server renews its registration; each renewal is bounded + /// by this too. + pub registration_renew_every: Duration, + /// How soon a server tries again after a renewal of its registration + /// failed. + pub registration_retry_every: Duration, /// How often a claimant tries again while another process holds the /// lock, and `migrate --yes` reads the registrations again. pub retry_every: Duration, @@ -66,7 +73,8 @@ pub struct Timing { impl Timing { /// The production timings: a 60 s lease renewed every 15 s with a 25 s - /// margin, and 45 s registrations renewed every 15 s. + /// margin, and 45 s registrations renewed every 5 s, tried again every + /// second after a failure, and lost after 30 s without a renewal. pub const PRODUCTION: Self = Self { lease: Duration::from_secs(60), renew_every: Duration::from_secs(15), @@ -74,6 +82,8 @@ impl Timing { idle_limit: Duration::from_secs(30), registration: Duration::from_secs(45), registration_deadline: Duration::from_secs(30), + registration_renew_every: Duration::from_secs(5), + registration_retry_every: Duration::from_secs(1), retry_every: Duration::from_millis(500), }; @@ -465,6 +475,8 @@ pub(super) mod tests { assert_eq!(timing.idle_limit, Duration::from_secs(30)); assert_eq!(timing.registration, Duration::from_secs(45)); assert_eq!(timing.registration_deadline, Duration::from_secs(30)); + assert_eq!(timing.registration_renew_every, Duration::from_secs(5)); + assert_eq!(timing.registration_retry_every, Duration::from_secs(1)); } /// A database that stopped answering: it accepts connections and never diff --git a/crates/storage-mongodb/src/migrations/mod.rs b/crates/storage-mongodb/src/migrations/mod.rs index 1b8714bec..3cd157908 100644 --- a/crates/storage-mongodb/src/migrations/mod.rs +++ b/crates/storage-mongodb/src/migrations/mod.rs @@ -27,6 +27,7 @@ mod backend; mod commands; mod guarded; mod locks; +mod registrar; mod registrations; // Frozen once released: neither rustfmt nor a later clippy lint may require // an edit that changes its checksum. @@ -52,6 +53,7 @@ pub(crate) use locks::{Lease, is_duplicate_key}; pub use locks::{MigrationLock, Timing}; #[cfg(feature = "test-util")] pub use postcheck::{Collection, Definitions, Index, describe_store}; +pub use registrar::ServerRegistration; pub use registrations::ServingRegistration; /// The catalog database every MongoDB deployment uses. @@ -264,7 +266,7 @@ pub async fn register_and_check( target: &MongoTarget, release: &str, timing: Timing, -) -> Result { +) -> Result { let registration = ServingRegistration::register(target, release, timing).await?; let status = match catalog_status(target).await { Ok(status) => status, @@ -274,19 +276,25 @@ pub async fn register_and_check( } }; match status { - CatalogStatus::Current { .. } => Ok(registration), + CatalogStatus::Current { .. } => {} CatalogStatus::AheadCompatible { .. } => { tracing::warn!( "{status}; serving it, because a binary may serve migrations it does not know \ when every one of them is additive" ); - Ok(registration) } status => { registration.release().await; - Err(MigrationError::Status(status)) + return Err(MigrationError::Status(status)); } } + Ok(ServerRegistration::new( + target, + release, + timing, + status, + registration, + )) } #[cfg(test)] diff --git a/crates/storage-mongodb/src/migrations/registrar.rs b/crates/storage-mongodb/src/migrations/registrar.rs new file mode 100644 index 000000000..9ff1d5cf5 --- /dev/null +++ b/crates/storage-mongodb/src/migrations/registrar.rs @@ -0,0 +1,122 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The registration a server holds for its life, and the backend's side of +//! registering again once it is lost ([`ServingRegistrar`]). +//! +//! A registration is lost when its document was deleted, or when no renewal +//! has succeeded for 30 s (a primary that stopped answering, an election +//! that did not finish, a partition), 15 s before the document expires. The +//! server then refuses requests and asks for a new registration: a new +//! document, then a read of the migration lock, so a migration that holds +//! it makes the attempt fail. The catalog's status is read again under the +//! new document, and the server serves again only when it is the status the +//! server started with; otherwise a migration ran in between, and the +//! server stops. + +use std::sync::{Arc, Mutex, PoisonError}; + +use async_trait::async_trait; +use extenddb_storage::hooks::{Reregistration, ServingRegistrar}; +use extenddb_storage::migrations::{CatalogStatus, MigrationError}; + +use super::locks::Timing; +use super::registrations::ServingRegistration; +use super::{MongoTarget, catalog_status}; + +/// The serving registration of a running server, with what it needs to +/// take it again: the catalog, the release it registers as, its timings and +/// the status the server started with. +pub struct ServerRegistration { + target: MongoTarget, + release: String, + timing: Timing, + started: CatalogStatus, + current: Mutex>, +} + +impl ServerRegistration { + pub(super) fn new( + target: &MongoTarget, + release: &str, + timing: Timing, + started: CatalogStatus, + registration: ServingRegistration, + ) -> Self { + Self { + target: target.clone(), + release: release.to_owned(), + timing, + started, + current: Mutex::new(Arc::new(registration)), + } + } + + /// The status the server started with. + #[must_use] + pub fn started(&self) -> &CatalogStatus { + &self.started + } + + /// The registration the server holds now. + #[must_use] + pub fn current(&self) -> Arc { + self.current + .lock() + .unwrap_or_else(PoisonError::into_inner) + .clone() + } +} + +#[async_trait] +impl ServingRegistrar for ServerRegistration { + async fn lost(&self) -> String { + let mut lost = self.current().lost(); + loop { + let reason = lost.borrow_and_update().clone(); + if let Some(reason) = reason { + return reason; + } + if lost.changed().await.is_err() { + // Released without being lost: nothing more is reported on + // this registration. + std::future::pending::<()>().await; + } + } + } + + async fn register_again(&self) -> Reregistration { + let registration = + match ServingRegistration::register(&self.target, &self.release, self.timing).await { + Ok(registration) => registration, + Err(MigrationError::Locked(why)) => return Reregistration::Refused(why), + Err(e) => return Reregistration::Unavailable(e.to_string()), + }; + let status = match catalog_status(&self.target).await { + Ok(status) => status, + Err(e) => { + registration.release().await; + return Reregistration::Unavailable(format!( + "the catalog's status could not be read: {e}" + )); + } + }; + if status != self.started { + registration.release().await; + return Reregistration::Refused(format!( + "the catalog's status changed while the server held no registration (when it \ + started: {}; now: {status}); start the server again to serve the catalog as it \ + is now", + self.started + )); + } + // Dropping the lost registration deletes its document, if it is + // still there, from a task of its own. + *self.current.lock().unwrap_or_else(PoisonError::into_inner) = Arc::new(registration); + Reregistration::Registered + } + + async fn release(&self) { + self.current().release().await; + } +} diff --git a/crates/storage-mongodb/src/migrations/registrations.rs b/crates/storage-mongodb/src/migrations/registrations.rs index ab994b10d..4440adeb9 100644 --- a/crates/storage-mongodb/src/migrations/registrations.rs +++ b/crates/storage-mongodb/src/migrations/registrations.rs @@ -6,6 +6,7 @@ //! `migrate --yes` waits on. The lock it is paired with is in `locks.rs`, //! whose module doc describes both. +use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Instant; use bson::{Document, doc}; @@ -63,12 +64,13 @@ fn host_name() -> String { } /// What a server holds while it serves: its document in `extenddb_servers`, -/// renewed every [`Timing::renew_every`]. +/// renewed every [`Timing::registration_renew_every`]. pub struct ServingRegistration { servers: mongodb::Collection, token: String, renewal: JoinHandle<()>, lost: watch::Receiver>, + released: AtomicBool, } impl ServingRegistration { @@ -113,6 +115,7 @@ impl ServingRegistration { token, renewal, lost, + released: AtomicBool::new(false), }; // The lock holds while its `expires_at` has not passed by the // database server's clock, as the claim decides. @@ -148,7 +151,8 @@ impl ServingRegistration { } /// Stop renewing and delete the registration. - pub async fn release(self) { + pub async fn release(&self) { + self.released.store(true, Ordering::Release); self.renewal.abort(); let _ = tokio::time::timeout( REQUEST_TIMEOUT, @@ -158,17 +162,35 @@ impl ServingRegistration { } } +/// A registration dropped without [`ServingRegistration::release`], as an +/// attempt to register again that the server drops at its bound, deletes +/// its document from a task of its own, so it leaves no registration +/// behind. impl Drop for ServingRegistration { fn drop(&mut self) { self.renewal.abort(); + if self.released.load(Ordering::Acquire) { + return; + } + let Ok(runtime) = tokio::runtime::Handle::try_current() else { + return; + }; + let (servers, token) = (self.servers.clone(), self.token.clone()); + runtime.spawn(async move { + let _ = + tokio::time::timeout(REQUEST_TIMEOUT, servers.delete_one(doc! { "_id": &token })) + .await; + }); } } -/// Renew a registration every `renew_every`. A renewal that matches no -/// document, or no successful renewal for `registration_deadline`, reports -/// the registration lost on `lost` and ends. No wait and no renewal outlasts -/// the deadline, so the loss is reported `registration_deadline` after the -/// send time of the last successful renewal, not a request timeout later. +/// Renew a registration every `registration_renew_every`, each renewal +/// bounded by that interval, and every `registration_retry_every` after a +/// renewal failed. A renewal that matches no document, or no successful +/// renewal for `registration_deadline`, reports the registration lost on +/// `lost` and ends. No wait and no renewal outlasts the deadline, so the +/// loss is reported `registration_deadline` after the send time of the last +/// successful renewal, not a request timeout later. fn spawn_registration_renewal( servers: mongodb::Collection, token: String, @@ -178,7 +200,7 @@ fn spawn_registration_renewal( ) -> JoinHandle<()> { tokio::spawn(async move { let mut renewed = registered; - let mut wait = timing.renew_every; + let mut wait = timing.registration_renew_every; let left = |renewed: Instant| { timing .registration_deadline @@ -191,18 +213,21 @@ fn spawn_registration_renewal( doc! { "_id": &token }, vec![doc! { "$set": { "expires_at": { "$add": ["$$NOW", millis(timing.registration)] } } }], ); - let answer = tokio::time::timeout(REQUEST_TIMEOUT.min(left(renewed)), renewal).await; + let bound = REQUEST_TIMEOUT + .min(timing.registration_renew_every) + .min(left(renewed)); + let answer = tokio::time::timeout(bound, renewal).await; let reason = match answer { Ok(Ok(result)) if result.matched_count == 1 => { renewed = sent; - wait = timing.renew_every; + wait = timing.registration_renew_every; continue; } Ok(Ok(_)) => { "this server's registration in extenddb_servers was deleted".to_owned() } Ok(Err(_)) | Err(_) if renewed.elapsed() < timing.registration_deadline => { - wait = timing.retry_every; + wait = timing.registration_retry_every; continue; } Ok(Err(e)) => format!( @@ -245,9 +270,9 @@ mod tests { .database("silent") .collection::("extenddb_servers"); let timing = Timing { - renew_every: Duration::from_millis(200), + registration_renew_every: Duration::from_millis(200), registration_deadline: Duration::from_secs(1), - retry_every: Duration::from_millis(50), + registration_retry_every: Duration::from_millis(50), ..Timing::PRODUCTION }; let (tx, mut lost) = watch::channel(None); diff --git a/crates/storage-mongodb/tests/migrations_holders.rs b/crates/storage-mongodb/tests/migrations_holders.rs index 8a7b90cdb..6ac5ccfb4 100644 --- a/crates/storage-mongodb/tests/migrations_holders.rs +++ b/crates/storage-mongodb/tests/migrations_holders.rs @@ -13,6 +13,7 @@ mod migrations_support; use std::time::{Duration, Instant}; use bson::{Document, doc}; +use extenddb_storage::hooks::ServingRegistrar; use extenddb_storage::management_store::OpError; use extenddb_storage::migrations::{ CatalogStatus, HasManagementStore, HeldLock, MigrationBackend, MigrationError, @@ -216,8 +217,8 @@ async fn a_renewed_registration_outlives_three_registration_lifetimes() { let timing = Timing { registration: Duration::from_secs(2), registration_deadline: Duration::from_millis(1500), - renew_every: Duration::from_millis(200), - retry_every: Duration::from_millis(100), + registration_renew_every: Duration::from_millis(200), + registration_retry_every: Duration::from_millis(100), ..Timing::PRODUCTION }; let serving = ServingRegistration::register(&d.target, "test", timing) diff --git a/crates/storage-mongodb/tests/migrations_locks.rs b/crates/storage-mongodb/tests/migrations_locks.rs index 7bcd81a38..a77aa8088 100644 --- a/crates/storage-mongodb/tests/migrations_locks.rs +++ b/crates/storage-mongodb/tests/migrations_locks.rs @@ -9,6 +9,7 @@ mod migrations_support; use std::time::{Duration, Instant}; use bson::{Document, doc}; +use extenddb_storage::hooks::ServingRegistrar; use extenddb_storage::migrations::{CatalogStatus, MigrationError, Runner}; use extenddb_storage_mongodb::migrations::{ MigrationLock, MongoMigrations, ServingRegistration, Timing, lock, migrate_holding, @@ -27,6 +28,8 @@ const SHORT: Timing = Timing { idle_limit: Duration::from_secs(30), registration: Duration::from_secs(6), registration_deadline: Duration::from_secs(4), + registration_renew_every: Duration::from_secs(1), + registration_retry_every: Duration::from_millis(100), retry_every: Duration::from_millis(100), }; @@ -272,3 +275,93 @@ async fn a_registration_that_cannot_be_renewed_is_reported_lost() { registration.release().await; d.drop_databases().await; } + +/// The ids of the registrations in `extenddb_servers`. +async fn registration_ids(d: &Deployment) -> Vec { + Deployment::documents(&d.catalog(), "extenddb_servers") + .await + .iter() + .map(|r| r.get_str("_id").expect("id").to_owned()) + .collect() +} + +/// A server whose registration was lost registers again: a new document, +/// the lock read, then the status, and it serves on when the status is the +/// one it started with. While a migration holds the lock, or once the +/// status changed, the attempt is refused and leaves no document. +#[tokio::test(flavor = "multi_thread")] +async fn a_server_registers_again_unless_a_migration_holds_the_catalog_or_its_status_changed() { + use extenddb_storage::hooks::Reregistration; + + let Some(uri) = server() else { return }; + let d = Deployment::new(&uri, "again").await; + d.create().await; + let serving = register_and_check(&d.target, "test", SHORT) + .await + .expect("serve"); + let first = registration_ids(&d).await; + assert_eq!(first.len(), 1, "{first:?}"); + d.catalog() + .collection::("extenddb_servers") + .delete_many(doc! {}) + .await + .expect("delete"); + let reason = tokio::time::timeout(Duration::from_secs(5), serving.lost()) + .await + .expect("reported within a renewal"); + assert!(reason.contains("deleted"), "{reason}"); + + assert_eq!(serving.register_again().await, Reregistration::Registered); + let again = registration_ids(&d).await; + assert_eq!(again.len(), 1, "{again:?}"); + assert_ne!(again, first); + + let held = MigrationLock::acquire(&d.target, Duration::from_secs(5), Timing::PRODUCTION) + .await + .expect("lock"); + let refused = serving.register_again().await; + assert!( + matches!(&refused, Reregistration::Refused(why) if why.contains("holds the migration lock")), + "{refused:?}" + ); + assert_eq!(registration_ids(&d).await, again); + held.release().await; + + d.catalog() + .collection::("settings") + .insert_one(doc! { "_id": "resource_tag_migration", "value": "applied" }) + .await + .expect("marker"); + let refused = serving.register_again().await; + assert!( + matches!(&refused, Reregistration::Refused(why) if why.contains("status changed")), + "{refused:?}" + ); + assert_eq!(registration_ids(&d).await, again); + serving.release().await; + assert_eq!(registration_ids(&d).await, Vec::::new()); + d.drop_databases().await; +} + +/// A registration dropped without a release, as an attempt the server drops +/// at its bound, deletes its document well before the document expires. +#[tokio::test(flavor = "multi_thread")] +async fn a_dropped_registration_deletes_its_document() { + let Some(uri) = server() else { return }; + let d = Deployment::new(&uri, "dropped").await; + d.create().await; + let registration = ServingRegistration::register(&d.target, "test", Timing::PRODUCTION) + .await + .expect("register"); + assert_eq!(registration_ids(&d).await.len(), 1); + drop(registration); + let started = Instant::now(); + while !registration_ids(&d).await.is_empty() { + assert!( + started.elapsed() < Duration::from_secs(5), + "the dropped registration's document is still there" + ); + tokio::time::sleep(Duration::from_millis(200)).await; + } + d.drop_databases().await; +} diff --git a/crates/storage/src/hooks.rs b/crates/storage/src/hooks.rs index a4a130930..a5b78d44a 100644 --- a/crates/storage/src/hooks.rs +++ b/crates/storage/src/hooks.rs @@ -79,7 +79,8 @@ pub trait ServerRuntimeHooks: Send + Sync { /// [`Reregistration::Refused`], or when 30 s pass without a /// registration, it stops as a shutdown signal stops it (the listener /// drains, [`WorkerContext::shutdown`] fires, the PID file is removed) - /// and exits 1. + /// and exits 1. On every shutdown it calls [`ServingRegistrar::release`] + /// once the background workers have stopped. fn serving_registrar(&self) -> Option> { None } @@ -110,6 +111,14 @@ pub trait ServingRegistrar: Send + Sync { /// the timeout passes, so a dropped attempt must leave no registration /// behind. async fn register_again(&self) -> Reregistration; + + /// Give up the registration the server holds, so a migration need not + /// wait for it to expire. `serve` calls it once, after the background + /// workers have stopped, so a migration cannot start while one of them + /// still writes to the catalog. The default does nothing, for a + /// registration that ends with the process, such as PostgreSQL's key, + /// which the database frees when the session closes. + async fn release(&self) {} } /// The outcome of one [`ServingRegistrar::register_again`]. diff --git a/docs/adr/0023-catalog-migrations.md b/docs/adr/0023-catalog-migrations.md index e3c57ca17..cd49d47eb 100644 --- a/docs/adr/0023-catalog-migrations.md +++ b/docs/adr/0023-catalog-migrations.md @@ -213,8 +213,8 @@ that each backend implements, as follows. |---|---|---| | PostgreSQL | A transaction-level advisory lock on the catalog database, on the same key that every release since v0.1.3 takes, held on a dedicated connection that runs `SELECT 1` every 15 s | `serve` holds a second advisory key in shared mode, on a connection whose TCP keepalive settings end the session within 30 s of the serving host dying; `migrate --yes` takes that key in exclusive mode | | SQLite | `flock` on `.lock`, retried every 250 ms up to the lock timeout | The same `flock`, which `serve` holds while it runs | - | Cassandra | A row in the catalog's `settings` table, written only by lightweight transactions at `SERIAL`, with a 60 s TTL renewed every 15 s | A row per server in `extenddb_servers`, renewed every 15 s with a 45 s TTL | - | MongoDB | A document in the `settings` collection, claimed by a conditional update with majority write concern and renewed by its owner | A document per server in `extenddb_servers` that expires 45 s after its last renewal | + | Cassandra | A row in the catalog's `settings` table, written only by lightweight transactions at `SERIAL`, with a 60 s TTL renewed every 15 s | A row per server in `extenddb_servers`, renewed every 5 s (every second after a failure) with a 45 s TTL, and lost after 30 s without a renewal | + | MongoDB | A document in the `settings` collection, claimed by a conditional update with majority write concern and renewed by its owner | A document per server in `extenddb_servers`, renewed every 5 s (every second after a failure), that expires 45 s after its last renewal and is lost after 30 s without one | On SQLite `.lock` is never removed, also not after a refused `serve` or `migrate --yes`. `flock` locks the open file: if one process removed diff --git a/docs/cassandra-setup.md b/docs/cassandra-setup.md index 5074e764a..52d61a1e0 100644 --- a/docs/cassandra-setup.md +++ b/docs/cassandra-setup.md @@ -250,6 +250,14 @@ change it, so your cluster's local datacenter has to be named `datacenter1` until that is configurable. ExtendDB creates its keyspaces with `NetworkTopologyStrategy` using these values. +With a replication factor of 1, the catalog has one replica, so a restart of +that node stops every request to it. A running server renews its serving +registration every 5 s and keeps serving while renewals fail. Once it has not +renewed for 30 s, it answers new requests with `ServiceUnavailable` and tries +to register again every second for up to 30 s. When the node answers in that +time, the server registers again and serves on; otherwise it stops with +exit 1. + ```toml [storage.cassandra] contact_points = ["cass-1:9042", "cass-2:9042", "cass-3:9042"] diff --git a/docs/manuals/07-upgrade-manual.md b/docs/manuals/07-upgrade-manual.md index 5f5176cba..51c3c38c8 100755 --- a/docs/manuals/07-upgrade-manual.md +++ b/docs/manuals/07-upgrade-manual.md @@ -93,9 +93,9 @@ A server notices at its next renewal, within 15 s, that the database ended the s The two transactions that hold the keys, the server's registration and the migration lock, turn off `idle_in_transaction_session_timeout` for themselves, and `transaction_timeout` on PostgreSQL 17 and later, so a role or database setting of either does not end them. PgBouncer's `idle_transaction_timeout` is a setting of PgBouncer, which a transaction cannot change. Each holder runs `SELECT 1` every 15 s, so set `idle_transaction_timeout` to 0, its default, or to 30 s or more. A value under 15 s closes the server's connection between two renewals, and the server then registers again every time. -On MongoDB, `migrate` connects with the connection string in the config file, which the server uses too. A user with the `readWrite` role on `extenddb_catalog` and on `extenddb_data` and no other role can run `init`, `migrate --yes`, `verify` and `serve`. `migrate --yes` takes the migration lock, a document in the catalog's `settings` collection, and then waits for the registration every running server keeps in the catalog's `extenddb_servers` collection, and after `--lock-timeout` exits 6 naming each server's host and pid. A server deletes its registration when it stops; a server that was killed leaves it until it expires, at most 45 s after its last renewal. A server that cannot renew its registration for 30 s, for example because the database stopped answering, exits 1 and removes its PID file. The lock and the registrations expire by the clock of the replica set's primary, so keep the clocks of its members in sync with NTP. +On MongoDB, `migrate` connects with the connection string in the config file, which the server uses too. A user with the `readWrite` role on `extenddb_catalog` and on `extenddb_data` and no other role can run `init`, `migrate --yes`, `verify` and `serve`. `migrate --yes` takes the migration lock, a document in the catalog's `settings` collection, and then waits for the registration every running server keeps in the catalog's `extenddb_servers` collection, and after `--lock-timeout` exits 6 naming each server's host and pid. A server renews its registration every 5 s, and every second after a renewal failed, and deletes it when it stops, after its background workers have stopped; a server that was killed leaves it until it expires, at most 45 s after its last renewal. A server whose registration was deleted, or that has not renewed it for 30 s, for example because the primary stopped answering, answers new requests with `ServiceUnavailable` and registers again, as a PostgreSQL server does after its session ends (above): it serves again once it holds a new registration and the catalog's status is the one it started with, and it stops with exit 1 when a migration holds the lock, when the status changed, or when 30 s pass without a registration. The lock and the registrations expire by the clock of the replica set's primary, so keep the clocks of its members in sync with NTP. -On Cassandra, `migrate` connects as the role the config file names, which the server connects as too. `--cassandra-user ` connects as another role, with its password in `EXTENDDB_CASSANDRA_PASSWORD` or `--cassandra-pass`; a wrong password is reported as a connection timeout. Migration 1 adopts a 0.1.x catalog: it keeps every setting except the `resource_tag_migration` row, which it deletes, and records itself in a ledger in the catalog keyspace and in every account keyspace. On a cluster that runs `CassandraAuthorizer`, `init` grants the application role SELECT, MODIFY, CREATE and ALTER on the catalog keyspace, those four and DROP on the default account's keyspace, and CREATE ON ALL KEYSPACES, which CreateAccount needs to create a keyspace per account and which also lets the role create tables in every keyspace on the cluster; a run of `migrate --yes --cassandra-user ` that creates a missing account keyspace grants the application role the five permissions on it. A 0.1.x `init` granted the application role nothing, so a deployment created before 1.0 on such a cluster needs the statements below before `migrate`. `migrate --yes` takes the migration lock, a row in the catalog's `settings` table written by lightweight transactions, and then waits for the registration every running server of this release keeps in the catalog's `extenddb_servers` table, and after `--lock-timeout` exits 6 naming each server. A server deletes its registration when it stops; a server that was killed leaves it until it expires, at most 45 s after its last renewal. A server that cannot renew its registration for 30 s exits 1. A migration needs every replica of the catalog keyspace reachable: the lock is written at `SERIAL`, the registrations are read at `ALL`, and after each schema statement the runner waits until every live node reports the same schema version. `serve` refuses a catalog with pending migrations, exiting 3 under `--foreground` with the pending migrations and the `extenddb migrate --config --yes` command to run. A 0.1.x server refuses a migrated catalog. +On Cassandra, `migrate` connects as the role the config file names, which the server connects as too. `--cassandra-user ` connects as another role, with its password in `EXTENDDB_CASSANDRA_PASSWORD` or `--cassandra-pass`; a wrong password is reported as a connection timeout. Migration 1 adopts a 0.1.x catalog: it keeps every setting except the `resource_tag_migration` row, which it deletes, and records itself in a ledger in the catalog keyspace and in every account keyspace. On a cluster that runs `CassandraAuthorizer`, `init` grants the application role SELECT, MODIFY, CREATE and ALTER on the catalog keyspace, those four and DROP on the default account's keyspace, and CREATE ON ALL KEYSPACES, which CreateAccount needs to create a keyspace per account and which also lets the role create tables in every keyspace on the cluster; a run of `migrate --yes --cassandra-user ` that creates a missing account keyspace grants the application role the five permissions on it. A 0.1.x `init` granted the application role nothing, so a deployment created before 1.0 on such a cluster needs the statements below before `migrate`. `migrate --yes` takes the migration lock, a row in the catalog's `settings` table written by lightweight transactions, and then waits for the registration every running server of this release keeps in the catalog's `extenddb_servers` table, and after `--lock-timeout` exits 6 naming each server. A server renews its registration every 5 s, and every second after a renewal failed, and deletes it when it stops, after its background workers have stopped; a server that was killed leaves it until it expires, at most 45 s after its last renewal. A server that has not renewed its registration for 30 s, for example while the only replica of the catalog restarts, answers new requests with `ServiceUnavailable` and registers again, as a PostgreSQL server does after its session ends (above), and stops with exit 1 when a migration holds the lock, when the status changed, or when 30 s pass without a registration. A migration needs every replica of the catalog keyspace reachable: the lock is written at `SERIAL`, the registrations are read at `ALL`, and after each schema statement the runner waits until every live node reports the same schema version. `serve` refuses a catalog with pending migrations, exiting 3 under `--foreground` with the pending migrations and the `extenddb migrate --config --yes` command to run. A 0.1.x server refuses a migrated catalog. On a deployment created before 1.0 on a cluster that runs `CassandraAuthorizer`, this release's `migrate`, `verify` and `serve`, connected as the application role, fail with `Unauthorized ... has no SELECT permission on
_catalog.accounts>` until the role holds its permissions. With every server stopped, run the statements below in `cqlsh` as a superuser; the application role cannot grant them to itself. `` and `` are `username` and `keyspace_prefix` under `[storage.cassandra]` in the config file. Each account has its own keyspace, so list the accounts first: From 69a1bbceab40b5dc3b222477770be155a49e9fb1 Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Sun, 1 Nov 2026 15:41:04 +0000 Subject: [PATCH 13/45] fix(storage-cassandra): bound the waits for the lock and the servers by one lock timeout migrate --yes on Cassandra gave the wait for the lock and the wait for the servers a --lock-timeout each, the second counted from when it took the lock, so a run could wait twice the timeout and then report only the second wait. Both waits now end at one deadline, the lock timeout counted from the start, as on PostgreSQL and MongoDB, and the refusal says how long the run waited for the lock and the servers. The upgrade manual says that one --lock-timeout bounds both waits, and that a server started while a migrator waits for the servers exits 6. Test: with a server registered, migrator A takes the lock and waits for the server; migrator B, started 1 s later with a 6 s timeout, gives up within 8 s of its start. With the server wait counted from when B took the lock, B waited 10.2 s. --- .../src/migrations/commands.rs | 9 ++-- .../src/migrations/registrations.rs | 16 +++--- .../tests/migrations/lock.rs | 49 +++++++++++++++++++ docs/manuals/07-upgrade-manual.md | 2 +- 4 files changed, 65 insertions(+), 11 deletions(-) diff --git a/crates/storage-cassandra/src/migrations/commands.rs b/crates/storage-cassandra/src/migrations/commands.rs index 9f315abe9..ebd7f0aa5 100644 --- a/crates/storage-cassandra/src/migrations/commands.rs +++ b/crates/storage-cassandra/src/migrations/commands.rs @@ -127,8 +127,10 @@ impl CatalogMigrations for CassandraCatalogMigrations { } /// Take the lock, then wait for every registered server to go away, - /// both within `timeout`: a migration needs every replica of the catalog - /// keyspace, and no server may run while the catalog changes. An + /// both within one `timeout` from the start: a migration needs every + /// replica of the catalog keyspace, and no server may run while the + /// catalog changes. A server that starts while this waits for the + /// servers sees the lock and exits 6. An /// uninitialized keyspace that holds another program's tables or /// `settings` rows is refused before anything is written to it, and /// named, since `init` refuses it too. A keyspace without the catalog's @@ -143,8 +145,9 @@ impl CatalogMigrations for CassandraCatalogMigrations { if !has_columns(&session, &catalog, "settings", &["key", "value"]).await? { return Err(MigrationError::Status(CatalogStatus::Uninitialized)); } + let started = tokio::time::Instant::now(); let lock = MigrationLock::acquire(session, &catalog, LeaseTimes::DEFAULT, timeout).await?; - if let Err(e) = wait_for_no_servers(&lock, timeout).await { + if let Err(e) = wait_for_no_servers(&lock, started, timeout).await { lock.release().await; return Err(e); } diff --git a/crates/storage-cassandra/src/migrations/registrations.rs b/crates/storage-cassandra/src/migrations/registrations.rs index 7fe3dd9c1..ede07b48b 100644 --- a/crates/storage-cassandra/src/migrations/registrations.rs +++ b/crates/storage-cassandra/src/migrations/registrations.rs @@ -106,18 +106,19 @@ pub(crate) async fn registered_servers( .collect() } -/// Wait, while holding `lock`, for every registered server to go away, for -/// up to `timeout`. +/// Wait, while holding `lock`, for every registered server to go away, +/// until `timeout` after `started`, the start of the run's wait for the +/// lock: one timeout bounds both waits. /// /// # Errors /// /// [`MigrationError::Locked`] (exit 6) naming the servers, or the error of -/// the read at `ALL`, when `timeout` passes first. +/// the read at `ALL`, when the deadline passes first. pub(crate) async fn wait_for_no_servers( lock: &MigrationLock, + started: Instant, timeout: Duration, ) -> Result<(), MigrationError> { - let started = Instant::now(); loop { lock.check()?; let read = registered_servers(lock.session(), lock.keyspace()).await; @@ -142,11 +143,12 @@ pub(crate) async fn wait_for_no_servers( }; if started.elapsed() >= timeout { return Err(MigrationError::Locked(format!( - "after waiting {} s, {outcome}", - timeout.as_secs() + "after waiting {} s for the lock and the servers, {outcome}", + started.elapsed().as_secs() ))); } - tokio::time::sleep(Duration::from_secs(1)).await; + tokio::time::sleep(Duration::from_secs(1).min(timeout.saturating_sub(started.elapsed()))) + .await; } } diff --git a/crates/storage-cassandra/tests/migrations/lock.rs b/crates/storage-cassandra/tests/migrations/lock.rs index 4387d7642..01d871d04 100644 --- a/crates/storage-cassandra/tests/migrations/lock.rs +++ b/crates/storage-cassandra/tests/migrations/lock.rs @@ -472,3 +472,52 @@ async fn a_registration_that_gets_no_answer_is_lost_before_its_row_expires() { tokio::time::sleep(Duration::from_millis(200)).await; } } + +/// One lock timeout bounds a migrator's wait for the lock and its wait for +/// the servers together. With a server registered, migrator A takes the +/// lock and waits for the server; migrator B, started 1 s later, waits for +/// the lock until A gives up and then for the server, and gives up at its +/// own timeout from its start, not one timeout after it took the lock. +#[tokio::test] +async fn one_lock_timeout_bounds_the_waits_for_the_lock_and_the_servers() { + if helpers::skip_without_cassandra() { + return; + } + let session = session().await; + let (prefix, _drops) = new_prefix("deadline"); + deployment(&session, &prefix).await; + let registration = register_server(session.clone(), &prefix, "127.0.0.1:7", "test") + .await + .expect("register"); + let target = target_of(&prefix); + let first = { + let target = target.clone(); + tokio::spawn(async move { + CassandraCatalogMigrations + .lock(&target, Duration::from_secs(4)) + .await + .map(drop) + }) + }; + tokio::time::sleep(Duration::from_secs(1)).await; + let started = Instant::now(); + let second = CassandraCatalogMigrations + .lock(&target, Duration::from_secs(6)) + .await + .map(drop); + let waited = started.elapsed(); + assert!( + matches!(&second, Err(MigrationError::Locked(m)) if m.contains("for the lock and the servers")), + "{second:?}" + ); + assert!( + waited < Duration::from_secs(8), + "B waited {waited:?} with a 6 s lock timeout" + ); + let first = first.await.expect("join"); + assert!( + matches!(&first, Err(MigrationError::Locked(_))), + "{first:?}" + ); + registration.release().await; +} diff --git a/docs/manuals/07-upgrade-manual.md b/docs/manuals/07-upgrade-manual.md index 51c3c38c8..a76dcc773 100755 --- a/docs/manuals/07-upgrade-manual.md +++ b/docs/manuals/07-upgrade-manual.md @@ -125,7 +125,7 @@ GRANT DROP ON KEYSPACE _account_ TO ''; A GRANT on an account keyspace that does not exist fails with `Resource ... doesn't exist`; skip that account, and `migrate --yes` creates its keyspace. A statement for a permission the role already holds prints `Role '' was already granted ...` and changes nothing, so a role granted some of these by hand runs the whole list. -Without `--yes`, `migrate` reads the [catalog status](#the-catalog-status), prints each pending migration, and changes nothing; it takes no lock, so it runs beside a server. While migration 2 is pending, the preview also prints the number of stored policies migration 2 rewrites and names the first 20 of them, each with its account, principal, policy name and number of keys, followed by the number of the others. To list them, it reads the catalog's policies a second time. With `--yes` it takes the migration lock, waiting up to `--lock-timeout` seconds (60 by default) for running servers and other migrations, reads the status again under the lock, and applies what is pending. It refuses a store that is not a catalog with exit status 1 and the advice to run `extenddb init`; on Cassandra it refuses a keyspace that holds another program's tables with a message that names them. The exit statuses are those in the status table; a script under `set -e` can run the preview as `extenddb migrate ... || [ $? -eq 3 ]`. +Without `--yes`, `migrate` reads the [catalog status](#the-catalog-status), prints each pending migration, and changes nothing; it takes no lock, so it runs beside a server. While migration 2 is pending, the preview also prints the number of stored policies migration 2 rewrites and names the first 20 of them, each with its account, principal, policy name and number of keys, followed by the number of the others. To list them, it reads the catalog's policies a second time. With `--yes` it takes the migration lock, waiting up to `--lock-timeout` seconds (60 by default) for running servers and other migrations, reads the status again under the lock, and applies what is pending. One `--lock-timeout`, counted from the start, bounds the wait for the lock and the wait for the servers together. On PostgreSQL, Cassandra and MongoDB it holds the migration lock while it waits for the servers, so a server that starts in that time exits 6. It refuses a store that is not a catalog with exit status 1 and the advice to run `extenddb init`; on Cassandra it refuses a keyspace that holds another program's tables with a message that names them. The exit statuses are those in the status table; a script under `set -e` can run the preview as `extenddb migrate ... || [ $? -eq 3 ]`. On PostgreSQL `migrate --yes` can also print: From 7fb3767a2ab7d39018bc2bb967880491e13a313d Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Sun, 1 Nov 2026 15:48:34 +0000 Subject: [PATCH 14/45] fix(storage-mongodb): delete the resource-tag marker with migration 1's ledger row Migration 1's catalog body deleted the resource_tag_migration settings document, which the 0.1.14 server needs, before the post-check and the recording of the step. When the post-check refused the catalog, the run recorded nothing and the document was gone, so neither 0.1.14 nor this release served the catalog. The backend now deletes the document in the transaction that records the catalog step, after the body and the post-check, and on every later run of the step, as a reconverge does. The upgrade manual's MongoDB paragraph now says that 0.1.12 and 0.1.13 servers read neither the document nor the catalog version, start on a migrated catalog and must not be run against it; that going back means restoring the backup taken before migrate --yes; and what a 0.1.x migrate does on a migrated catalog. Test: a 0.1.14 catalog whose post-check fails keeps the document and records nothing, and the run after the cause is removed deletes it. With the deletion back in the body, the document was gone after the failed run. --- .../storage-mongodb/src/migrations/backend.rs | 29 +++++++++++- .../src/migrations/m0001_baseline.rs | 19 +++----- .../tests/migrations_guards.rs | 46 +++++++++++++++++++ docs/manuals/07-upgrade-manual.md | 2 +- 4 files changed, 82 insertions(+), 14 deletions(-) diff --git a/crates/storage-mongodb/src/migrations/backend.rs b/crates/storage-mongodb/src/migrations/backend.rs index 31f4c36d9..6e0dee393 100644 --- a/crates/storage-mongodb/src/migrations/backend.rs +++ b/crates/storage-mongodb/src/migrations/backend.rs @@ -233,6 +233,23 @@ impl Writes<'_> { let mut session = self.session.lock().await; let _ = tokio::time::timeout(REQUEST_TIMEOUT, session.abort_transaction()).await; } + + /// Delete the `resource_tag_migration` settings row in the transaction. + /// Every released `serve` from v0.1.14 requires it, so deleting it makes + /// them refuse this catalog. + async fn delete_marker(&self) -> Result<(), MigrationError> { + let settings = self.db.collection::("settings"); + let mut session = self.session.lock().await; + self.stores + .request("delete the resource_tag_migration settings row", async { + settings + .delete_one(doc! { "_id": "resource_tag_migration" }) + .session(&mut *session) + .await + .map(|_| ()) + }) + .await + } } /// Create a database's ledger collection, accepting one that exists. @@ -466,9 +483,19 @@ impl MigrationBackend for MongoMigrations { db, session: tokio::sync::Mutex::new(session), }; + // Migration 1's catalog step deletes the marker last, after its body + // and the post-check, with its ledger row: a step that fails leaves + // the marker, which the 0.1.14 server needs. + let delete_marker = id == MigrationId::Baseline && store.kind == StoreKind::Catalog; in_transaction( writes.begin(), - record_rows(&writes, row, &done.notes, catalog_version), + async { + record_rows(&writes, row, &done.notes, catalog_version).await?; + if delete_marker { + writes.delete_marker().await?; + } + Ok(()) + }, writes.commit(), writes.abort(), ) diff --git a/crates/storage-mongodb/src/migrations/m0001_baseline.rs b/crates/storage-mongodb/src/migrations/m0001_baseline.rs index 6b900a47a..3ff88aab7 100644 --- a/crates/storage-mongodb/src/migrations/m0001_baseline.rs +++ b/crates/storage-mongodb/src/migrations/m0001_baseline.rs @@ -7,10 +7,13 @@ //! //! The catalog body creates the catalog collections and their indexes, the //! `extenddb_servers` collection the serving registration lives in, and the -//! legacy ledger document `001_initial` that every released `init` wrote, -//! and deletes the `resource_tag_migration` settings row every released -//! `init` seeded. The data body creates the data database's collections and -//! indexes, which before 1.0 only `init` created. Every request is safe to +//! legacy ledger document `001_initial` that every released `init` wrote. +//! The `resource_tag_migration` settings row every released `init` seeded +//! is deleted by the backend, in the transaction that records this body, +//! after the post-check: a body or post-check that fails leaves the row, so +//! the 0.1.14 server still serves the catalog. The data body creates the +//! data database's collections and indexes, which before 1.0 only `init` +//! created. Every request is safe to //! send again: creating a collection that exists with the same options //! succeeds, one that exists with other options fails with NamespaceExists //! (code 48), which is accepted by its code and left to the post-check, and @@ -201,14 +204,6 @@ impl CodeBody for CatalogBaseline { .map(|_| ()) }) .await?; - // Every released `serve` from v0.1.14 requires this row, so deleting - // it makes them refuse this catalog. - let settings = db.collection::("settings"); - stores - .request("delete the resource_tag_migration settings row", async { - settings.delete_one(doc! { "_id": "resource_tag_migration" }).await.map(|_| ()) - }) - .await?; Ok(Done::default()) } } diff --git a/crates/storage-mongodb/tests/migrations_guards.rs b/crates/storage-mongodb/tests/migrations_guards.rs index 2b460e741..039e76ec1 100644 --- a/crates/storage-mongodb/tests/migrations_guards.rs +++ b/crates/storage-mongodb/tests/migrations_guards.rs @@ -292,3 +292,49 @@ async fn a_lock_left_by_a_killed_holder_is_taken_once_it_expires() { again.release().await; d.drop_databases().await; } + +/// Migration 1 deletes the resource-tag marker, which the 0.1.14 server +/// needs, only with the catalog step's ledger row, after the post-check: +/// a 0.1.14 catalog whose post-check fails keeps the marker and records +/// nothing, so the 0.1.14 server still serves it. A run that passes deletes +/// it. +#[tokio::test(flavor = "multi_thread")] +async fn a_failed_post_check_keeps_the_marker_the_0_1_14_server_needs() { + let Some(uri) = server() else { return }; + let d = Deployment::new(&uri, "keepmarker").await; + fixture::load(&d, &fixture::read("0.1.14")).await; + assert_eq!( + d.setting("resource_tag_migration").await.as_deref(), + Some("applied") + ); + d.catalog() + .collection::("tags") + .drop() + .await + .expect("drop tags"); + d.catalog() + .run_command(doc! { "create": "tags", "capped": true, "size": 4096 }) + .await + .expect("capped"); + let error = d.migrate().await.expect_err("the post-check refuses"); + assert!(error.to_string().contains("the collection tags"), "{error}"); + assert_eq!( + d.setting("resource_tag_migration").await.as_deref(), + Some("applied"), + "the marker is gone after a failed run" + ); + assert!( + Deployment::documents(&d.catalog(), "extenddb_migrations") + .await + .is_empty() + ); + + d.catalog() + .collection::("tags") + .drop() + .await + .expect("drop tags"); + d.migrate().await.expect("migrate"); + assert_eq!(d.setting("resource_tag_migration").await, None); + d.drop_databases().await; +} diff --git a/docs/manuals/07-upgrade-manual.md b/docs/manuals/07-upgrade-manual.md index a76dcc773..752341276 100755 --- a/docs/manuals/07-upgrade-manual.md +++ b/docs/manuals/07-upgrade-manual.md @@ -93,7 +93,7 @@ A server notices at its next renewal, within 15 s, that the database ended the s The two transactions that hold the keys, the server's registration and the migration lock, turn off `idle_in_transaction_session_timeout` for themselves, and `transaction_timeout` on PostgreSQL 17 and later, so a role or database setting of either does not end them. PgBouncer's `idle_transaction_timeout` is a setting of PgBouncer, which a transaction cannot change. Each holder runs `SELECT 1` every 15 s, so set `idle_transaction_timeout` to 0, its default, or to 30 s or more. A value under 15 s closes the server's connection between two renewals, and the server then registers again every time. -On MongoDB, `migrate` connects with the connection string in the config file, which the server uses too. A user with the `readWrite` role on `extenddb_catalog` and on `extenddb_data` and no other role can run `init`, `migrate --yes`, `verify` and `serve`. `migrate --yes` takes the migration lock, a document in the catalog's `settings` collection, and then waits for the registration every running server keeps in the catalog's `extenddb_servers` collection, and after `--lock-timeout` exits 6 naming each server's host and pid. A server renews its registration every 5 s, and every second after a renewal failed, and deletes it when it stops, after its background workers have stopped; a server that was killed leaves it until it expires, at most 45 s after its last renewal. A server whose registration was deleted, or that has not renewed it for 30 s, for example because the primary stopped answering, answers new requests with `ServiceUnavailable` and registers again, as a PostgreSQL server does after its session ends (above): it serves again once it holds a new registration and the catalog's status is the one it started with, and it stops with exit 1 when a migration holds the lock, when the status changed, or when 30 s pass without a registration. The lock and the registrations expire by the clock of the replica set's primary, so keep the clocks of its members in sync with NTP. +On MongoDB, `migrate` connects with the connection string in the config file, which the server uses too. A user with the `readWrite` role on `extenddb_catalog` and on `extenddb_data` and no other role can run `init`, `migrate --yes`, `verify` and `serve`. `migrate --yes` takes the migration lock, a document in the catalog's `settings` collection, and then waits for the registration every running server keeps in the catalog's `extenddb_servers` collection, and after `--lock-timeout` exits 6 naming each server's host and pid. A server renews its registration every 5 s, and every second after a renewal failed, and deletes it when it stops, after its background workers have stopped; a server that was killed leaves it until it expires, at most 45 s after its last renewal. A server whose registration was deleted, or that has not renewed it for 30 s, for example because the primary stopped answering, answers new requests with `ServiceUnavailable` and registers again, as a PostgreSQL server does after its session ends (above): it serves again once it holds a new registration and the catalog's status is the one it started with, and it stops with exit 1 when a migration holds the lock, when the status changed, or when 30 s pass without a registration. The lock and the registrations expire by the clock of the replica set's primary, so keep the clocks of its members in sync with NTP. Migration 1 deletes the `resource_tag_migration` settings document in the transaction that records its catalog step, so a 0.1.14 server refuses the migrated catalog, and a run that fails before that transaction leaves the document in place. 0.1.12 and 0.1.13 servers read neither that document nor the catalog version, so they start on a migrated catalog and serve it: do not run them against it. To go back to a 0.1.x release, restore the backup taken before `migrate --yes`. A 0.1.x `migrate` on a migrated catalog reports a pending change, and its `--yes` writes the settings document back; this release's status then reports the catalog as written by an older release, and `migrate --yes --reconverge` repairs it. On Cassandra, `migrate` connects as the role the config file names, which the server connects as too. `--cassandra-user ` connects as another role, with its password in `EXTENDDB_CASSANDRA_PASSWORD` or `--cassandra-pass`; a wrong password is reported as a connection timeout. Migration 1 adopts a 0.1.x catalog: it keeps every setting except the `resource_tag_migration` row, which it deletes, and records itself in a ledger in the catalog keyspace and in every account keyspace. On a cluster that runs `CassandraAuthorizer`, `init` grants the application role SELECT, MODIFY, CREATE and ALTER on the catalog keyspace, those four and DROP on the default account's keyspace, and CREATE ON ALL KEYSPACES, which CreateAccount needs to create a keyspace per account and which also lets the role create tables in every keyspace on the cluster; a run of `migrate --yes --cassandra-user ` that creates a missing account keyspace grants the application role the five permissions on it. A 0.1.x `init` granted the application role nothing, so a deployment created before 1.0 on such a cluster needs the statements below before `migrate`. `migrate --yes` takes the migration lock, a row in the catalog's `settings` table written by lightweight transactions, and then waits for the registration every running server of this release keeps in the catalog's `extenddb_servers` table, and after `--lock-timeout` exits 6 naming each server. A server renews its registration every 5 s, and every second after a renewal failed, and deletes it when it stops, after its background workers have stopped; a server that was killed leaves it until it expires, at most 45 s after its last renewal. A server that has not renewed its registration for 30 s, for example while the only replica of the catalog restarts, answers new requests with `ServiceUnavailable` and registers again, as a PostgreSQL server does after its session ends (above), and stops with exit 1 when a migration holds the lock, when the status changed, or when 30 s pass without a registration. A migration needs every replica of the catalog keyspace reachable: the lock is written at `SERIAL`, the registrations are read at `ALL`, and after each schema statement the runner waits until every live node reports the same schema version. `serve` refuses a catalog with pending migrations, exiting 3 under `--foreground` with the pending migrations and the `extenddb migrate --config --yes` command to run. A 0.1.x server refuses a migrated catalog. From 4dfbea8d845651f8fc555fcf59b1373e85cffcc9 Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Sun, 1 Nov 2026 16:01:37 +0000 Subject: [PATCH 15/45] fix(storage-mongodb): tell a store that is not a catalog before the lock without reading every policy Before it took the lock, migrate --yes on MongoDB read the full catalog status to refuse a store that is not a catalog, and that status ran migration 2's detect, which reads the policies of every principal. On a catalog with 6,000 tagged users that read took 42 s on the test server, outside the lock timeout: against another holder of the lock, a run with --lock-timeout 2 gave up after 41 s. The check now reads the ledgers and the settings alone, through a new Runner::initialized, so the lock timeout bounds the wait, and the run reads the policies one time fewer: 24,030 finds instead of 36,081 on that catalog. The status under the lock and the runner's detect still read them. Test: with 6,000 tagged users and another holder of the lock, a run with a 2 s lock timeout gives up within 4 s. Without the change it waited 41.2 s. --- .../src/migrations/commands.rs | 12 ++++-- .../tests/migrations_holders.rs | 40 +++++++++++++++++++ crates/storage/src/migrations/runner.rs | 12 ++++++ 3 files changed, 60 insertions(+), 4 deletions(-) diff --git a/crates/storage-mongodb/src/migrations/commands.rs b/crates/storage-mongodb/src/migrations/commands.rs index 446c16358..e51c64d71 100644 --- a/crates/storage-mongodb/src/migrations/commands.rs +++ b/crates/storage-mongodb/src/migrations/commands.rs @@ -126,7 +126,9 @@ impl CatalogMigrations for MongoCatalogMigrations { /// Take the migration lock on `target`, waiting up to `timeout` for another /// migrator and then, within the same timeout, for running servers to stop. /// Unless `creating` (`init`), a store that is not a catalog is refused -/// first, with nothing written. +/// first, with nothing written, from its ledgers and settings alone: the +/// status read under the lock is the one that runs the pending migrations' +/// `detect`. /// /// # Errors /// @@ -140,9 +142,11 @@ pub async fn lock( timing: Timing, ) -> Result<(MigrationLock, MongoTarget), MigrationError> { if !creating { - let status = catalog_status(target).await?; - if status == CatalogStatus::Uninitialized { - return Err(MigrationError::Status(status)); + let migrations = MongoMigrations::open(target, None).await?; + let initialized = Runner::new(&migrations, "").initialized().await; + migrations.close().await; + if !initialized? { + return Err(MigrationError::Status(CatalogStatus::Uninitialized)); } } let started = Instant::now(); diff --git a/crates/storage-mongodb/tests/migrations_holders.rs b/crates/storage-mongodb/tests/migrations_holders.rs index 6ac5ccfb4..d663111dc 100644 --- a/crates/storage-mongodb/tests/migrations_holders.rs +++ b/crates/storage-mongodb/tests/migrations_holders.rs @@ -449,3 +449,43 @@ async fn an_undecodable_policy_is_named_and_the_others_are_rewritten() { assert_eq!(carol, ["b_tagged", ""]); d.drop_databases().await; } + +/// Before it takes the lock, `migrate --yes` decides whether the store is a +/// catalog from its ledgers and settings alone, without migration 2's +/// `detect` over every principal, so against another holder of the lock it +/// gives up at its lock timeout however many policies the catalog holds. +#[tokio::test(flavor = "multi_thread")] +async fn the_lock_timeout_bounds_migrate_yes_on_a_catalog_with_many_policies() { + let Some(uri) = server() else { return }; + let d = Deployment::new(&uri, "manypolicies").await; + fixture::load(&d, &fixture::read("0.1.14")).await; + let account = first_account(&d).await; + let tagged = doc! { + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Deny", "Action": "dynamodb:*", "Resource": "*", + "Condition": { "StringEquals": { "dynamodb:ResourceTag/env": "prod" } }, + }], + }; + add_users(&d, &account, "many", 6000, &tagged).await; + let started = Instant::now(); + let status = d.status().await; + let detect = started.elapsed(); + assert!(matches!(status, CatalogStatus::Behind { .. }), "{status:?}"); + let held = MigrationLock::acquire(&d.target, Duration::from_secs(5), Timing::PRODUCTION) + .await + .expect("another migrator's lock"); + let started = Instant::now(); + let refused = lock(&d.target, false, Duration::from_secs(2), Timing::PRODUCTION).await; + let waited = started.elapsed(); + held.release().await; + let error = refused + .err() + .expect("refused while another migrator holds the lock"); + assert_eq!(error.exit_code(), 6, "{error}"); + assert!( + waited < Duration::from_secs(4), + "waited {waited:?} with a 2 s lock timeout; the status with its detect took {detect:?}" + ); + d.drop_databases().await; +} diff --git a/crates/storage/src/migrations/runner.rs b/crates/storage/src/migrations/runner.rs index a9c431c95..6296d1a92 100644 --- a/crates/storage/src/migrations/runner.rs +++ b/crates/storage/src/migrations/runner.rs @@ -101,6 +101,18 @@ impl<'a, B: MigrationBackend> Runner<'a, B> { self.status_of(&snap).await } + /// Whether the store is a catalog, from its ledgers and settings alone: + /// the first of the status's tests, without the reads of the pending + /// migrations' `detect`, so a command can refuse a store that is not a + /// catalog before it takes the lock without reading every policy. + /// + /// # Errors + /// + /// A ledger or the settings could not be read. + pub async fn initialized(&self) -> Result { + Ok(self.read().await?.initialized()) + } + /// Apply every pending migration this binary carries. /// /// # Errors From 8ba58ee7834f9777206cbccc793c2813fbda4a0f Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Sun, 1 Nov 2026 16:42:56 +0000 Subject: [PATCH 16/45] fix(container): run migrate --yes in the compose bootstrap only when migrations are pending The bootstrap service of both compose stacks ran `migrate --yes` on every `up`. `migrate --yes` takes the migration lock and waits for every running server of the catalog to stop, so on a running stack with nothing to apply it waited the 60 s lock timeout and exited 6, and `docker compose up` failed. The PostgreSQL stack has done this since #666, and the MongoDB stack since its server registers. The bootstrap now runs the preview, which takes no lock, and runs `migrate --yes` only when the preview exits 3. Any other exit status ends the bootstrap with that status. The entrypoint in the deployment guide ran `migrate --yes` on every start and now runs it the same way, and the container README says what the bootstrap runs on later starts. --- docker-compose.mongodb.yml | 6 ++++++ docker-compose.yml | 6 ++++++ docker/README.md | 4 +++- docs/manuals/11-deployment-guide.md | 16 +++++++++++++--- 4 files changed, 28 insertions(+), 4 deletions(-) diff --git a/docker-compose.mongodb.yml b/docker-compose.mongodb.yml index fbfbb73bd..d9514a6bb 100644 --- a/docker-compose.mongodb.yml +++ b/docker-compose.mongodb.yml @@ -102,6 +102,12 @@ services: CONFIG=/var/lib/extenddb/extenddb.toml if [ -f "$$CONFIG" ]; then echo "bootstrap: existing config detected; running migrations" + # The preview takes no lock and exits 3 when migrations are + # pending. migrate --yes waits for a running server to stop, so it + # runs only then, and an `up` on a running stack exits 0 here. + rc=0 + extenddb migrate --config "$$CONFIG" || rc=$$? + [ "$$rc" -eq 3 ] || exit "$$rc" exec extenddb migrate --config "$$CONFIG" --yes fi cat > "$$CONFIG.bootstrap" < Date: Sun, 1 Nov 2026 16:42:56 +0000 Subject: [PATCH 17/45] fix(migrations): print a migration that does not apply as recorded, and skip it in a reconverge On Cassandra and MongoDB, migrations 3 and 4 do not apply. `init` and `migrate --yes` recorded their ledger rows and printed them as applied ("Applied 3 sort_key_definitions"), and `--reconverge` printed them as run again, though neither ran a body. Each step in a run's report now carries the outcome its ledger row records. A step of a migration that does not apply prints as "Recorded 3 sort_key_definitions as not applicable", with "migration" before the id in `init`'s output. A reconverge no longer plans a recorded row of such a migration, which has no body to run again; a store without the row still gets it. devtools/check-cassandra-catalog-lock forgot and applied migration 3 again, which runs no body on Cassandra. It now forgets and applies migration 2, and checks that the reconverge skips 2 and names neither 3 nor 4. --- crates/app/src/cmd_init.rs | 2 +- crates/app/src/cmd_migrate_runner.rs | 2 +- crates/app/src/util.rs | 101 +++++++++++++++--- .../storage-mongodb/tests/migrations_fresh.rs | 21 +++- .../src/migrations/refusal_tests.rs | 3 +- crates/storage/src/migrations/advice.rs | 3 +- .../src/migrations/reconverge_tests.rs | 24 +++-- crates/storage/src/migrations/runner.rs | 16 ++- crates/storage/src/migrations/status.rs | 4 + crates/storage/src/migrations/tests.rs | 24 +++++ devtools/check-cassandra-catalog-lock | 26 +++-- 11 files changed, 186 insertions(+), 40 deletions(-) diff --git a/crates/app/src/cmd_init.rs b/crates/app/src/cmd_init.rs index 87378db2d..c0f95f903 100755 --- a/crates/app/src/cmd_init.rs +++ b/crates/app/src/cmd_init.rs @@ -336,7 +336,7 @@ async fn init_on_runner( for note in &report.notes { println!(" {note}"); } - crate::util::print_applied(&report, " ", "Applied migration"); + crate::util::print_applied(&report, " ", "Applied", "migration"); bootstrapper.hold(&lock); let admin = write_bootstrap_rows(bootstrapper).await?; bootstrapper diff --git a/crates/app/src/cmd_migrate_runner.rs b/crates/app/src/cmd_migrate_runner.rs index 2c6cf07ba..744987b41 100644 --- a/crates/app/src/cmd_migrate_runner.rs +++ b/crates/app/src/cmd_migrate_runner.rs @@ -321,7 +321,7 @@ fn print_report(report: &RunReport, kind: RunKind) { RunKind::Create | RunKind::Migrate => "Applied", RunKind::Reconverge => "Ran again", }; - crate::util::print_applied(report, " ", verb); + crate::util::print_applied(report, " ", verb, ""); let show = |v: Option| v.map_or_else(|| "no ledger".to_owned(), |v| format!("migration {v}")); println!(); diff --git a/crates/app/src/util.rs b/crates/app/src/util.rs index 2ee12b99c..4358e5750 100755 --- a/crates/app/src/util.rs +++ b/crates/app/src/util.rs @@ -43,31 +43,52 @@ pub(crate) fn command_config( ) } -/// Print each step a run applied, as ` `, naming -/// the store only when the run touched more than one, and each step's notes -/// below it. +/// Print each step a run applied, as ` ` +/// (without ` ` when `noun` is empty), naming the store only when the +/// run touched more than one, and each step's notes below it. A step of a +/// migration that does not apply on this backend recorded its ledger row and +/// ran nothing, and is printed as `Recorded as not +/// applicable`. pub(crate) fn print_applied( report: &extenddb_storage::migrations::RunReport, indent: &str, verb: &str, + noun: &str, ) { + for line in applied_lines(report, verb, noun) { + println!("{indent}{line}"); + } +} + +/// The lines [`print_applied`] prints, without the indent. +fn applied_lines( + report: &extenddb_storage::migrations::RunReport, + verb: &str, + noun: &str, +) -> Vec { + use extenddb_storage::migrations::Outcome; + let several = report .applied .windows(2) .any(|w| w[0].store_name != w[1].store_name); + let mut lines = Vec::new(); for step in &report.applied { - if several { - println!( - "{indent}{verb} {} {} in {}", - step.id, step.name, step.store_name - ); + let mut what = if noun.is_empty() { + format!("{} {}", step.id, step.name) } else { - println!("{indent}{verb} {} {}", step.id, step.name); - } - for note in &step.notes { - println!("{indent} {note}"); + format!("{noun} {} {}", step.id, step.name) + }; + if several { + what = format!("{what} in {}", step.store_name); } + lines.push(match step.outcome { + Outcome::Applied => format!("{verb} {what}"), + Outcome::NotApplicable => format!("Recorded {what} as not applicable"), + }); + lines.extend(step.notes.iter().map(|note| format!(" {note}"))); } + lines } /// Append a secret to the backend argument vector when the corresponding CLI @@ -101,9 +122,63 @@ pub(crate) fn append_secret_arg( #[cfg(test)] mod tests { - use super::append_secret_arg; use std::ffi::OsString; + use extenddb_storage::migrations::{AppliedStep, Outcome, RunReport}; + + use super::{append_secret_arg, applied_lines}; + + fn step(id: u32, name: &'static str, store: &str, outcome: Outcome) -> AppliedStep { + AppliedStep { + id, + name, + store_name: store.to_owned(), + outcome, + notes: Vec::new(), + } + } + + /// A step of a migration that does not apply on the backend is printed + /// as recorded, with the verb's noun, never as applied or run again. + #[test] + fn a_step_that_does_not_apply_is_printed_as_recorded() { + let report = RunReport { + applied: vec![ + step(1, "baseline", "ks_catalog", Outcome::Applied), + step(1, "baseline", "ks_a1", Outcome::Applied), + step( + 3, + "sort_key_definitions", + "ks_catalog", + Outcome::NotApplicable, + ), + ], + ..RunReport::default() + }; + assert_eq!( + applied_lines(&report, "Ran again", ""), + [ + "Ran again 1 baseline in ks_catalog", + "Ran again 1 baseline in ks_a1", + "Recorded 3 sort_key_definitions in ks_catalog as not applicable", + ] + ); + let one_store = RunReport { + applied: vec![ + step(1, "baseline", "db", Outcome::Applied), + step(3, "sort_key_definitions", "db", Outcome::NotApplicable), + ], + ..RunReport::default() + }; + assert_eq!( + applied_lines(&one_store, "Applied", "migration"), + [ + "Applied migration 1 baseline", + "Recorded migration 3 sort_key_definitions as not applicable", + ] + ); + } + #[test] fn appends_environment_secret_when_flag_is_absent() { let mut args = vec!["extenddb".to_owned(), "migrate".to_owned()]; diff --git a/crates/storage-mongodb/tests/migrations_fresh.rs b/crates/storage-mongodb/tests/migrations_fresh.rs index 95e5393fe..dbf1c1e7d 100644 --- a/crates/storage-mongodb/tests/migrations_fresh.rs +++ b/crates/storage-mongodb/tests/migrations_fresh.rs @@ -110,6 +110,13 @@ async fn a_fresh_store_gets_every_migration_and_the_expected_definitions() { (4, d.target.catalog.clone()), ] ); + let not_applicable: Vec = report + .applied + .iter() + .filter(|s| s.outcome == Outcome::NotApplicable) + .map(|s| s.id) + .collect(); + assert_eq!(not_applicable, [3, 4], "{report:?}"); assert_eq!(d.status().await, CatalogStatus::Current { version: 4 }); let rows = d.catalog_ledger().await; let outcomes: Vec<(u32, Outcome)> = rows.iter().map(|r| (r.id, r.outcome)).collect(); @@ -165,10 +172,16 @@ async fn a_second_run_applies_nothing_and_reconverge_keeps_the_rows() { assert_eq!((again.from, again.to), (Some(4), Some(4))); let reconverged = d.reconverge().await.expect("reconverge"); - // Every step but migration 2's, which is not convergent. - assert_eq!(reconverged.applied.len(), 4, "{reconverged:?}"); - assert!( - reconverged.applied.iter().all(|step| step.id != 2), + // Migration 1's steps in the catalog and the data database: 2 is not + // convergent, and 3 and 4 do not apply on MongoDB. + let ran: Vec<(u32, String)> = reconverged + .applied + .iter() + .map(|step| (step.id, step.store_name.clone())) + .collect(); + assert_eq!( + ran, + [(1, d.target.catalog.clone()), (1, d.target.data.clone())], "{reconverged:?}" ); assert!( diff --git a/crates/storage-sqlite/src/migrations/refusal_tests.rs b/crates/storage-sqlite/src/migrations/refusal_tests.rs index a832ae224..457f93f1e 100644 --- a/crates/storage-sqlite/src/migrations/refusal_tests.rs +++ b/crates/storage-sqlite/src/migrations/refusal_tests.rs @@ -830,7 +830,7 @@ async fn migrate_yes_refuses_a_store_that_is_not_a_catalog() { /// it in the error; one that committed nothing removes it. #[tokio::test] async fn a_stopped_run_keeps_its_copy_only_after_a_commit() { - use extenddb_storage::migrations::{AppliedStep, RunReport}; + use extenddb_storage::migrations::{AppliedStep, Outcome, RunReport}; let db = temp_db("stopped-copy"); load_dump(&db, DEV_0_1_14).await; @@ -850,6 +850,7 @@ async fn a_stopped_run_keeps_its_copy_only_after_a_commit() { id: 1, name: "baseline", store_name: location.clone(), + outcome: Outcome::Applied, notes: Vec::new(), }]; diff --git a/crates/storage/src/migrations/advice.rs b/crates/storage/src/migrations/advice.rs index 594cb77e2..d9b189184 100644 --- a/crates/storage/src/migrations/advice.rs +++ b/crates/storage/src/migrations/advice.rs @@ -313,7 +313,7 @@ mod quoting_tests; mod tests { use super::{Advice, error_message, status_message}; use crate::migrations::{ - AppliedStep, CatalogStatus, Kind, MigrationError, MigrationId, Pending, RunReport, + AppliedStep, CatalogStatus, Kind, MigrationError, MigrationId, Outcome, Pending, RunReport, StoreKind, }; @@ -573,6 +573,7 @@ mod tests { id: 1, name: "baseline", store_name: "db".to_owned(), + outcome: Outcome::Applied, notes: Vec::new(), }], notes: vec!["copied the database to db.pre-migration-legacy-to-4-x".to_owned()], diff --git a/crates/storage/src/migrations/reconverge_tests.rs b/crates/storage/src/migrations/reconverge_tests.rs index cc57b1b50..880bba3ee 100644 --- a/crates/storage/src/migrations/reconverge_tests.rs +++ b/crates/storage/src/migrations/reconverge_tests.rs @@ -115,7 +115,8 @@ async fn a_ledger_without_migration_1_is_at_migration_0() { /// Id 4 is not convergent on the fake. While no store has a row for it, a /// reconverge applies it like any pending id. Once it is recorded, a /// reconverge leaves it as it is and names it in the report's notes, runs -/// the convergent ids again, and repairs the catalog. +/// the convergent ids again, and repairs the catalog. Id 3 does not apply on +/// the fake: a reconverge does not run its recorded row again. #[tokio::test] async fn reconverge_skips_a_recorded_id_that_is_not_convergent() { let fake = Fake::new(); @@ -129,12 +130,18 @@ async fn reconverge_skips_a_recorded_id_that_is_not_convergent() { s.traces.resource_tag_marker = true; }); }; + let steps_of = |id: MigrationId| steps_after(id.number() - 1) - steps_after(id.number()); + let three = MigrationId::SortKeyDefinitions; an_older_release_writes(); let report = runner(&fake).reconverge().await.unwrap(); + assert!( + report.applied.iter().all(|s| s.id != three.number()), + "{report:?}" + ); assert_eq!( report.applied.len(), - steps_after(0), - "every id ran: {report:?}" + steps_after(0) - steps_of(three), + "every id that applies ran: {report:?}" ); assert!( report.notes.iter().all(|n| !n.starts_with("Skipped")), @@ -147,13 +154,18 @@ async fn reconverge_skips_a_recorded_id_that_is_not_convergent() { an_older_release_writes(); let four = MigrationId::IndexTableBaseKey; - let steps_of_four = steps_after(four.number() - 1) - steps_after(four.number()); let report = runner(&fake).reconverge().await.unwrap(); assert!( - report.applied.iter().all(|s| s.id != four.number()), + report + .applied + .iter() + .all(|s| s.id != four.number() && s.id != three.number()), "{report:?}" ); - assert_eq!(report.applied.len(), steps_after(0) - steps_of_four); + assert_eq!( + report.applied.len(), + steps_after(0) - steps_of(four) - steps_of(three) + ); assert!( report.notes.contains(&format!( "Skipped {four}: it is not convergent on fake, so it does not run again over \ diff --git a/crates/storage/src/migrations/runner.rs b/crates/storage/src/migrations/runner.rs index 6296d1a92..10d4614ff 100644 --- a/crates/storage/src/migrations/runner.rs +++ b/crates/storage/src/migrations/runner.rs @@ -271,12 +271,15 @@ impl<'a, B: MigrationBackend> Runner<'a, B> { /// Every store without a row for an id up to `target`, in order: by id, /// then catalog first. With `rerun`, also every store with a row for a - /// convergent id. + /// convergent id that applies on this backend: one that does not has no + /// body to run again. fn planned(&self, snap: &Snapshot, target: MigrationId, rerun: bool) -> Vec { let mut steps = Vec::new(); for id in MigrationId::ALL.into_iter().filter(|id| *id <= target) { let migration = self.backend.migration(id); - let rerun = rerun && migration.convergent(); + let rerun = rerun + && migration.convergent() + && !matches!(migration, Migration::NotApplicable(_)); for kind in migration.store_kinds() { for store in snap.instances(kind) { let recorded = snap.has_row(store, id.number()); @@ -508,11 +511,12 @@ impl<'a, B: MigrationBackend> Runner<'a, B> { applied: &mut Vec, ) -> Result<(), MigrationError> { for (i, step) in steps.iter().enumerate() { - let done = self.apply_step(snap, step).await?; + let (done, outcome) = self.apply_step(snap, step).await?; applied.push(AppliedStep { id: step.id.number(), name: step.id.name(), store_name: step.store.name.clone(), + outcome, notes: done.notes, }); let last_of_id = steps.get(i + 1).is_none_or(|next| next.id != step.id); @@ -567,11 +571,13 @@ impl<'a, B: MigrationBackend> Runner<'a, B> { .collect() } + /// Apply `step` and return the backend's result with the outcome its + /// row records. async fn apply_step( &self, snap: &mut Snapshot, step: &Planned, - ) -> Result { + ) -> Result<(Done, Outcome), MigrationError> { let migration = self.backend.migration(step.id); let checksum = record_checksum( self.backend.released(), @@ -622,6 +628,6 @@ impl<'a, B: MigrationBackend> Runner<'a, B> { .apply(&step.store, step.id, action, row, catalog_version) .await?; *snap = after; - Ok(done) + Ok((done, outcome)) } } diff --git a/crates/storage/src/migrations/status.rs b/crates/storage/src/migrations/status.rs index c462c9c79..73af24c69 100644 --- a/crates/storage/src/migrations/status.rs +++ b/crates/storage/src/migrations/status.rs @@ -4,6 +4,7 @@ //! The catalog status, the runner's errors, what a run did, and the process //! exit codes. +use super::ledger::Outcome; use super::table::{Kind, StoreKind}; /// One step a run applied. @@ -12,6 +13,9 @@ pub struct AppliedStep { pub id: u32, pub name: &'static str, pub store_name: String, + /// `NotApplicable` for a migration that does not apply on this backend, + /// whose step recorded its ledger row and ran nothing. + pub outcome: Outcome, /// The body's notes from this run. A reconverge leaves the row's notes /// from the first application as they are, so these are printed only. pub notes: Vec, diff --git a/crates/storage/src/migrations/tests.rs b/crates/storage/src/migrations/tests.rs index 5f27a2ba2..9b07e980f 100644 --- a/crates/storage/src/migrations/tests.rs +++ b/crates/storage/src/migrations/tests.rs @@ -323,6 +323,30 @@ async fn a_fresh_store_gets_every_id_in_order_catalog_first() { ); } +/// A step of a migration that does not apply on the backend recorded its +/// row and ran nothing, and the report says so; every other step applied. +#[tokio::test] +async fn a_step_of_a_migration_that_does_not_apply_is_reported_as_not_applicable() { + let fake = Fake::new(); + let report = runner(&fake).migrate().await.unwrap(); + for step in &report.applied { + let id = MigrationId::from_number(step.id).expect("an id this binary carries"); + let expected = match table(id) { + Migration::NotApplicable(_) => Outcome::NotApplicable, + Migration::Applies { .. } => Outcome::Applied, + }; + assert_eq!(step.outcome, expected, "{step:?}"); + } + assert!( + report + .applied + .iter() + .any(|s| s.id == MigrationId::SortKeyDefinitions.number() + && s.outcome == Outcome::NotApplicable), + "{report:?}" + ); +} + #[tokio::test] async fn a_crash_between_stores_resumes_at_the_first_store_without_a_row() { let fake = Fake::new(); diff --git a/devtools/check-cassandra-catalog-lock b/devtools/check-cassandra-catalog-lock index c02ea4b27..6df71561d 100755 --- a/devtools/check-cassandra-catalog-lock +++ b/devtools/check-cassandra-catalog-lock @@ -28,8 +28,10 @@ with `init` and removes them at the end. - while another session holds the migration lock (the lock row, written by the script with a lightweight transaction as a migration would), serve and a second migrate --yes exit 6, and the preview exits 0; -- once the lock is free, migrate --yes --reconverge, migrate --forget 3 and - the migrate --yes that applies 3 again run, and serve runs; +- once the lock is free, migrate --yes --reconverge runs migration 1 again + and neither 2, which is not convergent, nor 3 and 4, which do not apply on + Cassandra; migrate --forget 2 and the migrate --yes that applies 2 again + run, and serve runs; - a server that cannot renew its registration for 30 s, with the node paused, stops with exit 1 and removes its PID file. @@ -250,21 +252,29 @@ def main(binary: str, container: str, contact_point: str, port: int) -> int: ) report(code == 0 and "True" in output, "the script releases the lock", output) - expect_exit( + output = expect_exit( 0, - "migrate --yes --reconverge once the lock is free, every body run again", + "migrate --yes --reconverge once the lock is free, migration 1 run again", migrate + ["--yes", "--reconverge", "--lock-timeout", "5"], env, contains="Ran again 1 baseline", ) - expect_exit(0, "migrate --forget 3", migrate + ["--forget", "3"], env) - expect_exit(3, "the preview lists migration 3 as pending", migrate, env, contains="3 sort_key_definitions") + report( + "Skipped 2 resource_tag_condition_keys: " in output + and not any(f"Ran again {n} " in output or f"Recorded {n} " in output for n in (2, 3, 4)), + "the reconverge skips migration 2 and does not name 3 or 4, which do not apply", + output, + ) + expect_exit(0, "migrate --forget 2", migrate + ["--forget", "2"], env) + expect_exit( + 3, "the preview lists migration 2 as pending", migrate, env, contains="2 resource_tag_condition_keys" + ) expect_exit( 0, - "migrate --yes applies migration 3 again", + "migrate --yes applies migration 2 again", migrate + ["--yes", "--lock-timeout", "5"], env, - contains="Applied 3 sort_key_definitions", + contains="Applied 2 resource_tag_condition_keys", ) server = start(serve, env, log) From 4dae6cf5530e6bc5245a407c3c4e9c13c171c078 Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Sun, 1 Nov 2026 16:13:25 +0000 Subject: [PATCH 18/45] fix(storage-cassandra): create an account's keyspace before its accounts row CreateAccount wrote the accounts row first and then created and migrated the account's keyspace. The catalog's status reads the account keyspaces from accounts, so a server killed between the two steps left a keyspace without migration 1 that the status counted, and later server starts refused the catalog until an operator ran migrate --yes. init's default account step wrote its row first too, and it generates a new account id each time it runs, so an init run again after that step would create a second default account. CreateAccount now creates the keyspace and runs the account migrations on it before it writes the row. A call stopped before the row leaves a keyspace that no status counts, so the catalog stays Current, and a call retried with the same account id completes the account. init's default account step creates and migrates the keyspace and grants the application role its permissions on it before it writes the row. When the catalog already has an account, the step writes no row and completes that account's keyspace instead. The message for a Behind catalog said "the catalog is at migration 0" when the catalog keyspace was at migration 4 and only an account keyspace lacked migration 1. When a store lacks a migration the catalog store has, the message and the status line now name each such store, with its kind and its migration, and give the catalog store's migration. Tests: a CreateAccount stopped by the test hook between its keyspace and its row leaves the keyspace migrated, no accounts row and a Current catalog, and the retried call writes the row. An account keyspace whose ledger lacks migration 1 is named in the Behind message. init's default account step run a second time keeps one account, and run after that account's keyspace was dropped creates and migrates it again. The shared crate's tests check the message for one store behind, for three stores of two kinds, and for an upgrade where every store waits for the same migration. --- crates/storage-cassandra/src/bootstrapper.rs | 146 ++++++++---- crates/storage-cassandra/src/catalog_store.rs | 24 +- .../src/management_store/accounts.rs | 17 +- crates/storage-cassandra/tests/migrations.rs | 221 ++++++++++++++++++ crates/storage/src/migrations/advice.rs | 104 ++++++++- crates/storage/src/migrations/status.rs | 68 +++++- 6 files changed, 509 insertions(+), 71 deletions(-) diff --git a/crates/storage-cassandra/src/bootstrapper.rs b/crates/storage-cassandra/src/bootstrapper.rs index 3113ee5f2..351014312 100644 --- a/crates/storage-cassandra/src/bootstrapper.rs +++ b/crates/storage-cassandra/src/bootstrapper.rs @@ -317,6 +317,86 @@ impl CassandraBootstrapper { foreign.join(", ") ))) } + + /// The lowest account id in the catalog's `accounts` table, read through + /// the held lease: the default account of an `init` that wrote its row + /// and stopped before its admin user. + async fn first_account(&self) -> OpResult> { + let keyspace = self.engine.catalog_keyspace(); + let rows = self + .held + .request( + "Read the accounts", + crate::cassandra_util::query_rows_quorum::( + &self.engine.session_arc(), + &format!("SELECT account_id FROM {keyspace}.accounts"), + cdrs_tokio::query_values!(), + "bootstrap_default_account", + ), + ) + .await?; + let ids = crate::cassandra_util::map_rows( + rows, + |row| { + crate::cassandra_util::get_column::( + row, + "account_id", + "bootstrap_default_account", + ) + }, + "bootstrap_default_account", + )?; + Ok(ids.into_iter().min()) + } + + /// Create `account_id`'s keyspace when it is missing, run every account + /// migration on it through the runner, as CreateAccount does, and grant + /// the application role its permissions on it, all under the lease of + /// the lock `init` holds: the CREATE KEYSPACE through the held lease with + /// the bound of a schema statement under the lock, and the account run + /// checking the lease before each write and recording progress on it. + /// Each step is safe to send again, so a resumed `init` completes a + /// keyspace an earlier one created. + async fn complete_account_keyspace(&self, account_id: &str) -> OpResult<()> { + let account_keyspace = self.engine.account_keyspace(account_id); + println!("--- Creating account keyspace..."); + self.held + .request_within( + "Create account keyspace", + crate::migrations::LOCKED_DDL_TIMEOUT, + self.engine.create_keyspace(&account_keyspace), + ) + .await?; + println!("--- Running the account migrations for the default account..."); + self.held.check("Migrate the default account's keyspace")?; + let session = self.engine.session_arc(); + let release = env!("CARGO_PKG_VERSION"); + let run = match self.held.handle() { + Some(lease) => { + crate::migrations::migrate_account_under( + session, + &self.keyspace_prefix, + &account_keyspace, + release, + lease, + ) + .await + } + None => { + crate::migrations::migrate_account( + session, + &self.keyspace_prefix, + &account_keyspace, + release, + ) + .await + } + }; + run.map_err(|e| OpError::Internal(format!("Migrate the default account's keyspace: {e}")))?; + // `init` created this keyspace as the admin role, so the application + // role holds no permission on it yet. + self.grant_app_role(&account_keyspace, true).await + } } #[async_trait] @@ -498,17 +578,28 @@ impl Bootstrapper for CassandraBootstrapper { Ok(()) } + /// Create the default account unless the catalog has an account, whose + /// keyspace is completed instead, so an `init` resumed after this step + /// creates no second account. The keyspace is created and migrated, and + /// the application role granted its permissions on it, before the + /// `accounts` row is written: the catalog's status reads the account + /// keyspaces from `accounts`, so an `init` stopped before the row leaves + /// a catalog whose status is `Current`, which a resumed `init` finishes. async fn bootstrap_default_account(&self) -> OpResult<()> { let keyspace = self.engine.catalog_keyspace(); + if let Some(account_id) = self.first_account().await? { + println!("--- Default account already exists; completing its keyspace."); + return self.complete_account_keyspace(&account_id).await; + } let account_id = generate_account_id(); let account_name = "default"; + self.complete_account_keyspace(&account_id).await?; let insert_cql = format!( "INSERT INTO {keyspace}.accounts (account_id, account_name, created_at) \ VALUES (?, ?, toTimestamp(now())) IF NOT EXISTS" ); - let applied = self .held .request( @@ -521,58 +612,11 @@ impl Bootstrapper for CassandraBootstrapper { ), ) .await?; - - if !applied { + if applied { + println!("--- Default account created"); + } else { println!("--- Default account already exists, skipping."); - return Ok(()); } - - println!("--- Default account created"); - - // The keyspace, then every account migration through the runner, - // as CreateAccount does, all under the lease of the lock `init` - // holds: the CREATE KEYSPACE through the held lease with the bound of - // a schema statement under the lock, and the account run checking - // the lease before each write and recording progress on it. - let account_keyspace = self.engine.account_keyspace(&account_id); - println!("--- Creating account keyspace..."); - self.held - .request_within( - "Create account keyspace", - crate::migrations::LOCKED_DDL_TIMEOUT, - self.engine.create_keyspace(&account_keyspace), - ) - .await?; - println!("--- Running the account migrations for the default account..."); - self.held.check("Migrate the default account's keyspace")?; - let session = self.engine.session_arc(); - let release = env!("CARGO_PKG_VERSION"); - let run = match self.held.handle() { - Some(lease) => { - crate::migrations::migrate_account_under( - session, - &self.keyspace_prefix, - &account_keyspace, - release, - lease, - ) - .await - } - None => { - crate::migrations::migrate_account( - session, - &self.keyspace_prefix, - &account_keyspace, - release, - ) - .await - } - }; - run.map_err(|e| OpError::Internal(format!("Migrate the default account's keyspace: {e}")))?; - // `init` created this keyspace as the admin role, so the application - // role holds no permission on it yet. - self.grant_app_role(&account_keyspace, true).await?; - Ok(()) } diff --git a/crates/storage-cassandra/src/catalog_store.rs b/crates/storage-cassandra/src/catalog_store.rs index 1b739647a..d9b5f97c7 100644 --- a/crates/storage-cassandra/src/catalog_store.rs +++ b/crates/storage-cassandra/src/catalog_store.rs @@ -101,13 +101,26 @@ impl CassandraCatalogStore { self.stop_after = Some(left); } - /// Fail a policy write once the crash-and-rerun tests' budget is spent, - /// as a migrator killed before the write stops. + /// Stop this store before its write after the first `writes` the test + /// hook counts, as a server killed at that point stops: each policy + /// write counts, and so does CreateAccount's `accounts` row, which is + /// written after the account's keyspace. For the tests of a + /// CreateAccount stopped between its keyspace and its row. + #[cfg(any(test, feature = "test-util"))] + #[must_use] + pub fn stopping_after(mut self, writes: usize) -> Self { + self.stop_after = Some(Arc::new(std::sync::atomic::AtomicUsize::new(writes))); + self + } + + /// Fail a policy write or CreateAccount's `accounts` row once the + /// crash-and-rerun tests' budget is spent, as a process killed before + /// the write stops. pub(crate) fn test_stop(&self) -> OpResult<()> { #[cfg(any(test, feature = "test-util"))] if crate::migrations::stopped(self.stop_after.as_deref()) { return Err(OpError::Internal( - "stopped by the test hook before a policy write".to_owned(), + "stopped by the test hook before a write".to_owned(), )); } Ok(()) @@ -168,8 +181,9 @@ impl CassandraCatalogStore { /// (idempotent): create it when missing, then run the account /// migrations on it through the migration runner, which waits for /// schema agreement after each statement and records each one in the - /// keyspace's ledger. A retried CreateAccount whose earlier call stopped - /// after the `accounts` row repairs the keyspace the same way. + /// keyspace's ledger. CreateAccount calls it before it writes the + /// `accounts` row, so a retried CreateAccount whose earlier call stopped + /// before the row completes the keyspace the same way. pub(crate) async fn ensure_account_keyspace(&self, account_id: &str) -> OpResult<()> { let keyspace_name = self.account_keyspace(account_id); let failed = |e: &dyn std::fmt::Display| { diff --git a/crates/storage-cassandra/src/management_store/accounts.rs b/crates/storage-cassandra/src/management_store/accounts.rs index 12423f34e..48fadc4b0 100644 --- a/crates/storage-cassandra/src/management_store/accounts.rs +++ b/crates/storage-cassandra/src/management_store/accounts.rs @@ -9,11 +9,22 @@ use extenddb_storage::management_store::{AccountDetail, OpError, OpResult}; use crate::catalog_store::CassandraCatalogStore; impl CassandraCatalogStore { + /// Create the account's keyspace and run every account migration on it, + /// then write its `accounts` row. The catalog's status reads the account + /// keyspaces from `accounts`, so a call that stops between the two (a + /// killed server) leaves a keyspace no status counts, and the status + /// stays `Current`. A retried call completes it: the keyspace is created + /// with `IF NOT EXISTS`, the migrations its ledger records do not run + /// again, and the row is written. A call for an account that exists + /// finds its keyspace complete and answers `AlreadyExists`. pub(crate) async fn create_account_impl( &self, account_id: &str, account_name: &str, ) -> OpResult<()> { + self.ensure_account_keyspace(account_id).await?; + self.test_stop()?; + let catalog_keyspace = self.catalog_keyspace(); let query = format!( "INSERT INTO {catalog_keyspace}.accounts (account_id, account_name, created_at) VALUES (?, ?, toTimestamp(now())) IF NOT EXISTS" @@ -44,12 +55,6 @@ impl CassandraCatalogStore { false }; - // Ensure account keyspace exists (whether account was just created OR already existed) - // This makes retries safe: if account creation succeeded but keyspace creation failed, - // the retry will create the keyspace before returning AlreadyExists error. - self.ensure_account_keyspace(account_id).await?; - - // Return AlreadyExists only AFTER keyspace is guaranteed to exist if !applied { return Err(OpError::AlreadyExists("Account already exists".to_owned())); } diff --git a/crates/storage-cassandra/tests/migrations.rs b/crates/storage-cassandra/tests/migrations.rs index 05f1d4f78..1e2fc9185 100644 --- a/crates/storage-cassandra/tests/migrations.rs +++ b/crates/storage-cassandra/tests/migrations.rs @@ -617,3 +617,224 @@ async fn create_account_runs_the_account_migrations_through_the_runner() { .is_err() ); } + +/// The ids recorded in a keyspace's ledger, sorted. +async fn ledger_ids(session: &CassandraSession, keyspace: &str) -> Vec { + use cdrs_tokio::types::IntoRustByName; + let rows = session + .query(format!("SELECT id FROM {keyspace}.extenddb_migrations")) + .await + .expect("read the ledger") + .response_body() + .expect("body") + .into_rows() + .unwrap_or_default(); + let mut ids: Vec = rows + .iter() + .map(|row| row.get_r_by_name("id").expect("id")) + .collect(); + ids.sort_unstable(); + ids +} + +/// CreateAccount creates and migrates the account's keyspace before it +/// writes the `accounts` row, from which the catalog's status reads the +/// account keyspaces. A call stopped before the row, as a server killed +/// there stops, leaves a complete keyspace no status counts, so the catalog +/// is still `Current`; the retried call writes the row. An account keyspace +/// whose ledger lacks migration 1, beside a catalog keyspace at migration 4, +/// is named in the status's message. +#[tokio::test] +async fn a_create_account_stopped_before_its_row_leaves_a_current_catalog_a_retry_completes() { + use extenddb_storage::management_store::OpError; + use extenddb_storage::migrations::advice::{Advice, status_message}; + + if helpers::skip_without_cassandra() { + return; + } + let session = session().await; + let (prefix, _drops) = new_prefix("stopacct"); + keyspace(&session, &format!("{prefix}_catalog")).await; + migrate_catalog_keyspace(session.clone(), &prefix, "test", TIMEOUT) + .await + .expect("migrate the catalog"); + let account = format!("{prefix}_account_111111111111"); + let store = + || CassandraCatalogStore::new(session.clone(), prefix.clone(), "datacenter1".to_owned(), 1); + + let error = store() + .stopping_after(0) + .create_account("111111111111", "one") + .await + .expect_err("the call stops before the row"); + assert!( + format!("{error:?}").contains("stopped by the test hook before a write"), + "{error:?}" + ); + assert_eq!( + ledger_ids(&session, &account).await, + vec![1], + "the keyspace is created and migrated before the row" + ); + assert_eq!( + store().list_all_accounts().await.expect("list"), + Vec::<(String, String)>::new(), + "no accounts row was written" + ); + assert_eq!( + catalog_status(session.clone(), &prefix) + .await + .expect("status"), + CatalogStatus::Current { version: 4 }, + "the keyspace without a row is not counted" + ); + + store() + .create_account("111111111111", "one") + .await + .expect("the retried call"); + assert_eq!( + store().list_all_accounts().await.expect("list"), + vec![("111111111111".to_owned(), "one".to_owned())] + ); + assert_eq!(ledger_ids(&session, &account).await, vec![1]); + assert_eq!( + catalog_status(session.clone(), &prefix) + .await + .expect("status"), + CatalogStatus::Current { version: 4 } + ); + assert!( + matches!( + store().create_account("111111111111", "one").await, + Err(OpError::AlreadyExists(_)) + ), + "a third call finds the account" + ); + + // A keyspace a run stopped in before migration 1's row. + cql(&session, &format!("TRUNCATE {account}.extenddb_migrations")).await; + let status = catalog_status(session.clone(), &prefix) + .await + .expect("status"); + assert!( + matches!( + status, + CatalogStatus::Behind { + version: Some(0), + .. + } + ), + "{status:?}" + ); + let advice = Advice { + release: "1.0.0-test", + location: "test", + backup: "back up the keyspaces", + config: None, + assignment: None, + }; + let message = status_message(&status, &advice); + assert!( + message.starts_with(&format!( + "account store {account} is at migration 0, behind the catalog store at migration \ + 4, and ExtendDB 1.0.0-test needs migration 4." + )), + "{message}" + ); + assert!( + !message.contains("the catalog is at migration 0"), + "{message}" + ); +} + +/// `init`'s bootstrapper for the deployment of `prefix`, connected as the +/// test node's superuser, which is also the application role here, so the +/// grants `init` sends name a role that exists. +async fn bootstrapper(prefix: &str) -> extenddb_storage_cassandra::CassandraBootstrapper { + let mut config = helpers::test_config(); + config.keyspace_prefix = prefix.to_owned(); + config.rebuild_cached_connection_string(); + let (host, port) = helpers::test_contact_point() + .split_once(':') + .map(|(h, p)| (h.to_owned(), p.parse().expect("a port"))) + .expect("host:port"); + let user = config.username.clone().unwrap_or_default(); + let password = config.password.clone().unwrap_or_default(); + let bootstrap = extenddb_storage::bootstrapper::BootstrapConfig { + host, + port, + admin_user: user.clone(), + admin_password: Some(password.clone()), + app_user: user, + app_password: password, + catalog_db: format!("{prefix}_catalog"), + data_db: String::new(), + }; + extenddb_storage_cassandra::CassandraBootstrapper::new(&config, bootstrap) + .await + .unwrap_or_else(|e| panic!("bootstrapper: {e:?}")) +} + +/// `init`'s default account step creates and migrates the account's +/// keyspace before it writes the `accounts` row, and a step run again on a +/// catalog that has an account creates no second account and completes that +/// account's keyspace: an `init` resumed after the step leaves one account +/// whose keyspace holds every account migration. +#[tokio::test] +async fn the_default_account_step_run_again_keeps_one_account_and_completes_its_keyspace() { + use extenddb_storage::bootstrapper::Bootstrapper; + + if helpers::skip_without_cassandra() { + return; + } + let session = session().await; + let (prefix, mut drops) = new_prefix("defacct"); + keyspace(&session, &format!("{prefix}_catalog")).await; + migrate_catalog_keyspace(session.clone(), &prefix, "test", TIMEOUT) + .await + .expect("migrate the catalog"); + let bootstrapper = bootstrapper(&prefix).await; + let store = CassandraCatalogStore::catalog_only(session.clone(), prefix.clone()); + + bootstrapper + .bootstrap_default_account() + .await + .expect("the default account"); + let accounts = store.list_all_accounts().await.expect("list"); + assert_eq!(accounts.len(), 1, "{accounts:?}"); + let (id, name) = accounts[0].clone(); + assert_eq!(name, "default"); + let account = format!("{prefix}_account_{id}"); + drops.0.push(account.clone()); + assert_eq!(ledger_ids(&session, &account).await, vec![1]); + + // The step again, as a resumed `init` runs it. + bootstrapper + .bootstrap_default_account() + .await + .expect("the step again"); + assert_eq!( + store.list_all_accounts().await.expect("list"), + vec![(id.clone(), "default".to_owned())], + "no second account" + ); + + // A keyspace that lost its migrations is completed by the step. + cql(&session, &format!("DROP KEYSPACE {account}")).await; + bootstrapper + .bootstrap_default_account() + .await + .expect("the step on a missing keyspace"); + assert_eq!(ledger_ids(&session, &account).await, vec![1]); + assert_eq!( + store.list_all_accounts().await.expect("list"), + vec![(id, "default".to_owned())] + ); + assert_eq!( + catalog_status(session.clone(), &prefix) + .await + .expect("status"), + CatalogStatus::Current { version: 4 } + ); +} diff --git a/crates/storage/src/migrations/advice.rs b/crates/storage/src/migrations/advice.rs index d9b189184..a93528b7a 100644 --- a/crates/storage/src/migrations/advice.rs +++ b/crates/storage/src/migrations/advice.rs @@ -7,7 +7,7 @@ use super::catalog::{CatalogMigrations, Target}; use super::registry::{MigrationId, RELEASED_THROUGH}; -use super::status::{CatalogStatus, MigrationError, Pending}; +use super::status::{CatalogStatus, MigrationError, Pending, behind_line, stores_behind}; /// What a message needs beyond the status. #[derive(Debug, Clone, Copy)] @@ -156,12 +156,18 @@ pub fn status_message(status: &CatalogStatus, advice: &Advice<'_>) -> String { let latest = MigrationId::LATEST.number(); match status { CatalogStatus::Behind { version, pending } => { - let (at, stop) = match version { - Some(v) => ( + let behind = stores_behind(pending); + let (at, stop) = match (behind, version) { + // A run stopped after the catalog store: name the stores it + // did not reach, since the catalog store is further on. + (Some((catalog_at, behind)), _) => { + (behind_line(catalog_at, &behind), "Stop the server.") + } + (None, Some(v)) => ( format!("the catalog is at migration {v}"), "Stop the server.", ), - None => ( + (None, None) => ( "the catalog has no migration ledger (a release before 1.0 created it)" .to_owned(), "Stop every server that uses this catalog: a server of a release before 1.0 \ @@ -430,6 +436,96 @@ mod tests { ); } + /// An account keyspace without migration 1 beside a catalog keyspace at + /// the latest migration puts the deployment at migration 0. The message + /// names that keyspace and the catalog store's migration instead of + /// saying the catalog is at migration 0. + #[test] + fn a_store_behind_the_catalog_store_is_named() { + let latest = MigrationId::LATEST.number(); + let baseline = MigrationId::ALL[0]; + let account = |name: &str, id: MigrationId| Pending { + id: id.number(), + name: id.name(), + store: StoreKind::Account, + store_name: name.to_owned(), + kind: Kind::Breaking, + }; + let status = CatalogStatus::Behind { + version: Some(0), + pending: vec![account("p_account_111111111111", baseline)], + }; + let message = status_message(&status, &ADVICE); + assert_eq!( + message.lines().next().unwrap_or_default(), + format!( + "account store p_account_111111111111 is at migration 0, behind the catalog \ + store at migration {latest}, and ExtendDB 1.0.1 needs migration {latest}." + ), + "{message}" + ); + assert!( + !message.contains("the catalog is at migration 0"), + "{message}" + ); + assert_eq!( + status.to_string(), + format!( + "account store p_account_111111111111 is at migration 0, behind the catalog \ + store at migration {latest}; pending: 1 baseline (account, breaking)" + ) + ); + + // Two stores behind, and a data store behind at another migration. + let pending = vec![ + account("p_account_1", baseline), + account("p_account_2", baseline), + Pending { + store: StoreKind::Data, + store_name: "extenddb_data".to_owned(), + ..account("", MigrationId::LATEST) + }, + ]; + let message = status_message( + &CatalogStatus::Behind { + version: Some(0), + pending, + }, + &ADVICE, + ); + assert!( + message.starts_with(&format!( + "account store p_account_1 (at migration 0), account store p_account_2 (at \ + migration 0) and data store extenddb_data (at migration {}) are behind the \ + catalog store at migration {latest}, and ExtendDB", + latest - 1 + )), + "{message}" + ); + + // An upgrade: every store waits for the same migration, so no store + // is behind the catalog store and the version is the catalog's. + let upgrade = CatalogStatus::Behind { + version: Some(latest - 1), + pending: vec![ + Pending { + store: StoreKind::Catalog, + store_name: "p_catalog".to_owned(), + ..account("", MigrationId::LATEST) + }, + account("p_account_1", MigrationId::LATEST), + ], + }; + let message = status_message(&upgrade, &ADVICE); + assert!( + message.starts_with(&format!( + "the catalog is at migration {}, and ExtendDB", + latest - 1 + )), + "{message}" + ); + } + #[test] fn mixed_kinds_are_named_per_migration() { let mut list = pending(Kind::Breaking); diff --git a/crates/storage/src/migrations/status.rs b/crates/storage/src/migrations/status.rs index 73af24c69..2ad04f0ba 100644 --- a/crates/storage/src/migrations/status.rs +++ b/crates/storage/src/migrations/status.rs @@ -5,6 +5,7 @@ //! exit codes. use super::ledger::Outcome; +use super::registry::MigrationId; use super::table::{Kind, StoreKind}; /// One step a run applied. @@ -63,6 +64,60 @@ pub struct Pending { pub kind: Kind, } +/// The stores of a `Behind` status's `pending` list whose ledger lacks a +/// migration the catalog store's ledger has, which a run stopped between +/// the catalog store and the others leaves: each with its own migration, +/// the one before its first pending id, in the order `pending` names them, +/// after the catalog store's migration. A store whose first pending id the +/// catalog store lacks too is not behind it, as on an upgrade, where every +/// store waits for the same migration. `None` when no store is behind. +#[must_use] +pub(crate) fn stores_behind(pending: &[Pending]) -> Option<(u32, Vec<(&Pending, u32)>)> { + let catalog_at = pending + .iter() + .filter(|p| p.store == StoreKind::Catalog) + .map(|p| p.id) + .min() + .map_or(MigrationId::LATEST.number(), |id| id.saturating_sub(1)); + let mut behind: Vec<(&Pending, u32)> = Vec::new(); + for p in pending.iter().filter(|p| p.store != StoreKind::Catalog) { + let at = p.id.saturating_sub(1); + match behind + .iter_mut() + .find(|(b, _)| b.store == p.store && b.store_name == p.store_name) + { + Some((_, lowest)) => *lowest = (*lowest).min(at), + None => behind.push((p, at)), + } + } + behind.retain(|(_, at)| *at < catalog_at); + (!behind.is_empty()).then_some((catalog_at, behind)) +} + +/// What a `Behind` status says first when [`stores_behind`] names stores: +/// `account store x is at migration 0, behind the catalog store at +/// migration 4`, or for several, `account store x (at migration 0) and data +/// store y (at migration 3) are behind the catalog store at migration 4`. +#[must_use] +pub(crate) fn behind_line(catalog_at: u32, behind: &[(&Pending, u32)]) -> String { + let named = |p: &Pending| format!("{} store {}", p.store.as_str(), p.store_name); + if let [(p, at)] = behind { + return format!( + "{} is at migration {at}, behind the catalog store at migration {catalog_at}", + named(p) + ); + } + let mut items: Vec = behind + .iter() + .map(|(p, at)| format!("{} (at migration {at})", named(p))) + .collect(); + let last = items.pop().unwrap_or_default(); + format!( + "{} and {last} are behind the catalog store at migration {catalog_at}", + items.join(", ") + ) +} + /// The status `serve`, `migrate` and `verify` act on. When several apply, it /// is the first in this order: `Changed`, `WrittenByOlderRelease`, /// `AheadIncompatible`, `NeedsAttention`, `Behind`, `AheadCompatible`, @@ -140,10 +195,13 @@ impl std::fmt::Display for CatalogStatus { match self { Self::Current { version } => write!(f, "the catalog is at migration {version}"), Self::Behind { version, pending } => { - let at = version.map_or_else( - || "has no migration ledger (a release before 1.0)".to_owned(), - |v| format!("is at migration {v}"), - ); + let at = match (stores_behind(pending), version) { + (Some((catalog_at, behind)), _) => behind_line(catalog_at, &behind), + (None, Some(v)) => format!("the catalog is at migration {v}"), + (None, None) => { + "the catalog has no migration ledger (a release before 1.0)".to_owned() + } + }; let list: Vec = pending .iter() .map(|p| { @@ -156,7 +214,7 @@ impl std::fmt::Display for CatalogStatus { ) }) .collect(); - write!(f, "the catalog {at}; pending: {}", list.join(", ")) + write!(f, "{at}; pending: {}", list.join(", ")) } Self::AheadCompatible { version, unknown } => write!( f, From 6dc1648ade1facf19caf50583eac603112a46ad6 Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Sun, 1 Nov 2026 16:30:01 +0000 Subject: [PATCH 19/45] fix(storage-mongodb): refuse init on a database holding another program's collections init on MongoDB took the migration lock on any catalog database that was not a catalog, and taking the lock writes its document into that database's settings collection. With extenddb_catalog holding another program's orders collection and a settings document with the _id theme, init exited 0, created the catalog collections beside orders and wrote its rows into that settings collection. The Cassandra backend refuses the same keyspace, naming its contents. Before init takes the lock on a database that is not a catalog (an empty ledger and no catalog_version document, as the status decides), it now reads what the database holds and refuses it, naming each item, when it finds a collection migration 1 does not create, a collection of a catalog collection's name with another type, other options or an index migration 1 does not create, a settings document other than the migration lock's, a schema_history document other than migration 1's 001_initial, or a document in any other catalog collection. init writes nothing to a refused database. A database an init was killed in before migration 1 recorded anything holds only migration 1's collections, the lock document and 001_initial, and init takes it. Tests: init is refused on a database holding an orders collection with a document and a settings document theme, the message names both, and the database keeps its collections and settings documents with no lock document added. A capped tags collection and a metrics collection holding a document are refused. A database holding migration 1's collections, the 001_initial document and an expired lock document is taken. The guard tests that run migration 1 over a database with a capped tags collection, a non-unique account_name index or a schema_history validator now take the lock directly, since init refuses those databases before it locks. --- .../src/migrations/commands.rs | 22 ++- .../storage-mongodb/src/migrations/foreign.rs | 154 ++++++++++++++++++ crates/storage-mongodb/src/migrations/mod.rs | 2 + .../storage-mongodb/tests/migrations_fresh.rs | 135 +++++++++++++++ .../tests/migrations_guards.rs | 16 +- 5 files changed, 320 insertions(+), 9 deletions(-) create mode 100644 crates/storage-mongodb/src/migrations/foreign.rs diff --git a/crates/storage-mongodb/src/migrations/commands.rs b/crates/storage-mongodb/src/migrations/commands.rs index e51c64d71..8344b8cb7 100644 --- a/crates/storage-mongodb/src/migrations/commands.rs +++ b/crates/storage-mongodb/src/migrations/commands.rs @@ -22,6 +22,7 @@ use extenddb_storage::migrations::{ RunKind, RunReport, Runner, Target, }; +use super::foreign::foreign_unless_a_catalog; use super::{ CATALOG_DATABASE, DATA_DATABASE, Lease, MigrationLock, MongoMigrations, MongoTarget, Timing, catalog_status, status_holding, @@ -128,11 +129,15 @@ impl CatalogMigrations for MongoCatalogMigrations { /// Unless `creating` (`init`), a store that is not a catalog is refused /// first, with nothing written, from its ledgers and settings alone: the /// status read under the lock is the one that runs the pending migrations' -/// `detect`. +/// `detect`. With `creating`, a store that is not a catalog and holds +/// another program's collections or `settings` documents is refused first, +/// naming them, with nothing written: taking the lock writes its document +/// into the catalog database's `settings`. /// /// # Errors /// -/// The store is not a catalog (`Uninitialized`, exit 1), the lock or a +/// The store is not a catalog (`Uninitialized`, exit 1), it holds another +/// program's contents ([`MigrationError::Refused`], exit 1), the lock or a /// server outlasted `timeout` ([`MigrationError::Locked`], exit 6), or a /// request failed. pub async fn lock( @@ -141,7 +146,18 @@ pub async fn lock( timeout: Duration, timing: Timing, ) -> Result<(MigrationLock, MongoTarget), MigrationError> { - if !creating { + if creating { + let foreign = foreign_unless_a_catalog(target).await?; + if !foreign.is_empty() { + return Err(MigrationError::Refused(format!( + "the database {} is not an ExtendDB catalog: it holds {}. Nothing was written \ + to it. ExtendDB keeps its catalog in a database of that name, so run init \ + against a MongoDB deployment where that database is empty or absent", + target.catalog, + foreign.join(", ") + ))); + } + } else { let migrations = MongoMigrations::open(target, None).await?; let initialized = Runner::new(&migrations, "").initialized().await; migrations.close().await; diff --git a/crates/storage-mongodb/src/migrations/foreign.rs b/crates/storage-mongodb/src/migrations/foreign.rs new file mode 100644 index 000000000..051aabd9f --- /dev/null +++ b/crates/storage-mongodb/src/migrations/foreign.rs @@ -0,0 +1,154 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! What a catalog database that is not a catalog holds that makes it another +//! program's, which `init` refuses before it writes anything. + +use bson::{Bson, Document, doc}; +use extenddb_storage::migrations::{MigrationBackend, MigrationError, StoreKind, StoreRef}; +use futures::TryStreamExt; + +use super::postcheck::{self, Definitions}; +use super::{CATALOG_EXPECTED, LEDGER, LOCK_ID, MongoMigrations, MongoStores, MongoTarget}; + +/// The `schema_history` document migration 1's catalog body writes. +const BASELINE_HISTORY: &str = "001_initial"; + +/// What the catalog database of `target` holds that makes it another +/// program's: the collections migration 1 does not create; a collection of +/// a catalog collection's name that is of another type, has options +/// migration 1 does not give it, or has an index migration 1 does not create +/// on it; `settings` documents other than the migration lock's; +/// `schema_history` documents other than the one migration 1 writes; and a +/// document in any other catalog collection. Empty for a database that does +/// not exist, and for one an `init` created and stopped in before migration +/// 1 recorded anything: that holds only collections migration 1 creates, +/// with their options and some of their indexes, at most the lock document +/// in `settings` and migration 1's `schema_history` document. A collection +/// name alone shows nothing: the catalog's names (`metrics`, `tags`, +/// `tables`, `accounts`) are common ones. Call it only for a database whose +/// status is `Uninitialized`, since a catalog's collections hold documents. +/// +/// # Errors +/// +/// A connection or a read failed. +pub async fn foreign_contents(target: &MongoTarget) -> Result, MigrationError> { + let migrations = MongoMigrations::open(target, None).await?; + let found = contents(migrations.stores()).await; + migrations.close().await; + found +} + +/// [`foreign_contents`] when the catalog database of `target` is not a +/// catalog, and nothing when it is one. It is not a catalog when its ledger +/// holds no row and its `settings` no `catalog_version` document, which is +/// how the status decides `Uninitialized`, read here without the status's +/// other reads: `init` calls this before it takes the lock. +/// +/// # Errors +/// +/// A connection or a read failed. +pub(crate) async fn foreign_unless_a_catalog( + target: &MongoTarget, +) -> Result, MigrationError> { + let migrations = MongoMigrations::open(target, None).await?; + let found = async { + let catalog = StoreRef { + kind: StoreKind::Catalog, + name: target.catalog.clone(), + }; + if !migrations.read_ledger(&catalog).await?.is_empty() + || migrations.read_traces().await?.catalog_version.is_some() + { + return Ok(Vec::new()); + } + contents(migrations.stores()).await + } + .await; + migrations.close().await; + found +} + +async fn contents(stores: &MongoStores) -> Result, MigrationError> { + let db = &stores.catalog; + let expected: Definitions = serde_json::from_str(CATALOG_EXPECTED) + .map_err(|e| MigrationError::Store(format!("read the expected definitions: {e}")))?; + let mut names = stores + .request(format!("list the collections of {}", db.name()), async { + db.list_collection_names().await + }) + .await?; + names.sort(); + // `system.views` exists when the database has a view; the view itself is + // named below when it has a catalog collection's name, and here when not. + let mut foreign: Vec = names + .iter() + .filter(|n| *n != LEDGER && !n.starts_with("system.") && !expected.contains_key(*n)) + .map(|n| format!("the collection {n}")) + .collect(); + let ours: Vec = names + .into_iter() + .filter(|n| expected.contains_key(n)) + .collect(); + let actual = postcheck::describe(stores, db, &ours).await?; + for (name, have) in &actual { + let Some(want) = expected.get(name) else { + continue; + }; + if have.kind != want.kind || have.options != want.options { + foreign.push(format!( + "the collection {name}, a {} with options {} that migration 1 does not create", + have.kind, have.options + )); + continue; + } + if let Some(index) = have + .indexes + .iter() + .find(|(index, definition)| want.indexes.get(*index) != Some(*definition)) + .map(|(index, _)| index) + { + foreign.push(format!( + "the collection {name}, with the index {index} migration 1 does not create" + )); + continue; + } + let ours_alone = match name.as_str() { + "settings" => Some(LOCK_ID), + "schema_history" => Some(BASELINE_HISTORY), + _ => None, + }; + let filter = ours_alone.map_or_else(Document::new, |id| doc! { "_id": { "$ne": id } }); + let collection = db.collection::(name); + let found = stores + .request( + format!("read the documents of {}.{name}", db.name()), + async { + collection + .find(filter) + .projection(doc! { "_id": 1 }) + .limit(3) + .await? + .try_collect::>() + .await + }, + ) + .await?; + if found.is_empty() { + continue; + } + if ours_alone.is_none() { + foreign.push(format!("documents in the collection {name}")); + continue; + } + for document in &found { + let id = match document.get("_id") { + Some(Bson::String(id)) => id.clone(), + Some(other) => other.to_string(), + None => String::new(), + }; + foreign.push(format!("the {name} document '{id}'")); + } + } + Ok(foreign) +} diff --git a/crates/storage-mongodb/src/migrations/mod.rs b/crates/storage-mongodb/src/migrations/mod.rs index 3cd157908..12698fc45 100644 --- a/crates/storage-mongodb/src/migrations/mod.rs +++ b/crates/storage-mongodb/src/migrations/mod.rs @@ -25,6 +25,7 @@ mod backend; mod commands; +mod foreign; mod guarded; mod locks; mod registrar; @@ -49,6 +50,7 @@ use extenddb_storage::migrations::{ pub use backend::{MongoMigrations, MongoStores}; pub(crate) use commands::lease_of; pub use commands::{MongoCatalogMigrations, lock}; +pub use foreign::foreign_contents; pub(crate) use locks::{Lease, is_duplicate_key}; pub use locks::{MigrationLock, Timing}; #[cfg(feature = "test-util")] diff --git a/crates/storage-mongodb/tests/migrations_fresh.rs b/crates/storage-mongodb/tests/migrations_fresh.rs index dbf1c1e7d..7866ab002 100644 --- a/crates/storage-mongodb/tests/migrations_fresh.rs +++ b/crates/storage-mongodb/tests/migrations_fresh.rs @@ -89,6 +89,141 @@ async fn stores_that_are_not_catalogs_are_uninitialized_and_migrate_writes_nothi } } +/// `init` takes the migration lock on a store that is not a catalog, and +/// taking it writes the lock document into the catalog database's +/// `settings`. A database that holds another program's collections or +/// `settings` documents is refused before that write, naming them, and +/// nothing is written; so is one with a catalog collection of another shape +/// or holding documents. A database an `init` stopped in before migration 1 +/// recorded anything (the lock document, migration 1's collections and its +/// `schema_history` document) is taken. +#[tokio::test(flavor = "multi_thread")] +async fn init_refuses_another_programs_database_before_it_writes_anything() { + let Some(uri) = server() else { return }; + let other = Deployment::new(&uri, "foreign").await; + other + .catalog() + .collection::("orders") + .insert_one(doc! { "order": 1 }) + .await + .expect("insert"); + other + .catalog() + .collection::("settings") + .insert_one(doc! { "_id": "theme", "value": "dark" }) + .await + .expect("insert"); + let before = Deployment::documents(&other.catalog(), "settings").await; + let refused = lock( + &other.target, + true, + Duration::from_secs(2), + Timing::PRODUCTION, + ) + .await + .err() + .expect("refused"); + let message = refused.to_string(); + assert!( + message.contains(&format!( + "the database {} is not an ExtendDB catalog: it holds the collection orders, the \ + settings document 'theme'. Nothing was written to it.", + other.target.catalog + )), + "{message}" + ); + assert_eq!(refused.exit_code(), exit::ERROR); + assert_eq!( + Deployment::collections(&other.catalog()).await, + ["orders", "settings"] + ); + assert_eq!( + Deployment::documents(&other.catalog(), "settings").await, + before, + "no lock document was written" + ); + assert!(Deployment::collections(&other.data()).await.is_empty()); + + // Catalog names alone: a capped `tags` and a `metrics` with a document. + let named = Deployment::new(&uri, "named").await; + named + .catalog() + .create_collection("tags") + .capped(true) + .size(4096) + .await + .expect("create"); + named + .catalog() + .collection::("metrics") + .insert_one(doc! { "reading": 1.5 }) + .await + .expect("insert"); + let message = lock( + &named.target, + true, + Duration::from_secs(2), + Timing::PRODUCTION, + ) + .await + .err() + .expect("refused") + .to_string(); + assert!( + message.contains("documents in the collection metrics") + && message.contains("the collection tags, a collection with options"), + "{message}" + ); + assert!( + Deployment::documents(&named.catalog(), "settings") + .await + .is_empty() + ); + + // What an `init` killed before migration 1's ledger row leaves, once its + // lock's lease has passed. + let stopped = Deployment::new(&uri, "stopped").await; + stopped + .catalog() + .collection::("settings") + .insert_one(doc! { + "_id": "migration_lock", + "owner": "a killed init", + "expires_at": bson::DateTime::from_millis(0), + }) + .await + .expect("insert"); + let expected: Definitions = serde_json::from_str(CATALOG_EXPECTED).expect("expected"); + for name in expected.keys() { + if name != "settings" { + stopped + .catalog() + .create_collection(name) + .await + .expect("create"); + } + } + stopped + .catalog() + .collection::("schema_history") + .insert_one(doc! { "_id": "001_initial" }) + .await + .expect("insert"); + assert_eq!(stopped.status().await, CatalogStatus::Uninitialized); + let (held, _) = lock( + &stopped.target, + true, + Duration::from_secs(60), + Timing::PRODUCTION, + ) + .await + .expect("init takes the database"); + held.release().await; + for d in [other, named, stopped] { + d.drop_databases().await; + } +} + #[tokio::test(flavor = "multi_thread")] async fn a_fresh_store_gets_every_migration_and_the_expected_definitions() { let Some(uri) = server() else { return }; diff --git a/crates/storage-mongodb/tests/migrations_guards.rs b/crates/storage-mongodb/tests/migrations_guards.rs index 039e76ec1..24bce7466 100644 --- a/crates/storage-mongodb/tests/migrations_guards.rs +++ b/crates/storage-mongodb/tests/migrations_guards.rs @@ -83,10 +83,12 @@ async fn a_store_whose_objects_differ_records_nothing_and_names_the_object() { .run_command(doc! { "create": "tags", "capped": true, "size": 4096 }) .await .expect("capped"); - let (held, target) = lock(&d.target, true, Duration::from_secs(5), Timing::PRODUCTION) + // `init` refuses this database before it takes the lock + // (tests/migrations_fresh.rs); the runner's own check is tested here. + let held = MigrationLock::acquire(&d.target, Duration::from_secs(5), Timing::PRODUCTION) .await .expect("lock"); - let migrations = MongoMigrations::open(&target, Some(&held)) + let migrations = MongoMigrations::open(&d.target, Some(&held)) .await .expect("open"); let error = Runner::new(&migrations, "test") @@ -116,10 +118,10 @@ async fn a_store_whose_objects_differ_records_nothing_and_names_the_object() { ) .await .expect("index"); - let (held, target) = lock(&d.target, true, Duration::from_secs(5), Timing::PRODUCTION) + let held = MigrationLock::acquire(&d.target, Duration::from_secs(5), Timing::PRODUCTION) .await .expect("lock"); - let migrations = MongoMigrations::open(&target, Some(&held)) + let migrations = MongoMigrations::open(&d.target, Some(&held)) .await .expect("open"); let error = Runner::new(&migrations, "test") @@ -147,10 +149,12 @@ async fn a_failed_write_stops_the_run_instead_of_being_discarded() { }) .await .expect("validator"); - let (held, target) = lock(&d.target, true, Duration::from_secs(5), Timing::PRODUCTION) + // `init` refuses this database before it takes the lock; the runner's + // handling of the refused write is tested here. + let held = MigrationLock::acquire(&d.target, Duration::from_secs(5), Timing::PRODUCTION) .await .expect("lock"); - let migrations = MongoMigrations::open(&target, Some(&held)) + let migrations = MongoMigrations::open(&d.target, Some(&held)) .await .expect("open"); let error = Runner::new(&migrations, "test") From 835f9ce01b365efec6786bf7f637c2ba95a791be Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Sun, 1 Nov 2026 16:53:10 +0000 Subject: [PATCH 20/45] fix(init): finish a catalog an init stopped in before its admin user An init that stopped after its migrations and before its bootstrap rows (a kill, a lost connection, a request timeout or a lapsed lease) left a catalog whose status is Current and that has no admin user. The next init refused it as already initialized and pointed at extenddb migrate, which writes no encryption key, and the server then exits because the key is missing. init writes the config file after the rows, so migrate and destroy had no config file to read either. init now reads the catalog's status under the migration lock, then asks the backend whether the catalog has an admin user. A catalog whose status is Current and that has no admin user is finished: init runs no migration, writes the bootstrap rows the catalog lacks (the encryption key only when there is none, the default account only when there is none, then the admin user), the data connection rows and the config file. A catalog with an admin user, and one whose status is anything other than Current or Uninitialized, is refused with the same message as before. init reads the status once and hands it to the run. On Cassandra and MongoDB init takes an existing catalog store before the lock, so the next init finishes the catalog. On SQLite and PostgreSQL init still refuses an existing database file or catalog database before it takes the lock. Bootstrapper gains has_admin_user, which SQLite, PostgreSQL, Cassandra and MongoDB answer from their admin_users table or collection, Cassandra at LOCAL_QUORUM. Its default answers true, so a backend that does not implement it keeps the old refusal. The bootstrap steps move to write_bootstrap_rows in the storage crate, beside finishes_a_stopped_init, so the backends' tests run the steps init runs. Each backend's steps skip a row the catalog has, Cassandra's default account step since it creates the account's keyspace before the row. MongoDB's bootstrapper takes its database names from a field, which a test-only constructor sets. Tests: with fakes, init on an Uninitialized store runs the migrations and then the rows; on a Current catalog without an admin user it runs no migration and writes the rows; on a Current catalog with an admin user, or a Behind one, it is refused with the same text and writes nothing. SQLite, PostgreSQL, MongoDB and Cassandra each finish a catalog an init left after its migrations and its encryption key (Cassandra and SQLite also after the default account), keep the key and the account, write the admin user, and then report an admin user, so init refuses the catalog; on Cassandra a new bootstrapper first takes the existing catalog keyspace as init does. The storage crate checks that the default has_admin_user keeps init refusing a Current catalog. --- crates/app/src/cmd_init.rs | 65 +++-- crates/app/src/cmd_init_tests.rs | 235 ++++++++++++++++++ crates/storage-cassandra/src/bootstrapper.rs | 20 ++ crates/storage-cassandra/tests/migrations.rs | 111 +++++++++ crates/storage-mongodb/src/bootstrapper.rs | 44 +++- .../storage-mongodb/tests/migrations_fresh.rs | 74 ++++++ crates/storage-postgres/src/bootstrapper.rs | 8 + .../tests/migrations_fresh.rs | 73 ++++++ crates/storage-sqlite/src/bootstrapper.rs | 148 +++++++++++ crates/storage/src/bootstrapper.rs | 72 +++++- crates/storage/src/migrations/catalog.rs | 19 +- crates/storage/src/migrations/mod.rs | 2 +- 12 files changed, 829 insertions(+), 42 deletions(-) create mode 100644 crates/app/src/cmd_init_tests.rs diff --git a/crates/app/src/cmd_init.rs b/crates/app/src/cmd_init.rs index c0f95f903..cb370dfd2 100755 --- a/crates/app/src/cmd_init.rs +++ b/crates/app/src/cmd_init.rs @@ -309,10 +309,13 @@ pub async fn run(args: InitArgs, release: &str) -> anyhow::Result { } /// `init` on a backend on the migration runner, all under the migration -/// lock: the catalog's status, which must be `Uninitialized` (another `init` -/// may have created the catalog while this one waited), then every -/// migration, then the rows a server needs before it can start, then the -/// data connection rows. Holding the lock through the rows keeps a server +/// lock: the catalog's status, then every migration on a store that is not +/// a catalog (`Uninitialized`), then the rows a server needs before it can +/// start, then the data connection rows. A catalog whose migrations are all +/// applied (`Current`) and that has no admin user, which an `init` that +/// stopped after its migrations leaves, gets no migration and the rows it +/// lacks; any other catalog is refused (another `init` may have created it +/// while this one waited). Holding the lock through the rows keeps a server /// from starting on a catalog that has no encryption key yet. The target /// carries what the bootstrapper resolved from the command line, so the /// backend reads no flag itself. @@ -321,7 +324,8 @@ async fn init_on_runner( migrations: &dyn extenddb_storage::migrations::CatalogMigrations, release: &str, ) -> anyhow::Result { - use extenddb_storage::migrations::{RunKind, run_locked}; + use extenddb_storage::bootstrapper::{finishes_a_stopped_init, write_bootstrap_rows}; + use extenddb_storage::migrations::{RunKind, run_with_status}; let target = bootstrapper.init_target(); let lock = migrations @@ -329,16 +333,29 @@ async fn init_on_runner( .await .map_err(|e| anyhow::anyhow!("{e}"))?; let result = async { - println!("--- Creating the catalog through its migrations..."); - let report = run_locked(migrations, &target, &lock, release, RunKind::Create) + let status = migrations + .status_under_lock(&lock, &target) .await .map_err(|e| anyhow::anyhow!("{e}"))?; - for note in &report.notes { - println!(" {note}"); + if !finishes_a_stopped_init(bootstrapper, &status) + .await + .map_err(crate::cmd_migrate::op_err)? + { + println!("--- Creating the catalog through its migrations..."); + let report = run_with_status(migrations, &status, &lock, release, RunKind::Create) + .await + .map_err(|e| anyhow::anyhow!("{e}"))?; + for note in &report.notes { + println!(" {note}"); + } + crate::util::print_applied(&report, " ", "Applied", "migration"); } - crate::util::print_applied(&report, " ", "Applied", "migration"); bootstrapper.hold(&lock); - let admin = write_bootstrap_rows(bootstrapper).await?; + let env_user = std::env::var("EXTENDDB_ADMIN_USER").ok(); + let env_pass = std::env::var("EXTENDDB_ADMIN_PASSWORD").ok(); + let admin = write_bootstrap_rows(bootstrapper, env_user.as_deref(), env_pass.as_deref()) + .await + .map_err(crate::cmd_migrate::op_err)?; bootstrapper .record_data_connection() .await @@ -350,29 +367,9 @@ async fn init_on_runner( result } -/// The rows a server needs before it can start: encryption key, default -/// account, admin user. -async fn write_bootstrap_rows( - bootstrapper: &dyn extenddb_storage::bootstrapper::Bootstrapper, -) -> anyhow::Result { - bootstrapper - .bootstrap_encryption_key() - .await - .map_err(crate::cmd_migrate::op_err)?; // REQ-AUTH-010 - - bootstrapper - .bootstrap_default_account() - .await - .map_err(crate::cmd_migrate::op_err)?; - - // REQ-AUTH-003 - let env_user = std::env::var("EXTENDDB_ADMIN_USER").ok(); - let env_pass = std::env::var("EXTENDDB_ADMIN_PASSWORD").ok(); - bootstrapper - .bootstrap_admin_user(env_user.as_deref(), env_pass.as_deref()) - .await - .map_err(crate::cmd_migrate::op_err) -} +#[cfg(test)] +#[path = "cmd_init_tests.rs"] +mod runner_tests; #[cfg(test)] mod tests { diff --git a/crates/app/src/cmd_init_tests.rs b/crates/app/src/cmd_init_tests.rs new file mode 100644 index 000000000..30c8ecf07 --- /dev/null +++ b/crates/app/src/cmd_init_tests.rs @@ -0,0 +1,235 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `init` under the migration lock: a store that is not a catalog is created +//! through its migrations, a catalog whose migrations are applied and that +//! has no admin user is finished with the rows it lacks, and a catalog with +//! an admin user is refused. +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use async_trait::async_trait; +use extenddb_storage::bootstrapper::{AdminBootstrapResult, Bootstrapper}; +use extenddb_storage::management_store::OpResult; +use extenddb_storage::migrations::{ + CatalogMigrations, CatalogStatus, HeldLock, Kind, LedgerRow, Listed, MigrationError, + MigrationId, Pending, RunKind, RunReport, StoreKind, Target, +}; + +use super::init_on_runner; + +type Log = Arc>>; + +fn record(log: &Log, entry: &str) { + log.lock().unwrap().push(entry.to_owned()); +} + +struct FakeMigrations { + status: CatalogStatus, + log: Log, +} + +#[async_trait] +impl CatalogMigrations for FakeMigrations { + fn listed(&self, _id: MigrationId) -> Listed { + Listed::NotApplicable("fake") + } + fn backup_advice(&self, target: &Target) -> String { + format!("back up {}", target.location) + } + async fn status(&self, _target: &Target) -> Result { + record(&self.log, "status"); + Ok(self.status.clone()) + } + async fn catalog_ledger(&self, _: &Target) -> Result, MigrationError> { + Ok(Vec::new()) + } + async fn lock(&self, target: &Target, _: Duration) -> Result { + record(&self.log, "lock"); + Ok(HeldLock::new(target.location.as_str(), ())) + } + async fn run( + &self, + _lock: &HeldLock, + _release: &str, + kind: RunKind, + ) -> Result { + assert_eq!(kind, RunKind::Create); + record(&self.log, "run the migrations"); + Ok(RunReport::default()) + } + async fn forget(&self, _lock: &HeldLock, _id: u32) -> Result<(), MigrationError> { + unreachable!("init forgets nothing") + } + async fn release(&self, _lock: HeldLock) { + record(&self.log, "release"); + } +} + +struct FakeBootstrapper { + admin: bool, + log: Log, +} + +#[async_trait] +impl Bootstrapper for FakeBootstrapper { + async fn ensure_app_user(&self) -> OpResult<()> { + Ok(()) + } + async fn grant_app_role_to_admin(&self) -> OpResult<()> { + Ok(()) + } + async fn create_catalog_db(&self) -> OpResult<()> { + Ok(()) + } + async fn create_data_db(&self) -> OpResult<()> { + Ok(()) + } + async fn record_data_connection(&self) -> OpResult<()> { + record(&self.log, "data connection"); + Ok(()) + } + async fn bootstrap_encryption_key(&self) -> OpResult<()> { + record(&self.log, "encryption key"); + Ok(()) + } + async fn bootstrap_default_account(&self) -> OpResult<()> { + record(&self.log, "default account"); + Ok(()) + } + async fn bootstrap_admin_user( + &self, + _env_user: Option<&str>, + _env_password: Option<&str>, + ) -> OpResult { + record(&self.log, "admin user"); + Ok(AdminBootstrapResult { + username: "admin".to_owned(), + generated_password: Some("generated".to_owned()), + already_existed: false, + from_env: false, + }) + } + async fn has_admin_user(&self) -> OpResult { + record(&self.log, "has an admin user?"); + Ok(self.admin) + } + async fn list_table_names(&self) -> OpResult> { + Ok(Vec::new()) + } + async fn get_data_db_name(&self) -> OpResult> { + Ok(None) + } + async fn drop_databases(&self, _data_db: &str) -> OpResult<()> { + unreachable!("init drops nothing") + } + fn catalog_database_name(&self) -> String { + "fake".to_owned() + } + fn endpoint_info(&self) -> String { + "fake".to_owned() + } + fn catalog_connection_url(&self) -> String { + "fake://catalog".to_owned() + } + fn hold(&self, _lock: &HeldLock) { + record(&self.log, "hold"); + } + fn generate_backend_config_section(&self) -> String { + String::new() + } +} + +/// `init_on_runner` on a catalog of `status` whose admin users `admin` +/// says exist: its result and every call it made, in order. +async fn init(status: CatalogStatus, admin: bool) -> (anyhow::Result, Vec) { + let log = Log::default(); + let migrations = FakeMigrations { + status, + log: Arc::clone(&log), + }; + let bootstrapper = FakeBootstrapper { + admin, + log: Arc::clone(&log), + }; + let result = init_on_runner(&bootstrapper, &migrations, "1.0.0-test") + .await + .map(|admin| admin.username); + let calls = log.lock().unwrap().clone(); + (result, calls) +} + +const ROWS: [&str; 5] = [ + "hold", + "encryption key", + "default account", + "admin user", + "data connection", +]; + +fn calls(first: &[&str], last: &[&str]) -> Vec { + first.iter().chain(last).map(|c| (*c).to_owned()).collect() +} + +#[tokio::test] +async fn a_store_that_is_not_a_catalog_is_created_then_given_its_rows() { + let (result, log) = init(CatalogStatus::Uninitialized, false).await; + assert_eq!(result.expect("init"), "admin"); + let mut expected = calls(&["lock", "status", "run the migrations"], &ROWS); + expected.push("release".to_owned()); + assert_eq!(log, expected); +} + +/// What an `init` stopped after its migrations leaves: every migration is +/// recorded and there is no admin user. `init` runs no migration and writes +/// the rows, each of which the backend writes only when it is missing. +#[tokio::test] +async fn a_current_catalog_without_an_admin_user_is_finished_without_a_migration() { + let current = CatalogStatus::Current { + version: MigrationId::LATEST.number(), + }; + let (result, log) = init(current, false).await; + assert_eq!(result.expect("init"), "admin"); + let mut expected = calls(&["lock", "status", "has an admin user?"], &ROWS); + expected.push("release".to_owned()); + assert_eq!(log, expected); +} + +/// A catalog with an admin user, and one whose migrations are not all +/// applied, are refused as initialized, with nothing run or written and the +/// lock released. +#[tokio::test] +async fn a_catalog_with_an_admin_user_or_pending_migrations_is_refused() { + let current = CatalogStatus::Current { + version: MigrationId::LATEST.number(), + }; + let (result, log) = init(current, true).await; + let message = result.expect_err("refused").to_string(); + assert_eq!( + message, + format!( + "the catalog is already initialized (the catalog is at migration {}); use \ + 'extenddb migrate' for pending migrations", + MigrationId::LATEST.number() + ) + ); + assert_eq!( + log, + calls(&["lock", "status", "has an admin user?"], &["release"]) + ); + + let behind = CatalogStatus::Behind { + version: Some(1), + pending: vec![Pending { + id: 2, + name: "resource_tag_condition_keys", + store: StoreKind::Catalog, + store_name: "fake".to_owned(), + kind: Kind::Breaking, + }], + }; + let (result, log) = init(behind, false).await; + let message = result.expect_err("refused").to_string(); + assert!(message.contains("already initialized"), "{message}"); + assert_eq!(log, calls(&["lock", "status"], &["release"])); +} diff --git a/crates/storage-cassandra/src/bootstrapper.rs b/crates/storage-cassandra/src/bootstrapper.rs index 351014312..5c7bff97e 100644 --- a/crates/storage-cassandra/src/bootstrapper.rs +++ b/crates/storage-cassandra/src/bootstrapper.rs @@ -680,6 +680,26 @@ impl Bootstrapper for CassandraBootstrapper { }) } + /// Read at `LOCAL_QUORUM`, since `init` finishes a catalog this answers + /// `false` for: at `ONE` a replica that missed the admin user's write + /// would read as empty. + async fn has_admin_user(&self) -> OpResult { + let keyspace = self.engine.catalog_keyspace(); + let rows = self + .held + .request( + "Read the admin users", + crate::cassandra_util::query_rows_quorum::( + &self.engine.session_arc(), + &format!("SELECT admin_name FROM {keyspace}.admin_users LIMIT 1"), + cdrs_tokio::query_values!(), + "has_admin_user", + ), + ) + .await?; + Ok(!rows.is_empty()) + } + async fn list_table_names(&self) -> OpResult> { let keyspace = self.engine.catalog_keyspace(); diff --git a/crates/storage-cassandra/tests/migrations.rs b/crates/storage-cassandra/tests/migrations.rs index 1e2fc9185..6d65a5120 100644 --- a/crates/storage-cassandra/tests/migrations.rs +++ b/crates/storage-cassandra/tests/migrations.rs @@ -838,3 +838,114 @@ async fn the_default_account_step_run_again_keeps_one_account_and_completes_its_ CatalogStatus::Current { version: 4 } ); } + +/// An `init` killed after its default account, before its admin user, +/// leaves a catalog whose status is `Current` and that has no admin user, +/// since the account's keyspace is migrated before its row. The next +/// `init`, a new process, takes the keyspace as `init` does, and its steps +/// under the lock run no migration, keep the key, create no second account +/// and write the admin user; the `init` after that is refused as +/// initialized. +#[tokio::test] +async fn init_finishes_a_catalog_stopped_before_its_admin_user() { + use extenddb_storage::bootstrapper::{ + Bootstrapper, finishes_a_stopped_init, write_bootstrap_rows, + }; + use extenddb_storage::migrations::run_with_status; + + if helpers::skip_without_cassandra() { + return; + } + let session = session().await; + let (prefix, mut drops) = new_prefix("finish"); + let migrations = CassandraCatalogMigrations; + let store = CassandraCatalogStore::catalog_only(session.clone(), prefix.clone()); + + // The stopped init. + let first = bootstrapper(&prefix).await; + let target = first.init_target(); + first + .create_catalog_db() + .await + .expect("the catalog keyspace"); + let lock = migrations.lock(&target, TIMEOUT).await.expect("lock"); + let status = migrations + .status_under_lock(&lock, &target) + .await + .expect("status"); + run_with_status(&migrations, &status, &lock, "test", RunKind::Create) + .await + .expect("create"); + first.hold(&lock); + first.bootstrap_encryption_key().await.expect("key"); + first.bootstrap_default_account().await.expect("account"); + migrations.release(lock).await; + let key = store.get_setting("encryption_key").await.expect("read"); + let accounts = store.list_all_accounts().await.expect("list"); + assert_eq!(accounts.len(), 1, "{accounts:?}"); + drops.0.push(format!("{prefix}_account_{}", accounts[0].0)); + // `first` holds the lease of the lock it released, so its requests are + // refused; a bootstrapper of its own reads the admin users. + assert!( + !bootstrapper(&prefix) + .await + .has_admin_user() + .await + .expect("read") + ); + + // The next init finishes it. + let next = bootstrapper(&prefix).await; + next.create_catalog_db() + .await + .expect("init takes the existing catalog keyspace"); + let lock = migrations.lock(&target, TIMEOUT).await.expect("lock"); + let status = migrations + .status_under_lock(&lock, &target) + .await + .expect("status"); + assert_eq!(status, CatalogStatus::Current { version: 4 }); + assert!( + finishes_a_stopped_init(&next, &status) + .await + .expect("decided") + ); + next.hold(&lock); + let admin = write_bootstrap_rows(&next, None, None).await.expect("rows"); + next.record_data_connection() + .await + .expect("data connection"); + migrations.release(lock).await; + assert_eq!(admin.username, "admin"); + assert!(!admin.already_existed && admin.generated_password.is_some()); + assert_eq!( + store.get_setting("encryption_key").await.expect("read"), + key + ); + assert_eq!(store.list_all_accounts().await.expect("list"), accounts); + + // The init after that is refused. + let last = bootstrapper(&prefix).await; + let lock = migrations.lock(&target, TIMEOUT).await.expect("lock"); + let status = migrations + .status_under_lock(&lock, &target) + .await + .expect("status"); + assert!( + !finishes_a_stopped_init(&last, &status) + .await + .expect("decided") + ); + let refused = run_with_status(&migrations, &status, &lock, "test", RunKind::Create) + .await + .expect_err("refused"); + migrations.release(lock).await; + assert!( + refused + .to_string() + .starts_with("the catalog is already initialized"), + "{refused}" + ); + drop(drops); + helpers::flush_pending_drops(&session).await; +} diff --git a/crates/storage-mongodb/src/bootstrapper.rs b/crates/storage-mongodb/src/bootstrapper.rs index 7719717fd..85b474d1e 100644 --- a/crates/storage-mongodb/src/bootstrapper.rs +++ b/crates/storage-mongodb/src/bootstrapper.rs @@ -13,12 +13,16 @@ use extenddb_storage::error::StorageError; use extenddb_storage::management_store::{OpError, OpResult}; use extenddb_storage::migrations::HeldLock; -use crate::migrations::{Lease, REQUEST_TIMEOUT}; +use crate::migrations::{CATALOG_DATABASE, DATA_DATABASE, Lease, REQUEST_TIMEOUT}; /// `MongoDB` bootstrapper for init/destroy/migrate operations. pub struct MongoBootstrapper { client: mongodb::Client, connection_string: String, + /// The catalog database, [`CATALOG_DATABASE`] outside the tests. + catalog: String, + /// The data database, [`DATA_DATABASE`] outside the tests. + data: String, /// The lease of the migration lock `init` holds while it writes the /// bootstrap rows, set by [`Bootstrapper::hold`]; every row's requests /// check it first. @@ -58,12 +62,34 @@ impl MongoBootstrapper { Ok(Self { client, connection_string, + catalog: CATALOG_DATABASE.to_owned(), + data: DATA_DATABASE.to_owned(), + lease: OnceLock::new(), + }) + } + + /// A bootstrapper for the deployment `target` names, whose databases + /// may have other names, for tests that share one server. + /// + /// # Errors + /// + /// The client cannot be built from the connection string. + #[cfg(feature = "test-util")] + pub async fn for_target(target: &crate::migrations::MongoTarget) -> Result { + let client = crate::migrations::connect(&target.connection_string) + .await + .map_err(|e| StorageError::Connection(e.to_string()))?; + Ok(Self { + client, + connection_string: target.connection_string.clone(), + catalog: target.catalog.clone(), + data: target.data.clone(), lease: OnceLock::new(), }) } fn catalog_db(&self) -> mongodb::Database { - self.client.database("extenddb_catalog") + self.client.database(&self.catalog) } /// Send one of `init`'s bootstrap requests as the runner sends a @@ -95,7 +121,7 @@ impl MongoBootstrapper { } fn data_db(&self) -> mongodb::Database { - self.client.database("extenddb_data") + self.client.database(&self.data) } } @@ -293,6 +319,14 @@ impl Bootstrapper for MongoBootstrapper { }) } + async fn has_admin_user(&self) -> OpResult { + let admin_users = self.catalog_db().collection::("admin_users"); + let found = self + .request("check for an admin user", admin_users.find_one(doc! {})) + .await?; + Ok(found.is_some()) + } + async fn list_table_names(&self) -> OpResult> { use futures::TryStreamExt; @@ -350,7 +384,7 @@ impl Bootstrapper for MongoBootstrapper { } fn catalog_database_name(&self) -> String { - "extenddb_catalog".to_string() + self.catalog.clone() } fn endpoint_info(&self) -> String { @@ -358,7 +392,7 @@ impl Bootstrapper for MongoBootstrapper { } fn catalog_connection_url(&self) -> String { - format!("{}/extenddb_catalog", self.connection_string) + format!("{}/{}", self.connection_string, self.catalog) } /// The connection string the config file names, which the server uses diff --git a/crates/storage-mongodb/tests/migrations_fresh.rs b/crates/storage-mongodb/tests/migrations_fresh.rs index 7866ab002..cad96bcd6 100644 --- a/crates/storage-mongodb/tests/migrations_fresh.rs +++ b/crates/storage-mongodb/tests/migrations_fresh.rs @@ -224,6 +224,80 @@ async fn init_refuses_another_programs_database_before_it_writes_anything() { } } +/// An `init` stopped after its migrations and its encryption key, before +/// its default account and its admin user, leaves a catalog whose status is +/// `Current` and that has no admin user. The next `init`'s steps under the +/// lock run no migration, keep the key and write the account and the admin +/// user; after that a catalog with an admin user is not finished, and +/// `init` refuses it as initialized. +#[tokio::test(flavor = "multi_thread")] +async fn init_finishes_a_catalog_stopped_before_its_admin_user() { + use extenddb_storage::bootstrapper::{ + Bootstrapper, finishes_a_stopped_init, write_bootstrap_rows, + }; + use extenddb_storage_mongodb::MongoBootstrapper; + use extenddb_storage_mongodb::migrations::status_holding; + + let Some(uri) = server() else { return }; + let d = Deployment::new(&uri, "finish").await; + let bootstrapper = MongoBootstrapper::for_target(&d.target) + .await + .expect("bootstrapper"); + d.create().await; + bootstrapper.bootstrap_encryption_key().await.expect("key"); + let key = d.setting("encryption_key").await.expect("a key"); + assert!(!bootstrapper.has_admin_user().await.expect("read")); + + let (held, target) = lock(&d.target, true, Duration::from_secs(10), Timing::PRODUCTION) + .await + .expect("lock"); + let status = status_holding(&target, &held).await.expect("status"); + assert_eq!( + status, + CatalogStatus::Current { + version: MigrationId::LATEST.number() + } + ); + assert!( + finishes_a_stopped_init(&bootstrapper, &status) + .await + .expect("decided") + ); + let admin = write_bootstrap_rows(&bootstrapper, None, None) + .await + .expect("rows"); + bootstrapper + .record_data_connection() + .await + .expect("data connection"); + held.release().await; + assert_eq!(admin.username, "admin"); + assert!(!admin.already_existed && admin.generated_password.is_some()); + assert_eq!( + d.setting("encryption_key").await, + Some(key), + "the key is kept" + ); + assert_eq!( + Deployment::documents(&d.catalog(), "accounts").await.len(), + 1 + ); + assert_eq!( + Deployment::documents(&d.catalog(), "admin_users") + .await + .len(), + 1 + ); + assert!(bootstrapper.has_admin_user().await.expect("read")); + assert!( + !finishes_a_stopped_init(&bootstrapper, &d.status().await) + .await + .expect("decided"), + "a catalog with an admin user is refused" + ); + d.drop_databases().await; +} + #[tokio::test(flavor = "multi_thread")] async fn a_fresh_store_gets_every_migration_and_the_expected_definitions() { let Some(uri) = server() else { return }; diff --git a/crates/storage-postgres/src/bootstrapper.rs b/crates/storage-postgres/src/bootstrapper.rs index d081b1f0c..7b3eed02d 100755 --- a/crates/storage-postgres/src/bootstrapper.rs +++ b/crates/storage-postgres/src/bootstrapper.rs @@ -347,6 +347,14 @@ impl Bootstrapper for PostgresBootstrapper { }) } + async fn has_admin_user(&self) -> OpResult { + let pool = self.app_pool(&self.config.catalog_db).await?; + sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM admin_users)") + .fetch_one(&pool) + .await + .map_err(|e| OpError::Internal(format!("Check admin users: {e}"))) + } + async fn list_table_names(&self) -> OpResult> { let Ok(pool) = self.app_pool(&self.config.catalog_db).await else { return Ok(Vec::new()); diff --git a/crates/storage-postgres/tests/migrations_fresh.rs b/crates/storage-postgres/tests/migrations_fresh.rs index 2542bb88b..c088c8d84 100644 --- a/crates/storage-postgres/tests/migrations_fresh.rs +++ b/crates/storage-postgres/tests/migrations_fresh.rs @@ -380,3 +380,76 @@ async fn the_post_check_stops_a_different_shape_and_accepts_the_pre_release_one( assert_eq!(s.status().await, CatalogStatus::Current { version: LATEST }); catalog.close().await; } + +/// An `init` stopped after its migrations and its encryption key leaves a +/// catalog whose status is `Current` and that has no admin user. The next +/// `init`'s steps keep the key and write the default account and the admin +/// user; after that the catalog is not finished again, and `init` refuses it +/// as initialized. +#[tokio::test] +async fn init_finishes_a_catalog_stopped_before_its_admin_user() { + use extenddb_storage::bootstrapper::{ + BootstrapConfig, Bootstrapper, finishes_a_stopped_init, write_bootstrap_rows, + }; + use extenddb_storage_postgres::PostgresBootstrapper; + + let Some(base) = base_conn() else { + return skip("init finishes a stopped init"); + }; + let s = Scratch::new(&base, "pgm_finish").await; + let options = s.options(); + let user = options.get_username().to_owned(); + let password = url_password(&base); + let bootstrapper = PostgresBootstrapper::new(BootstrapConfig { + host: options.get_host().to_owned(), + port: options.get_port(), + admin_user: user.clone(), + admin_password: password.clone(), + app_user: user, + app_password: password.unwrap_or_default(), + catalog_db: s.catalog_name.clone(), + data_db: s.data_name.clone(), + }); + migrate(&s.target(), RELEASE, std::time::Duration::from_secs(5)) + .await + .expect("the migrations"); + bootstrapper.bootstrap_encryption_key().await.expect("key"); + let catalog = s.catalog().await; + let key = settings(&catalog).await.get("encryption_key").cloned(); + assert!(key.is_some()); + assert!(!bootstrapper.has_admin_user().await.expect("read")); + + let status = s.status().await; + assert_eq!(status, CatalogStatus::Current { version: LATEST }); + assert!( + finishes_a_stopped_init(&bootstrapper, &status) + .await + .expect("decided") + ); + let admin = write_bootstrap_rows(&bootstrapper, None, None) + .await + .expect("rows"); + bootstrapper + .record_data_connection() + .await + .expect("data connection"); + assert_eq!(admin.username, "admin"); + assert!(!admin.already_existed && admin.generated_password.is_some()); + assert_eq!(settings(&catalog).await.get("encryption_key").cloned(), key); + assert_eq!(count(&catalog, "SELECT COUNT(*) FROM accounts").await, 1); + assert_eq!(count(&catalog, "SELECT COUNT(*) FROM admin_users").await, 1); + assert!( + !finishes_a_stopped_init(&bootstrapper, &s.status().await) + .await + .expect("decided"), + "a catalog with an admin user is refused" + ); + catalog.close().await; +} + +/// The password in a `postgresql://user:password@host:port` URL. +fn url_password(url: &str) -> Option { + let rest = url.split_once("://")?.1; + let (userinfo, _) = rest.split_once('@')?; + userinfo.split_once(':').map(|(_, p)| p.to_owned()) +} diff --git a/crates/storage-sqlite/src/bootstrapper.rs b/crates/storage-sqlite/src/bootstrapper.rs index 5374db1db..d00c89424 100644 --- a/crates/storage-sqlite/src/bootstrapper.rs +++ b/crates/storage-sqlite/src/bootstrapper.rs @@ -294,6 +294,14 @@ impl Bootstrapper for SqliteBootstrapper { }) } + async fn has_admin_user(&self) -> OpResult { + let pool = self.pool().await?; + sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM admin_users)") + .fetch_one(&pool) + .await + .map_err(|e| OpError::Internal(format!("check admin users: {e}"))) + } + async fn list_table_names(&self) -> OpResult> { let Ok(pool) = self.pool().await else { return Ok(Vec::new()); @@ -398,3 +406,143 @@ mod lock_tests { let _ = std::fs::remove_dir_all(&dir); } } + +#[cfg(test)] +mod init_tests { + use std::time::Duration; + + use super::SqliteBootstrapper; + use extenddb_storage::bootstrapper::{ + Bootstrapper, finishes_a_stopped_init, write_bootstrap_rows, + }; + use extenddb_storage::migrations::{ + CatalogMigrations, CatalogStatus, MigrationId, RunKind, run_with_status, + }; + + use crate::migrations::SqliteCatalogMigrations; + + async fn rows(path: &str, sql: &str) -> Vec { + let pool = sqlx::sqlite::SqlitePoolOptions::new() + .max_connections(1) + .connect(&crate::sqlite_util::sqlite_url(path)) + .await + .expect("open"); + let found: Vec = sqlx::query_scalar(sql).fetch_all(&pool).await.expect(sql); + pool.close().await; + found + } + + /// `init` stopped after its migrations, its encryption key and its + /// default account leaves a catalog whose status is `Current` and that + /// has no admin user. The next `init`'s steps under the lock run no + /// migration, keep the key and the account, and write the admin user; + /// the `init` after that is refused as initialized. + #[tokio::test] + async fn init_finishes_a_catalog_stopped_before_its_admin_user_and_refuses_it_after() { + let dir = std::env::temp_dir().join(format!( + "extenddb-init-finish-{}", + uuid::Uuid::new_v4().simple() + )); + let path = dir.join("extenddb.sqlite").to_string_lossy().into_owned(); + let bootstrapper = SqliteBootstrapper::new(path.clone()); + let migrations = SqliteCatalogMigrations; + let target = bootstrapper.init_target(); + let timeout = Duration::from_secs(10); + + // The stopped init. + bootstrapper.create_catalog_db().await.expect("file"); + let lock = migrations.lock(&target, timeout).await.expect("lock"); + let status = migrations + .status_under_lock(&lock, &target) + .await + .expect("status"); + assert!( + !finishes_a_stopped_init(&bootstrapper, &status) + .await + .expect("decided") + ); + run_with_status(&migrations, &status, &lock, "1.0.0-test", RunKind::Create) + .await + .expect("create"); + bootstrapper.bootstrap_encryption_key().await.expect("key"); + bootstrapper + .bootstrap_default_account() + .await + .expect("account"); + migrations.release(lock).await; + let key = rows( + &path, + "SELECT value FROM settings WHERE key = 'encryption_key'", + ) + .await; + let accounts = rows(&path, "SELECT account_id FROM accounts").await; + assert_eq!((key.len(), accounts.len()), (1, 1)); + assert!(!bootstrapper.has_admin_user().await.expect("read")); + + // The next init finishes it. + let lock = migrations.lock(&target, timeout).await.expect("lock"); + let status = migrations + .status_under_lock(&lock, &target) + .await + .expect("status"); + assert_eq!( + status, + CatalogStatus::Current { + version: MigrationId::LATEST.number() + } + ); + assert!( + finishes_a_stopped_init(&bootstrapper, &status) + .await + .expect("decided") + ); + bootstrapper.hold(&lock); + let admin = write_bootstrap_rows(&bootstrapper, None, None) + .await + .expect("rows"); + bootstrapper + .record_data_connection() + .await + .expect("data connection"); + migrations.release(lock).await; + assert_eq!(admin.username, "admin"); + assert!(!admin.already_existed && admin.generated_password.is_some()); + assert!(bootstrapper.has_admin_user().await.expect("read")); + assert_eq!( + rows( + &path, + "SELECT value FROM settings WHERE key = 'encryption_key'" + ) + .await, + key, + "the key a server may already have used is kept" + ); + assert_eq!( + rows(&path, "SELECT account_id FROM accounts").await, + accounts + ); + + // The init after that is refused. + let lock = migrations.lock(&target, timeout).await.expect("lock"); + let status = migrations + .status_under_lock(&lock, &target) + .await + .expect("status"); + assert!( + !finishes_a_stopped_init(&bootstrapper, &status) + .await + .expect("decided") + ); + let refused = run_with_status(&migrations, &status, &lock, "1.0.0-test", RunKind::Create) + .await + .expect_err("refused"); + migrations.release(lock).await; + assert!( + refused + .to_string() + .starts_with("the catalog is already initialized"), + "{refused}" + ); + let _ = std::fs::remove_dir_all(&dir); + } +} diff --git a/crates/storage/src/bootstrapper.rs b/crates/storage/src/bootstrapper.rs index 500803ec5..440ec7af8 100755 --- a/crates/storage/src/bootstrapper.rs +++ b/crates/storage/src/bootstrapper.rs @@ -77,6 +77,16 @@ pub trait Bootstrapper: Send + Sync { env_password: Option<&str>, ) -> OpResult; + /// Whether the catalog's `admin_users` table or collection holds a user. + /// `init` writes the admin user last, so a catalog whose migrations are + /// all applied and that has none is one an `init` stopped in, which + /// [`finishes_a_stopped_init`] has `init` finish. The default answers + /// `true`, so on a backend that does not read the table `init` refuses + /// such a catalog as an initialized one. + async fn has_admin_user(&self) -> OpResult { + Ok(true) + } + /// List table names in the catalog (for destroy confirmation display). async fn list_table_names(&self) -> OpResult>; @@ -115,6 +125,50 @@ pub trait Bootstrapper: Send + Sync { fn generate_backend_config_section(&self) -> String; } +/// Whether `init`, holding the migration lock on a catalog whose status it +/// read under the lock as `status`, finishes that catalog instead of +/// refusing it as initialized: every migration is applied (`Current`) and +/// the catalog has no admin user, the last of the rows `init` writes, which +/// is what an `init` that stopped after its migrations leaves. `init` then +/// runs no migration and writes the bootstrap rows the catalog lacks +/// ([`write_bootstrap_rows`]). A catalog with an admin user is refused. +/// +/// # Errors +/// +/// The admin users cannot be read. +pub async fn finishes_a_stopped_init( + bootstrapper: &dyn Bootstrapper, + status: &crate::migrations::CatalogStatus, +) -> OpResult { + if !matches!(status, crate::migrations::CatalogStatus::Current { .. }) { + return Ok(false); + } + Ok(!bootstrapper.has_admin_user().await?) +} + +/// The rows a server needs before it can start, in the order `init` writes +/// them: the encryption key, the default account, the admin user. Each step +/// writes its row only when the catalog lacks it, so on a catalog an earlier +/// `init` stopped in this writes the rows that `init` did not. +/// `admin_user` and `admin_password` come from `EXTENDDB_ADMIN_USER` and +/// `EXTENDDB_ADMIN_PASSWORD`. +/// +/// # Errors +/// +/// As the backend's bootstrap steps. +pub async fn write_bootstrap_rows( + bootstrapper: &dyn Bootstrapper, + admin_user: Option<&str>, + admin_password: Option<&str>, +) -> OpResult { + bootstrapper.bootstrap_encryption_key().await?; // REQ-AUTH-010 + bootstrapper.bootstrap_default_account().await?; + // REQ-AUTH-003 + bootstrapper + .bootstrap_admin_user(admin_user, admin_password) + .await +} + use std::future::Future; use std::pin::Pin; @@ -532,7 +586,8 @@ pub mod helpers { /// new required method is added, or a defaulted one loses its default, this /// stops compiling, which is the same breakage an out-of-tree backend crate /// would hit. It also pins object safety by boxing as `dyn Bootstrapper`, and -/// checks that the defaulted `hold` is a usable no-op. +/// checks that the defaulted `hold` is a usable no-op and that the defaulted +/// `has_admin_user` keeps `init` refusing an initialized catalog. #[cfg(test)] mod out_of_tree_compat_tests { use super::*; @@ -598,6 +653,21 @@ mod out_of_tree_compat_tests { } } + /// A backend that does not read its admin users takes the default + /// `has_admin_user`, under which `init` refuses a catalog whose + /// migrations are applied instead of finishing it. + #[tokio::test] + async fn has_admin_user_defaults_to_true_so_init_refuses_a_current_catalog() { + let bootstrapper = MinimalBootstrapper; + assert!(bootstrapper.has_admin_user().await.expect("answered")); + let current = crate::migrations::CatalogStatus::Current { version: 4 }; + assert!( + !finishes_a_stopped_init(&bootstrapper, ¤t) + .await + .expect("decided") + ); + } + /// A backend that keeps no lease takes the default `hold`, which keeps /// nothing and needs no lock of its own kind. #[test] diff --git a/crates/storage/src/migrations/catalog.rs b/crates/storage/src/migrations/catalog.rs index b67daf710..4c85389c8 100644 --- a/crates/storage/src/migrations/catalog.rs +++ b/crates/storage/src/migrations/catalog.rs @@ -376,7 +376,24 @@ pub async fn run_locked( kind: RunKind, ) -> Result { let status = migrations.status_under_lock(lock, target).await?; - if let Some(refusal) = refusal(&status, kind) { + run_with_status(migrations, &status, lock, release, kind).await +} + +/// [`run_locked`] for a caller that has read the status under `lock` +/// itself, as `status`: `init`, which reads it first to decide whether it +/// finishes a catalog an earlier `init` stopped in, and so reads it once. +/// +/// # Errors +/// +/// As [`run_locked`]. +pub async fn run_with_status( + migrations: &dyn CatalogMigrations, + status: &CatalogStatus, + lock: &HeldLock, + release: &str, + kind: RunKind, +) -> Result { + if let Some(refusal) = refusal(status, kind) { return Err(refusal); } migrations.run(lock, release, kind).await diff --git a/crates/storage/src/migrations/mod.rs b/crates/storage/src/migrations/mod.rs index d84fb6fb7..f46d6a3ed 100644 --- a/crates/storage/src/migrations/mod.rs +++ b/crates/storage/src/migrations/mod.rs @@ -41,7 +41,7 @@ pub use backend::{MigrationBackend, Step, StoreRef}; pub use catalog::{ AdminRole, CatalogMigrations, HeldLock, Listed, RunKind, StatusNote, Target, before_lock, binary_catalog_label, forget_locked, installed, policy_rewrite_lines, policy_rewrite_pending, - run_locked, + run_locked, run_with_status, }; #[cfg(any(test, feature = "test-util"))] pub use checksum::check_released; From fd19d0bc7ff70e996593f325dc4ad1cc4bdaa9da Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Sun, 1 Nov 2026 16:55:32 +0000 Subject: [PATCH 21/45] fix(init): say so when init finishes a stopped init When init finishes a catalog an earlier init stopped in, it printed nothing before the bootstrap rows, so nothing in its output said why no migration ran. It now prints one line before the rows: --- Finishing a stopped init: the catalog is at migration 4 and has no admin user. A catalog with an admin user is refused with the same output as before: the line before the migrations, then the refusal that the catalog is already initialized. init_on_runner takes the function that prints its own lines, so the tests read them in order with the calls it makes. Tests: with fakes, the finishing line comes after the status and the admin user check and before the first bootstrap row; a store that is not a catalog gets the line before the migrations, then the migrations and the rows; a catalog with an admin user, and a Behind one, get the line before the migrations and the refusal text unchanged. --- crates/app/src/cmd_init.rs | 20 ++++++++++++----- crates/app/src/cmd_init_tests.rs | 38 +++++++++++++++++++++++--------- 2 files changed, 42 insertions(+), 16 deletions(-) diff --git a/crates/app/src/cmd_init.rs b/crates/app/src/cmd_init.rs index cb370dfd2..5ec78a390 100755 --- a/crates/app/src/cmd_init.rs +++ b/crates/app/src/cmd_init.rs @@ -254,7 +254,10 @@ pub async fn run(args: InitArgs, release: &str) -> anyhow::Result { let migrations = extenddb_storage::migrations::installed() .ok_or_else(|| anyhow::anyhow!("no storage backend is installed"))?; - let admin_result = init_on_runner(bootstrapper.as_ref(), migrations, release).await?; + let admin_result = init_on_runner(bootstrapper.as_ref(), migrations, release, &mut |line| { + println!("{line}"); + }) + .await?; if admin_result.already_existed { // Already printed by the bootstrap store. @@ -318,11 +321,14 @@ pub async fn run(args: InitArgs, release: &str) -> anyhow::Result { /// while this one waited). Holding the lock through the rows keeps a server /// from starting on a catalog that has no encryption key yet. The target /// carries what the bootstrapper resolved from the command line, so the -/// backend reads no flag itself. +/// backend reads no flag itself. `say` prints this function's own lines: +/// the one before the migrations, their notes, and the one that says a +/// stopped `init` is being finished, before its rows. async fn init_on_runner( bootstrapper: &dyn extenddb_storage::bootstrapper::Bootstrapper, migrations: &dyn extenddb_storage::migrations::CatalogMigrations, release: &str, + say: &mut dyn FnMut(&str), ) -> anyhow::Result { use extenddb_storage::bootstrapper::{finishes_a_stopped_init, write_bootstrap_rows}; use extenddb_storage::migrations::{RunKind, run_with_status}; @@ -337,16 +343,20 @@ async fn init_on_runner( .status_under_lock(&lock, &target) .await .map_err(|e| anyhow::anyhow!("{e}"))?; - if !finishes_a_stopped_init(bootstrapper, &status) + if finishes_a_stopped_init(bootstrapper, &status) .await .map_err(crate::cmd_migrate::op_err)? { - println!("--- Creating the catalog through its migrations..."); + say(&format!( + "--- Finishing a stopped init: {status} and has no admin user." + )); + } else { + say("--- Creating the catalog through its migrations..."); let report = run_with_status(migrations, &status, &lock, release, RunKind::Create) .await .map_err(|e| anyhow::anyhow!("{e}"))?; for note in &report.notes { - println!(" {note}"); + say(&format!(" {note}")); } crate::util::print_applied(&report, " ", "Applied", "migration"); } diff --git a/crates/app/src/cmd_init_tests.rs b/crates/app/src/cmd_init_tests.rs index 30c8ecf07..20ad1ccdc 100644 --- a/crates/app/src/cmd_init_tests.rs +++ b/crates/app/src/cmd_init_tests.rs @@ -141,7 +141,8 @@ impl Bootstrapper for FakeBootstrapper { } /// `init_on_runner` on a catalog of `status` whose admin users `admin` -/// says exist: its result and every call it made, in order. +/// says exist: its result and every call it made and line it printed +/// (`say: `), in order. async fn init(status: CatalogStatus, admin: bool) -> (anyhow::Result, Vec) { let log = Log::default(); let migrations = FakeMigrations { @@ -152,13 +153,17 @@ async fn init(status: CatalogStatus, admin: bool) -> (anyhow::Result, Ve admin, log: Arc::clone(&log), }; - let result = init_on_runner(&bootstrapper, &migrations, "1.0.0-test") + let said = Arc::clone(&log); + let mut say = move |line: &str| record(&said, &format!("say: {line}")); + let result = init_on_runner(&bootstrapper, &migrations, "1.0.0-test", &mut say) .await .map(|admin| admin.username); let calls = log.lock().unwrap().clone(); (result, calls) } +const CREATING: &str = "say: --- Creating the catalog through its migrations..."; + const ROWS: [&str; 5] = [ "hold", "encryption key", @@ -175,29 +180,37 @@ fn calls(first: &[&str], last: &[&str]) -> Vec { async fn a_store_that_is_not_a_catalog_is_created_then_given_its_rows() { let (result, log) = init(CatalogStatus::Uninitialized, false).await; assert_eq!(result.expect("init"), "admin"); - let mut expected = calls(&["lock", "status", "run the migrations"], &ROWS); + let mut expected = calls(&["lock", "status", CREATING, "run the migrations"], &ROWS); expected.push("release".to_owned()); assert_eq!(log, expected); } /// What an `init` stopped after its migrations leaves: every migration is -/// recorded and there is no admin user. `init` runs no migration and writes +/// recorded and there is no admin user. `init` runs no migration, prints +/// one line saying it finishes a stopped `init` before the rows, and writes /// the rows, each of which the backend writes only when it is missing. #[tokio::test] async fn a_current_catalog_without_an_admin_user_is_finished_without_a_migration() { - let current = CatalogStatus::Current { - version: MigrationId::LATEST.number(), - }; + let latest = MigrationId::LATEST.number(); + let current = CatalogStatus::Current { version: latest }; let (result, log) = init(current, false).await; assert_eq!(result.expect("init"), "admin"); - let mut expected = calls(&["lock", "status", "has an admin user?"], &ROWS); + let finishing = format!( + "say: --- Finishing a stopped init: the catalog is at migration {latest} and has no \ + admin user." + ); + let mut expected = calls( + &["lock", "status", "has an admin user?", finishing.as_str()], + &ROWS, + ); expected.push("release".to_owned()); assert_eq!(log, expected); } /// A catalog with an admin user, and one whose migrations are not all /// applied, are refused as initialized, with nothing run or written and the -/// lock released. +/// lock released. The output is what `init` printed before it finished +/// stopped catalogs: the line before the migrations, then the refusal. #[tokio::test] async fn a_catalog_with_an_admin_user_or_pending_migrations_is_refused() { let current = CatalogStatus::Current { @@ -215,7 +228,10 @@ async fn a_catalog_with_an_admin_user_or_pending_migrations_is_refused() { ); assert_eq!( log, - calls(&["lock", "status", "has an admin user?"], &["release"]) + calls( + &["lock", "status", "has an admin user?", CREATING], + &["release"] + ) ); let behind = CatalogStatus::Behind { @@ -231,5 +247,5 @@ async fn a_catalog_with_an_admin_user_or_pending_migrations_is_refused() { let (result, log) = init(behind, false).await; let message = result.expect_err("refused").to_string(); assert!(message.contains("already initialized"), "{message}"); - assert_eq!(log, calls(&["lock", "status"], &["release"])); + assert_eq!(log, calls(&["lock", "status", CREATING], &["release"])); } From 206b090b6e12d407072ff883442bad285449d9f9 Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Sun, 1 Nov 2026 17:28:58 +0000 Subject: [PATCH 22/45] fix(app): name the stores behind the catalog store in the migrate preview A run stopped after the catalog store leaves a deployment at migration 0 while its catalog store is at the latest migration, for example an account keyspace without migration 1. The status line and the message that serve and verify print name the stores behind the catalog store, but the migrate preview still said "The catalog is at migration 0; this binary carries 4." The preview's first line for such a catalog now names them in the same words: "Account store is at migration 0, behind the catalog store at migration 4; this binary carries 4." An upgrade, where every store waits for the same migration, still prints the catalog's migration. --- crates/app/src/cmd_migrate_runner.rs | 35 ++++++++++++----- crates/app/src/cmd_migrate_runner_tests.rs | 44 ++++++++++++++++++++++ crates/storage/src/migrations/mod.rs | 5 ++- crates/storage/src/migrations/status.rs | 4 +- 4 files changed, 75 insertions(+), 13 deletions(-) diff --git a/crates/app/src/cmd_migrate_runner.rs b/crates/app/src/cmd_migrate_runner.rs index 744987b41..ec4d4f35f 100644 --- a/crates/app/src/cmd_migrate_runner.rs +++ b/crates/app/src/cmd_migrate_runner.rs @@ -22,7 +22,8 @@ use extenddb_config as config; use extenddb_storage::migrations::advice::{self, Advice, AdviceFor}; use extenddb_storage::migrations::{ AdminRole, CatalogMigrations, CatalogStatus, Listed, MigrationError, MigrationId, Pending, - RunKind, RunReport, Target, before_lock, exit, forget_locked, run_locked, + RunKind, RunReport, Target, before_lock, behind_line, exit, forget_locked, run_locked, + stores_behind, }; use crate::cmd_migrate::MigrateArgs; @@ -227,15 +228,7 @@ async fn preview( release before 1.0 is not detected while it runs." ); } - match version { - Some(v) => { - println!("The catalog is at migration {v}; this binary carries {latest}.") - } - None => println!( - "The catalog has no migration ledger (a release before 1.0 created it); this \ - binary carries migrations 1 to {latest}." - ), - } + println!("{}", behind_summary(*version, pending, latest)); println!("Pending:"); for line in pending_lines(pending) { println!("{line}"); @@ -263,6 +256,28 @@ async fn preview( Ok(status.exit_code()) } +/// The preview's first line for a `Behind` status: the stores a stopped run +/// left behind the catalog store, when there are any, else the catalog's +/// migration, then the latest migration this binary carries. +fn behind_summary(version: Option, pending: &[Pending], latest: u32) -> String { + match (stores_behind(pending), version) { + (Some((catalog_at, behind)), _) => { + let mut line = behind_line(catalog_at, &behind); + if let Some(first) = line.get(..1).map(str::to_uppercase) { + line.replace_range(..1, &first); + } + format!("{line}; this binary carries {latest}.") + } + (None, Some(v)) => { + format!("The catalog is at migration {v}; this binary carries {latest}.") + } + (None, None) => format!( + "The catalog has no migration ledger (a release before 1.0 created it); this binary \ + carries migrations 1 to {latest}." + ), + } +} + /// The preview's lines for `pending`, one per id, kind of store and kind: /// a body that runs in several stores of one kind, as in each of /// Cassandra's account keyspaces, is one line with the number of stores. diff --git a/crates/app/src/cmd_migrate_runner_tests.rs b/crates/app/src/cmd_migrate_runner_tests.rs index 0bee9a140..feed9cb4a 100644 --- a/crates/app/src/cmd_migrate_runner_tests.rs +++ b/crates/app/src/cmd_migrate_runner_tests.rs @@ -529,3 +529,47 @@ fn the_preview_lists_a_body_for_many_stores_once() { "{lines:?}" ); } + +/// A run stopped after the catalog store leaves stores behind it, and the +/// preview's first line names them with the catalog store's migration, +/// where the catalog's own migration (0 here) would read as nothing +/// applied. An upgrade, where every store waits for the same migration, +/// keeps the catalog's migration. +#[test] +fn the_preview_names_the_stores_behind_the_catalog_store() { + let pending_in = |id: MigrationId, store: StoreKind, store_name: &str| Pending { + id: id.number(), + name: id.name(), + store, + store_name: store_name.to_owned(), + kind: Kind::Breaking, + }; + let latest = MigrationId::LATEST.number(); + let stopped = [pending_in( + MigrationId::Baseline, + StoreKind::Account, + "ks_account_1", + )]; + assert_eq!( + super::behind_summary(Some(0), &stopped, latest), + format!( + "Account store ks_account_1 is at migration 0, behind the catalog store at \ + migration {latest}; this binary carries {latest}." + ) + ); + let upgrade = [ + pending_in(MigrationId::LATEST, StoreKind::Catalog, "ks_catalog"), + pending_in(MigrationId::LATEST, StoreKind::Account, "ks_account_1"), + ]; + assert_eq!( + super::behind_summary(Some(latest - 1), &upgrade, latest), + format!( + "The catalog is at migration {}; this binary carries {latest}.", + latest - 1 + ) + ); + assert!( + super::behind_summary(None, &upgrade, latest) + .starts_with("The catalog has no migration ledger"), + ); +} diff --git a/crates/storage/src/migrations/mod.rs b/crates/storage/src/migrations/mod.rs index f46d6a3ed..aca1136e2 100644 --- a/crates/storage/src/migrations/mod.rs +++ b/crates/storage/src/migrations/mod.rs @@ -51,7 +51,10 @@ pub use ledger::{CatalogTraces, LedgerColumns, LedgerRow, Outcome}; pub use record::{LedgerWrites, in_transaction, record_rows, sql_ledger_ddl}; pub use registry::{MigrationId, RELEASED_THROUGH}; pub use runner::Runner; -pub use status::{AppliedStep, CatalogStatus, MigrationError, Pending, RunReport, exit}; +pub use status::{ + AppliedStep, CatalogStatus, MigrationError, Pending, RunReport, behind_line, exit, + stores_behind, +}; #[cfg(any(test, feature = "test-util"))] pub use table::refuse_additive; pub use table::{Body, Kind, Migration, StoreBody, StoreKind}; diff --git a/crates/storage/src/migrations/status.rs b/crates/storage/src/migrations/status.rs index 2ad04f0ba..8f8c8b07c 100644 --- a/crates/storage/src/migrations/status.rs +++ b/crates/storage/src/migrations/status.rs @@ -72,7 +72,7 @@ pub struct Pending { /// catalog store lacks too is not behind it, as on an upgrade, where every /// store waits for the same migration. `None` when no store is behind. #[must_use] -pub(crate) fn stores_behind(pending: &[Pending]) -> Option<(u32, Vec<(&Pending, u32)>)> { +pub fn stores_behind(pending: &[Pending]) -> Option<(u32, Vec<(&Pending, u32)>)> { let catalog_at = pending .iter() .filter(|p| p.store == StoreKind::Catalog) @@ -99,7 +99,7 @@ pub(crate) fn stores_behind(pending: &[Pending]) -> Option<(u32, Vec<(&Pending, /// migration 4`, or for several, `account store x (at migration 0) and data /// store y (at migration 3) are behind the catalog store at migration 4`. #[must_use] -pub(crate) fn behind_line(catalog_at: u32, behind: &[(&Pending, u32)]) -> String { +pub fn behind_line(catalog_at: u32, behind: &[(&Pending, u32)]) -> String { let named = |p: &Pending| format!("{} store {}", p.store.as_str(), p.store_name); if let [(p, at)] = behind { return format!( From 9e87767c822ff2ce813f33d31b382d5198f9c363 Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Sun, 1 Nov 2026 18:52:05 +0000 Subject: [PATCH 23/45] test(devtools): check that a server with a lost registration answers 503 before it expires check-cassandra-catalog-lock and check-mongodb-catalog-lock paused the database and required the server to exit 1 within 45 s and 31 s, the rule before servers on these backends registered again. A server now answers ServiceUnavailable once its registration has not been renewed for 30 s, tries to register again for 30 s, and then stops, so both checks failed on a server that behaved as the upgrade manual says. Both checks now poll /health while the database is paused and require a 503 before the registration's 45 s expiry, the point from which a migrator could find no server, then exit 1 and the PID file removed. A helper in catalog_checks.py polls the server for both. --- devtools/catalog_checks.py | 22 ++++++++++++++ devtools/check-cassandra-catalog-lock | 41 ++++++++++++++++----------- devtools/check-mongodb-catalog-lock | 41 +++++++++++++++++---------- 3 files changed, 72 insertions(+), 32 deletions(-) diff --git a/devtools/catalog_checks.py b/devtools/catalog_checks.py index ddfa8d94b..b687def35 100644 --- a/devtools/catalog_checks.py +++ b/devtools/catalog_checks.py @@ -86,6 +86,28 @@ def healthy(url: str, proc: subprocess.Popen | None = None, seconds: float = 60) return False +def watch_lost_registration( + proc: subprocess.Popen, url: str, seconds: float +) -> tuple[float | None, int | None, float]: + """Poll `url` (a server's /health) every 0.5 s, while the server's + database does not answer, until `proc` exits or `seconds` pass. Returns + the seconds to the first 503 (None without one), the exit code (None + while the server runs) and the seconds waited.""" + started = time.monotonic() + first_503 = None + while time.monotonic() - started < seconds and proc.poll() is None: + probe = subprocess.run( + ["curl", "-sk", "-o", "/dev/null", "-w", "%{http_code}", "--max-time", "2", url], + capture_output=True, + text=True, + check=False, + ) + if first_503 is None and probe.stdout.strip() == "503": + first_503 = time.monotonic() - started + time.sleep(0.5) + return first_503, proc.poll(), time.monotonic() - started + + def start(cmd: list[str], env: dict | None, log: str, cwd: str | None = None) -> subprocess.Popen: """Start `cmd` in the background with its output in the file `log`.""" with open(log, "w", encoding="utf-8") as out: diff --git a/devtools/check-cassandra-catalog-lock b/devtools/check-cassandra-catalog-lock index 6df71561d..aef75773d 100755 --- a/devtools/check-cassandra-catalog-lock +++ b/devtools/check-cassandra-catalog-lock @@ -32,8 +32,9 @@ with `init` and removes them at the end. and neither 2, which is not convergent, nor 3 and 4, which do not apply on Cassandra; migrate --forget 2 and the migrate --yes that applies 2 again run, and serve runs; -- a server that cannot renew its registration for 30 s, with the node - paused, stops with exit 1 and removes its PID file. +- with the node paused, a server that cannot renew its registration answers + 503 before the registration's 45 s TTL ends, then tries to register again + for 30 s, stops with exit 1 and removes its PID file. init's HOME points into a temporary directory, so its TLS certificate is not written to the caller's home. The server binds 127.0.0.1 only. @@ -47,7 +48,7 @@ import tempfile import time import tomllib -from catalog_checks import expect_exit, finish, healthy, report, run, start, stop +from catalog_checks import expect_exit, finish, healthy, report, run, start, stop, watch_lost_registration ADMIN_USER = os.environ.get("CASSANDRA_ADMIN_USER", "cassandra") ADMIN_PASSWORD = os.environ.get("CASSANDRA_ADMIN_PASSWORD", "cassandra") @@ -282,25 +283,31 @@ def main(binary: str, container: str, contact_point: str, port: int) -> int: if not report(healthy(f"https://127.0.0.1:{port}/health", server), "serve once the lock is free"): return finish() report(os.path.exists(pid_file), f"the server wrote its PID file {pid_file}") - # The node stops answering: the server cannot renew its registration - # and must stop before the row's 45 s TTL ends. + # The node stops answering, so the server cannot renew its + # registration. It must answer 503 before the row's 45 s TTL ends, + # when a migrator could find no server, and stop with exit 1 once it + # cannot register again. subprocess.run(["docker", "pause", container], check=True, capture_output=True) paused = True - started = time.monotonic() - try: - code = server.wait(timeout=60) - except subprocess.TimeoutExpired: - code = None - waited = time.monotonic() - started + first_503, code, waited = watch_lost_registration( + server, f"https://127.0.0.1:{port}/health", 120 + ) subprocess.run(["docker", "unpause", container], check=True, capture_output=True) paused = False with open(log, encoding="utf-8") as f: - report( - code == 1 and waited < 45, - f"a server that cannot renew its registration stops with exit {code} after " - f"{waited:.0f} s", - f.read()[-3000:], - ) + output = f.read()[-3000:] + report( + first_503 is not None and first_503 < 45, + "the server answered 503 " + + ("never" if first_503 is None else f"{first_503:.0f} s after the node stopped answering") + + ", before its registration's 45 s TTL ends", + output, + ) + report( + code == 1, + f"a server that cannot register again stops with exit {code} after {waited:.0f} s", + output, + ) report(not os.path.exists(pid_file), "the stopped server removed its PID file") finally: if paused: diff --git a/devtools/check-mongodb-catalog-lock b/devtools/check-mongodb-catalog-lock index c369a858f..a81bc6f64 100755 --- a/devtools/check-mongodb-catalog-lock +++ b/devtools/check-mongodb-catalog-lock @@ -21,9 +21,10 @@ extenddb-check-lock-, published on 127.0.0.1 only: prints, run exactly as printed, applies what is missing; - a server killed with SIGKILL keeps migrate --yes waiting until its registration expires, at most 45 s after its last renewal; -- a server whose database stops answering exits 1 within 30 s, because its - registration counts as lost 30 s after its last renewal, which it sent - before the database stopped answering, and removes its PID file. +- a server whose database stops answering answers 503 before its + registration's 45 s expiry, since the registration counts as lost 30 s + after its last successful renewal, then tries to register again for 30 s, + exits 1 and removes its PID file. Everything runs in a new temporary directory, with HOME pointed into it so init's TLS certificate is not written to the caller's home. The server @@ -34,12 +35,22 @@ import shlex import shutil import signal import socket -import subprocess import sys import tempfile import time -from catalog_checks import docker, expect_exit, finish, healthy, mongosh, report, start, start_mongo, stop +from catalog_checks import ( + docker, + expect_exit, + finish, + healthy, + mongosh, + report, + start, + start_mongo, + stop, + watch_lost_registration, +) def live_servers(container: str, port: int) -> int: @@ -141,24 +152,24 @@ def main(binary: str, mongo_port: int, api_port: int) -> int: else: report(False, "the server did not start for the kill check") - # A server whose database stops answering exits 1 and removes its - # PID file. + # A server whose database stops answering answers 503 before its + # registration's 45 s expiry, when a migrator could find no server, + # then exits 1 once it cannot register again, and removes its PID + # file. server = start(serve + ["--write-pid-file"], env, "serve-paused.log") servers.append(server) pid_file = os.path.join(work, "run", f"extenddb-{api_port}.pid") if healthy(url, server, 60) and report(os.path.exists(pid_file), "the server wrote its PID file"): docker("pause", container) - paused = time.monotonic() - try: - server.wait(timeout=90) - except subprocess.TimeoutExpired: - pass - took = time.monotonic() - paused + first_503, code, took = watch_lost_registration(server, url, 120) docker("unpause", container) with open("serve-paused.log", encoding="utf-8") as f: log = f.read() - report(server.returncode == 1 and took <= 31, - f"the server exited {server.returncode} {took:.0f} s after its database stopped answering", log) + report(first_503 is not None and first_503 < 45, + "the server answered 503 " + + ("never" if first_503 is None else f"{first_503:.0f} s after its database stopped answering") + + ", before its registration's 45 s expiry", log) + report(code == 1, f"the server exited {code} {took:.0f} s after its database stopped answering", log) report("registration" in log, "its last line names the lost registration", log) report(not os.path.exists(pid_file), "the server removed its PID file") else: From c67275f50940a81bb03d1dcc65129756ab3a6436 Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Sun, 1 Nov 2026 19:22:31 +0000 Subject: [PATCH 24/45] test(storage-cassandra): keep other account runs out of the init finish test The test that finishes a stopped init runs the default account's migrations under the migration lock's lease, as init does. A process runs its account migrations one at a time (#453), and the tests of the migrations binary run at once, so the test's account run waits for other tests' account runs. That wait records no progress on the lease, so a wait longer than the lease's 30 s stall stops the renewal. In the full binary on two nodes, with each account run's wait printed, the test's run waited 87.3 s and found its lease lapsed, and the test failed with "the migration lock ... was not renewed in time". Run alone on two nodes, it passed in 48 s. The tests that run an account migration now take a lock of the test binary for reading, and this test takes it for writing: it starts once the account runs in flight have ended, and no other starts until it ends. init's own process runs no other account migration, so the product code is unchanged. The comment on migrate_account_under now says that its wait for another account run records no progress on the lease. On a two-node cluster, tests/migrations.rs had 17 passed and 2 failed before (this test, and the timing test of a killed holder's lock, which passed alone), and 19 passed with this commit. --- crates/storage-cassandra/src/migrations/mod.rs | 4 +++- crates/storage-cassandra/tests/migrations.rs | 17 +++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/crates/storage-cassandra/src/migrations/mod.rs b/crates/storage-cassandra/src/migrations/mod.rs index aa272e30a..79054a703 100644 --- a/crates/storage-cassandra/src/migrations/mod.rs +++ b/crates/storage-cassandra/src/migrations/mod.rs @@ -187,7 +187,9 @@ pub async fn migrate_account( /// [`migrate_account`] under the lease of the migration lock the caller /// holds: `init`'s default account. Every write is checked against the /// lease, every completed request records progress on it, and each schema -/// statement gets the bound of one sent under the lock. +/// statement gets the bound of one sent under the lock. The run first waits +/// for any other account run of this process, and that wait records no +/// progress on the lease; `init`'s process runs no other. /// /// # Errors /// diff --git a/crates/storage-cassandra/tests/migrations.rs b/crates/storage-cassandra/tests/migrations.rs index 6d65a5120..99ca888c6 100644 --- a/crates/storage-cassandra/tests/migrations.rs +++ b/crates/storage-cassandra/tests/migrations.rs @@ -33,6 +33,15 @@ use extenddb_storage_cassandra::migrations::{ }; use migrations_support::{TIMEOUT, cql, keyspace, new_prefix, pinned_session, session}; +/// Taken for reading by each test that runs an account migration, and for +/// writing by the test that runs `init`'s default account under the +/// migration lock. A process runs its account migrations one at a time +/// (#453) and the tests of this binary run at once, so a run under the +/// lock's lease would otherwise wait for other tests' account runs, and the +/// wait records no progress on the lease: on two nodes it outlasted the +/// lease's 30 s stall. `init`'s own process runs no other account migration. +static ACCOUNT_RUNS: tokio::sync::RwLock<()> = tokio::sync::RwLock::const_new(()); + /// A catalog keyspace at migration 4 and one account keyspace. async fn deployment(session: &Arc, prefix: &str) -> String { keyspace(session, &format!("{prefix}_catalog")).await; @@ -49,9 +58,11 @@ async fn deployment(session: &Arc, prefix: &str) -> String { ), ) .await; + let runs = ACCOUNT_RUNS.read().await; migrate_account(session.clone(), prefix, &account, "test") .await .expect("migrate the account"); + drop(runs); account } @@ -552,6 +563,7 @@ async fn create_account_runs_the_account_migrations_through_the_runner() { if helpers::skip_without_cassandra() { return; } + let _runs = ACCOUNT_RUNS.read().await; let session = session().await; let (prefix, _drops) = new_prefix("account"); keyspace(&session, &format!("{prefix}_catalog")).await; @@ -652,6 +664,7 @@ async fn a_create_account_stopped_before_its_row_leaves_a_current_catalog_a_retr if helpers::skip_without_cassandra() { return; } + let _runs = ACCOUNT_RUNS.read().await; let session = session().await; let (prefix, _drops) = new_prefix("stopacct"); keyspace(&session, &format!("{prefix}_catalog")).await; @@ -788,6 +801,7 @@ async fn the_default_account_step_run_again_keeps_one_account_and_completes_its_ if helpers::skip_without_cassandra() { return; } + let _runs = ACCOUNT_RUNS.read().await; let session = session().await; let (prefix, mut drops) = new_prefix("defacct"); keyspace(&session, &format!("{prefix}_catalog")).await; @@ -856,6 +870,9 @@ async fn init_finishes_a_catalog_stopped_before_its_admin_user() { if helpers::skip_without_cassandra() { return; } + // Both inits run the default account's migrations under the lock's + // lease, so no other test's account run may be in flight. + let _runs = ACCOUNT_RUNS.write().await; let session = session().await; let (prefix, mut drops) = new_prefix("finish"); let migrations = CassandraCatalogMigrations; From ca132671e61b94b9ec4d0e2c418040168724d4ff Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Sun, 1 Nov 2026 21:12:00 +0000 Subject: [PATCH 25/45] fix(server): stop the background workers while the serving registration is lost Since 1c3b2bf60 a Cassandra or MongoDB server whose registration counts as lost refuses new requests and registers again for up to 30 s, but its background workers went on: they stopped only when the server gave up, up to 30 s after the loss, while the registration's row or document expires 45 s after the last renewal, at most 15 s after the loss is reported. A migration that took the catalog then could run beside a worker whose writes still landed, for example when the server's registration requests cannot reach the database and the engine's connections can. A PostgreSQL server's workers went on the same way, and the database frees its key as soon as the session ends. The server's workers and the backend's now run as one set on a token of their own. At a loss the registration supervisor stops the set before it tries to register again: the token fires, and the workers still running after 5 s are aborted. Once the server holds a new registration and the catalog's status is the one it started with, it starts a new set and serves again. A worker that an abort cannot end, one that blocks its thread, stops the server instead. The stop at shutdown works the same way, and the registration is released only when every worker has ended; otherwise it is left to expire. MongoDB's four workers and Cassandra's GSI propagation, control-plane, GSI delay, transaction recovery and GSI backfill recovery workers took no shutdown token, so the drain timed out after 5 s and the release came while they ran. Each of them now returns when its token fires, and the GSI propagation workers are among the drained handles. The upgrade manual says what a server stops while it holds no registration. Tests: the supervisor stops the workers at a loss before its first attempt and starts them after a new registration; a worker that does not stop stops the server with no attempt; a stop aborts a worker that ignores its token before the release, and keeps the registration when a worker blocks its thread; a stopped set starts again. Each fails with its part of the change reverted. --- crates/server/src/registration.rs | 190 +++++-- crates/server/src/serve.rs | 488 ++++++++++++++---- crates/storage-cassandra/src/lib.rs | 38 +- crates/storage-cassandra/src/workers.rs | 91 ++-- .../storage-mongodb/src/backup_live_tests.rs | 5 +- crates/storage-mongodb/src/lib.rs | 14 +- crates/storage-mongodb/src/snapshot_engine.rs | 7 +- crates/storage-mongodb/src/ttl_worker.rs | 47 +- crates/storage/src/hooks.rs | 39 +- docs/manuals/07-upgrade-manual.md | 6 +- 10 files changed, 694 insertions(+), 231 deletions(-) diff --git a/crates/server/src/registration.rs b/crates/server/src/registration.rs index bcdc5d075..18383752f 100644 --- a/crates/server/src/registration.rs +++ b/crates/server/src/registration.rs @@ -7,17 +7,18 @@ //! A migration can start on the catalog once the registration is gone, so //! from that moment the server admits no new request: each one gets //! `ServiceUnavailable` (HTTP 503), which the AWS SDKs retry, and `/health` -//! answers the same. Requests already running go on. The server asks the -//! backend to register again at once and then every second, and serves -//! again when the backend reports a new registration with the catalog's -//! status unchanged. When the backend refuses (a migration holds the -//! migration lock or waits for the servers to stop, or the status changed), -//! or 30 s pass without a registration, the server asks for a stop: the -//! workers' shutdown token fires at once, and `serve` then stops the way a -//! shutdown signal stops it (the listener drains, letting requests already -//! running finish, the PID file is removed) and exits 1. A signal that came -//! first decides the exit status, also when a stop is asked for during its -//! drain. +//! answers the same. It also stops its background workers: their token +//! fires, and the ones still running after 5 s are aborted. Requests already +//! running go on. The server then asks the backend to register again at once +//! and then every second, and when the backend reports a new registration +//! with the catalog's status unchanged, it starts a new set of workers and +//! serves again. When the backend refuses (a migration holds the migration +//! lock or waits for the servers to stop, or the status changed), when 30 s +//! pass without a registration, or when a worker could not be stopped, the +//! server asks for a stop: `serve` then stops the way a shutdown signal +//! stops it (the listener drains, letting requests already running finish, +//! the PID file is removed) and exits 1. A signal that came first decides +//! the exit status, also when a stop is asked for during its drain. //! //! The server learns of a loss at its backend's next renewal, and serves as //! before until then; a migration that takes the catalog in that time runs @@ -38,6 +39,18 @@ use extenddb_storage::hooks::{Reregistration, ServingRegistrar}; /// the Cassandra and MongoDB backends give a registration they cannot renew. pub(crate) const WINDOW: Duration = Duration::from_secs(30); +/// The server's background workers, which the supervisor stops while the +/// server holds no registration and starts again once it holds a new one. +#[async_trait::async_trait] +pub(crate) trait WorkerControl: Send + Sync { + /// Stop the workers that run now, and report whether every worker the + /// server started has ended. + async fn stop(&self) -> bool; + + /// Start a new set of workers. + async fn start(&self); +} + /// The pause between two attempts to register again. const RETRY: Duration = Duration::from_secs(1); @@ -126,13 +139,15 @@ pub(crate) async fn refuse_while_unregistered( } /// Watch the registration until `shutdown` fires: on each loss refuse new -/// requests and register again within [`WINDOW`], or ask for a stop. +/// requests, stop the workers and register again within [`WINDOW`], or ask +/// for a stop. pub(crate) async fn supervise( registrar: Arc, admission: Admission, shutdown: CancellationToken, + workers: Arc, ) { - supervise_within(registrar, admission, shutdown, WINDOW, RETRY).await; + supervise_within(registrar, admission, shutdown, workers, WINDOW, RETRY).await; } /// How one loss ended. @@ -148,6 +163,7 @@ async fn supervise_within( registrar: Arc, admission: Admission, shutdown: CancellationToken, + workers: Arc, window: Duration, retry: Duration, ) { @@ -158,28 +174,41 @@ async fn supervise_within( }; admission.refuse(); tracing::warn!( - "{reason}; new requests get ServiceUnavailable while the server registers again, for \ - up to {} s", + "{reason}; new requests get ServiceUnavailable and the background workers stop while \ + the server registers again, for up to {} s", window.as_secs() ); + // A migration can take the catalog once the lost registration + // expires, so the workers stop now, before it can, and start again + // only under a new registration. + if !workers.stop().await { + ask_for_stop( + &admission, + &shutdown, + format!( + "the server stops: {reason}, and a background worker did not stop, so it does \ + not register again" + ), + ); + return; + } match register_again(registrar.as_ref(), &shutdown, window, retry).await { Outcome::Registered { attempts, after } => { + workers.start().await; admission.admit(); tracing::info!( "registered again with the catalog after {} ms and {attempts} attempt(s), and \ - its status is the one the server started with; serving requests again", + its status is the one the server started with; the background workers run \ + again and requests are served", after.as_millis() ); } Outcome::Stop(why) => { - let message = - format!("the server stops: {reason}, and it could not register again: {why}"); - tracing::error!("{message}"); - // A migration may hold the catalog now: the background - // workers stop at once, before the listener's drain, which - // lets requests already running finish. - shutdown.cancel(); - admission.stop(message); + ask_for_stop( + &admission, + &shutdown, + format!("the server stops: {reason}, and it could not register again: {why}"), + ); return; } Outcome::Shutdown => return, @@ -187,6 +216,13 @@ async fn supervise_within( } } +/// Log `message`, fire the shutdown token and ask `serve` to stop. +fn ask_for_stop(admission: &Admission, shutdown: &CancellationToken, message: String) { + tracing::error!("{message}"); + shutdown.cancel(); + admission.stop(message); +} + /// Attempts until one registers, the backend refuses, `window` passes or /// `shutdown` fires. Each attempt is bounded by the time left. async fn register_again( @@ -252,7 +288,47 @@ mod tests { use tokio::sync::Notify; use tower::ServiceExt; - use super::{Admission, refuse_while_unregistered, supervise_within}; + use super::{Admission, WorkerControl, refuse_while_unregistered, supervise_within}; + + /// Workers that record when the supervisor stops and starts them, and + /// whether the server was refusing requests then; `stops` says whether + /// a stop ends them. + struct Recorded { + events: Mutex>, + admission: OnceLock, + stops: bool, + } + + impl Recorded { + fn new(stops: bool) -> Arc { + Arc::new(Self { + events: Mutex::new(Vec::new()), + admission: OnceLock::new(), + stops, + }) + } + + fn record(&self, event: &'static str) { + let refusing = self.admission.get().is_some_and(Admission::refusing); + self.events.lock().unwrap().push((event, refusing)); + } + + fn events(&self) -> Vec<(&'static str, bool)> { + self.events.lock().unwrap().clone() + } + } + + #[async_trait] + impl WorkerControl for Recorded { + async fn stop(&self) -> bool { + self.record("stop"); + self.stops + } + + async fn start(&self) { + self.record("start"); + } + } /// A registrar whose losses a test fires, and whose attempts answer from /// a script; an empty script answers `Unavailable`, and `None` in it is @@ -315,14 +391,24 @@ mod tests { fn start( registrar: &Arc, window: Duration, + ) -> (Admission, CancellationToken, tokio::task::JoinHandle<()>) { + start_with(registrar, window, &Recorded::new(true)) + } + + fn start_with( + registrar: &Arc, + window: Duration, + workers: &Arc, ) -> (Admission, CancellationToken, tokio::task::JoinHandle<()>) { let admission = Admission::default(); let _ = registrar.admission.set(admission.clone()); + let _ = workers.admission.set(admission.clone()); let shutdown = CancellationToken::new(); let task = tokio::spawn(supervise_within( Arc::clone(registrar) as Arc, admission.clone(), shutdown.clone(), + Arc::clone(workers) as Arc, window, Duration::from_millis(20), )); @@ -336,7 +422,8 @@ mod tests { Some(Reregistration::Unavailable("connection refused".to_owned())), Some(Reregistration::Registered), ]); - let (admission, shutdown, task) = start(®istrar, Duration::from_secs(5)); + let workers = Recorded::new(true); + let (admission, shutdown, task) = start_with(®istrar, Duration::from_secs(5), &workers); assert!(!admission.refusing()); registrar.lose.notify_one(); until("three attempts", || registrar.attempts() == 3).await; @@ -346,6 +433,9 @@ mod tests { [true, true, true], "requests were admitted while the server held no registration" ); + // The workers stopped while new requests were refused, before the + // first attempt, and started again under the new registration. + assert_eq!(workers.events(), [("stop", true), ("start", true)]); assert_eq!(admission.stop_reason(), None); // A second loss is handled the same way. registrar @@ -360,27 +450,61 @@ mod tests { registrar.refusing_at_attempts.lock().unwrap()[3], "requests were admitted while the server held no registration" ); + assert_eq!( + workers.events(), + [ + ("stop", true), + ("start", true), + ("stop", true), + ("start", true) + ] + ); shutdown.cancel(); task.await.unwrap(); assert_eq!(admission.stop_reason(), None); } + /// A worker that a stop could not end may still write when a migration + /// takes the catalog, so the server stops instead of registering again. + #[tokio::test] + async fn a_worker_that_does_not_stop_stops_the_server_without_an_attempt() { + let registrar = Scripted::answering(vec![Some(Reregistration::Registered)]); + let workers = Recorded::new(false); + let (admission, shutdown, task) = start_with(®istrar, Duration::from_secs(5), &workers); + registrar.lose.notify_one(); + tokio::time::timeout(Duration::from_secs(10), admission.stop_requested()) + .await + .expect("a stop is asked for"); + task.await.unwrap(); + assert!(shutdown.is_cancelled()); + assert_eq!(registrar.attempts(), 0, "it registered again"); + assert_eq!(workers.events(), [("stop", true)]); + let reason = admission.stop_reason().unwrap(); + assert!( + reason.contains("the session ended") && reason.contains("did not stop"), + "{reason}" + ); + assert!( + admission.refusing(), + "admitting after a worker kept running" + ); + } + #[tokio::test] async fn a_refusal_asks_for_a_stop_naming_it() { let registrar = Scripted::answering(vec![Some(Reregistration::Refused( "a migration is running on this catalog".to_owned(), ))]); - let (admission, shutdown, task) = start(®istrar, Duration::from_secs(5)); + let workers = Recorded::new(true); + let (admission, shutdown, task) = start_with(®istrar, Duration::from_secs(5), &workers); registrar.lose.notify_one(); tokio::time::timeout(Duration::from_secs(10), admission.stop_requested()) .await .expect("a stop is asked for"); - // A migration may hold the catalog by now: the background workers' - // token fires with the stop, before the listener drains. - assert!( - shutdown.is_cancelled(), - "the workers run on beside the migration" - ); + // The workers stopped at the loss and do not start again, and the + // shutdown token fires with the stop, before the listener drains. + assert_eq!(workers.events(), [("stop", true)]); + assert!(shutdown.is_cancelled(), "the shutdown token did not fire"); task.await.unwrap(); let reason = admission.stop_reason().unwrap(); assert!( diff --git a/crates/server/src/serve.rs b/crates/server/src/serve.rs index 68bd54a28..37cc11f0d 100644 --- a/crates/server/src/serve.rs +++ b/crates/server/src/serve.rs @@ -13,6 +13,7 @@ use std::net::TcpListener; use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; use std::path::PathBuf; @@ -27,6 +28,7 @@ use tracing_subscriber::{ }; use crate::AppState; +use crate::registration::WorkerControl; use crate::workers; /// Build provenance of the deployed binary. @@ -665,66 +667,89 @@ async fn serve_inner(params: ServeParams, port: u16) -> anyhow::Result<()> { admission: admission.clone(), }; - // Workers run until `shutdown` is cancelled, which happens after the HTTP - // server stops accepting. Handles are collected so shutdown can drain them - // (the metrics flush worker persists its final bucket on the way out) - // instead of leaving the work to a runtime drop. + // The server's own workers and the backend's run as one set, each set on + // a child token of `shutdown`, which is cancelled after the HTTP server + // stops accepting. The registration supervisor stops the set while the + // server holds no registration and starts a new one once it holds one. + // Handles are kept so a stop can drain them (the metrics flush worker + // persists its final bucket on the way out) instead of leaving the work + // to a runtime drop. let shutdown = CancellationToken::new(); - let mut worker_handles = vec![ - // D-22: Poll log_level from the settings table. - tokio::spawn(workers::poll_log_level( - catalog_store.clone(), - reload_handle.clone(), - app_config.logging.level.clone(), - shutdown.clone(), - )), - // Poll the throttling_enabled runtime setting. - tokio::spawn(workers::poll_throttling_enabled( - catalog_store.clone(), - throttle, - config_throttling, - shutdown.clone(), - )), - // Metrics pruning and flushing. - tokio::spawn(workers::metrics_prune_worker( - metrics.clone(), - shutdown.clone(), - )), - tokio::spawn(workers::metrics_flush_worker( - metrics.clone(), - catalog_store.clone(), - shutdown.clone(), - )), - // Clean up old login attempt records. - tokio::spawn(workers::login_attempt_cleanup_worker( - catalog_store.clone(), - shutdown.clone(), - )), - // Phase 11a: Warn about approximate consumed capacity. - tokio::spawn(workers::capacity_warning_worker(shutdown.clone())), - ]; - - // Spawn backend-specific workers via runtime hooks - if let Some(hooks) = runtime_hooks { - let worker_ctx = extenddb_storage::WorkerContext { - metrics: metrics.clone(), - catalog_store: catalog_store.clone(), - reload_handle: reload_handle.clone(), - config_log_level: app_config.logging.level.clone(), - shutdown: shutdown.clone(), - }; - worker_handles.extend(hooks.spawn_workers(&worker_ctx).await); - } + let hooks: Option> = runtime_hooks.map(Arc::from); + let spawn: SpawnWorkers = { + let catalog_store = catalog_store.clone(); + let reload_handle = reload_handle.clone(); + let config_log_level = app_config.logging.level.clone(); + let metrics = metrics.clone(); + Box::new(move |token: CancellationToken| { + let catalog_store = catalog_store.clone(); + let reload_handle = reload_handle.clone(); + let config_log_level = config_log_level.clone(); + let throttle = throttle.clone(); + let metrics = metrics.clone(); + let hooks = hooks.clone(); + futures::FutureExt::boxed(async move { + let mut handles = vec![ + // D-22: Poll log_level from the settings table. + tokio::spawn(workers::poll_log_level( + catalog_store.clone(), + reload_handle.clone(), + config_log_level.clone(), + token.clone(), + )), + // Poll the throttling_enabled runtime setting. + tokio::spawn(workers::poll_throttling_enabled( + catalog_store.clone(), + throttle, + config_throttling, + token.clone(), + )), + // Metrics pruning and flushing. + tokio::spawn(workers::metrics_prune_worker( + metrics.clone(), + token.clone(), + )), + tokio::spawn(workers::metrics_flush_worker( + metrics.clone(), + catalog_store.clone(), + token.clone(), + )), + // Clean up old login attempt records. + tokio::spawn(workers::login_attempt_cleanup_worker( + catalog_store.clone(), + token.clone(), + )), + // Phase 11a: Warn about approximate consumed capacity. + tokio::spawn(workers::capacity_warning_worker(token.clone())), + ]; + // Backend-specific workers via runtime hooks. + if let Some(hooks) = hooks { + let worker_ctx = extenddb_storage::WorkerContext { + metrics, + catalog_store, + reload_handle, + config_log_level, + shutdown: token, + }; + handles.extend(hooks.spawn_workers(&worker_ctx).await); + } + handles + }) + }) + }; + let background = Arc::new(BackgroundWorkers::new(spawn, &shutdown)); + background.start().await; // A backend whose registration the database can end: the supervisor - // refuses requests while it is lost, and asks for a stop when it cannot - // be taken again. It is drained with the other workers. - if let Some(registrar) = registrar { - worker_handles.push(tokio::spawn(crate::registration::supervise( + // refuses requests and stops the workers while it is lost, and asks for + // a stop when it cannot be taken again. + let supervisor = registrar.map(|registrar| { + tokio::spawn(crate::registration::supervise( registrar, admission.clone(), shutdown.clone(), - ))); - } + Arc::clone(&background) as Arc, + )) + }); let tls_config = if tls_enabled { let cert_path = config::expand_tilde(&app_config.server.tls.cert_path); @@ -739,10 +764,22 @@ async fn serve_inner(params: ServeParams, port: u16) -> anyhow::Result<()> { let server_result = crate::start_server(listener, state, pid_file, tls_config).await; - // The HTTP server has stopped accepting; drain the workers so in-flight - // cycles finish and the metrics flush worker writes its final bucket, - // then give up the serving registration. - drain_then_release(&shutdown, worker_handles, held.as_deref()).await; + // The HTTP server has stopped accepting. The supervisor returns once the + // token fires; then the workers stop, so in-flight cycles finish and the + // metrics flush worker writes its final bucket, and the serving + // registration is given up once every one of them has ended. + shutdown.cancel(); + if let Some(supervisor) = supervisor + && tokio::time::timeout(SUPERVISOR_TIMEOUT, supervisor) + .await + .is_err() + { + tracing::warn!( + "The registration supervisor did not return within {}s", + SUPERVISOR_TIMEOUT.as_secs() + ); + } + stop_then_release(&background, held.as_deref()).await; server_result?; @@ -758,22 +795,168 @@ async fn serve_inner(params: ServeParams, port: u16) -> anyhow::Result<()> { Ok(()) } -/// Drain the workers, then release the serving registration: a migration -/// that waits for the servers starts once the registration is gone, so -/// releasing it first would let the migration run beside a worker that -/// still writes to the catalog. The release is bounded, like the drain. -async fn drain_then_release( - shutdown: &CancellationToken, +/// How long `serve` waits for the registration supervisor once the shutdown +/// token fires: longer than a stop of the workers, which the supervisor may +/// be running then. +const SUPERVISOR_TIMEOUT: Duration = Duration::from_secs(10); + +/// How long a stop waits for the workers after their token fires before it +/// aborts the ones still running. +const DRAIN_TIMEOUT: Duration = Duration::from_secs(5); + +/// How long a stop waits for the aborted workers to end. +const ABORT_TIMEOUT: Duration = Duration::from_secs(1); + +/// Starts one set of background workers on the token it is given, and +/// returns their handles. +type SpawnWorkers = Box< + dyn Fn( + CancellationToken, + ) -> futures::future::BoxFuture<'static, Vec>> + + Send + + Sync, +>; + +/// The set of workers that runs now, with its token. +struct WorkerSet { + token: CancellationToken, handles: Vec>, +} + +/// The server's background workers, its own and the backend's, which start +/// and stop as one set. `serve` starts the first set and stops the one that +/// runs at shutdown; the registration supervisor stops the set while the +/// server holds no registration and starts a new one once it holds one. +/// Each set runs on a child token of the server's shutdown token. +struct BackgroundWorkers { + spawn: SpawnWorkers, + shutdown: CancellationToken, + running: tokio::sync::Mutex>, + /// A stop could not end every worker of its set. + left_running: AtomicBool, + drain: Duration, + abort: Duration, +} + +impl BackgroundWorkers { + fn new(spawn: SpawnWorkers, shutdown: &CancellationToken) -> Self { + Self::with_bounds(spawn, shutdown, DRAIN_TIMEOUT, ABORT_TIMEOUT) + } + + fn with_bounds( + spawn: SpawnWorkers, + shutdown: &CancellationToken, + drain: Duration, + abort: Duration, + ) -> Self { + Self { + spawn, + shutdown: shutdown.clone(), + running: tokio::sync::Mutex::new(None), + left_running: AtomicBool::new(false), + drain, + abort, + } + } +} + +#[async_trait::async_trait] +impl WorkerControl for BackgroundWorkers { + /// Cancel the running set's token and wait for its workers, abort those + /// still running after the drain bound, and wait for them to end. + /// Reports whether every worker of every set has ended; a worker that + /// outlives its abort (one that blocks its thread) is reported once and + /// for the rest of the process. + async fn stop(&self) -> bool { + let mut running = self.running.lock().await; + if let Some(set) = running.take() + && !stop_set(set, self.drain, self.abort).await + { + self.left_running.store(true, Ordering::Release); + } + !self.left_running.load(Ordering::Acquire) + } + + async fn start(&self) { + let mut running = self.running.lock().await; + if running.is_some() { + return; + } + let token = self.shutdown.child_token(); + let handles = (self.spawn)(token.clone()).await; + *running = Some(WorkerSet { token, handles }); + } +} + +/// Stop one set of workers; whether every one of them ended. +async fn stop_set(set: WorkerSet, drain: Duration, abort: Duration) -> bool { + set.token.cancel(); + let count = set.handles.len(); + let aborts: Vec<_> = set + .handles + .iter() + .map(tokio::task::JoinHandle::abort_handle) + .collect(); + let mut joined = futures::future::join_all(set.handles); + if let Ok(results) = tokio::time::timeout(drain, &mut joined).await { + let panicked = results.iter().filter(|r| r.is_err()).count(); + if panicked > 0 { + tracing::warn!("{panicked} of {count} background worker(s) ended abnormally"); + } else { + tracing::info!("{count} background worker(s) drained"); + } + return true; + } + let left = aborts.iter().filter(|handle| !handle.is_finished()).count(); + for handle in &aborts { + handle.abort(); + } + if tokio::time::timeout(abort, joined).await.is_ok() { + tracing::warn!( + "{left} of {count} background worker(s) did not stop within {}s and were aborted", + drain.as_secs() + ); + true + } else { + tracing::error!( + "{left} of {count} background worker(s) did not stop within {}s, and some still ran \ + {}s after they were aborted", + drain.as_secs(), + abort.as_secs() + ); + false + } +} + +/// Stop the workers, then release the serving registration: a migration +/// that waits for the servers starts once the registration is gone, so +/// releasing it while a worker runs would let the migration run beside a +/// worker that still writes to the catalog. When a worker could not be +/// stopped, the registration is not released and expires by itself. The +/// stop and the release are bounded. +async fn stop_then_release( + workers: &BackgroundWorkers, registrar: Option<&dyn extenddb_storage::hooks::ServingRegistrar>, ) { const RELEASE_TIMEOUT: Duration = Duration::from_secs(10); - drain_workers(shutdown, handles).await; - if let Some(registrar) = registrar - && tokio::time::timeout(RELEASE_TIMEOUT, registrar.release()) - .await - .is_err() + let bound = workers.drain + workers.abort + SUPERVISOR_TIMEOUT; + let ended = tokio::time::timeout(bound, workers.stop()) + .await + .unwrap_or(false); + let Some(registrar) = registrar else { + return; + }; + if !ended { + tracing::warn!( + "A background worker may still run, so the serving registration is not released; it \ + expires by itself" + ); + return; + } + if tokio::time::timeout(RELEASE_TIMEOUT, registrar.release()) + .await + .is_err() { tracing::warn!( "The serving registration was not released within {}s; it expires by itself", @@ -782,33 +965,6 @@ async fn drain_then_release( } } -/// Cancel the shutdown token and wait for every worker to return. -/// -/// Bounded by `DRAIN_TIMEOUT` so a worker stuck inside a backend call cannot -/// hold the process open; a timeout is logged and the remaining tasks are -/// dropped, which is the pre-drain behavior. -async fn drain_workers(shutdown: &CancellationToken, handles: Vec>) { - const DRAIN_TIMEOUT: Duration = Duration::from_secs(5); - - shutdown.cancel(); - let count = handles.len(); - let drain = futures::future::join_all(handles); - match tokio::time::timeout(DRAIN_TIMEOUT, drain).await { - Ok(results) => { - let panicked = results.iter().filter(|r| r.is_err()).count(); - if panicked > 0 { - tracing::warn!("{panicked} of {count} background worker(s) ended abnormally"); - } else { - tracing::info!("{count} background worker(s) drained"); - } - } - Err(_) => tracing::warn!( - "Background workers did not drain within {}s; shutting down anyway", - DRAIN_TIMEOUT.as_secs() - ), - } -} - /// P57 Bug 7: Best-effort raw syslog write for fatal errors and panics. /// /// Used when the tracing subscriber may not be initialized — during early @@ -840,14 +996,19 @@ pub fn log_to_syslog_raw(msg: &str) { #[cfg(test)] mod tests { + use std::sync::atomic::{AtomicU32, Ordering}; use std::sync::{Arc, Mutex}; use std::time::Duration; use extenddb_storage::CancellationToken; use extenddb_storage::hooks::{Reregistration, ServingRegistrar}; + use super::{BackgroundWorkers, SpawnWorkers, WorkerControl}; + + type Events = Arc>>; + /// A registrar that records when it is released. - struct Recorder(Arc>>); + struct Recorder(Events); #[async_trait::async_trait] impl ServingRegistrar for Recorder { @@ -864,24 +1025,137 @@ mod tests { } } + /// Workers whose sets each run `worker` once, with short stop bounds. + fn workers(shutdown: &CancellationToken, worker: F) -> BackgroundWorkers + where + F: Fn(CancellationToken) -> Fut + Send + Sync + 'static, + Fut: std::future::Future + Send + 'static, + { + let spawn: SpawnWorkers = Box::new(move |token| { + let handle = tokio::spawn(worker(token)); + futures::FutureExt::boxed(async move { vec![handle] }) + }); + BackgroundWorkers::with_bounds( + spawn, + shutdown, + Duration::from_millis(200), + Duration::from_millis(300), + ) + } + /// The registration is released after every worker has returned, so a /// migration that waits for the servers cannot start while a worker /// still writes to the catalog. #[tokio::test] async fn the_registration_is_released_after_the_workers_stop() { - let events = Arc::new(Mutex::new(Vec::new())); + let events: Events = Arc::new(Mutex::new(Vec::new())); let shutdown = CancellationToken::new(); - let worker = { - let (events, shutdown) = (events.clone(), shutdown.clone()); - tokio::spawn(async move { - shutdown.cancelled().await; - // A last write after the shutdown signal. - tokio::time::sleep(Duration::from_millis(200)).await; - events.lock().unwrap().push("worker stopped"); + let workers = { + let events = events.clone(); + workers(&shutdown, move |token| { + let events = events.clone(); + async move { + token.cancelled().await; + // A last write after the token fired. + tokio::time::sleep(Duration::from_millis(50)).await; + events.lock().unwrap().push("worker stopped"); + } }) }; + workers.start().await; let registrar = Recorder(events.clone()); - super::drain_then_release(&shutdown, vec![worker], Some(®istrar)).await; + shutdown.cancel(); + super::stop_then_release(&workers, Some(®istrar)).await; assert_eq!(*events.lock().unwrap(), ["worker stopped", "released"]); } + + /// A worker that does not return when its token fires is aborted at the + /// drain bound, and the registration is released only after that. + #[tokio::test] + async fn a_worker_that_ignores_its_token_is_aborted_before_the_release() { + let events: Events = Arc::new(Mutex::new(Vec::new())); + let ticks = Arc::new(AtomicU32::new(0)); + let shutdown = CancellationToken::new(); + let workers = { + let ticks = ticks.clone(); + workers(&shutdown, move |_token| { + let ticks = ticks.clone(); + async move { + loop { + tokio::time::sleep(Duration::from_millis(5)).await; + ticks.fetch_add(1, Ordering::SeqCst); + } + } + }) + }; + workers.start().await; + let registrar = Recorder(events.clone()); + shutdown.cancel(); + super::stop_then_release(&workers, Some(®istrar)).await; + assert_eq!(*events.lock().unwrap(), ["released"]); + let at_release = ticks.load(Ordering::SeqCst); + tokio::time::sleep(Duration::from_millis(50)).await; + assert_eq!( + ticks.load(Ordering::SeqCst), + at_release, + "the worker ran on after the release" + ); + } + + /// A worker that an abort cannot end, one that blocks its thread, keeps + /// the registration: it is left to expire, and every later stop reports + /// that a worker may still run. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn a_worker_that_blocks_its_thread_keeps_the_registration() { + let events: Events = Arc::new(Mutex::new(Vec::new())); + let shutdown = CancellationToken::new(); + let workers = workers(&shutdown, |_token| async { + std::thread::sleep(Duration::from_millis(1500)); + }); + workers.start().await; + // Let the worker reach its blocking call before the stop. + tokio::time::sleep(Duration::from_millis(50)).await; + let registrar = Recorder(events.clone()); + shutdown.cancel(); + super::stop_then_release(&workers, Some(®istrar)).await; + assert!(events.lock().unwrap().is_empty(), "released"); + assert!(!workers.stop().await, "a later stop forgot the worker"); + } + + /// A stopped set starts again on a new token: the supervisor stops the + /// workers at a loss and starts them after a new registration. + #[tokio::test] + async fn the_workers_start_again_after_a_stop() { + let started = Arc::new(AtomicU32::new(0)); + let stopped = Arc::new(AtomicU32::new(0)); + let shutdown = CancellationToken::new(); + let workers = { + let (started, stopped) = (started.clone(), stopped.clone()); + workers(&shutdown, move |token| { + let (started, stopped) = (started.clone(), stopped.clone()); + async move { + started.fetch_add(1, Ordering::SeqCst); + token.cancelled().await; + stopped.fetch_add(1, Ordering::SeqCst); + } + }) + }; + workers.start().await; + tokio::time::sleep(Duration::from_millis(20)).await; + assert!(workers.stop().await); + assert_eq!( + ( + started.load(Ordering::SeqCst), + stopped.load(Ordering::SeqCst) + ), + (1, 1) + ); + assert!(!shutdown.is_cancelled(), "a stop cancelled the server"); + workers.start().await; + tokio::time::sleep(Duration::from_millis(20)).await; + assert_eq!(started.load(Ordering::SeqCst), 2, "no new set started"); + shutdown.cancel(); + assert!(workers.stop().await); + assert_eq!(stopped.load(Ordering::SeqCst), 2); + } } diff --git a/crates/storage-cassandra/src/lib.rs b/crates/storage-cassandra/src/lib.rs index 40b36427b..099e99562 100644 --- a/crates/storage-cassandra/src/lib.rs +++ b/crates/storage-cassandra/src/lib.rs @@ -60,7 +60,6 @@ use std::sync::Arc; struct CassandraRuntimeHooks { engine: Arc, control_plane_notify: Arc, - gsi_worker_guard: std::sync::OnceLock, /// The registration this server holds while it serves, which the /// server takes again when it is lost and releases after its workers /// stop. @@ -70,23 +69,32 @@ struct CassandraRuntimeHooks { #[async_trait::async_trait] impl ServerRuntimeHooks for CassandraRuntimeHooks { async fn spawn_workers(&self, ctx: &WorkerContext) -> Vec> { - // Backend-specific workers that need Cassandra internals. - // Each handle is returned so `serve` can drain them on shutdown. + // Backend-specific workers that need Cassandra internals. Each one + // returns once `ctx.shutdown` fires, and its handle is returned so + // `serve` can drain it. let storage_for_poller = self.engine.clone(); let cp_notify = self.control_plane_notify.clone(); let catalog_store = ctx.catalog_store.clone(); + let token = ctx.shutdown.clone(); let control_plane = tokio::spawn(async move { - workers::poll_control_plane_transitions(storage_for_poller, cp_notify, catalog_store) - .await; + workers::poll_control_plane_transitions( + storage_for_poller, + cp_notify, + catalog_store, + token, + ) + .await; }); let engine_for_recovery = self.engine.clone(); + let token = ctx.shutdown.clone(); let transaction_recovery = tokio::spawn(async move { workers::poll_transaction_recovery( engine_for_recovery, std::time::Duration::from_secs(60), std::time::Duration::from_secs(30), + token, ) .await; }); @@ -113,24 +121,25 @@ impl ServerRuntimeHooks for CassandraRuntimeHooks { gsi_delay.store(ms, std::sync::atomic::Ordering::Relaxed); } let catalog_store_for_gsi = ctx.catalog_store.clone(); - let gsi_delay_poller = - tokio::spawn( - async move { workers::poll_gsi_delay(catalog_store_for_gsi, gsi_delay).await }, - ); + let token = ctx.shutdown.clone(); + let gsi_delay_poller = tokio::spawn(async move { + workers::poll_gsi_delay(catalog_store_for_gsi, gsi_delay, token).await; + }); // GSI propagation workers (one per partition). - let guard = workers::spawn_gsi_workers(self.engine.clone()); - let _ = self.gsi_worker_guard.set(guard); + let gsi_propagation = workers::spawn_gsi_workers_until(self.engine.clone(), &ctx.shutdown); // GSI backfill recovery worker. let engine_for_gsi_recovery = self.engine.clone(); let gsi_backfill_notify = self.engine.gsi_backfill_notify.clone(); + let token = ctx.shutdown.clone(); let gsi_backfill_recovery = tokio::spawn(async move { workers::poll_gsi_backfill_recovery( engine_for_gsi_recovery, gsi_backfill_notify, std::time::Duration::from_secs(120), // stale threshold std::time::Duration::from_secs(60), // scan interval backstop + token, ) .await; }); @@ -170,7 +179,7 @@ impl ServerRuntimeHooks for CassandraRuntimeHooks { .await; }); - vec![ + let mut handles = vec![ control_plane, transaction_recovery, gsi_delay_poller, @@ -179,7 +188,9 @@ impl ServerRuntimeHooks for CassandraRuntimeHooks { ttl_repair, ttl_audit, restore_sweep, - ] + ]; + handles.extend(gsi_propagation); + handles } fn backend_info(&self) -> Option { @@ -401,7 +412,6 @@ fn server_components_factory( let runtime_hooks = Box::new(CassandraRuntimeHooks { engine: engine.clone(), control_plane_notify, - gsi_worker_guard: std::sync::OnceLock::new(), registration, }); diff --git a/crates/storage-cassandra/src/workers.rs b/crates/storage-cassandra/src/workers.rs index 2bd5996e7..b6efb5381 100644 --- a/crates/storage-cassandra/src/workers.rs +++ b/crates/storage-cassandra/src/workers.rs @@ -9,20 +9,21 @@ use std::time::Duration; use std::sync::atomic::AtomicU64; use extenddb_storage::management_store::SettingsStore; +use extenddb_storage::{CancellationToken, sleep_or_shutdown}; use crate::CassandraEngine; -/// Poll `index_propagation_delay_ms` from settings every 30 seconds and update -/// the in-memory atomic used by `put_item/update_item/delete_item`. +/// Poll `index_propagation_delay_ms` from settings every 5 seconds and update +/// the in-memory atomic used by `put_item/update_item/delete_item`, until +/// `shutdown` fires. pub(crate) async fn poll_gsi_delay( store: Arc, gsi_delay: Arc, + shutdown: CancellationToken, ) { const POLL_INTERVAL: Duration = Duration::from_secs(5); - loop { - tokio::time::sleep(POLL_INTERVAL).await; - + while sleep_or_shutdown(&shutdown, POLL_INTERVAL).await { match store.get_setting("index_propagation_delay_ms").await { Ok(Some(val)) => { if let Ok(ms) = val.parse::() { @@ -51,6 +52,7 @@ pub(crate) async fn poll_control_plane_transitions( storage: Arc, notify: Arc, settings: Arc, + shutdown: CancellationToken, ) { const ACTIVE_POLL: Duration = Duration::from_secs(1); const IDLE_TIMEOUT: Duration = Duration::from_secs(60); @@ -58,7 +60,10 @@ pub(crate) async fn poll_control_plane_transitions( loop { // Idle: wait for a wake signal or timeout (defensive sweep) - let _ = tokio::time::timeout(IDLE_TIMEOUT, notify.notified()).await; + tokio::select! { + () = shutdown.cancelled() => return, + _ = tokio::time::timeout(IDLE_TIMEOUT, notify.notified()) => {} + } // Read control_plane_delay_seconds from settings to compute active window let delay_secs = read_control_plane_delay(&*settings).await; @@ -82,7 +87,9 @@ pub(crate) async fn poll_control_plane_transitions( if tokio::time::Instant::now() >= deadline { break; } - tokio::time::sleep(ACTIVE_POLL).await; + if !sleep_or_shutdown(&shutdown, ACTIVE_POLL).await { + return; + } } } } @@ -91,15 +98,15 @@ pub(crate) async fn poll_control_plane_transitions( /// /// Scans every account keyspace for transactions older than `timeout` and /// resumes COMMIT or executes ROLLBACK as appropriate. Runs every -/// `scan_interval` seconds for the lifetime of the process. +/// `scan_interval` seconds until `shutdown` fires, and stops before the next +/// keyspace once it has. pub(crate) async fn poll_transaction_recovery( engine: Arc, timeout: Duration, scan_interval: Duration, + shutdown: CancellationToken, ) { - loop { - tokio::time::sleep(scan_interval).await; - + while sleep_or_shutdown(&shutdown, scan_interval).await { #[allow(clippy::cast_possible_truncation)] let cutoff = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) @@ -116,6 +123,9 @@ pub(crate) async fn poll_transaction_recovery( }; for keyspace in &keyspaces { + if shutdown.is_cancelled() { + return; + } let entries = match engine.scan_old_transactions(keyspace, cutoff).await { Ok(e) => e, Err(e) => { @@ -186,13 +196,12 @@ async fn read_control_plane_delay(store: &S) -> f64 { /// Handle that stops GSI workers when dropped. pub struct GsiWorkerGuard { - shutdown: Arc, + shutdown: CancellationToken, } impl Drop for GsiWorkerGuard { fn drop(&mut self) { - self.shutdown - .store(true, std::sync::atomic::Ordering::Relaxed); + self.shutdown.cancel(); } } @@ -201,28 +210,35 @@ impl Drop for GsiWorkerGuard { /// Returns a `GsiWorkerGuard`; workers stop when it is dropped. #[must_use] pub fn spawn_gsi_workers(engine: Arc) -> GsiWorkerGuard { - let shutdown = Arc::new(std::sync::atomic::AtomicBool::new(false)); - for worker_id in 0..crate::gsi_queue::NUM_WORKERS { - let engine = engine.clone(); - let shutdown = shutdown.clone(); - tokio::spawn(async move { - gsi_worker(worker_id, engine, shutdown).await; - }); - } + let shutdown = CancellationToken::new(); + drop(spawn_gsi_workers_until(engine, &shutdown)); GsiWorkerGuard { shutdown } } -async fn gsi_worker( - worker_id: u64, +/// Spawn GSI propagation workers, one per partition, that return once +/// `shutdown` fires; their handles, for the server's drain. +pub(crate) fn spawn_gsi_workers_until( engine: Arc, - shutdown: Arc, -) { + shutdown: &CancellationToken, +) -> Vec> { + (0..crate::gsi_queue::NUM_WORKERS) + .map(|worker_id| { + let engine = engine.clone(); + let shutdown = shutdown.clone(); + tokio::spawn(async move { + gsi_worker(worker_id, engine, shutdown).await; + }) + }) + .collect() +} + +async fn gsi_worker(worker_id: u64, engine: Arc, shutdown: CancellationToken) { const MAX_IDLE: Duration = Duration::from_secs(1); tracing::debug!("GSI worker {worker_id} started"); loop { - if shutdown.load(std::sync::atomic::Ordering::Relaxed) { + if shutdown.is_cancelled() { tracing::debug!("GSI worker {worker_id} shutting down"); return; } @@ -233,14 +249,15 @@ async fn gsi_worker( .await .unwrap_or(MAX_IDLE) .min(MAX_IDLE); - tokio::time::timeout(wait, engine.gsi_queue.notify.notified()) - .await - .ok(); + tokio::select! { + () = shutdown.cancelled() => {} + _ = tokio::time::timeout(wait, engine.gsi_queue.notify.notified()) => {} + } } Ok(_) => continue, Err(e) => { tracing::error!("GSI worker {worker_id}: {e}"); - tokio::time::sleep(MAX_IDLE).await; + let _ = sleep_or_shutdown(&shutdown, MAX_IDLE).await; } } } @@ -450,16 +467,21 @@ async fn gsi_apply_index( /// Woken immediately when `apply_table_update` commits a new `CREATING` index, /// and also sweeps periodically as a backstop. Treats a NULL `backfill_heartbeat` /// (never started) and a heartbeat older than `stale_threshold` (crashed) the -/// same way: drop+recreate the data table and rerun the backfill. +/// same way: drop+recreate the data table and rerun the backfill. Returns once +/// `shutdown` fires, before the next keyspace. pub(crate) async fn poll_gsi_backfill_recovery( engine: Arc, notify: Arc, stale_threshold: Duration, scan_interval: Duration, + shutdown: CancellationToken, ) { loop { // Wait for an explicit wake or the periodic backstop timeout. - let _ = tokio::time::timeout(scan_interval, notify.notified()).await; + tokio::select! { + () = shutdown.cancelled() => return, + _ = tokio::time::timeout(scan_interval, notify.notified()) => {} + } let keyspaces = match list_account_keyspaces(&engine).await { Ok(ks) => ks, @@ -470,6 +492,9 @@ pub(crate) async fn poll_gsi_backfill_recovery( }; for account_ks in &keyspaces { + if shutdown.is_cancelled() { + return; + } if let Err(e) = recover_stale_backfills(&engine, account_ks, stale_threshold).await { tracing::warn!("gsi_recovery: {account_ks}: {e}"); } diff --git a/crates/storage-mongodb/src/backup_live_tests.rs b/crates/storage-mongodb/src/backup_live_tests.rs index 75ed23761..2536d1d55 100644 --- a/crates/storage-mongodb/src/backup_live_tests.rs +++ b/crates/storage-mongodb/src/backup_live_tests.rs @@ -121,7 +121,10 @@ async fn engine_on(catalog: &str, data: &str, liveness: BackupLiveness) -> Mongo fn spawn_worker(engine: &MongoEngine) -> tokio::task::JoinHandle<()> { let engine = Arc::new(engine.clone()); - tokio::spawn(crate::ttl_worker::gsi_backfill_worker(engine)) + tokio::spawn(crate::ttl_worker::gsi_backfill_worker( + engine, + extenddb_storage::CancellationToken::new(), + )) } fn key(name: &str, key_type: KeyType) -> KeySchemaElement { diff --git a/crates/storage-mongodb/src/lib.rs b/crates/storage-mongodb/src/lib.rs index c199de6f8..787f503dc 100644 --- a/crates/storage-mongodb/src/lib.rs +++ b/crates/storage-mongodb/src/lib.rs @@ -54,23 +54,29 @@ struct MongoRuntimeHooks { #[async_trait::async_trait] impl ServerRuntimeHooks for MongoRuntimeHooks { + /// Each worker returns once `ctx.shutdown` fires, and its handle is + /// returned so `serve` can drain it. async fn spawn_workers(&self, ctx: &WorkerContext) -> Vec> { let storage_for_ttl = self.engine.clone(); let metrics = ctx.metrics.clone(); + let token = ctx.shutdown.clone(); let ttl = tokio::spawn(async move { - ttl_worker::ttl_cleanup_worker(storage_for_ttl, metrics).await; + ttl_worker::ttl_cleanup_worker(storage_for_ttl, metrics, token).await; }); let storage_for_stream = self.engine.clone(); + let token = ctx.shutdown.clone(); let stream = tokio::spawn(async move { - ttl_worker::stream_record_cleanup_worker(storage_for_stream).await; + ttl_worker::stream_record_cleanup_worker(storage_for_stream, token).await; }); let storage_for_backfill = self.engine.clone(); + let token = ctx.shutdown.clone(); let backfill = tokio::spawn(async move { - ttl_worker::gsi_backfill_worker(storage_for_backfill).await; + ttl_worker::gsi_backfill_worker(storage_for_backfill, token).await; }); let storage_for_control_plane = self.engine.clone(); + let token = ctx.shutdown.clone(); let control_plane = tokio::spawn(async move { - ttl_worker::control_plane_worker(storage_for_control_plane).await; + ttl_worker::control_plane_worker(storage_for_control_plane, token).await; }); tracing::info!( "MongoDB backend: TTL, stream cleanup, GSI backfill, and control-plane workers spawned" diff --git a/crates/storage-mongodb/src/snapshot_engine.rs b/crates/storage-mongodb/src/snapshot_engine.rs index c28c55397..fc8eefa48 100644 --- a/crates/storage-mongodb/src/snapshot_engine.rs +++ b/crates/storage-mongodb/src/snapshot_engine.rs @@ -483,9 +483,10 @@ mod tests { restore_server_slots: Arc::new(tokio::sync::Semaphore::new(slots)), restore_account_slots: crate::backup_engine::RestoreAccountSlots::default(), }; - let worker = tokio::spawn(crate::ttl_worker::gsi_backfill_worker(Arc::new( - engine.clone(), - ))); + let worker = tokio::spawn(crate::ttl_worker::gsi_backfill_worker( + Arc::new(engine.clone()), + extenddb_storage::CancellationToken::new(), + )); Fixture { engine: Some(engine), worker, diff --git a/crates/storage-mongodb/src/ttl_worker.rs b/crates/storage-mongodb/src/ttl_worker.rs index e9cd41941..9d309f575 100644 --- a/crates/storage-mongodb/src/ttl_worker.rs +++ b/crates/storage-mongodb/src/ttl_worker.rs @@ -10,7 +10,10 @@ use bson::{Document, doc}; use extenddb_core::metrics::MetricsCollector; use extenddb_core::types::{KeySchemaElement, Projection, ProjectionType, UserIdentity}; use extenddb_storage::error::StorageError; -use extenddb_storage::{DataEngine, MetadataEngine, StreamEngine, TableEngine, WorkerStore}; +use extenddb_storage::{ + CancellationToken, DataEngine, MetadataEngine, StreamEngine, TableEngine, WorkerStore, + sleep_or_shutdown, +}; use futures::TryStreamExt; use crate::MongoEngine; @@ -27,11 +30,16 @@ const GSI_BACKFILL_BATCH: i64 = 500; /// without busy-spinning. const CONTROL_PLANE_POLL_INTERVAL: Duration = Duration::from_millis(250); -pub(crate) async fn ttl_cleanup_worker(storage: Arc, metrics: Arc) { +/// Retry pending TTL indexes and delete expired items every +/// [`SCAN_INTERVAL`], until `shutdown` fires. +pub(crate) async fn ttl_cleanup_worker( + storage: Arc, + metrics: Arc, + shutdown: CancellationToken, +) { let region_arc: Arc = Arc::from(storage.region.as_str()); - loop { - tokio::time::sleep(SCAN_INTERVAL).await; + while sleep_or_shutdown(&shutdown, SCAN_INTERVAL).await { retry_pending_indexes(&storage).await; sweep_expired_items(&storage, &metrics, ®ion_arc).await; } @@ -40,10 +48,13 @@ pub(crate) async fn ttl_cleanup_worker(storage: Arc, metrics: Arc) { - loop { - tokio::time::sleep(STREAM_CLEANUP_INTERVAL).await; +/// missing (init predates the schema change) or lagging. Returns once +/// `shutdown` fires. +pub(crate) async fn stream_record_cleanup_worker( + storage: Arc, + shutdown: CancellationToken, +) { + while sleep_or_shutdown(&shutdown, STREAM_CLEANUP_INTERVAL).await { match StreamEngine::cleanup_expired_stream_records(&*storage, STREAM_RETENTION_HOURS).await { Ok(0) => {} @@ -81,12 +92,11 @@ pub(crate) async fn stream_record_cleanup_worker(storage: Arc) { /// left by a process that died is removed within about /// `stale_after + heartbeat_interval + GSI_BACKFILL_INTERVAL` without a restart /// of this one. The sweep removes only state whose heartbeat stopped, so it is -/// safe while other processes share the catalog. -pub(crate) async fn gsi_backfill_worker(storage: Arc) { +/// safe while other processes share the catalog. Returns once `shutdown` +/// fires, at its next pause between cycles. +pub(crate) async fn gsi_backfill_worker(storage: Arc, shutdown: CancellationToken) { let mut last_backup_sweep = std::time::Instant::now(); - loop { - tokio::time::sleep(GSI_BACKFILL_INTERVAL).await; - + while sleep_or_shutdown(&shutdown, GSI_BACKFILL_INTERVAL).await { if last_backup_sweep.elapsed() >= storage.backup_liveness.heartbeat_interval { last_backup_sweep = std::time::Instant::now(); match storage.sweep_stale_backup_claims().await { @@ -119,6 +129,9 @@ pub(crate) async fn gsi_backfill_worker(storage: Arc) { }; for job in jobs { + if shutdown.is_cancelled() { + return; + } if let Err(e) = run_gsi_backfill_job(&storage, &job).await { tracing::warn!( "GSI backfill worker: job failed for index_id={}: {e}", @@ -551,10 +564,10 @@ fn build_ttl_condition( /// Background poller that flips tables out of the transient `CREATING` state /// once their scheduled `status_transition_at` has passed. See -/// [`crate::worker_store`] for how rows enter `CREATING`. -pub(crate) async fn control_plane_worker(storage: Arc) { - loop { - tokio::time::sleep(CONTROL_PLANE_POLL_INTERVAL).await; +/// [`crate::worker_store`] for how rows enter `CREATING`. Returns once +/// `shutdown` fires. +pub(crate) async fn control_plane_worker(storage: Arc, shutdown: CancellationToken) { + while sleep_or_shutdown(&shutdown, CONTROL_PLANE_POLL_INTERVAL).await { match WorkerStore::process_control_plane_transitions(&*storage).await { Ok(t) if t.is_empty() => {} Ok(transitions) => { diff --git a/crates/storage/src/hooks.rs b/crates/storage/src/hooks.rs index a5b78d44a..55d73b834 100644 --- a/crates/storage/src/hooks.rs +++ b/crates/storage/src/hooks.rs @@ -52,9 +52,14 @@ pub trait ServerRuntimeHooks: Send + Sync { /// state (connection pools, notify handles, etc.). /// /// Return the spawned tasks' join handles so the server can await them - /// during shutdown after cancelling [`WorkerContext::shutdown`]. Returning - /// an empty vector opts out of the drain (the tasks are then dropped when - /// the runtime shuts down). + /// after cancelling [`WorkerContext::shutdown`]; the server aborts the + /// ones still running 5 s later. Every worker this spawns must be among + /// the handles and return once that token fires. + /// + /// Called again for each new set of workers: a server whose serving + /// registration is lost stops its workers, and starts a new set with a + /// new token once it registers again (see + /// [`ServerRuntimeHooks::serving_registrar`]). async fn spawn_workers(&self, ctx: &WorkerContext) -> Vec>; /// Get backend-specific info for logging (optional). @@ -73,14 +78,15 @@ pub trait ServerRuntimeHooks: Send + Sync { /// /// `serve` calls this once, before [`ServerRuntimeHooks::spawn_workers`]. /// When [`ServingRegistrar::lost`] returns, the server answers every new - /// request with `ServiceUnavailable` and calls - /// [`ServingRegistrar::register_again`] at once and then every second. - /// It serves again on [`Reregistration::Registered`]. On - /// [`Reregistration::Refused`], or when 30 s pass without a - /// registration, it stops as a shutdown signal stops it (the listener - /// drains, [`WorkerContext::shutdown`] fires, the PID file is removed) - /// and exits 1. On every shutdown it calls [`ServingRegistrar::release`] - /// once the background workers have stopped. + /// request with `ServiceUnavailable`, stops its background workers, and + /// then calls [`ServingRegistrar::register_again`] at once and every + /// second after. On [`Reregistration::Registered`] it starts a new set of + /// workers and serves again. On [`Reregistration::Refused`], when 30 s + /// pass without a registration, or when a worker could not be stopped, it + /// stops as a shutdown signal stops it (the listener drains, the PID file + /// is removed) and exits 1. On every shutdown it calls + /// [`ServingRegistrar::release`] once every background worker has ended, + /// and not at all when one could not be stopped. fn serving_registrar(&self) -> Option> { None } @@ -113,11 +119,12 @@ pub trait ServingRegistrar: Send + Sync { async fn register_again(&self) -> Reregistration; /// Give up the registration the server holds, so a migration need not - /// wait for it to expire. `serve` calls it once, after the background - /// workers have stopped, so a migration cannot start while one of them - /// still writes to the catalog. The default does nothing, for a - /// registration that ends with the process, such as PostgreSQL's key, - /// which the database frees when the session closes. + /// wait for it to expire. `serve` calls it at most once, after every + /// background worker has ended, so a migration cannot start while one of + /// them still writes to the catalog; when a worker could not be stopped + /// it does not call it, and the registration expires. The default does + /// nothing, for a registration that ends with the process, such as + /// PostgreSQL's key, which the database frees when the session closes. async fn release(&self) {} } diff --git a/docs/manuals/07-upgrade-manual.md b/docs/manuals/07-upgrade-manual.md index 752341276..32d735057 100755 --- a/docs/manuals/07-upgrade-manual.md +++ b/docs/manuals/07-upgrade-manual.md @@ -89,13 +89,13 @@ The PostgreSQL commands and `serve` work through PgBouncer in transaction mode a - Behind PgBouncer, the TCP keepalives the serving session sets (30 s) run on the database's connection to PgBouncer, which stays up when a serving host dies. The database keeps that host's key until PgBouncer closes the dead host's connection, which PgBouncer's own keepalive settings decide: by default the operating system's, two hours idle on Linux. Until then `migrate --yes` exits 6 naming the dead server. Set `tcp_keepalive = 1`, `tcp_keepidle = 10`, `tcp_keepintvl = 5` and `tcp_keepcnt = 4` in PgBouncer's `[pgbouncer]` section for the same 30 s, or end the session with `SELECT pg_terminate_backend()` on the pid `migrate` names. - After a failover of the catalog's server, the new primary holds no server's key, while a server that served through the old primary goes on serving through connections that reconnect to the new primary, until it counts its key as lost, at most 30 s after its last renewal, and registers again on the new primary. A `migrate --yes` in those 30 s does not see that server. Wait 30 s after a failover before you run `migrate --yes`, or stop the servers first. -A server notices at its next renewal, within 15 s, that the database ended the session holding its key: a restart or failover of the catalog's server, `SELECT pg_terminate_backend()`, or a pooler that closed the connection. From then on it answers each new request, `/health` included, with `ServiceUnavailable` (HTTP 503), which the AWS SDKs retry, and lets the requests already running finish. It tries to register again on a new session at once and then every second, and serves again once it holds the key and the catalog's status is the one it started with. When a migration holds the migration lock or waits for the servers to stop, when the status changed, or when 30 s pass without a registration, the server stops its background workers at once and then stops the way SIGTERM stops it, finishing the requests in flight and removing its PID file, and exits 1. Until the renewal that finds the session gone, the server serves as before, so a `migrate --yes` that was waiting for the servers can start beside it in those seconds: stop every server before you migrate, as the steps above say. +A server notices at its next renewal, within 15 s, that the database ended the session holding its key: a restart or failover of the catalog's server, `SELECT pg_terminate_backend()`, or a pooler that closed the connection. From then on it answers each new request, `/health` included, with `ServiceUnavailable` (HTTP 503), which the AWS SDKs retry, lets the requests already running finish, and stops its background workers, aborting the ones still running after 5 s. It tries to register again on a new session at once and then every second, and once it holds the key and the catalog's status is the one it started with, it starts its background workers again and serves. When a migration holds the migration lock or waits for the servers to stop, when the status changed, when 30 s pass without a registration, or when a background worker could not be stopped, the server stops the way SIGTERM stops it, finishing the requests in flight and removing its PID file, and exits 1. Until the renewal that finds the session gone, the server serves as before, so a `migrate --yes` that was waiting for the servers can start beside it in those seconds: stop every server before you migrate, as the steps above say. The two transactions that hold the keys, the server's registration and the migration lock, turn off `idle_in_transaction_session_timeout` for themselves, and `transaction_timeout` on PostgreSQL 17 and later, so a role or database setting of either does not end them. PgBouncer's `idle_transaction_timeout` is a setting of PgBouncer, which a transaction cannot change. Each holder runs `SELECT 1` every 15 s, so set `idle_transaction_timeout` to 0, its default, or to 30 s or more. A value under 15 s closes the server's connection between two renewals, and the server then registers again every time. -On MongoDB, `migrate` connects with the connection string in the config file, which the server uses too. A user with the `readWrite` role on `extenddb_catalog` and on `extenddb_data` and no other role can run `init`, `migrate --yes`, `verify` and `serve`. `migrate --yes` takes the migration lock, a document in the catalog's `settings` collection, and then waits for the registration every running server keeps in the catalog's `extenddb_servers` collection, and after `--lock-timeout` exits 6 naming each server's host and pid. A server renews its registration every 5 s, and every second after a renewal failed, and deletes it when it stops, after its background workers have stopped; a server that was killed leaves it until it expires, at most 45 s after its last renewal. A server whose registration was deleted, or that has not renewed it for 30 s, for example because the primary stopped answering, answers new requests with `ServiceUnavailable` and registers again, as a PostgreSQL server does after its session ends (above): it serves again once it holds a new registration and the catalog's status is the one it started with, and it stops with exit 1 when a migration holds the lock, when the status changed, or when 30 s pass without a registration. The lock and the registrations expire by the clock of the replica set's primary, so keep the clocks of its members in sync with NTP. Migration 1 deletes the `resource_tag_migration` settings document in the transaction that records its catalog step, so a 0.1.14 server refuses the migrated catalog, and a run that fails before that transaction leaves the document in place. 0.1.12 and 0.1.13 servers read neither that document nor the catalog version, so they start on a migrated catalog and serve it: do not run them against it. To go back to a 0.1.x release, restore the backup taken before `migrate --yes`. A 0.1.x `migrate` on a migrated catalog reports a pending change, and its `--yes` writes the settings document back; this release's status then reports the catalog as written by an older release, and `migrate --yes --reconverge` repairs it. +On MongoDB, `migrate` connects with the connection string in the config file, which the server uses too. A user with the `readWrite` role on `extenddb_catalog` and on `extenddb_data` and no other role can run `init`, `migrate --yes`, `verify` and `serve`. `migrate --yes` takes the migration lock, a document in the catalog's `settings` collection, and then waits for the registration every running server keeps in the catalog's `extenddb_servers` collection, and after `--lock-timeout` exits 6 naming each server's host and pid. A server renews its registration every 5 s, and every second after a renewal failed, and deletes it when it stops, once every background worker has ended (it leaves it to expire when one could not be stopped); a server that was killed leaves it until it expires, at most 45 s after its last renewal. A server whose registration was deleted, or that has not renewed it for 30 s, for example because the primary stopped answering, answers new requests with `ServiceUnavailable` and registers again, as a PostgreSQL server does after its session ends (above): it serves again once it holds a new registration and the catalog's status is the one it started with, and it stops with exit 1 when a migration holds the lock, when the status changed, or when 30 s pass without a registration. The lock and the registrations expire by the clock of the replica set's primary, so keep the clocks of its members in sync with NTP. Migration 1 deletes the `resource_tag_migration` settings document in the transaction that records its catalog step, so a 0.1.14 server refuses the migrated catalog, and a run that fails before that transaction leaves the document in place. 0.1.12 and 0.1.13 servers read neither that document nor the catalog version, so they start on a migrated catalog and serve it: do not run them against it. To go back to a 0.1.x release, restore the backup taken before `migrate --yes`. A 0.1.x `migrate` on a migrated catalog reports a pending change, and its `--yes` writes the settings document back; this release's status then reports the catalog as written by an older release, and `migrate --yes --reconverge` repairs it. -On Cassandra, `migrate` connects as the role the config file names, which the server connects as too. `--cassandra-user ` connects as another role, with its password in `EXTENDDB_CASSANDRA_PASSWORD` or `--cassandra-pass`; a wrong password is reported as a connection timeout. Migration 1 adopts a 0.1.x catalog: it keeps every setting except the `resource_tag_migration` row, which it deletes, and records itself in a ledger in the catalog keyspace and in every account keyspace. On a cluster that runs `CassandraAuthorizer`, `init` grants the application role SELECT, MODIFY, CREATE and ALTER on the catalog keyspace, those four and DROP on the default account's keyspace, and CREATE ON ALL KEYSPACES, which CreateAccount needs to create a keyspace per account and which also lets the role create tables in every keyspace on the cluster; a run of `migrate --yes --cassandra-user ` that creates a missing account keyspace grants the application role the five permissions on it. A 0.1.x `init` granted the application role nothing, so a deployment created before 1.0 on such a cluster needs the statements below before `migrate`. `migrate --yes` takes the migration lock, a row in the catalog's `settings` table written by lightweight transactions, and then waits for the registration every running server of this release keeps in the catalog's `extenddb_servers` table, and after `--lock-timeout` exits 6 naming each server. A server renews its registration every 5 s, and every second after a renewal failed, and deletes it when it stops, after its background workers have stopped; a server that was killed leaves it until it expires, at most 45 s after its last renewal. A server that has not renewed its registration for 30 s, for example while the only replica of the catalog restarts, answers new requests with `ServiceUnavailable` and registers again, as a PostgreSQL server does after its session ends (above), and stops with exit 1 when a migration holds the lock, when the status changed, or when 30 s pass without a registration. A migration needs every replica of the catalog keyspace reachable: the lock is written at `SERIAL`, the registrations are read at `ALL`, and after each schema statement the runner waits until every live node reports the same schema version. `serve` refuses a catalog with pending migrations, exiting 3 under `--foreground` with the pending migrations and the `extenddb migrate --config --yes` command to run. A 0.1.x server refuses a migrated catalog. +On Cassandra, `migrate` connects as the role the config file names, which the server connects as too. `--cassandra-user ` connects as another role, with its password in `EXTENDDB_CASSANDRA_PASSWORD` or `--cassandra-pass`; a wrong password is reported as a connection timeout. Migration 1 adopts a 0.1.x catalog: it keeps every setting except the `resource_tag_migration` row, which it deletes, and records itself in a ledger in the catalog keyspace and in every account keyspace. On a cluster that runs `CassandraAuthorizer`, `init` grants the application role SELECT, MODIFY, CREATE and ALTER on the catalog keyspace, those four and DROP on the default account's keyspace, and CREATE ON ALL KEYSPACES, which CreateAccount needs to create a keyspace per account and which also lets the role create tables in every keyspace on the cluster; a run of `migrate --yes --cassandra-user ` that creates a missing account keyspace grants the application role the five permissions on it. A 0.1.x `init` granted the application role nothing, so a deployment created before 1.0 on such a cluster needs the statements below before `migrate`. `migrate --yes` takes the migration lock, a row in the catalog's `settings` table written by lightweight transactions, and then waits for the registration every running server of this release keeps in the catalog's `extenddb_servers` table, and after `--lock-timeout` exits 6 naming each server. A server renews its registration every 5 s, and every second after a renewal failed, and deletes it when it stops, once every background worker has ended (it leaves it to expire when one could not be stopped); a server that was killed leaves it until it expires, at most 45 s after its last renewal. A server that has not renewed its registration for 30 s, for example while the only replica of the catalog restarts, answers new requests with `ServiceUnavailable` and registers again, as a PostgreSQL server does after its session ends (above), and stops with exit 1 when a migration holds the lock, when the status changed, or when 30 s pass without a registration. A migration needs every replica of the catalog keyspace reachable: the lock is written at `SERIAL`, the registrations are read at `ALL`, and after each schema statement the runner waits until every live node reports the same schema version. `serve` refuses a catalog with pending migrations, exiting 3 under `--foreground` with the pending migrations and the `extenddb migrate --config --yes` command to run. A 0.1.x server refuses a migrated catalog. On a deployment created before 1.0 on a cluster that runs `CassandraAuthorizer`, this release's `migrate`, `verify` and `serve`, connected as the application role, fail with `Unauthorized ... has no SELECT permission on
_catalog.accounts>` until the role holds its permissions. With every server stopped, run the statements below in `cqlsh` as a superuser; the application role cannot grant them to itself. `` and `` are `username` and `keyspace_prefix` under `[storage.cassandra]` in the config file. Each account has its own keyspace, so list the accounts first: From 6282493a75d983cadbd532ae2541e06d9d360d2d Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Sun, 1 Nov 2026 21:12:00 +0000 Subject: [PATCH 26/45] fix(storage-cassandra): bound a registration renewal by the time left to its deadline A renewal was bounded by the 5 s renewal interval wherever it was sent, so one sent less than 5 s before the deadline, 30 s after the last successful renewal, timed out after it, and the last attempt went out late: with a node that does not answer, the loss was reported up to 40 s after the last renewal, 5 s before the row expires, where the module documentation said 35 s. A renewal sent before the deadline is now bounded by the time left to it as well, as MongoDB's renewal is, and the last attempt, at the deadline, by 5 s, so the loss is reported at most 35 s after the last successful renewal was sent. Test: with every renewal after the first unanswered, on tokio's paused clock, the loss is reported 30 to 35 s after the first renewal; with the old bound it is reported 39 s after it. --- crates/storage-cassandra/Cargo.toml | 1 + .../src/migrations/registrations.rs | 115 +++++++++++++++--- 2 files changed, 97 insertions(+), 19 deletions(-) diff --git a/crates/storage-cassandra/Cargo.toml b/crates/storage-cassandra/Cargo.toml index 234d2bf9a..cb3cf4bba 100644 --- a/crates/storage-cassandra/Cargo.toml +++ b/crates/storage-cassandra/Cargo.toml @@ -37,3 +37,4 @@ test-util = ["extenddb-storage/test-util"] [dev-dependencies] extenddb-storage = { workspace = true, features = ["test-util"] } extenddb-storage-cassandra = { path = ".", features = ["test-util"] } +tokio = { workspace = true, features = ["test-util"] } diff --git a/crates/storage-cassandra/src/migrations/registrations.rs b/crates/storage-cassandra/src/migrations/registrations.rs index ede07b48b..8e8235973 100644 --- a/crates/storage-cassandra/src/migrations/registrations.rs +++ b/crates/storage-cassandra/src/migrations/registrations.rs @@ -7,9 +7,12 @@ //! with a TTL of 45 s and again every 5 s at `LOCAL_QUORUM`. A renewal that //! fails is tried again every second. The registration counts as lost only //! when no renewal has succeeded for 30 s, after a last attempt at that -//! deadline, so the server stops admitting requests at least 10 s before its +//! deadline. Each attempt is bounded by 5 s, and one sent before the +//! deadline also by the time left to it, so the loss is reported at most +//! 35 s after the last successful renewal was sent, at least 10 s before its //! row expires and `migrate --yes`, which waits while a row exists, can see -//! it gone. The server then registers again with a new row +//! it gone. The server then stops admitting requests and stops its +//! background workers, and registers again with a new row //! ([`super::registrar`]). use std::sync::Arc; @@ -35,9 +38,11 @@ pub struct RegistrationTimes { pub renew_every: Duration, /// How soon a renewal that failed is tried again. pub retry_every: Duration, - /// How long after the last successful renewal the registration counts - /// as lost. Each renewal is bounded by `renew_every`, so the loss is - /// reported by `lost_after + renew_every`, which must stay under the TTL. + /// How long after the last successful renewal was sent the registration + /// counts as lost. A renewal sent before then is bounded by the time + /// left, and the last attempt, at that deadline, by `renew_every`, so + /// the loss is reported by `lost_after + renew_every`, which must stay + /// under the TTL. pub lost_after: Duration, } @@ -308,10 +313,10 @@ async fn delete_row( .map(drop) } -/// Write the row again every `renew_every`, each write bounded by -/// `renew_every`; after a failure try again every `retry_every`. Report the -/// registration lost once the attempt at `lost_after` after the last -/// successful renewal fails too. +/// Write the row again every `renew_every`; after a failure try again every +/// `retry_every`. Report the registration lost once the attempt at the +/// deadline, `lost_after` after the last successful renewal was sent, fails +/// too. async fn renew_registration( session: Arc, cql: String, @@ -319,12 +324,7 @@ async fn renew_registration( times: RegistrationTimes, lost: watch::Sender>, ) { - let mut last_ok = Instant::now(); - let mut next = last_ok + times.renew_every; - loop { - let deadline = last_ok + times.lost_after; - tokio::time::sleep_until(next.min(deadline)).await; - let sent = Instant::now(); + renew_with(times, lost, || { // LOCAL_QUORUM, so a running server stays up while a remote // datacenter is unreachable. let renewal = send( @@ -335,14 +335,44 @@ async fn renew_registration( None, "renew the serving registration", ); - let why = match tokio::time::timeout(times.renew_every, renewal).await { - Ok(Ok(_)) => { + async move { renewal.await.map(drop).map_err(|e| e.to_string()) } + }) + .await; +} + +/// [`renew_registration`]'s schedule, with the renewal given. An attempt +/// sent before the deadline is bounded by `renew_every` and by the time left +/// to the deadline, so the attempt at the deadline is never late; that last +/// attempt is bounded by `renew_every`. The loss is therefore reported at +/// most `lost_after + renew_every` after the last successful renewal was +/// sent. +async fn renew_with( + times: RegistrationTimes, + lost: watch::Sender>, + mut renew: F, +) where + F: FnMut() -> Fut, + Fut: std::future::Future>, +{ + let mut last_ok = Instant::now(); + let mut next = last_ok + times.renew_every; + loop { + let deadline = last_ok + times.lost_after; + tokio::time::sleep_until(next.min(deadline)).await; + let sent = Instant::now(); + let bound = if sent < deadline { + times.renew_every.min(deadline - sent) + } else { + times.renew_every + }; + let why = match tokio::time::timeout(bound, renew()).await { + Ok(Ok(())) => { last_ok = sent; next = sent + times.renew_every; continue; } - Ok(Err(e)) => e.to_string(), - Err(_) => format!("no answer within {} s", times.renew_every.as_secs()), + Ok(Err(e)) => e, + Err(_) => format!("no answer within {} ms", bound.as_millis()), }; if sent >= deadline { let _ = lost.send(Some(format!( @@ -358,8 +388,13 @@ async fn renew_registration( #[cfg(test)] mod tests { + use std::sync::Arc; + use std::sync::atomic::{AtomicU32, Ordering}; use std::time::Duration; + use tokio::sync::watch; + use tokio::time::Instant; + /// The loss is reported by `lost_after + renew_every`, the last /// attempt's bound, before the row's TTL ends. #[test] @@ -371,6 +406,48 @@ mod tests { assert!(times.renew_every < times.lost_after); } + /// With every renewal after the first unanswered, the loss is reported + /// no earlier than `lost_after` and no later than `lost_after + + /// renew_every` after the last successful renewal was sent: the attempt + /// sent just before the deadline ends at it, so the last attempt starts + /// on time. + #[tokio::test(start_paused = true)] + async fn the_loss_is_reported_one_renewal_bound_after_the_deadline() { + let times = super::RegistrationTimes::DEFAULT; + let calls = Arc::new(AtomicU32::new(0)); + let (lost, mut reported) = watch::channel(None); + let started = Instant::now(); + let renew = { + let calls = calls.clone(); + move || { + let first = calls.fetch_add(1, Ordering::SeqCst) == 0; + async move { + if first { + Ok(()) + } else { + std::future::pending().await + } + } + } + }; + tokio::spawn(super::renew_with(times, lost, renew)); + reported.changed().await.expect("reported"); + let lost_at = started.elapsed(); + // The first renewal, the one that succeeded, was sent one interval + // after the start. + let last_ok = times.renew_every; + assert!( + lost_at >= last_ok + times.lost_after, + "lost {lost_at:?} after the start" + ); + assert!( + lost_at <= last_ok + times.lost_after + times.renew_every, + "lost {lost_at:?} after the start" + ); + let reason = reported.borrow().clone().expect("a reason"); + assert!(reason.contains("not renewed for 30 s"), "{reason}"); + } + /// The times ADR-0023 (decision 5) and the module documentation give the /// lock and the registration: a 60 s lock row renewed every 15 s, acted /// on until 25 s before it expires and no longer renewed after 30 s From 9d15bc7ec67a469c2507966fbb8cea27fab7e832 Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Sun, 1 Nov 2026 21:12:00 +0000 Subject: [PATCH 27/45] test(devtools): measure the lock checks' bounds from the last renewal Both lock checks required the server's first 503 within 45 s of the pause, but the registration expires 45 s after its last renewal, which came up to 5 s before the pause, so a server that answered its first 503 42 s after the pause passed with its row or document already gone. They now require the 503 within 40 s of the pause. They also required exit 1 at any time within 120 s, which did not check that the server tries to register again: they now require it 25 to 50 s after the first 503, which holds the 30 s of attempts and the release's 10 s bound after them. With this branch's release binaries, the Cassandra server answered 503 35.2 s after the pause and exited 1 at 80 s, and the MongoDB server answered 503 at 30.2 s and exited 1 at 75 s; both checks pass. --- devtools/check-cassandra-catalog-lock | 24 +++++++++++++--------- devtools/check-mongodb-catalog-lock | 29 ++++++++++++++++----------- 2 files changed, 31 insertions(+), 22 deletions(-) diff --git a/devtools/check-cassandra-catalog-lock b/devtools/check-cassandra-catalog-lock index aef75773d..d27061206 100755 --- a/devtools/check-cassandra-catalog-lock +++ b/devtools/check-cassandra-catalog-lock @@ -33,8 +33,9 @@ with `init` and removes them at the end. Cassandra; migrate --forget 2 and the migrate --yes that applies 2 again run, and serve runs; - with the node paused, a server that cannot renew its registration answers - 503 before the registration's 45 s TTL ends, then tries to register again - for 30 s, stops with exit 1 and removes its PID file. + 503 within 40 s of the pause, before its row can expire (45 s after its + last renewal, which came up to 5 s before the pause), then tries to + register again for 30 s, stops with exit 1 and removes its PID file. init's HOME points into a temporary directory, so its TLS certificate is not written to the caller's home. The server binds 127.0.0.1 only. @@ -284,9 +285,11 @@ def main(binary: str, container: str, contact_point: str, port: int) -> int: return finish() report(os.path.exists(pid_file), f"the server wrote its PID file {pid_file}") # The node stops answering, so the server cannot renew its - # registration. It must answer 503 before the row's 45 s TTL ends, - # when a migrator could find no server, and stop with exit 1 once it - # cannot register again. + # registration. Its row expires 45 s after its last renewal, which + # came up to 5 s before the pause, so it must answer 503 within 40 s + # of the pause, before a migrator could find no server. It then tries + # to register again for 30 s and stops with exit 1: after the window + # and at most its 10 s bound on the release later. subprocess.run(["docker", "pause", container], check=True, capture_output=True) paused = True first_503, code, waited = watch_lost_registration( @@ -297,15 +300,16 @@ def main(binary: str, container: str, contact_point: str, port: int) -> int: with open(log, encoding="utf-8") as f: output = f.read()[-3000:] report( - first_503 is not None and first_503 < 45, + first_503 is not None and first_503 < 40, "the server answered 503 " - + ("never" if first_503 is None else f"{first_503:.0f} s after the node stopped answering") - + ", before its registration's 45 s TTL ends", + + ("never" if first_503 is None else f"{first_503:.1f} s after the node stopped answering") + + ", within 40 s, before its row can expire", output, ) report( - code == 1, - f"a server that cannot register again stops with exit {code} after {waited:.0f} s", + code == 1 and first_503 is not None and first_503 + 25 <= waited <= first_503 + 50, + f"a server that cannot register again stops with exit {code} {waited:.0f} s after the " + "pause, 25 to 50 s after its first 503", output, ) report(not os.path.exists(pid_file), "the stopped server removed its PID file") diff --git a/devtools/check-mongodb-catalog-lock b/devtools/check-mongodb-catalog-lock index a81bc6f64..25ef8e9e8 100755 --- a/devtools/check-mongodb-catalog-lock +++ b/devtools/check-mongodb-catalog-lock @@ -21,10 +21,11 @@ extenddb-check-lock-, published on 127.0.0.1 only: prints, run exactly as printed, applies what is missing; - a server killed with SIGKILL keeps migrate --yes waiting until its registration expires, at most 45 s after its last renewal; -- a server whose database stops answering answers 503 before its - registration's 45 s expiry, since the registration counts as lost 30 s - after its last successful renewal, then tries to register again for 30 s, - exits 1 and removes its PID file. +- a server whose database stops answering answers 503 within 40 s, before + its registration can expire (45 s after its last renewal, which came up to + 5 s before the pause), since the registration counts as lost 30 s after + its last successful renewal, then tries to register again for 30 s, exits + 1 and removes its PID file. Everything runs in a new temporary directory, with HOME pointed into it so init's TLS certificate is not written to the caller's home. The server @@ -152,10 +153,12 @@ def main(binary: str, mongo_port: int, api_port: int) -> int: else: report(False, "the server did not start for the kill check") - # A server whose database stops answering answers 503 before its - # registration's 45 s expiry, when a migrator could find no server, - # then exits 1 once it cannot register again, and removes its PID - # file. + # A server whose database stops answering answers 503 within 40 s, + # before its registration can expire (45 s after its last renewal, + # up to 5 s before the pause) and a migrator could find no server, + # then tries to register again for 30 s and exits 1, after the window + # and at most its 10 s bound on the release later, and removes its + # PID file. server = start(serve + ["--write-pid-file"], env, "serve-paused.log") servers.append(server) pid_file = os.path.join(work, "run", f"extenddb-{api_port}.pid") @@ -165,11 +168,13 @@ def main(binary: str, mongo_port: int, api_port: int) -> int: docker("unpause", container) with open("serve-paused.log", encoding="utf-8") as f: log = f.read() - report(first_503 is not None and first_503 < 45, + report(first_503 is not None and first_503 < 40, "the server answered 503 " - + ("never" if first_503 is None else f"{first_503:.0f} s after its database stopped answering") - + ", before its registration's 45 s expiry", log) - report(code == 1, f"the server exited {code} {took:.0f} s after its database stopped answering", log) + + ("never" if first_503 is None else f"{first_503:.1f} s after its database stopped answering") + + ", within 40 s, before its registration can expire", log) + report(code == 1 and first_503 is not None and first_503 + 25 <= took <= first_503 + 50, + f"the server exited {code} {took:.0f} s after its database stopped answering, " + "25 to 50 s after its first 503", log) report("registration" in log, "its last line names the lost registration", log) report(not os.path.exists(pid_file), "the server removed its PID file") else: From a16bfdd93176348d6bf991d02ce4572f5b1e53e8 Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Sun, 1 Nov 2026 21:12:00 +0000 Subject: [PATCH 28/45] fix(storage-cassandra): refuse a catalog-named table that lacks a column migration 1 creates foreign_contents took a table with a catalog table's name for the catalog's when each of its columns is one migration 1 gives that table, so an empty table of another program whose columns are a part of a catalog table's, such as accounts (account_id text PRIMARY KEY), was taken as the catalog's. init then created the other catalog tables beside it, and migration 1's ALTER TABLE ... ADD statements changed it. Such a table now belongs to another program unless every column it lacks is one that migration 1 adds with ALTER TABLE ... ADD, which a table created before that statement lacks; those columns are read from migration 1's catalog body. Tests: empty accounts and tables tables with a part of the catalog's columns are refused, naming the columns they lack and none that migration 1 adds with ALTER; a unit test reads the eight ALTER TABLE ... ADD columns from the body. A new test runs init's first step, ensure_app_user, on a keyspace of another program's tables and finds no row for the application role in system_auth.roles afterwards; with the refusal removed from that step, the step creates the role. Each fails with its part of the change reverted. --- .../storage-cassandra/src/migrations/mod.rs | 32 ++++- .../src/migrations/postcheck.rs | 15 ++- .../storage-cassandra/src/migrations/tests.rs | 29 +++++ crates/storage-cassandra/tests/migrations.rs | 114 +++++++++++++++++- 4 files changed, 184 insertions(+), 6 deletions(-) diff --git a/crates/storage-cassandra/src/migrations/mod.rs b/crates/storage-cassandra/src/migrations/mod.rs index 79054a703..863d02360 100644 --- a/crates/storage-cassandra/src/migrations/mod.rs +++ b/crates/storage-cassandra/src/migrations/mod.rs @@ -402,7 +402,8 @@ pub async fn create_lock_table( /// What an uninitialized `keyspace` holds that makes it another program's: /// the tables migration 1 does not create; a table of a catalog table's -/// name with a column migration 1 does not give it, or defines otherwise; +/// name with a column migration 1 does not give it, or defines otherwise, +/// or without a column migration 1 creates it with; /// a `settings` table without the catalog's `key` and `value` columns; /// `settings` rows other than the migration lock's; `schema_history` rows /// other than the ones migration 1 inserts; and a row in any other table. @@ -441,6 +442,7 @@ pub async fn foreign_contents( .map(|t| format!("the table {t}")) .collect(); let actual = postcheck::describe(session, keyspace).await?; + let added = baseline_added_columns(); for table in tables.iter().filter(|t| ours.contains(t.as_str())) { let differing = postcheck::columns_not_in(&expected, &actual, table); if table == "settings" @@ -457,6 +459,19 @@ pub async fn foreign_contents( )); continue; } + // Migration 1 creates each table with all its columns but the ones + // a later `ALTER TABLE ... ADD` adds, so only those may be missing. + let missing: Vec = postcheck::columns_missing(&expected, &actual, table) + .into_iter() + .filter(|column| !added.contains(&(table.as_str(), column.as_str()))) + .collect(); + if !missing.is_empty() { + foreign.push(format!( + "the table {table}, which lacks the column(s) {} migration 1 creates it with", + missing.join(", ") + )); + continue; + } let context = format!("read the rows of {keyspace}.{table}"); match table.as_str() { "settings" => { @@ -514,6 +529,21 @@ pub async fn foreign_contents( /// The `schema_history` rows migration 1's catalog body inserts, as /// `(version, description)`. +/// The (table, column) pairs migration 1 adds to the catalog keyspace with +/// `ALTER TABLE ... ADD IF NOT EXISTS`: a table an `init` or a 0.1.x release +/// created before such a statement lacks them, and migration 1 adds them. +fn baseline_added_columns() -> std::collections::BTreeSet<(&'static str, &'static str)> { + CATALOG_BASELINE + .lines() + .filter_map(|line| { + let rest = line.trim().strip_prefix("ALTER TABLE ")?; + let (table, rest) = rest.split_once(' ')?; + let (column, _) = rest.strip_prefix("ADD IF NOT EXISTS ")?.split_once(' ')?; + Some((table, column)) + }) + .collect() +} + fn baseline_history() -> Vec<(i32, &'static str)> { CATALOG_BASELINE .lines() diff --git a/crates/storage-cassandra/src/migrations/postcheck.rs b/crates/storage-cassandra/src/migrations/postcheck.rs index 96280b8a2..3119d2776 100644 --- a/crates/storage-cassandra/src/migrations/postcheck.rs +++ b/crates/storage-cassandra/src/migrations/postcheck.rs @@ -154,9 +154,22 @@ pub fn compare(expected: &Definitions, actual: &Definitions) -> Vec { differences } +/// The columns of `table` that `expected` has and `actual` lacks, by name. +#[must_use] +pub fn columns_missing(expected: &Definitions, actual: &Definitions, table: &str) -> Vec { + expected + .columns + .keys() + .filter_map(|key| { + let (t, name) = key.split_once('.')?; + (t == table && !actual.columns.contains_key(key)).then(|| name.to_owned()) + }) + .collect() +} + /// The columns of `table` in `actual` that `expected` does not have, or /// defines otherwise, by name. A column `expected` has and `actual` lacks -/// is not one of them. +/// is not one of them ([`columns_missing`] names those). #[must_use] pub fn columns_not_in(expected: &Definitions, actual: &Definitions, table: &str) -> Vec { actual diff --git a/crates/storage-cassandra/src/migrations/tests.rs b/crates/storage-cassandra/src/migrations/tests.rs index e2943deb5..037323f88 100644 --- a/crates/storage-cassandra/src/migrations/tests.rs +++ b/crates/storage-cassandra/src/migrations/tests.rs @@ -22,6 +22,35 @@ fn no_migration_is_additive_before_the_old_binary_job() { refuse_additive("cassandra", migration).unwrap_or_else(|e| panic!("{e}")); } +/// Every `ALTER TABLE ... ADD` of migration 1's catalog body is read as a +/// column that may be missing from a table an `init` left, and each one is +/// a column of the v1.0 schema. +#[test] +fn the_columns_migration_1_adds_with_alter_are_read_from_its_body() { + let added = super::baseline_added_columns(); + let statements = CATALOG_BASELINE + .lines() + .filter(|line| line.trim_start().starts_with("ALTER TABLE")) + .count(); + assert_eq!(added.len(), statements, "{added:?}"); + assert!( + added.contains(&("tables", "ttl_backfill_cursor")), + "{added:?}" + ); + assert!( + added.contains(&("backups_by_arn", "definition")), + "{added:?}" + ); + let expected: super::postcheck::Definitions = + serde_json::from_str(super::CATALOG_EXPECTED).expect("expected definitions"); + for (table, column) in &added { + assert!( + expected.columns.contains_key(&format!("{table}.{column}")), + "{table}.{column} is not in the v1.0 schema" + ); + } +} + #[test] fn every_migration_records_a_row_in_the_catalog_and_names_its_stores() { for id in MigrationId::ALL { diff --git a/crates/storage-cassandra/tests/migrations.rs b/crates/storage-cassandra/tests/migrations.rs index 99ca888c6..9a70c2e53 100644 --- a/crates/storage-cassandra/tests/migrations.rs +++ b/crates/storage-cassandra/tests/migrations.rs @@ -372,6 +372,37 @@ async fn tables_with_catalog_names_and_other_columns_or_rows_are_refused() { "nothing was created" ); + // Empty tables whose columns are a part of a catalog table's, with the + // same names and types: migration 1 creates each of these tables with + // the columns they lack, so another program made them. + let (subset, _subset_drops) = new_prefix("subset"); + let catalog = format!("{subset}_catalog"); + keyspace(&session, &catalog).await; + for statement in [ + format!("CREATE TABLE {catalog}.accounts (account_id text PRIMARY KEY)"), + format!( + "CREATE TABLE {catalog}.tables (account_id text, table_name text, PRIMARY KEY \ + ((account_id), table_name))" + ), + ] { + cql(&session, &statement).await; + } + match migrations.lock(&target_of(&subset), TIMEOUT).await { + Err(MigrationError::Refused(message)) => assert!( + message.contains("is not an ExtendDB catalog") + && message.contains("the table accounts, which lacks the column(s) account_name") + && message.contains("the table tables, which lacks the column(s)") + && !message.contains("ttl_backfill_cursor"), + "{message}" + ), + other => panic!("{:?}", other.err()), + } + assert_eq!( + helpers_tables(&session, &catalog).await, + ["accounts", "tables"], + "nothing was created" + ); + let (history, _history_drops) = new_prefix("history"); let catalog = format!("{history}_catalog"); keyspace(&session, &catalog).await; @@ -413,6 +444,68 @@ async fn tables_with_catalog_names_and_other_columns_or_rows_are_refused() { } } +/// `init` refuses another program's keyspace in its first step, before it +/// creates the application role, so a refused `init` leaves no role behind: +/// the keyspace of `metrics` and `tags` with a row each, as in +/// `tables_with_catalog_names_and_other_columns_or_rows_are_refused`. +#[tokio::test] +async fn init_refuses_a_foreign_keyspace_before_it_creates_its_role() { + use extenddb_storage::bootstrapper::Bootstrapper; + use extenddb_storage::management_store::OpError; + + if helpers::skip_without_cassandra() { + return; + } + let session = session().await; + let (prefix, _drops) = new_prefix("norole"); + let catalog = format!("{prefix}_catalog"); + keyspace(&session, &catalog).await; + for statement in [ + format!("CREATE TABLE {catalog}.metrics (id int PRIMARY KEY, reading double)"), + format!("CREATE TABLE {catalog}.tags (name text PRIMARY KEY, color text)"), + format!("INSERT INTO {catalog}.metrics (id, reading) VALUES (1, 2.5)"), + format!("INSERT INTO {catalog}.tags (name, color) VALUES ('urgent', 'red')"), + ] { + cql(&session, &statement).await; + } + let role = format!("{prefix}_app"); + let bootstrapper = bootstrapper_as(&prefix, &role, "a-new-password").await; + let refused = bootstrapper.ensure_app_user().await; + let roles = role_rows(&session, &role).await; + if roles > 0 { + cql(&session, &format!("DROP ROLE IF EXISTS {role}")).await; + } + match refused { + Err(OpError::AlreadyExists(message)) => assert!( + message.contains("is not an ExtendDB catalog") && message.contains("the table metrics"), + "{message}" + ), + other => panic!("{other:?}"), + } + assert_eq!(roles, 0, "the refused init created the role {role}"); + assert_eq!( + helpers_tables(&session, &catalog).await, + ["metrics", "tags"], + "nothing was created" + ); +} + +/// The rows `system_auth.roles` holds for `role`. +async fn role_rows(session: &CassandraSession, role: &str) -> usize { + session + .query_with_values( + "SELECT role FROM system_auth.roles WHERE role = ?", + cdrs_tokio::query_values!(role), + ) + .await + .expect("read system_auth.roles") + .response_body() + .expect("body") + .into_rows() + .unwrap_or_default() + .len() +} + /// The tables of `keyspace`, in name order. async fn helpers_tables(session: &CassandraSession, keyspace: &str) -> Vec { use cdrs_tokio::types::IntoRustByName; @@ -765,6 +858,19 @@ async fn a_create_account_stopped_before_its_row_leaves_a_current_catalog_a_retr /// test node's superuser, which is also the application role here, so the /// grants `init` sends name a role that exists. async fn bootstrapper(prefix: &str) -> extenddb_storage_cassandra::CassandraBootstrapper { + let config = helpers::test_config(); + let user = config.username.clone().unwrap_or_default(); + let password = config.password.clone().unwrap_or_default(); + bootstrapper_as(prefix, &user, &password).await +} + +/// `init`'s bootstrapper for the deployment of `prefix`, connected as the +/// test node's superuser, with `app_user` as the application role. +async fn bootstrapper_as( + prefix: &str, + app_user: &str, + app_password: &str, +) -> extenddb_storage_cassandra::CassandraBootstrapper { let mut config = helpers::test_config(); config.keyspace_prefix = prefix.to_owned(); config.rebuild_cached_connection_string(); @@ -777,10 +883,10 @@ async fn bootstrapper(prefix: &str) -> extenddb_storage_cassandra::CassandraBoot let bootstrap = extenddb_storage::bootstrapper::BootstrapConfig { host, port, - admin_user: user.clone(), - admin_password: Some(password.clone()), - app_user: user, - app_password: password, + admin_user: user, + admin_password: Some(password), + app_user: app_user.to_owned(), + app_password: app_password.to_owned(), catalog_db: format!("{prefix}_catalog"), data_db: String::new(), }; From fa4f16a142d4261bbe0f9b05e599577820a28372 Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Sun, 1 Nov 2026 21:12:00 +0000 Subject: [PATCH 29/45] fix(storage-cassandra): tell a keyspace that is not a catalog from its ledgers and settings before the lock Before it took the lock, migrate --yes on Cassandra read the full catalog status to decide whether the keyspace is a catalog, and that status runs migration 2's detect, which reads every principal's policies, outside --lock-timeout. It now decides from the ledgers and settings alone, through Runner::initialized, as MongoDB does since 4dfbea8d8; the status under the lock still runs the detect. Test: with migration 2 pending and the iam_policies table dropped, the lock is taken and the status under the lock is the read that fails; with the old check before the lock, taking the lock fails on that read. --- .../src/migrations/commands.rs | 15 ++++-- crates/storage-cassandra/tests/migrations.rs | 49 +++++++++++++++++++ 2 files changed, 61 insertions(+), 3 deletions(-) diff --git a/crates/storage-cassandra/src/migrations/commands.rs b/crates/storage-cassandra/src/migrations/commands.rs index ebd7f0aa5..f78dc4441 100644 --- a/crates/storage-cassandra/src/migrations/commands.rs +++ b/crates/storage-cassandra/src/migrations/commands.rs @@ -133,15 +133,24 @@ impl CatalogMigrations for CassandraCatalogMigrations { /// servers sees the lock and exits 6. An /// uninitialized keyspace that holds another program's tables or /// `settings` rows is refused before anything is written to it, and - /// named, since `init` refuses it too. A keyspace without the catalog's + /// named, since `init` refuses it too; whether it is uninitialized is + /// read from its ledgers and settings alone, as on MongoDB, so nothing + /// before the lock reads the policies. A keyspace without the catalog's /// `settings` table is not a catalog; `init` creates it before it takes /// the lock. async fn lock(&self, target: &Target, timeout: Duration) -> Result { let deployment = Deployment::from_target(target)?; let session = deployment.connect().await?; let catalog = deployment.catalog_keyspace(); - let status = catalog_status(session.clone(), &deployment.keyspace_prefix).await?; - refuse_foreign(&session, &deployment, &status).await?; + // Whether this is a catalog, from its ledgers and settings alone: + // the full status runs the pending migrations' `detect`, which for + // migration 2 reads every principal's policies, and would run here + // outside `timeout`. The status under the lock runs it. + let migrations = + CassandraMigrations::for_status(session.clone(), &deployment.keyspace_prefix); + if !Runner::new(&migrations, "").initialized().await? { + refuse_foreign(&session, &deployment, &CatalogStatus::Uninitialized).await?; + } if !has_columns(&session, &catalog, "settings", &["key", "value"]).await? { return Err(MigrationError::Status(CatalogStatus::Uninitialized)); } diff --git a/crates/storage-cassandra/tests/migrations.rs b/crates/storage-cassandra/tests/migrations.rs index 9a70c2e53..27558c177 100644 --- a/crates/storage-cassandra/tests/migrations.rs +++ b/crates/storage-cassandra/tests/migrations.rs @@ -490,6 +490,55 @@ async fn init_refuses_a_foreign_keyspace_before_it_creates_its_role() { ); } +/// `migrate --yes` tells whether the keyspace is a catalog before it takes +/// the lock from the ledgers and settings alone, so the status's reads for +/// the pending migrations, which for migration 2 read every principal's +/// policies, run under the lock and inside `--lock-timeout`: with migration +/// 2 pending and the policy table gone, the lock is taken, and the status +/// under the lock is the read that fails. +#[tokio::test] +async fn the_check_before_the_lock_reads_no_policy() { + if helpers::skip_without_cassandra() { + return; + } + let session = session().await; + let (prefix, _drops) = new_prefix("prelock"); + deployment(&session, &prefix).await; + let migrations = CassandraCatalogMigrations; + let target = target_of(&prefix); + let lock = migrations.lock(&target, TIMEOUT).await.expect("lock"); + migrations + .forget(&lock, 2) + .await + .expect("forget migration 2"); + migrations.release(lock).await; + // A user, whose policies migration 2's read looks up. + cql( + &session, + &format!( + "INSERT INTO {prefix}_catalog.iam_users (account_id, user_name, user_arn, created_at) \ + VALUES ('111111111111', 'reader', 'arn:aws:iam::111111111111:user/reader', \ + toTimestamp(now()))" + ), + ) + .await; + cql( + &session, + &format!("DROP TABLE {prefix}_catalog.iam_policies"), + ) + .await; + let lock = migrations + .lock(&target, TIMEOUT) + .await + .expect("the lock, taken before any policy is read"); + let status = migrations.status_under_lock(&lock, &target).await; + migrations.release(lock).await; + match status { + Err(e) => assert!(e.to_string().contains("iam_policies"), "{e}"), + Ok(status) => panic!("the status under the lock read no policy: {status:?}"), + } +} + /// The rows `system_auth.roles` holds for `role`. async fn role_rows(session: &CassandraSession, role: &str) -> usize { session From f6d64b9c018f0b206760cd6f018b103e76ab4573 Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Sun, 1 Nov 2026 21:12:00 +0000 Subject: [PATCH 30/45] fix(storage-mongodb): keep the driver's text in migration 2's policy reads list_policy_documents and boundary_document, which only migration 2 calls, answered a driver error with "Database error", so migration 2's error named a policy read that failed without saying why, for example a primary that did not answer. They now keep the driver's text, as the Cassandra store does. --- crates/storage-mongodb/src/management_store.rs | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/crates/storage-mongodb/src/management_store.rs b/crates/storage-mongodb/src/management_store.rs index 35db6b722..93e18cfe8 100644 --- a/crates/storage-mongodb/src/management_store.rs +++ b/crates/storage-mongodb/src/management_store.rs @@ -1473,14 +1473,11 @@ impl ManagementStore for MongoCatalogStore { }) .with_options(opts) .await - .map_err(|e| { - tracing::error!("list_policy_documents: {e}"); - OpError::Internal("Database error".to_owned()) - })?; - let docs: Vec = cursor.try_collect().await.map_err(|e| { - tracing::error!("list_policy_documents cursor: {e}"); - OpError::Internal("Database error".to_owned()) - })?; + .map_err(|e| OpError::Internal(format!("list_policy_documents: {e}")))?; + let docs: Vec = cursor + .try_collect() + .await + .map_err(|e| OpError::Internal(format!("list_policy_documents: {e}")))?; Ok(docs .iter() .map(|d| { @@ -1653,10 +1650,7 @@ impl ManagementStore for MongoCatalogStore { "principal_name": &principal_name, }) .await - .map_err(|e| { - tracing::error!("boundary_document: {e}"); - OpError::Internal("Database error".to_owned()) - })?; + .map_err(|e| OpError::Internal(format!("boundary_document: {e}")))?; Ok(doc.as_ref().map(stored_document)) }) } From 935c42202ff5f59c7200d3c8e29b48ba93cbf7a2 Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Sun, 1 Nov 2026 21:12:00 +0000 Subject: [PATCH 31/45] fix(storage): name five stores in the behind line and count the others The line a Behind status starts with named every store behind the catalog store, so a run stopped after migration 1's catalog step on a deployment with 1,000 accounts printed 1,000 keyspaces in one sentence in serve, verify and the migrate preview. The line now names the first five and counts the others: "... and 995 more stores are behind the catalog store at migration 4". --- crates/storage/src/migrations/status.rs | 61 ++++++++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/crates/storage/src/migrations/status.rs b/crates/storage/src/migrations/status.rs index 8f8c8b07c..b511763c2 100644 --- a/crates/storage/src/migrations/status.rs +++ b/crates/storage/src/migrations/status.rs @@ -94,10 +94,16 @@ pub fn stores_behind(pending: &[Pending]) -> Option<(u32, Vec<(&Pending, u32)>)> (!behind.is_empty()).then_some((catalog_at, behind)) } +/// How many stores [`behind_line`] names before it counts the others. +pub const BEHIND_NAMED: usize = 5; + /// What a `Behind` status says first when [`stores_behind`] names stores: /// `account store x is at migration 0, behind the catalog store at /// migration 4`, or for several, `account store x (at migration 0) and data /// store y (at migration 3) are behind the catalog store at migration 4`. +/// Past [`BEHIND_NAMED`] stores, the others are counted: `account store a +/// (at migration 0), ..., account store e (at migration 0) and 995 more +/// stores are behind the catalog store at migration 4`. #[must_use] pub fn behind_line(catalog_at: u32, behind: &[(&Pending, u32)]) -> String { let named = |p: &Pending| format!("{} store {}", p.store.as_str(), p.store_name); @@ -109,9 +115,15 @@ pub fn behind_line(catalog_at: u32, behind: &[(&Pending, u32)]) -> String { } let mut items: Vec = behind .iter() + .take(BEHIND_NAMED) .map(|(p, at)| format!("{} (at migration {at})", named(p))) .collect(); - let last = items.pop().unwrap_or_default(); + let more = behind.len().saturating_sub(BEHIND_NAMED); + let last = match more { + 0 => items.pop().unwrap_or_default(), + 1 => "1 more store".to_owned(), + n => format!("{n} more stores"), + }; format!( "{} and {last} are behind the catalog store at migration {catalog_at}", items.join(", ") @@ -344,3 +356,50 @@ impl MigrationError { } } } + +#[cfg(test)] +mod tests { + use super::{BEHIND_NAMED, Pending, behind_line}; + use crate::migrations::table::{Kind, StoreKind}; + + fn account(n: usize) -> Pending { + Pending { + id: 1, + name: "baseline", + store: StoreKind::Account, + store_name: format!("ks_account_{n}"), + kind: Kind::Breaking, + } + } + + /// Two stores are both named; past `BEHIND_NAMED` the others are + /// counted, as on a deployment with many account keyspaces behind. + #[test] + fn the_behind_line_names_a_few_stores_and_counts_the_others() { + let two: Vec = (1..=2).map(account).collect(); + let behind: Vec<(&Pending, u32)> = two.iter().map(|p| (p, 0)).collect(); + assert_eq!( + behind_line(4, &behind), + "account store ks_account_1 (at migration 0) and account store ks_account_2 (at \ + migration 0) are behind the catalog store at migration 4" + ); + + let many: Vec = (1..=1000).map(account).collect(); + let behind: Vec<(&Pending, u32)> = many.iter().map(|p| (p, 0)).collect(); + let line = behind_line(4, &behind); + assert!( + line.starts_with("account store ks_account_1 (at migration 0), ") + && line.contains("ks_account_5 (at migration 0) and 995 more stores are behind") + && !line.contains("ks_account_6"), + "{line}" + ); + + let one_more: Vec = (1..=BEHIND_NAMED + 1).map(account).collect(); + let behind: Vec<(&Pending, u32)> = one_more.iter().map(|p| (p, 0)).collect(); + assert!( + behind_line(4, &behind).contains(" and 1 more store are behind"), + "{}", + behind_line(4, &behind) + ); + } +} From 5a09aa94590b0ff26124c0bb5b26b4d642dc50f2 Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Sun, 1 Nov 2026 21:12:00 +0000 Subject: [PATCH 32/45] test(storage-cassandra): take ACCOUNT_RUNS in the lapsed-lease account run test an_account_run_under_a_lapsed_lease_writes_nothing runs an account migration and took no ACCOUNT_RUNS guard, against what c67275f50 says of the tests that run one. Its run stops at its first write, so it held the process's account mutex for no measurable time, but with the guard the rule has no exception. --- crates/storage-cassandra/tests/migrations/lock.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/storage-cassandra/tests/migrations/lock.rs b/crates/storage-cassandra/tests/migrations/lock.rs index 01d871d04..b56c2d441 100644 --- a/crates/storage-cassandra/tests/migrations/lock.rs +++ b/crates/storage-cassandra/tests/migrations/lock.rs @@ -255,6 +255,7 @@ async fn an_account_run_under_a_lapsed_lease_writes_nothing() { if helpers::skip_without_cassandra() { return; } + let _runs = super::ACCOUNT_RUNS.read().await; let session = session().await; let (prefix, _drops) = new_prefix("lapsed"); let catalog = format!("{prefix}_catalog"); From 9c6c248854e674d213cd4c8ff2db8b2990a06146 Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Sun, 1 Nov 2026 21:12:00 +0000 Subject: [PATCH 33/45] docs(cassandra): say what decides recovery after a node restart, and what a stopped init leaves docs/cassandra-setup.md said that a server registers again when the node answers within its 30 s window. After a docker restart of a single node, the server registered again in 1 run of 4 and stopped with exit 1 in the other 3, while its sessions still had no connection to the node that had answered: whether it comes back depends on the Cassandra driver reconnecting in time. The page now says so, that the server also stops its background workers while it holds no registration, and that it should run under a supervisor that starts it again. It also says that an init stopped while it created the default account's keyspace leaves that keyspace with no accounts row, that the next init does not use it, and how to drop it. --- docs/cassandra-setup.md | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/docs/cassandra-setup.md b/docs/cassandra-setup.md index 52d61a1e0..db794a1a5 100644 --- a/docs/cassandra-setup.md +++ b/docs/cassandra-setup.md @@ -134,6 +134,14 @@ init creates: Further account keyspaces are created when accounts are added. +An `init` that stopped after its migrations and before the admin user is +finished by the next `init`, which writes the rows the catalog lacks. One that +stopped while it created the default account's keyspace, before the account's +row in `_catalog.accounts`, leaves that keyspace with no account: the +next `init` creates another default account and keyspace, and the first +keyspace, whose id is in no `accounts` row, stays until you drop it with +`DROP KEYSPACE _account_`. + > **Save the admin credentials.** They are printed once and cannot be retrieved > later. @@ -253,10 +261,13 @@ until that is configurable. ExtendDB creates its keyspaces with With a replication factor of 1, the catalog has one replica, so a restart of that node stops every request to it. A running server renews its serving registration every 5 s and keeps serving while renewals fail. Once it has not -renewed for 30 s, it answers new requests with `ServiceUnavailable` and tries -to register again every second for up to 30 s. When the node answers in that -time, the server registers again and serves on; otherwise it stops with -exit 1. +renewed for 30 s, it answers new requests with `ServiceUnavailable`, stops its +background workers, and tries to register again every second for up to 30 s. +It registers again, starts its workers again and serves on only when the +Cassandra driver has reconnected to the node within that time, which after a +restart of 30 s or more it often has not; otherwise it stops with exit 1. Run +the server under a supervisor that starts it again, such as systemd or +Kubernetes. ```toml [storage.cassandra] From a0a902b89af856899eb89b2054654feffed10364 Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Sun, 1 Nov 2026 21:12:00 +0000 Subject: [PATCH 34/45] test(storage-mongodb): a primary that hangs before a renewal and answers again keeps the registration D3 asked for a test that freezes the primary before a renewal is due. The MongoDB CI jobs run a single-member set, so the test puts a proxy in front of it that holds every byte while it is paused: the silence starts 50 ms before the first renewal is due and lasts 1.2 s, inside the 2 s deadline the test gives the registration, and no loss is reported in the 3 s after it ends. With a failed renewal counted as a loss at once, the test fails on the renewal sent into the silence. --- .../tests/migrations_holders.rs | 43 ++++++- .../tests/migrations_support/mod.rs | 1 + .../tests/migrations_support/proxy.rs | 105 ++++++++++++++++++ 3 files changed, 147 insertions(+), 2 deletions(-) create mode 100644 crates/storage-mongodb/tests/migrations_support/proxy.rs diff --git a/crates/storage-mongodb/tests/migrations_holders.rs b/crates/storage-mongodb/tests/migrations_holders.rs index d663111dc..5128bc1c0 100644 --- a/crates/storage-mongodb/tests/migrations_holders.rs +++ b/crates/storage-mongodb/tests/migrations_holders.rs @@ -19,9 +19,10 @@ use extenddb_storage::migrations::{ CatalogStatus, HasManagementStore, HeldLock, MigrationBackend, MigrationError, }; use extenddb_storage_mongodb::migrations::{ - MigrationLock, MongoMigrations, ServingRegistration, Timing, lock, migrate_holding, - register_and_check, status_holding, + MigrationLock, MongoMigrations, MongoTarget, ServingRegistration, Timing, lock, + migrate_holding, register_and_check, status_holding, }; +use migrations_support::proxy::PausingProxy; use migrations_support::{Deployment, fixture, server}; /// The id of the deployment's first account: `account_id`, or `_id` for the @@ -489,3 +490,41 @@ async fn the_lock_timeout_bounds_migrate_yes_on_a_catalog_with_many_policies() { ); d.drop_databases().await; } + +/// A primary that stops answering just before a renewal is due, and answers +/// again before the registration's deadline, does not take the registration: +/// the renewal sent into the silence is bounded, the next ones go out every +/// retry interval, and one gets through once the primary answers. The +/// primary's silence is a proxy that holds every byte while it is paused. +#[tokio::test(flavor = "multi_thread")] +async fn a_primary_that_hangs_before_a_renewal_and_answers_again_keeps_the_registration() { + let Some(uri) = server() else { return }; + let d = Deployment::new(&uri, "hang").await; + let proxy = PausingProxy::in_front_of(&uri).await; + let through_proxy = + MongoTarget::with_databases(proxy.uri(), d.target.catalog.clone(), d.target.data.clone()); + let timing = Timing { + registration_renew_every: Duration::from_millis(400), + registration_deadline: Duration::from_secs(2), + registration_retry_every: Duration::from_millis(50), + ..Timing::PRODUCTION + }; + let registration = ServingRegistration::register(&through_proxy, "test", timing) + .await + .expect("register"); + let mut lost = registration.lost(); + // The first renewal is due 400 ms after the registration: the silence + // starts 50 ms before it and lasts 1.2 s, inside the 2 s deadline. + tokio::time::sleep(Duration::from_millis(350)).await; + proxy.pause(); + tokio::time::sleep(Duration::from_millis(1200)).await; + proxy.resume(); + let changed = tokio::time::timeout(Duration::from_secs(3), lost.changed()).await; + let reason = lost.borrow().clone(); + registration.release().await; + d.drop_databases().await; + assert!( + changed.is_err() && reason.is_none(), + "the registration was lost: {reason:?}" + ); +} diff --git a/crates/storage-mongodb/tests/migrations_support/mod.rs b/crates/storage-mongodb/tests/migrations_support/mod.rs index 567687a41..06bddd927 100644 --- a/crates/storage-mongodb/tests/migrations_support/mod.rs +++ b/crates/storage-mongodb/tests/migrations_support/mod.rs @@ -12,6 +12,7 @@ #![allow(dead_code)] pub mod fixture; +pub mod proxy; use std::sync::atomic::{AtomicU32, Ordering}; use std::time::Duration; diff --git a/crates/storage-mongodb/tests/migrations_support/proxy.rs b/crates/storage-mongodb/tests/migrations_support/proxy.rs new file mode 100644 index 000000000..b2e4244aa --- /dev/null +++ b/crates/storage-mongodb/tests/migrations_support/proxy.rs @@ -0,0 +1,105 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! A TCP proxy in front of the test server that stops passing bytes while it +//! is paused and passes them again once it is resumed, as a primary that +//! hangs and then answers again does. Bytes that arrive while it is paused +//! are held, not dropped, and connections stay open. + +use std::net::SocketAddr; + +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::tcp::{OwnedReadHalf, OwnedWriteHalf}; +use tokio::net::{TcpListener, TcpStream}; +use tokio::sync::watch; +use tokio::task::JoinHandle; + +pub struct PausingProxy { + pub address: SocketAddr, + paused: watch::Sender, + accepting: JoinHandle<()>, +} + +impl PausingProxy { + /// A proxy on a port of 127.0.0.1 in front of the first host of `uri`. + /// + /// # Panics + /// + /// When the URI names no host or the port cannot be bound. + pub async fn in_front_of(uri: &str) -> Self { + let upstream = uri + .strip_prefix("mongodb://") + .and_then(|rest| rest.split(['/', '?', ',']).next()) + .map(|host| host.rsplit_once('@').map_or(host, |(_, h)| h).to_owned()) + .expect("a host in the URI"); + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); + let address = listener.local_addr().expect("address"); + let (paused, watching) = watch::channel(false); + let accepting = tokio::spawn(async move { + while let Ok((client, _)) = listener.accept().await { + let Ok(server) = TcpStream::connect(&upstream).await else { + continue; + }; + let (client_read, client_write) = client.into_split(); + let (server_read, server_write) = server.into_split(); + tokio::spawn(copy_unless_paused( + client_read, + server_write, + watching.clone(), + )); + tokio::spawn(copy_unless_paused( + server_read, + client_write, + watching.clone(), + )); + } + }); + Self { + address, + paused, + accepting, + } + } + + /// The connection string of the proxy, for a direct connection. + #[must_use] + pub fn uri(&self) -> String { + format!("mongodb://{}/?directConnection=true", self.address) + } + + pub fn pause(&self) { + self.paused.send_replace(true); + } + + pub fn resume(&self) { + self.paused.send_replace(false); + } +} + +impl Drop for PausingProxy { + fn drop(&mut self) { + self.accepting.abort(); + } +} + +/// Copy `from` to `to`, holding what is read while the proxy is paused +/// until it is resumed. +async fn copy_unless_paused( + mut from: OwnedReadHalf, + mut to: OwnedWriteHalf, + mut paused: watch::Receiver, +) { + let mut buffer = vec![0_u8; 16 * 1024]; + loop { + let read = match from.read(&mut buffer).await { + Ok(0) | Err(_) => return, + Ok(n) => n, + }; + if paused.wait_for(|paused| !*paused).await.is_err() { + return; + } + if to.write_all(&buffer[..read]).await.is_err() { + return; + } + } +} From 82a76e67b5a469e8f04993dc8242157e0c36145f Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Sun, 1 Nov 2026 21:46:53 +0000 Subject: [PATCH 35/45] docs(storage-cassandra): put baseline_history's doc comment back on it a16bfdd93 added baseline_added_columns between baseline_history's doc comment and the function, so the new function carried both comments and baseline_history had none. --- crates/storage-cassandra/src/migrations/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/storage-cassandra/src/migrations/mod.rs b/crates/storage-cassandra/src/migrations/mod.rs index 863d02360..ad5aa82c7 100644 --- a/crates/storage-cassandra/src/migrations/mod.rs +++ b/crates/storage-cassandra/src/migrations/mod.rs @@ -527,8 +527,6 @@ pub async fn foreign_contents( Ok(foreign) } -/// The `schema_history` rows migration 1's catalog body inserts, as -/// `(version, description)`. /// The (table, column) pairs migration 1 adds to the catalog keyspace with /// `ALTER TABLE ... ADD IF NOT EXISTS`: a table an `init` or a 0.1.x release /// created before such a statement lacks them, and migration 1 adds them. @@ -544,6 +542,8 @@ fn baseline_added_columns() -> std::collections::BTreeSet<(&'static str, &'stati .collect() } +/// The `schema_history` rows migration 1's catalog body inserts, as +/// `(version, description)`. fn baseline_history() -> Vec<(i32, &'static str)> { CATALOG_BASELINE .lines() From 21ff73c629db1a2647bf62170b19c65c1e92669a Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Sun, 1 Nov 2026 21:46:53 +0000 Subject: [PATCH 36/45] fix(storage-cassandra): count the first registration deadline from the INSERT's send renew_with set last_ok when the renewal task started, after register_timed had the INSERT's answer. A replica can start the row's TTL as soon as it applies the INSERT, before that answer, so the loss came up to 35 s after the answer, and with a slow answer less than 10 s before the row could expire. register_timed now reads the clock before it sends the INSERT and passes that time in as the first last_ok, as MongoDB's registration does. The module and RegistrationTimes docs say the deadline counts from the last successful write of the row, the registering INSERT included. the_first_deadline_counts_from_the_registrations_send answers the INSERT 10 s after its send and no renewal after it, and requires the loss within 35 s of the send. With last_ok set at the task's start, the loss came 45 s after the send and the test failed. --- .../src/migrations/registrations.rs | 68 ++++++++++++++----- 1 file changed, 51 insertions(+), 17 deletions(-) diff --git a/crates/storage-cassandra/src/migrations/registrations.rs b/crates/storage-cassandra/src/migrations/registrations.rs index 8e8235973..da0b0b693 100644 --- a/crates/storage-cassandra/src/migrations/registrations.rs +++ b/crates/storage-cassandra/src/migrations/registrations.rs @@ -6,12 +6,12 @@ //! A server registers in `extenddb_servers`: one row, written at `QUORUM` //! with a TTL of 45 s and again every 5 s at `LOCAL_QUORUM`. A renewal that //! fails is tried again every second. The registration counts as lost only -//! when no renewal has succeeded for 30 s, after a last attempt at that -//! deadline. Each attempt is bounded by 5 s, and one sent before the +//! when no write of the row has succeeded for 30 s, after a last attempt at +//! that deadline. Each attempt is bounded by 5 s, and one sent before the //! deadline also by the time left to it, so the loss is reported at most -//! 35 s after the last successful renewal was sent, at least 10 s before its -//! row expires and `migrate --yes`, which waits while a row exists, can see -//! it gone. The server then stops admitting requests and stops its +//! 35 s after the last successful write was sent (the INSERT that +//! registered, then each renewal), at least 10 s before its row expires and +//! `migrate --yes`, which waits while a row exists, can see it gone. The server then stops admitting requests and stops its //! background workers, and registers again with a new row //! ([`super::registrar`]). @@ -38,11 +38,12 @@ pub struct RegistrationTimes { pub renew_every: Duration, /// How soon a renewal that failed is tried again. pub retry_every: Duration, - /// How long after the last successful renewal was sent the registration - /// counts as lost. A renewal sent before then is bounded by the time - /// left, and the last attempt, at that deadline, by `renew_every`, so - /// the loss is reported by `lost_after + renew_every`, which must stay - /// under the TTL. + /// How long after the last successful write of the row was sent (the + /// INSERT that registered, then each renewal) the registration counts as + /// lost. A renewal sent before then is bounded by the time left, and the + /// last attempt, at that deadline, by `renew_every`, so the loss is + /// reported by `lost_after + renew_every`, which must stay under the + /// TTL. pub lost_after: Duration, } @@ -224,6 +225,10 @@ impl ServingRegistration { let values = || { cdrs_tokio::query_values!(server_id.as_str(), host, pid, release, started_at.as_str()) }; + // The row's TTL can start when a replica applies the INSERT, which + // can be as early as its send, so the first deadline counts from + // here and not from the answer. + let sent = Instant::now(); send( &session, &cql, @@ -239,6 +244,7 @@ impl ServingRegistration { cql, values(), times, + sent, lost_tx, )); Ok(Self { @@ -315,16 +321,18 @@ async fn delete_row( /// Write the row again every `renew_every`; after a failure try again every /// `retry_every`. Report the registration lost once the attempt at the -/// deadline, `lost_after` after the last successful renewal was sent, fails -/// too. +/// deadline, `lost_after` after the last successful write of the row was +/// sent (`registered`, the send of the INSERT that registered, until a +/// renewal succeeds), fails too. async fn renew_registration( session: Arc, cql: String, values: cdrs_tokio::query::QueryValues, times: RegistrationTimes, + registered: Instant, lost: watch::Sender>, ) { - renew_with(times, lost, || { + renew_with(times, registered, lost, || { // LOCAL_QUORUM, so a running server stays up while a remote // datacenter is unreachable. let renewal = send( @@ -344,17 +352,18 @@ async fn renew_registration( /// sent before the deadline is bounded by `renew_every` and by the time left /// to the deadline, so the attempt at the deadline is never late; that last /// attempt is bounded by `renew_every`. The loss is therefore reported at -/// most `lost_after + renew_every` after the last successful renewal was -/// sent. +/// most `lost_after + renew_every` after the last successful write of the +/// row was sent, the first of them at `registered`. async fn renew_with( times: RegistrationTimes, + registered: Instant, lost: watch::Sender>, mut renew: F, ) where F: FnMut() -> Fut, Fut: std::future::Future>, { - let mut last_ok = Instant::now(); + let mut last_ok = registered; let mut next = last_ok + times.renew_every; loop { let deadline = last_ok + times.lost_after; @@ -430,7 +439,7 @@ mod tests { } } }; - tokio::spawn(super::renew_with(times, lost, renew)); + tokio::spawn(super::renew_with(times, started, lost, renew)); reported.changed().await.expect("reported"); let lost_at = started.elapsed(); // The first renewal, the one that succeeded, was sent one interval @@ -448,6 +457,31 @@ mod tests { assert!(reason.contains("not renewed for 30 s"), "{reason}"); } + /// The first deadline counts from when the INSERT that registered was + /// sent: with that INSERT answered 10 s after its send and no renewal + /// answered, the loss is still reported by `lost_after + renew_every` + /// after the send, before a row whose TTL started at the send expires. + #[tokio::test(start_paused = true)] + async fn the_first_deadline_counts_from_the_registrations_send() { + let times = super::RegistrationTimes::DEFAULT; + let registered = Instant::now(); + tokio::time::advance(Duration::from_secs(10)).await; + let (lost, mut reported) = watch::channel(None); + tokio::spawn(super::renew_with(times, registered, lost, || { + std::future::pending::>() + })); + reported.changed().await.expect("reported"); + let lost_at = registered.elapsed(); + assert!( + lost_at >= times.lost_after, + "lost {lost_at:?} after the INSERT was sent" + ); + assert!( + lost_at <= times.lost_after + times.renew_every, + "lost {lost_at:?} after the INSERT was sent" + ); + } + /// The times ADR-0023 (decision 5) and the module documentation give the /// lock and the registration: a 60 s lock row renewed every 15 s, acted /// on until 25 s before it expires and no longer renewed after 30 s From d8fc8d75b9a8621ee650eceac31bb8355c5eb5cb Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Sun, 1 Nov 2026 21:59:21 +0000 Subject: [PATCH 37/45] fix(storage-cassandra): stop the GSI backfills the recovery worker started with its set The GSI backfill recovery worker ran each backfill in a task of its own, started with tokio::spawn and given no token, and every new index's backfill runs that way. ca132671e stops a set's workers at a lost registration and releases the registration once they have ended, but these tasks were not among the set's handles: a backfill went on writing index rows, its heartbeat and the index's ACTIVE status after the stop and after the release. The worker now keeps its backfills in a JoinSet and runs each one until the set's token fires. Once it fires the backfill stops at its next request and the task writes nothing more: the index stays CREATING under its propagation hold, its heartbeat goes stale, and a later pass runs the backfill again, as after a crash. A backfill that has finished its scan completes the activation and the hold's release. Once the token has fired the worker takes no further index, and it returns only after its backfills have ended, so the set's drain and abort cover them. a_backfill_the_recovery_worker_started_ends_before_the_worker_returns adds an index to a table of 20,000 items, fires the worker's token after the backfill's first heartbeat, and requires the index to stay CREATING with the same row count 2 s and 4 s after the worker returned. It passed with 1,000 rows written. With the backfill detached and given no token, as before, and with it in the JoinSet but given no token, the index was ACTIVE with 20,000 rows and the test failed. --- crates/storage-cassandra/src/workers.rs | 100 ++++++-- .../storage-cassandra/tests/direct/index.rs | 224 ++++++++++++++++++ 2 files changed, 302 insertions(+), 22 deletions(-) diff --git a/crates/storage-cassandra/src/workers.rs b/crates/storage-cassandra/src/workers.rs index b6efb5381..49935f530 100644 --- a/crates/storage-cassandra/src/workers.rs +++ b/crates/storage-cassandra/src/workers.rs @@ -467,8 +467,9 @@ async fn gsi_apply_index( /// Woken immediately when `apply_table_update` commits a new `CREATING` index, /// and also sweeps periodically as a backstop. Treats a NULL `backfill_heartbeat` /// (never started) and a heartbeat older than `stale_threshold` (crashed) the -/// same way: drop+recreate the data table and rerun the backfill. Returns once -/// `shutdown` fires, before the next keyspace. +/// same way: drop+recreate the data table and rerun the backfill. Once +/// `shutdown` fires it takes no further index, and it returns only after the +/// backfills it started have ended, so the server's stop covers them too. pub(crate) async fn poll_gsi_backfill_recovery( engine: Arc, notify: Arc, @@ -476,12 +477,15 @@ pub(crate) async fn poll_gsi_backfill_recovery( scan_interval: Duration, shutdown: CancellationToken, ) { - loop { + let mut backfills = tokio::task::JoinSet::new(); + 'scan: loop { // Wait for an explicit wake or the periodic backstop timeout. tokio::select! { - () = shutdown.cancelled() => return, + () = shutdown.cancelled() => break, _ = tokio::time::timeout(scan_interval, notify.notified()) => {} } + // Collect the backfills that have ended. + while backfills.try_join_next().is_some() {} let keyspaces = match list_account_keyspaces(&engine).await { Ok(ks) => ks, @@ -493,19 +497,55 @@ pub(crate) async fn poll_gsi_backfill_recovery( for account_ks in &keyspaces { if shutdown.is_cancelled() { - return; + break 'scan; } - if let Err(e) = recover_stale_backfills(&engine, account_ks, stale_threshold).await { + if let Err(e) = recover_stale_backfills( + &engine, + account_ks, + stale_threshold, + &shutdown, + &mut backfills, + ) + .await + { tracing::warn!("gsi_recovery: {account_ks}: {e}"); } } } + // A backfill stops at its next request once `shutdown` fires; one that + // has finished its scan completes the activation and the hold's release. + while backfills.join_next().await.is_some() {} +} + +/// The GSI backfill recovery worker `serve` runs, with the stale threshold +/// and scan interval given, until `shutdown` fires; for tests. +#[cfg(any(test, feature = "test-util"))] +#[doc(hidden)] +#[must_use] +pub fn spawn_gsi_backfill_recovery_for_tests( + engine: Arc, + stale_threshold: Duration, + scan_interval: Duration, + shutdown: CancellationToken, +) -> tokio::task::JoinHandle<()> { + let notify = engine.gsi_backfill_notify.clone(); + tokio::spawn(poll_gsi_backfill_recovery( + engine, + notify, + stale_threshold, + scan_interval, + shutdown, + )) } +/// Rerun the stale backfills of `account_ks`, each in a task of +/// `backfills` that stops once `shutdown` fires. async fn recover_stale_backfills( engine: &Arc, account_ks: &str, stale_threshold: Duration, + shutdown: &CancellationToken, + backfills: &mut tokio::task::JoinSet<()>, ) -> Result<(), extenddb_storage::error::StorageError> { use cdrs_tokio::types::IntoRustByName as _; @@ -531,6 +571,9 @@ async fn recover_stale_backfills( .await?; for row in rows { + if shutdown.is_cancelled() { + return Ok(()); + } let heartbeat_ms: Option = row.get_by_name("backfill_heartbeat").ok().flatten(); let is_stale = heartbeat_ms.is_none_or(|hb| hb < cutoff_ms); if !is_stale { @@ -651,7 +694,8 @@ async fn recover_stale_backfills( continue; } - // Spawn the backfill so the recovery loop is not blocked. + // Run the backfill in a task of its own so the recovery loop is not + // blocked, and in `backfills` so the worker returns only after it. let session = engine.session_arc(); let gsi_queue = Arc::clone(&engine.gsi_queue); let account_ks_owned = account_ks.to_owned(); @@ -659,21 +703,33 @@ async fn recover_stale_backfills( let table_id_owned = table_id.clone(); let index_id_owned = index_id.clone(); let index_name_owned = index_name.clone(); - tokio::spawn(async move { - let result = crate::data::ddl::backfill_gsi( - &session, - &account_ks_owned, - &catalog_ks_owned, - &table_id_owned, - &index_id_owned, - &index_name_owned, - &index_key_schema, - &base_attr_defs, - &base_key_schema, - &base_attr_defs, - &projection, - ) - .await; + let shutdown = shutdown.clone(); + backfills.spawn(async move { + // Once `shutdown` fires the backfill stops at its next request + // and the task writes nothing more: the index stays CREATING + // under its hold, its heartbeat goes stale, and a later pass runs + // the backfill again, as after a crash. + let Some(result) = shutdown + .run_until_cancelled(crate::data::ddl::backfill_gsi( + &session, + &account_ks_owned, + &catalog_ks_owned, + &table_id_owned, + &index_id_owned, + &index_name_owned, + &index_key_schema, + &base_attr_defs, + &base_key_schema, + &base_attr_defs, + &projection, + )) + .await + else { + tracing::info!( + "gsi_recovery: backfill for '{index_name_owned}' stopped by the shutdown" + ); + return; + }; if let Err(e) = result { tracing::error!( diff --git a/crates/storage-cassandra/tests/direct/index.rs b/crates/storage-cassandra/tests/direct/index.rs index ddb4ac02c..0d5423a05 100644 --- a/crates/storage-cassandra/tests/direct/index.rs +++ b/crates/storage-cassandra/tests/direct/index.rs @@ -897,3 +897,227 @@ async fn test_ttl_on_a_table_with_an_async_gsi_is_refused_as_a_validation_error( } panic!("table {name} never became ACTIVE"); } + +/// The index's status, backfill heartbeat and id in the catalog. +async fn index_state( + engine: &CassandraEngine, + table_id: &str, + index_name: &str, +) -> (String, Option, String) { + use cdrs_tokio::types::IntoRustByName as _; + + let catalog_keyspace = engine.catalog_keyspace(); + let rows = extenddb_storage_cassandra::cassandra_util::query_rows::< + extenddb_storage::error::StorageError, + >( + &engine.session_arc(), + &format!( + "SELECT index_status, backfill_heartbeat, index_id FROM {catalog_keyspace}.indexes \ + WHERE table_id = ? AND index_name = ?" + ), + cdrs_tokio::query_values!(table_id, index_name), + "index_state", + ) + .await + .expect("read the index row"); + let row = rows.first().expect("the index row"); + let status: String = row + .get_by_name("index_status") + .ok() + .flatten() + .unwrap_or_default(); + let heartbeat: Option = row.get_by_name("backfill_heartbeat").ok().flatten(); + let index_id: String = row + .get_by_name("index_id") + .ok() + .flatten() + .unwrap_or_default(); + (status, heartbeat, index_id) +} + +/// The rows of an index's data table. +async fn index_rows(engine: &CassandraEngine, account_keyspace: &str, index_id: &str) -> i64 { + use cdrs_tokio::types::IntoRustByName as _; + + let rows = extenddb_storage_cassandra::cassandra_util::query_rows::< + extenddb_storage::error::StorageError, + >( + &engine.session_arc(), + &format!( + "SELECT COUNT(*) FROM {account_keyspace}.{}", + extenddb_storage_cassandra::data::ddl::index_table_name(index_id) + ), + cdrs_tokio::query_values!(), + "index_rows", + ) + .await + .expect("count the index rows"); + let count: Option = rows + .first() + .and_then(|row| row.get_by_name("count").ok().flatten()); + count.expect("a count") +} + +/// A backfill the GSI recovery worker started ends before the worker +/// returns: once the worker's token fires, the backfill writes no more index +/// rows and does not mark the index ACTIVE, so the server's stop at a lost +/// registration, which waits for its workers, covers it. The table holds 40 +/// pages, so the backfill is still running when the token fires after its +/// first page; the test fails, rather than passing, if it is not. +#[tokio::test] +async fn a_backfill_the_recovery_worker_started_ends_before_the_worker_returns() { + use extenddb_core::types::{CreateGsiAction, GlobalSecondaryIndexUpdate, UpdateTableInput}; + use extenddb_storage::TableEngine as _; + use std::time::Duration; + + const ITEMS: usize = 20_000; + + if crate::helpers::skip_without_cassandra() { + return; + } + let config = test_config(); + let engine = Arc::new(CassandraEngine::new(&config, "us-east-1").await.unwrap()); + let table = crate::helpers::TestTable::new(&engine, "BackfillStopTable", false).await; + tokio::time::sleep(Duration::from_millis(350)).await; + engine + .process_control_plane_transitions() + .await + .expect("activate the table"); + + let mut writes = tokio::task::JoinSet::new(); + for i in 0..ITEMS { + if writes.len() >= 64 { + writes + .join_next() + .await + .expect("a write") + .expect("put_item"); + } + let engine = engine.clone(); + let key_info = table.key_info.clone(); + writes.spawn(async move { + let mut item = Item::new(); + item.insert("id".to_owned(), AttributeValue::S(format!("item-{i}"))); + item.insert( + "status".to_owned(), + AttributeValue::S(format!("s{}", i % 64)), + ); + engine + .put_item( + &key_info, + item, + false, + None, + &ExpressionMaps::default(), + None, + ) + .await + .expect("put_item"); + }); + } + while let Some(write) = writes.join_next().await { + write.expect("put_item"); + } + + let shutdown = extenddb_storage::CancellationToken::new(); + let worker = extenddb_storage_cassandra::workers::spawn_gsi_backfill_recovery_for_tests( + engine.clone(), + Duration::from_secs(120), + Duration::from_secs(60), + shutdown.clone(), + ); + engine + .update_table( + &table.key_info.account_id, + UpdateTableInput { + table_name: table.key_info.table_name.clone(), + billing_mode: None, + table_throughput_mode: None, + provisioned_throughput: None, + deletion_protection_enabled: None, + global_secondary_index_updates: Some(vec![GlobalSecondaryIndexUpdate { + create: Some(CreateGsiAction { + index_name: "StatusIndex".to_owned(), + key_schema: vec![KeySchemaElement { + attribute_name: "status".to_owned(), + key_type: KeyType::Hash, + }], + projection: Projection { + projection_type: ProjectionType::All, + non_key_attributes: None, + }, + provisioned_throughput: None, + }), + update: None, + delete: None, + }]), + attribute_definitions: Some(vec![ + AttributeDefinition { + attribute_name: "id".to_owned(), + attribute_type: ScalarAttributeType::S, + }, + AttributeDefinition { + attribute_name: "status".to_owned(), + attribute_type: ScalarAttributeType::S, + }, + ]), + stream_specification: None, + table_class: None, + on_demand_throughput: None, + vector_index_updates: None, + }, + ) + .await + .expect("add the index"); + + // The backfill writes its heartbeat after its first page. + let table_id = table.key_info.table_id.clone(); + let deadline = tokio::time::Instant::now() + Duration::from_secs(120); + let index_id = loop { + let (status, heartbeat, index_id) = index_state(&engine, &table_id, "StatusIndex").await; + if heartbeat.is_some() { + assert_eq!( + status, "CREATING", + "the backfill of {ITEMS} items ended before its first heartbeat was read" + ); + break index_id; + } + assert!( + tokio::time::Instant::now() < deadline, + "the backfill wrote no heartbeat within 120 s" + ); + tokio::time::sleep(Duration::from_millis(10)).await; + }; + shutdown.cancel(); + tokio::time::timeout(Duration::from_secs(30), worker) + .await + .expect("the worker returns within 30 s of its token") + .expect("the worker does not panic"); + + // A batch sent before the token fired can land after the task ended; + // give it time before the first count. + let account_keyspace = engine.account_keyspace(&table.key_info.account_id); + tokio::time::sleep(Duration::from_secs(2)).await; + let (status, _, _) = index_state(&engine, &table_id, "StatusIndex").await; + let written = index_rows(&engine, &account_keyspace, &index_id).await; + tokio::time::sleep(Duration::from_secs(2)).await; + let (status_later, _, _) = index_state(&engine, &table_id, "StatusIndex").await; + let written_later = index_rows(&engine, &account_keyspace, &index_id).await; + println!( + "index rows 2 s after the worker returned: {written} ({status}); 2 s later: \ + {written_later} ({status_later})" + ); + assert_eq!( + status, "CREATING", + "the index is {status} 2 s after the worker returned, with {written} rows" + ); + assert!( + usize::try_from(written).expect("a count") < ITEMS, + "the backfill wrote all {ITEMS} index rows: the token did not stop it" + ); + assert_eq!( + (status_later.as_str(), written_later), + ("CREATING", written), + "the backfill went on after the worker returned" + ); +} From 51ab5ea4630ad39c9bf2f803692ee3d9a43683fe Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Sun, 1 Nov 2026 22:02:47 +0000 Subject: [PATCH 38/45] fix(storage-cassandra): bound the GSI delay read before a set's workers start spawn_workers read index_propagation_delay_ms, and then gsi_propagation_delay_ms, with no bound before it spawned a worker. Since ca132671e a set starts after every registration again, under the set's lock and before requests are admitted again, so a node that answered the registration and then stopped answering held the server there: it refused every request, ran no worker, and acted on no loss of its new registration until the read returned. The read now goes through read_gsi_delay, which gives up after 2 s and leaves the atomic at its value; the GSI delay poller reads the setting again every 5 s. The comment the call replaces gave the poller's interval as 30 s. a_read_of_the_gsi_delay_that_gets_no_answer_ends_at_its_bound reads from a store whose reads never answer and requires the read to end at the bound; with the bound removed it failed. the_gsi_delay_is_read_from_the_setting_or_its_legacy_name pins the two keys and their order. --- crates/storage-cassandra/src/lib.rs | 23 ++--- crates/storage-cassandra/src/workers.rs | 111 ++++++++++++++++++++++++ 2 files changed, 117 insertions(+), 17 deletions(-) diff --git a/crates/storage-cassandra/src/lib.rs b/crates/storage-cassandra/src/lib.rs index 099e99562..46fe89cda 100644 --- a/crates/storage-cassandra/src/lib.rs +++ b/crates/storage-cassandra/src/lib.rs @@ -99,24 +99,13 @@ impl ServerRuntimeHooks for CassandraRuntimeHooks { .await; }); - // Read the initial GSI delay immediately so the atomic is correct from - // the first request, not after the first 30s sleep. + // Read the GSI delay before requests are admitted, so the atomic is + // correct from the first request and not from the poller's first + // read 5 s later. Bounded by `INITIAL_GSI_DELAY_READ`; past it the + // atomic keeps its value until the poller's read. let gsi_delay = self.engine.gsi_default_delay_ms.clone(); - let initial_delay = match ctx - .catalog_store - .get_setting("index_propagation_delay_ms") - .await - { - Ok(Some(v)) => Some(v), - _ => ctx - .catalog_store - .get_setting("gsi_propagation_delay_ms") - .await - .ok() - .flatten(), - }; - if let Some(val) = initial_delay - && let Ok(ms) = val.parse::() + if let Some(ms) = + workers::read_gsi_delay(&*ctx.catalog_store, workers::INITIAL_GSI_DELAY_READ).await { gsi_delay.store(ms, std::sync::atomic::Ordering::Relaxed); } diff --git a/crates/storage-cassandra/src/workers.rs b/crates/storage-cassandra/src/workers.rs index 49935f530..1e225d409 100644 --- a/crates/storage-cassandra/src/workers.rs +++ b/crates/storage-cassandra/src/workers.rs @@ -16,6 +16,33 @@ use crate::CassandraEngine; /// Poll `index_propagation_delay_ms` from settings every 5 seconds and update /// the in-memory atomic used by `put_item/update_item/delete_item`, until /// `shutdown` fires. +/// How long `spawn_workers` waits for the GSI delay setting before it spawns +/// the workers. A set starts after every registration again, while the +/// server admits nothing and acts on no loss, so the read is bounded; the +/// poller reads the setting again every 5 s. +pub(crate) const INITIAL_GSI_DELAY_READ: Duration = Duration::from_secs(2); + +/// The GSI delay setting, `index_propagation_delay_ms` or else the legacy +/// `gsi_propagation_delay_ms`, read within `bound`: `None` when neither is +/// set to a number, a read fails, or the bound passes first. +pub(crate) async fn read_gsi_delay( + store: &S, + bound: Duration, +) -> Option { + let read = async { + let value = match store.get_setting("index_propagation_delay_ms").await { + Ok(Some(v)) => Some(v), + _ => store + .get_setting("gsi_propagation_delay_ms") + .await + .ok() + .flatten(), + }; + value.and_then(|v| v.parse::().ok()) + }; + tokio::time::timeout(bound, read).await.ok().flatten() +} + pub(crate) async fn poll_gsi_delay( store: Arc, gsi_delay: Arc, @@ -781,3 +808,87 @@ pub(crate) fn is_table_not_found(err: &extenddb_storage::error::StorageError) -> _ => false, } } + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use extenddb_storage::management_store::{OpResult, SettingsStore}; + use futures::future::BoxFuture; + use tokio::time::Instant; + + /// A settings store holding `rows`, or one whose reads never answer. + struct Settings { + rows: Vec<(&'static str, &'static str)>, + hang: bool, + } + + impl SettingsStore for Settings { + fn get_setting(&self, key: &str) -> BoxFuture<'_, OpResult>> { + let value = self + .rows + .iter() + .find(|(k, _)| *k == key) + .map(|(_, v)| (*v).to_owned()); + let hang = self.hang; + Box::pin(async move { + if hang { + std::future::pending::<()>().await; + } + Ok(value) + }) + } + fn set_setting(&self, _key: &str, _value: &str) -> BoxFuture<'_, OpResult<()>> { + Box::pin(async { Ok(()) }) + } + fn list_settings(&self) -> BoxFuture<'_, OpResult>> { + Box::pin(async { Ok(Vec::new()) }) + } + } + + /// A read of the GSI delay that gets no answer ends at its bound, so a + /// set started after a registration again spawns its workers. + #[tokio::test(start_paused = true)] + async fn a_read_of_the_gsi_delay_that_gets_no_answer_ends_at_its_bound() { + let store = Settings { + rows: Vec::new(), + hang: true, + }; + let started = Instant::now(); + let read = tokio::time::timeout( + Duration::from_secs(60), + super::read_gsi_delay(&store, super::INITIAL_GSI_DELAY_READ), + ) + .await; + assert_eq!(read, Ok(None), "the read was not bounded"); + assert_eq!(started.elapsed(), super::INITIAL_GSI_DELAY_READ); + } + + /// The delay is read from `index_propagation_delay_ms`, else from the + /// legacy `gsi_propagation_delay_ms`; a value that is not a number reads + /// as none. + #[tokio::test] + async fn the_gsi_delay_is_read_from_the_setting_or_its_legacy_name() { + let bound = super::INITIAL_GSI_DELAY_READ; + let read = |rows| async move { + super::read_gsi_delay(&Settings { rows, hang: false }, bound).await + }; + assert_eq!( + read(vec![ + ("index_propagation_delay_ms", "250"), + ("gsi_propagation_delay_ms", "40") + ]) + .await, + Some(250) + ); + assert_eq!( + read(vec![("gsi_propagation_delay_ms", "40")]).await, + Some(40) + ); + assert_eq!( + read(vec![("index_propagation_delay_ms", "soon")]).await, + None + ); + assert_eq!(read(Vec::new()).await, None); + } +} From 921e69f174f84debc1ff771e3d09b55b6eafe581 Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Sun, 1 Nov 2026 22:06:37 +0000 Subject: [PATCH 39/45] test(devtools): the MongoDB lock check reads the stopped server's last line "its last line names the lost registration" tested the whole log, and since 1c3b2bf60 the log names the registration at every failed renewal and every attempt to register again, so the check passed whatever the server's last line said. It now tests the last non-empty line, and prints it. With the working tree's MongoDB binary the check passed, and the last line was the fatal message: "extenddb fatal: the server stops: this server could not renew its registration for 30 s (no answer), and it could not register again: no attempt succeeded within 30 s (3 attempt(s); ...)". On a log whose earlier lines name the registration and whose last line does not, the old test passes and the new one fails. --- devtools/check-mongodb-catalog-lock | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/devtools/check-mongodb-catalog-lock b/devtools/check-mongodb-catalog-lock index 25ef8e9e8..f13f764e6 100755 --- a/devtools/check-mongodb-catalog-lock +++ b/devtools/check-mongodb-catalog-lock @@ -175,7 +175,9 @@ def main(binary: str, mongo_port: int, api_port: int) -> int: report(code == 1 and first_503 is not None and first_503 + 25 <= took <= first_503 + 50, f"the server exited {code} {took:.0f} s after its database stopped answering, " "25 to 50 s after its first 503", log) - report("registration" in log, "its last line names the lost registration", log) + last = next((line for line in reversed(log.splitlines()) if line.strip()), "") + report("registration" in last, + f"its last line names the lost registration: {last.strip()[-300:]!r}", log) report(not os.path.exists(pid_file), "the server removed its PID file") else: report(False, "the server did not start for the pause check") From 448c36f39f6e8e2080d5645373ae5c52185354eb Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Sun, 1 Nov 2026 22:11:08 +0000 Subject: [PATCH 40/45] test(storage-mongodb): the proxy test's silent connections never answer a0a902b89's proxy held the bytes it read while paused and delivered them when it resumed, so a renewal that waited through the silence was answered. The test therefore passed with b8584d590's renewal bound, the time left to the deadline, which is the defect it was added for: a renewal sent to a primary that hangs spends its whole bound, and the next attempt comes too late. A connection that carries bytes while the proxy is paused now passes nothing from then on, in either direction, and stays open, as a connection to a primary that hung does. Connections that carried nothing during the silence, and connections opened after it, pass bytes. A connection the driver opens during the silence never finishes its handshake, and the driver opens at most two at a time by default, so the proxy's connection string sets maxConnecting=20. The test passed twice as committed, in 4.70 s and 4.76 s. With each renewal bounded by REQUEST_TIMEOUT.min(left(renewed)), as at b8584d590, it failed twice: "this server could not renew its registration for 2 s (no answer)". --- .../tests/migrations_holders.rs | 9 ++-- .../tests/migrations_support/proxy.rs | 47 ++++++++++++++----- 2 files changed, 40 insertions(+), 16 deletions(-) diff --git a/crates/storage-mongodb/tests/migrations_holders.rs b/crates/storage-mongodb/tests/migrations_holders.rs index 5128bc1c0..db0d4205e 100644 --- a/crates/storage-mongodb/tests/migrations_holders.rs +++ b/crates/storage-mongodb/tests/migrations_holders.rs @@ -493,9 +493,12 @@ async fn the_lock_timeout_bounds_migrate_yes_on_a_catalog_with_many_policies() { /// A primary that stops answering just before a renewal is due, and answers /// again before the registration's deadline, does not take the registration: -/// the renewal sent into the silence is bounded, the next ones go out every -/// retry interval, and one gets through once the primary answers. The -/// primary's silence is a proxy that holds every byte while it is paused. +/// the renewal sent into the silence is bounded by the renewal interval, the +/// next ones go out every retry interval, and one sent after the silence gets +/// through. The silence is a proxy on which a connection that carried bytes +/// during it never answers, as a connection to a primary that hung does, so +/// a renewal bounded only by the time left to the deadline waits on its +/// connection until the deadline, and the registration is lost. #[tokio::test(flavor = "multi_thread")] async fn a_primary_that_hangs_before_a_renewal_and_answers_again_keeps_the_registration() { let Some(uri) = server() else { return }; diff --git a/crates/storage-mongodb/tests/migrations_support/proxy.rs b/crates/storage-mongodb/tests/migrations_support/proxy.rs index b2e4244aa..345cb9171 100644 --- a/crates/storage-mongodb/tests/migrations_support/proxy.rs +++ b/crates/storage-mongodb/tests/migrations_support/proxy.rs @@ -1,12 +1,17 @@ // Copyright 2026 ExtendDB contributors // SPDX-License-Identifier: Apache-2.0 -//! A TCP proxy in front of the test server that stops passing bytes while it -//! is paused and passes them again once it is resumed, as a primary that -//! hangs and then answers again does. Bytes that arrive while it is paused -//! are held, not dropped, and connections stay open. +//! A TCP proxy in front of the test server that passes no bytes while it is +//! paused, as a primary that hangs does. A connection that carried bytes +//! during the silence stays silent after it: what it held is never +//! delivered, and it stays open, as a connection to a primary that never +//! answers what was sent to it. Connections that carried nothing during the +//! silence, and connections opened after it, pass bytes again once it is +//! resumed, as connections to the primary that answers do. use std::net::SocketAddr; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::tcp::{OwnedReadHalf, OwnedWriteHalf}; @@ -42,15 +47,18 @@ impl PausingProxy { }; let (client_read, client_write) = client.into_split(); let (server_read, server_write) = server.into_split(); - tokio::spawn(copy_unless_paused( + let silent = Arc::new(AtomicBool::new(false)); + tokio::spawn(copy_unless_silent( client_read, server_write, watching.clone(), + silent.clone(), )); - tokio::spawn(copy_unless_paused( + tokio::spawn(copy_unless_silent( server_read, client_write, watching.clone(), + silent, )); } }); @@ -61,10 +69,16 @@ impl PausingProxy { } } - /// The connection string of the proxy, for a direct connection. + /// The connection string of the proxy, for a direct connection. A + /// connection the driver opens during the silence never finishes its + /// handshake, and the driver opens at most two at a time by default, so + /// the string lets it open more. #[must_use] pub fn uri(&self) -> String { - format!("mongodb://{}/?directConnection=true", self.address) + format!( + "mongodb://{}/?directConnection=true&maxConnecting=20", + self.address + ) } pub fn pause(&self) { @@ -82,12 +96,14 @@ impl Drop for PausingProxy { } } -/// Copy `from` to `to`, holding what is read while the proxy is paused -/// until it is resumed. -async fn copy_unless_paused( +/// Copy `from` to `to`. Bytes read while the proxy is paused make the +/// connection `silent`, both directions: from then on it passes nothing, and +/// its halves are kept so the connection stays open. +async fn copy_unless_silent( mut from: OwnedReadHalf, mut to: OwnedWriteHalf, - mut paused: watch::Receiver, + paused: watch::Receiver, + silent: Arc, ) { let mut buffer = vec![0_u8; 16 * 1024]; loop { @@ -95,7 +111,12 @@ async fn copy_unless_paused( Ok(0) | Err(_) => return, Ok(n) => n, }; - if paused.wait_for(|paused| !*paused).await.is_err() { + if *paused.borrow() { + silent.store(true, Ordering::SeqCst); + } + if silent.load(Ordering::SeqCst) { + let _open = (from, to); + std::future::pending::<()>().await; return; } if to.write_all(&buffer[..read]).await.is_err() { From e8f507f9f04e7fd00c52f0c5f7cee82ddd38782c Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Sun, 1 Nov 2026 22:15:17 +0000 Subject: [PATCH 41/45] docs: say what goes on while a server holds no registration The registration module and the upgrade manual said that a server stops its background workers at a lost registration, and left out what does not stop. Requests already running go on, and so does the work they left running in the background, until it ends or the process exits: a RestoreTableFromBackup's item copy on every backend, an UpdateTimeToLive backfill on Cassandra and PostgreSQL, and a GSI or vector index build on PostgreSQL. PostgreSQL's GSI propagation workers, which the engine starts with itself, write only the data database and go on too. registration.rs and the manual's PostgreSQL paragraph now say so, and hooks.rs requires a task a worker starts to end before the worker returns, as d8fc8d75b makes Cassandra's GSI backfill recovery do. The Cassandra setup page and the upgrade manual said a server answers ServiceUnavailable once it has not renewed for 30 s. It tries its renewals for 30 s after the last one that succeeded, and the attempt at that deadline is bounded by 5 s, so it keeps serving through an outage of 25 to 30 s, depending on when the outage starts, and answers 503 30 to 35 s after its last successful renewal. Both pages now say that. The manual's PgBouncer paragraph says that with idle_transaction_timeout under 15 s the server registers again at every renewal. Since ca132671e each time also stops the background workers and starts a new set, whose workers wait their interval before their first run, so the TTL scan, the table size refresh and the idempotency token and stream record cleanups never run. The paragraph now says so. The Drop docs of both registrations said that an attempt to register again the server drops leaves no registration behind. That holds once its INSERT or insert has returned. An attempt dropped while that write is in flight leaves a row or document that expires by itself, and migrate --yes counts it as a running server until then. --- crates/server/src/registration.rs | 31 ++++++++++++------- .../src/migrations/registrations.rs | 7 +++-- .../src/migrations/registrations.rs | 8 +++-- crates/storage/src/hooks.rs | 3 +- docs/cassandra-setup.md | 9 ++++-- docs/manuals/07-upgrade-manual.md | 6 ++-- 6 files changed, 40 insertions(+), 24 deletions(-) diff --git a/crates/server/src/registration.rs b/crates/server/src/registration.rs index 18383752f..e8022c5f7 100644 --- a/crates/server/src/registration.rs +++ b/crates/server/src/registration.rs @@ -7,18 +7,25 @@ //! A migration can start on the catalog once the registration is gone, so //! from that moment the server admits no new request: each one gets //! `ServiceUnavailable` (HTTP 503), which the AWS SDKs retry, and `/health` -//! answers the same. It also stops its background workers: their token -//! fires, and the ones still running after 5 s are aborted. Requests already -//! running go on. The server then asks the backend to register again at once -//! and then every second, and when the backend reports a new registration -//! with the catalog's status unchanged, it starts a new set of workers and -//! serves again. When the backend refuses (a migration holds the migration -//! lock or waits for the servers to stop, or the status changed), when 30 s -//! pass without a registration, or when a worker could not be stopped, the -//! server asks for a stop: `serve` then stops the way a shutdown signal -//! stops it (the listener drains, letting requests already running finish, -//! the PID file is removed) and exits 1. A signal that came first decides -//! the exit status, also when a stop is asked for during its drain. +//! answers the same. It also stops its background workers and the tasks +//! they started: their token fires, and the ones still running after 5 s are +//! aborted. What requests started goes on until it ends or the process +//! exits: the requests already running, and the work they left running in +//! the background, which is a `RestoreTableFromBackup`'s item copy on every +//! backend, an `UpdateTimeToLive` backfill on Cassandra and PostgreSQL, and +//! a GSI or vector index build on PostgreSQL. PostgreSQL's GSI propagation +//! workers, which the engine starts with itself and which write only the +//! data database, go on too. The server then asks the backend to register +//! again at once and then every second, and when the backend reports a new +//! registration with the catalog's status unchanged, it starts a new set of +//! workers and serves again. When the backend refuses (a migration holds the +//! migration lock or waits for the servers to stop, or the status changed), +//! when 30 s pass without a registration, or when a worker could not be +//! stopped, the server asks for a stop: `serve` then stops the way a +//! shutdown signal stops it (the listener drains, letting requests already +//! running finish, the PID file is removed) and exits 1. A signal that came +//! first decides the exit status, also when a stop is asked for during its +//! drain. //! //! The server learns of a loss at its backend's next renewal, and serves as //! before until then; a migration that takes the catalog in that time runs diff --git a/crates/storage-cassandra/src/migrations/registrations.rs b/crates/storage-cassandra/src/migrations/registrations.rs index da0b0b693..33d011215 100644 --- a/crates/storage-cassandra/src/migrations/registrations.rs +++ b/crates/storage-cassandra/src/migrations/registrations.rs @@ -277,8 +277,11 @@ impl ServingRegistration { } /// A registration dropped without [`ServingRegistration::release`], as an -/// attempt to register again that the server drops at its bound, deletes -/// its row from a task of its own, so it leaves no registration behind. +/// attempt to register again that the server drops at its bound after the +/// INSERT returned, deletes its row from a task of its own. An attempt +/// dropped while its INSERT is in flight has no registration yet: a row +/// that INSERT writes stays until its TTL ends, and `migrate --yes` counts +/// it as a running server until then. impl Drop for ServingRegistration { fn drop(&mut self) { self.renewal.abort(); diff --git a/crates/storage-mongodb/src/migrations/registrations.rs b/crates/storage-mongodb/src/migrations/registrations.rs index 4440adeb9..8fd1c29e8 100644 --- a/crates/storage-mongodb/src/migrations/registrations.rs +++ b/crates/storage-mongodb/src/migrations/registrations.rs @@ -163,9 +163,11 @@ impl ServingRegistration { } /// A registration dropped without [`ServingRegistration::release`], as an -/// attempt to register again that the server drops at its bound, deletes -/// its document from a task of its own, so it leaves no registration -/// behind. +/// attempt to register again that the server drops at its bound after its +/// insert returned, deletes its document from a task of its own. An attempt +/// dropped while its insert is in flight has no registration yet: a +/// document that insert writes stays until it expires, and `migrate --yes` +/// counts it as a running server until then. impl Drop for ServingRegistration { fn drop(&mut self) { self.renewal.abort(); diff --git a/crates/storage/src/hooks.rs b/crates/storage/src/hooks.rs index 55d73b834..3d3318052 100644 --- a/crates/storage/src/hooks.rs +++ b/crates/storage/src/hooks.rs @@ -54,7 +54,8 @@ pub trait ServerRuntimeHooks: Send + Sync { /// Return the spawned tasks' join handles so the server can await them /// after cancelling [`WorkerContext::shutdown`]; the server aborts the /// ones still running 5 s later. Every worker this spawns must be among - /// the handles and return once that token fires. + /// the handles and return once that token fires, and a task a worker + /// starts must end before the worker returns, so the stop covers it too. /// /// Called again for each new set of workers: a server whose serving /// registration is lost stops its workers, and starts a new set with a diff --git a/docs/cassandra-setup.md b/docs/cassandra-setup.md index db794a1a5..3ad0074bf 100644 --- a/docs/cassandra-setup.md +++ b/docs/cassandra-setup.md @@ -260,9 +260,12 @@ until that is configurable. ExtendDB creates its keyspaces with With a replication factor of 1, the catalog has one replica, so a restart of that node stops every request to it. A running server renews its serving -registration every 5 s and keeps serving while renewals fail. Once it has not -renewed for 30 s, it answers new requests with `ServiceUnavailable`, stops its -background workers, and tries to register again every second for up to 30 s. +registration every 5 s and keeps serving while renewals fail. It tries its +renewals for 30 s after the last one that succeeded, so it keeps serving +through an outage of 25 to 30 s, depending on when the outage starts. After +that, 30 to 35 s after its last successful renewal, it answers new requests +with `ServiceUnavailable`, stops its background workers, and tries to register +again every second for up to 30 s. It registers again, starts its workers again and serves on only when the Cassandra driver has reconnected to the node within that time, which after a restart of 30 s or more it often has not; otherwise it stops with exit 1. Run diff --git a/docs/manuals/07-upgrade-manual.md b/docs/manuals/07-upgrade-manual.md index 32d735057..2832d45da 100755 --- a/docs/manuals/07-upgrade-manual.md +++ b/docs/manuals/07-upgrade-manual.md @@ -89,13 +89,13 @@ The PostgreSQL commands and `serve` work through PgBouncer in transaction mode a - Behind PgBouncer, the TCP keepalives the serving session sets (30 s) run on the database's connection to PgBouncer, which stays up when a serving host dies. The database keeps that host's key until PgBouncer closes the dead host's connection, which PgBouncer's own keepalive settings decide: by default the operating system's, two hours idle on Linux. Until then `migrate --yes` exits 6 naming the dead server. Set `tcp_keepalive = 1`, `tcp_keepidle = 10`, `tcp_keepintvl = 5` and `tcp_keepcnt = 4` in PgBouncer's `[pgbouncer]` section for the same 30 s, or end the session with `SELECT pg_terminate_backend()` on the pid `migrate` names. - After a failover of the catalog's server, the new primary holds no server's key, while a server that served through the old primary goes on serving through connections that reconnect to the new primary, until it counts its key as lost, at most 30 s after its last renewal, and registers again on the new primary. A `migrate --yes` in those 30 s does not see that server. Wait 30 s after a failover before you run `migrate --yes`, or stop the servers first. -A server notices at its next renewal, within 15 s, that the database ended the session holding its key: a restart or failover of the catalog's server, `SELECT pg_terminate_backend()`, or a pooler that closed the connection. From then on it answers each new request, `/health` included, with `ServiceUnavailable` (HTTP 503), which the AWS SDKs retry, lets the requests already running finish, and stops its background workers, aborting the ones still running after 5 s. It tries to register again on a new session at once and then every second, and once it holds the key and the catalog's status is the one it started with, it starts its background workers again and serves. When a migration holds the migration lock or waits for the servers to stop, when the status changed, when 30 s pass without a registration, or when a background worker could not be stopped, the server stops the way SIGTERM stops it, finishing the requests in flight and removing its PID file, and exits 1. Until the renewal that finds the session gone, the server serves as before, so a `migrate --yes` that was waiting for the servers can start beside it in those seconds: stop every server before you migrate, as the steps above say. +A server notices at its next renewal, within 15 s, that the database ended the session holding its key: a restart or failover of the catalog's server, `SELECT pg_terminate_backend()`, or a pooler that closed the connection. From then on it answers each new request, `/health` included, with `ServiceUnavailable` (HTTP 503), which the AWS SDKs retry, lets the requests already running finish, and stops its background workers, aborting the ones still running after 5 s. Work that requests started in the background goes on until it ends or the server exits: a RestoreTableFromBackup's item copy on every backend, an UpdateTimeToLive backfill on Cassandra and PostgreSQL, and a GSI or vector index build on PostgreSQL. It tries to register again on a new session at once and then every second, and once it holds the key and the catalog's status is the one it started with, it starts its background workers again and serves. When a migration holds the migration lock or waits for the servers to stop, when the status changed, when 30 s pass without a registration, or when a background worker could not be stopped, the server stops the way SIGTERM stops it, finishing the requests in flight and removing its PID file, and exits 1. Until the renewal that finds the session gone, the server serves as before, so a `migrate --yes` that was waiting for the servers can start beside it in those seconds: stop every server before you migrate, as the steps above say. -The two transactions that hold the keys, the server's registration and the migration lock, turn off `idle_in_transaction_session_timeout` for themselves, and `transaction_timeout` on PostgreSQL 17 and later, so a role or database setting of either does not end them. PgBouncer's `idle_transaction_timeout` is a setting of PgBouncer, which a transaction cannot change. Each holder runs `SELECT 1` every 15 s, so set `idle_transaction_timeout` to 0, its default, or to 30 s or more. A value under 15 s closes the server's connection between two renewals, and the server then registers again every time. +The two transactions that hold the keys, the server's registration and the migration lock, turn off `idle_in_transaction_session_timeout` for themselves, and `transaction_timeout` on PostgreSQL 17 and later, so a role or database setting of either does not end them. PgBouncer's `idle_transaction_timeout` is a setting of PgBouncer, which a transaction cannot change. Each holder runs `SELECT 1` every 15 s, so set `idle_transaction_timeout` to 0, its default, or to 30 s or more. A value under 15 s closes the server's connection between two renewals, and the server then registers again every time. Each time it also stops its background workers and starts a new set, and each worker waits its interval before its first run, so a worker whose interval is longer than the 15 s between two renewals never runs: among them the TTL scan (60 s), the table size refresh (300 s), and the idempotency token and stream record cleanups (600 s and 3600 s). On MongoDB, `migrate` connects with the connection string in the config file, which the server uses too. A user with the `readWrite` role on `extenddb_catalog` and on `extenddb_data` and no other role can run `init`, `migrate --yes`, `verify` and `serve`. `migrate --yes` takes the migration lock, a document in the catalog's `settings` collection, and then waits for the registration every running server keeps in the catalog's `extenddb_servers` collection, and after `--lock-timeout` exits 6 naming each server's host and pid. A server renews its registration every 5 s, and every second after a renewal failed, and deletes it when it stops, once every background worker has ended (it leaves it to expire when one could not be stopped); a server that was killed leaves it until it expires, at most 45 s after its last renewal. A server whose registration was deleted, or that has not renewed it for 30 s, for example because the primary stopped answering, answers new requests with `ServiceUnavailable` and registers again, as a PostgreSQL server does after its session ends (above): it serves again once it holds a new registration and the catalog's status is the one it started with, and it stops with exit 1 when a migration holds the lock, when the status changed, or when 30 s pass without a registration. The lock and the registrations expire by the clock of the replica set's primary, so keep the clocks of its members in sync with NTP. Migration 1 deletes the `resource_tag_migration` settings document in the transaction that records its catalog step, so a 0.1.14 server refuses the migrated catalog, and a run that fails before that transaction leaves the document in place. 0.1.12 and 0.1.13 servers read neither that document nor the catalog version, so they start on a migrated catalog and serve it: do not run them against it. To go back to a 0.1.x release, restore the backup taken before `migrate --yes`. A 0.1.x `migrate` on a migrated catalog reports a pending change, and its `--yes` writes the settings document back; this release's status then reports the catalog as written by an older release, and `migrate --yes --reconverge` repairs it. -On Cassandra, `migrate` connects as the role the config file names, which the server connects as too. `--cassandra-user ` connects as another role, with its password in `EXTENDDB_CASSANDRA_PASSWORD` or `--cassandra-pass`; a wrong password is reported as a connection timeout. Migration 1 adopts a 0.1.x catalog: it keeps every setting except the `resource_tag_migration` row, which it deletes, and records itself in a ledger in the catalog keyspace and in every account keyspace. On a cluster that runs `CassandraAuthorizer`, `init` grants the application role SELECT, MODIFY, CREATE and ALTER on the catalog keyspace, those four and DROP on the default account's keyspace, and CREATE ON ALL KEYSPACES, which CreateAccount needs to create a keyspace per account and which also lets the role create tables in every keyspace on the cluster; a run of `migrate --yes --cassandra-user ` that creates a missing account keyspace grants the application role the five permissions on it. A 0.1.x `init` granted the application role nothing, so a deployment created before 1.0 on such a cluster needs the statements below before `migrate`. `migrate --yes` takes the migration lock, a row in the catalog's `settings` table written by lightweight transactions, and then waits for the registration every running server of this release keeps in the catalog's `extenddb_servers` table, and after `--lock-timeout` exits 6 naming each server. A server renews its registration every 5 s, and every second after a renewal failed, and deletes it when it stops, once every background worker has ended (it leaves it to expire when one could not be stopped); a server that was killed leaves it until it expires, at most 45 s after its last renewal. A server that has not renewed its registration for 30 s, for example while the only replica of the catalog restarts, answers new requests with `ServiceUnavailable` and registers again, as a PostgreSQL server does after its session ends (above), and stops with exit 1 when a migration holds the lock, when the status changed, or when 30 s pass without a registration. A migration needs every replica of the catalog keyspace reachable: the lock is written at `SERIAL`, the registrations are read at `ALL`, and after each schema statement the runner waits until every live node reports the same schema version. `serve` refuses a catalog with pending migrations, exiting 3 under `--foreground` with the pending migrations and the `extenddb migrate --config --yes` command to run. A 0.1.x server refuses a migrated catalog. +On Cassandra, `migrate` connects as the role the config file names, which the server connects as too. `--cassandra-user ` connects as another role, with its password in `EXTENDDB_CASSANDRA_PASSWORD` or `--cassandra-pass`; a wrong password is reported as a connection timeout. Migration 1 adopts a 0.1.x catalog: it keeps every setting except the `resource_tag_migration` row, which it deletes, and records itself in a ledger in the catalog keyspace and in every account keyspace. On a cluster that runs `CassandraAuthorizer`, `init` grants the application role SELECT, MODIFY, CREATE and ALTER on the catalog keyspace, those four and DROP on the default account's keyspace, and CREATE ON ALL KEYSPACES, which CreateAccount needs to create a keyspace per account and which also lets the role create tables in every keyspace on the cluster; a run of `migrate --yes --cassandra-user ` that creates a missing account keyspace grants the application role the five permissions on it. A 0.1.x `init` granted the application role nothing, so a deployment created before 1.0 on such a cluster needs the statements below before `migrate`. `migrate --yes` takes the migration lock, a row in the catalog's `settings` table written by lightweight transactions, and then waits for the registration every running server of this release keeps in the catalog's `extenddb_servers` table, and after `--lock-timeout` exits 6 naming each server. A server renews its registration every 5 s, and every second after a renewal failed, and deletes it when it stops, once every background worker has ended (it leaves it to expire when one could not be stopped); a server that was killed leaves it until it expires, at most 45 s after its last renewal. A server tries its renewals for 30 s after the last one that succeeded, so it keeps serving through an outage of 25 to 30 s, depending on when the outage starts, for example while the only replica of the catalog restarts. After that, 30 to 35 s after its last successful renewal, it answers new requests with `ServiceUnavailable` and registers again, as a PostgreSQL server does after its session ends (above), and stops with exit 1 when a migration holds the lock, when the status changed, or when 30 s pass without a registration. A migration needs every replica of the catalog keyspace reachable: the lock is written at `SERIAL`, the registrations are read at `ALL`, and after each schema statement the runner waits until every live node reports the same schema version. `serve` refuses a catalog with pending migrations, exiting 3 under `--foreground` with the pending migrations and the `extenddb migrate --config --yes` command to run. A 0.1.x server refuses a migrated catalog. On a deployment created before 1.0 on a cluster that runs `CassandraAuthorizer`, this release's `migrate`, `verify` and `serve`, connected as the application role, fail with `Unauthorized ... has no SELECT permission on
_catalog.accounts>` until the role holds its permissions. With every server stopped, run the statements below in `cqlsh` as a superuser; the application role cannot grant them to itself. `` and `` are `username` and `keyspace_prefix` under `[storage.cassandra]` in the config file. Each account has its own keyspace, so list the accounts first: From aeeeacad99c407884fe701605f3c3f5f092d07aa Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Sun, 1 Nov 2026 23:37:45 +0000 Subject: [PATCH 42/45] test(storage-mongodb): leave the proxy test time for attempts after the silence 448c36f39's proxy test failed in every run of the whole migrations_holders binary, whose other tests keep the server busy, and passed alone. The renewal task starts when the registration's insert is answered, which came about 200 ms after its send, so the renewals went out at 604, 1057 and 1510 ms after the send, all three into the silence, and the first one after it at 1962 ms, bounded by the 38 ms left to the 2 s deadline. On a new connection it needed 152 ms. The deadline is now 4 s, and the test waits 4 s after the silence for a loss. In the whole binary the first attempt after the silence renewed after 152 ms, and the test passed in 3 runs of 3, and alone in 2 of 2. With each renewal bounded only by the time left to the deadline, as at b8584d590, the renewal sent into the silence waited 3.1 s and 3.6 s for the deadline and the test failed, in the whole binary and alone. --- crates/storage-mongodb/tests/migrations_holders.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/crates/storage-mongodb/tests/migrations_holders.rs b/crates/storage-mongodb/tests/migrations_holders.rs index db0d4205e..b235e1a16 100644 --- a/crates/storage-mongodb/tests/migrations_holders.rs +++ b/crates/storage-mongodb/tests/migrations_holders.rs @@ -508,7 +508,7 @@ async fn a_primary_that_hangs_before_a_renewal_and_answers_again_keeps_the_regis MongoTarget::with_databases(proxy.uri(), d.target.catalog.clone(), d.target.data.clone()); let timing = Timing { registration_renew_every: Duration::from_millis(400), - registration_deadline: Duration::from_secs(2), + registration_deadline: Duration::from_secs(4), registration_retry_every: Duration::from_millis(50), ..Timing::PRODUCTION }; @@ -516,13 +516,17 @@ async fn a_primary_that_hangs_before_a_renewal_and_answers_again_keeps_the_regis .await .expect("register"); let mut lost = registration.lost(); - // The first renewal is due 400 ms after the registration: the silence - // starts 50 ms before it and lasts 1.2 s, inside the 2 s deadline. + // The first renewal is due 400 ms after the registration's insert is + // answered, about when `register` returns: the silence starts 50 ms + // before it and lasts 1.2 s. The deadline counts from the insert's send, + // and on a busy server the insert and an attempt on a new connection + // after the silence can each take a few hundred milliseconds, so the + // deadline is 4 s. tokio::time::sleep(Duration::from_millis(350)).await; proxy.pause(); tokio::time::sleep(Duration::from_millis(1200)).await; proxy.resume(); - let changed = tokio::time::timeout(Duration::from_secs(3), lost.changed()).await; + let changed = tokio::time::timeout(Duration::from_secs(4), lost.changed()).await; let reason = lost.borrow().clone(); registration.release().await; d.drop_databases().await; From dd9d97bae99b60efe16451a6ad7d8e2c5479a066 Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Mon, 2 Nov 2026 12:13:38 +0000 Subject: [PATCH 43/45] docs(server): name MongoDB's key repair pass among what goes on unregistered The pass #473 added runs from the server's start in a task of its own, outside the worker set, so it goes on while the server holds no registration. It writes only the data database. --- crates/server/src/registration.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/server/src/registration.rs b/crates/server/src/registration.rs index e8022c5f7..a763f774b 100644 --- a/crates/server/src/registration.rs +++ b/crates/server/src/registration.rs @@ -14,8 +14,9 @@ //! the background, which is a `RestoreTableFromBackup`'s item copy on every //! backend, an `UpdateTimeToLive` backfill on Cassandra and PostgreSQL, and //! a GSI or vector index build on PostgreSQL. PostgreSQL's GSI propagation -//! workers, which the engine starts with itself and which write only the -//! data database, go on too. The server then asks the backend to register +//! workers, which the engine starts with itself, and MongoDB's startup pass +//! that rebuilds documents stored without their key (#473) write only the +//! data database, and go on too. The server then asks the backend to register //! again at once and then every second, and when the backend reports a new //! registration with the catalog's status unchanged, it starts a new set of //! workers and serves again. When the backend refuses (a migration holds the From 6bb8b8b080120224a9c9ff2dfc75cd2e8efa7b07 Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Mon, 2 Nov 2026 21:58:17 +0000 Subject: [PATCH 44/45] test(migrations): expect migration 5 in the Cassandra and MongoDB live tests The Cassandra test of a stopped CreateAccount expects the status message to name the latest migration, which is now 5. The MongoDB reconverge test expects migration 5 to run again in the catalog, since it is convergent and applies on MongoDB, and the rows that stand for a later binary's migration use id 6, the id after the latest. --- crates/storage-cassandra/tests/migrations.rs | 3 ++- .../storage-mongodb/tests/migrations_fresh.rs | 19 ++++++++++++------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/crates/storage-cassandra/tests/migrations.rs b/crates/storage-cassandra/tests/migrations.rs index 718e79965..3a2349632 100644 --- a/crates/storage-cassandra/tests/migrations.rs +++ b/crates/storage-cassandra/tests/migrations.rs @@ -905,7 +905,8 @@ async fn a_create_account_stopped_before_its_row_leaves_a_current_catalog_a_retr assert!( message.starts_with(&format!( "account store {account} is at migration 0, behind the catalog store at migration \ - 4, and ExtendDB 1.0.0-test needs migration 4." + {latest}, and ExtendDB 1.0.0-test needs migration {latest}.", + latest = MigrationId::LATEST.number() )), "{message}" ); diff --git a/crates/storage-mongodb/tests/migrations_fresh.rs b/crates/storage-mongodb/tests/migrations_fresh.rs index 51ffb6127..12cf1838e 100644 --- a/crates/storage-mongodb/tests/migrations_fresh.rs +++ b/crates/storage-mongodb/tests/migrations_fresh.rs @@ -392,8 +392,9 @@ async fn a_second_run_applies_nothing_and_reconverge_keeps_the_rows() { assert_eq!((again.from, again.to), (Some(5), Some(5))); let reconverged = d.reconverge().await.expect("reconverge"); - // Migration 1's steps in the catalog and the data database: 2 is not - // convergent, and 3 and 4 do not apply on MongoDB. + // Migration 1's steps in the catalog and the data database, and migration + // 5's in the catalog: 2 is not convergent, and 3 and 4 do not apply on + // MongoDB. let ran: Vec<(u32, String)> = reconverged .applied .iter() @@ -401,7 +402,11 @@ async fn a_second_run_applies_nothing_and_reconverge_keeps_the_rows() { .collect(); assert_eq!( ran, - [(1, d.target.catalog.clone()), (1, d.target.data.clone())], + [ + (1, d.target.catalog.clone()), + (1, d.target.data.clone()), + (5, d.target.catalog.clone()), + ], "{reconverged:?}" ); assert!( @@ -468,19 +473,19 @@ async fn a_changed_checksum_a_newer_catalog_and_an_older_release_are_reported() doc! { "_id": id, "name": "later", "kind": kind, "checksum": "c", "outcome": "applied", "notes": "", "applied_at": bson::DateTime::now(), "applied_by": "9.9.9" } }; - ledger.insert_one(row(5, "breaking")).await.expect("insert"); + ledger.insert_one(row(6, "breaking")).await.expect("insert"); let status = d.status().await; assert_eq!(status.label(), "AheadIncompatible"); assert_eq!(status.exit_code(), exit::AHEAD_INCOMPATIBLE); ledger - .delete_one(doc! { "_id": 5_i64 }) + .delete_one(doc! { "_id": 6_i64 }) .await .expect("delete"); - ledger.insert_one(row(5, "additive")).await.expect("insert"); + ledger.insert_one(row(6, "additive")).await.expect("insert"); assert_eq!(d.status().await.label(), "AheadCompatible"); assert!(d.migrate().await.expect("migrate").applied.is_empty()); ledger - .delete_one(doc! { "_id": 5_i64 }) + .delete_one(doc! { "_id": 6_i64 }) .await .expect("delete"); From 0ccc53b522d422a0467132903f520d66f0cd1cdc Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Mon, 2 Nov 2026 22:06:12 +0000 Subject: [PATCH 45/45] docs(schema): drop continuous_backups from the Cassandra and MongoDB dumps Migration 5 drops the table on Cassandra and the collection on MongoDB, so a fresh install no longer has either, and the catalog jobs compare the committed dumps with a fresh install. --- docs/schema/cassandra.txt | 4 ---- docs/schema/mongodb.txt | 2 -- 2 files changed, 6 deletions(-) diff --git a/docs/schema/cassandra.txt b/docs/schema/cassandra.txt index e4f20e073..bc7ee91cc 100644 --- a/docs/schema/cassandra.txt +++ b/docs/schema/cassandra.txt @@ -5,7 +5,6 @@ table admin_users table backups_by_account table backups_by_arn table backups_by_table -table continuous_backups table extenddb_migrations table extenddb_servers table iam_group_members @@ -72,9 +71,6 @@ column backups_by_table.backup_name: text regular column backups_by_table.backup_size_bytes: bigint regular column backups_by_table.table_arn: text regular column backups_by_table.table_id: text regular -column continuous_backups.account_id: text partition_key 0 -column continuous_backups.table_name: text clustering 0 asc -column continuous_backups.pitr_enabled: boolean regular column extenddb_migrations.id: int partition_key 0 column extenddb_migrations.applied_at: timestamp regular column extenddb_migrations.applied_by: text regular diff --git a/docs/schema/mongodb.txt b/docs/schema/mongodb.txt index 44474e255..490403643 100644 --- a/docs/schema/mongodb.txt +++ b/docs/schema/mongodb.txt @@ -9,8 +9,6 @@ index extenddb_catalog.admin_users _id_ key {"_id":1} options {} collection extenddb_catalog.backups type collection options {} index extenddb_catalog.backups _id_ key {"_id":1} options {} index extenddb_catalog.backups account_id_1_table_name_1 key {"account_id":1,"table_name":1} options {} -collection extenddb_catalog.continuous_backups type collection options {} -index extenddb_catalog.continuous_backups _id_ key {"_id":1} options {} collection extenddb_catalog.extenddb_migrations type collection options {} index extenddb_catalog.extenddb_migrations _id_ key {"_id":1} options {} collection extenddb_catalog.extenddb_servers type collection options {}