diff --git a/crates/core/src/types/table.rs b/crates/core/src/types/table.rs index fdebcab30..fa6817ff6 100755 --- a/crates/core/src/types/table.rs +++ b/crates/core/src/types/table.rs @@ -815,8 +815,20 @@ impl TableDescription { /// table is ACTIVE (measured on DynamoDB, and still absent 30 s later). /// Backends report the summary whenever the table has a registered /// restore. + /// - A disabled `StreamSpecification` is omitted: the service carries the + /// member only while a stream is enabled (measured 2026-10-24 on the + /// disable response and on `DescribeTable` afterwards). Backends keep + /// the disabled specification stored, with the closed stream's view + /// type, so `DescribeStream` of that stream can still report it. pub fn prepare_for_response(&mut self) { self.populate_table_throughput_mode_summary(); + if self + .stream_specification + .as_ref() + .is_some_and(|s| !s.stream_enabled) + { + self.stream_specification = None; + } if self.table_status != TableStatus::Creating { self.restore_summary = None; } else if let Some(summary) = self.restore_summary.as_mut() { diff --git a/crates/core/src/validation/mod.rs b/crates/core/src/validation/mod.rs index a849f2c95..da5fe6a0c 100755 --- a/crates/core/src/validation/mod.rs +++ b/crates/core/src/validation/mod.rs @@ -660,15 +660,21 @@ fn validate_index_projections(input: &CreateTableInput) -> Result<(), DynamoDbEr /// # Errors /// /// Returns `DynamoDbError::ValidationException` when `StreamEnabled` is false -/// but a `StreamViewType` is present. +/// but a `StreamViewType` is present, or `StreamEnabled` is true without one +/// (measured 2026-10-24; its order against the other CreateTable checks is not +/// measured, so it sits with the other stream check). fn validate_stream_specification(input: &CreateTableInput) -> Result<(), DynamoDbError> { - if let Some(stream) = &input.stream_specification - && !stream.stream_enabled - && stream.stream_view_type.is_some() - { - return Err(DynamoDbError::ValidationException( - "One or more parameter values were invalid: Table is being created with a stream disabled, UpdateViewType should not be specified".to_owned(), - )); + if let Some(stream) = &input.stream_specification { + if !stream.stream_enabled && stream.stream_view_type.is_some() { + return Err(DynamoDbError::ValidationException( + "One or more parameter values were invalid: Table is being created with a stream disabled, UpdateViewType should not be specified".to_owned(), + )); + } + if stream.stream_enabled && stream.stream_view_type.is_none() { + return Err(DynamoDbError::ValidationException( + "One or more parameter values were invalid: Table is being created with a stream enabled, UpdateViewType is required".to_owned(), + )); + } } Ok(()) } @@ -4622,6 +4628,17 @@ mod tests { stream_view_type: Some(StreamViewType::NewImage), }); assert!(validate_create_table(&input, &LimitsConfig::default()).is_ok()); + + // Enabled without a view type is refused (measured 2026-10-24). + input.stream_specification = Some(StreamSpecification { + stream_enabled: true, + stream_view_type: None, + }); + let err = validate_create_table(&input, &LimitsConfig::default()).unwrap_err(); + assert_eq!( + err.to_string(), + "One or more parameter values were invalid: Table is being created with a stream enabled, UpdateViewType is required" + ); } #[test] diff --git a/crates/engine/src/backup.rs b/crates/engine/src/backup.rs index 56445e86b..3ed2117c4 100755 --- a/crates/engine/src/backup.rs +++ b/crates/engine/src/backup.rs @@ -22,7 +22,7 @@ use extenddb_core::limits::LimitsConfig; use extenddb_core::types::{ BackupDescription, BackupSummary, BillingMode, CreateTableInput, GlobalSecondaryIndexInfo, LocalSecondaryIndexInfo, ProvisionedThroughput, RestoreSummary, SourceTableFeatureDetails, - SseDescription, TableDescription, TableStatus, + SseDescription, StreamSpecification, TableDescription, TableStatus, }; use extenddb_storage::backup_catalog::{ BackupKey, BackupRow, BackupStatus, DeleteStart, TimeBounds, is_backup_not_found, @@ -717,7 +717,23 @@ fn apply_source_to_description( }) .collect() }), - stream_description: d.stream_specification.clone(), + // A backup of a table whose stream was disabled carries the + // stored `{StreamEnabled: false}`, and DescribeBackup shows it + // as `StreamDescription: {StreamEnabled: false}`, as it did + // before the view type was kept in the stored specification; + // the service's answer for that backup is not measured. The + // view type the backends now keep for the closed stream is + // dropped here so the answer stays the one before. + stream_description: d.stream_specification.clone().map(|s| { + if s.stream_enabled { + s + } else { + StreamSpecification { + stream_enabled: false, + stream_view_type: None, + } + } + }), time_to_live_description: d.time_to_live.clone(), sse_description: SseDescription::from_specification( d.sse_specification.as_ref(), diff --git a/crates/engine/src/create_table.rs b/crates/engine/src/create_table.rs index ff51fac17..25b92ab64 100755 --- a/crates/engine/src/create_table.rs +++ b/crates/engine/src/create_table.rs @@ -29,6 +29,7 @@ pub async fn handle_create_table( ], )?; + crate::require_stream_enabled_member(&body)?; let mut input: CreateTableInput = serde_json::from_value(body).map_err(|e| { let msg = e.to_string(); if msg.contains("validation error detected") diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 3f8828d0b..992173e9f 100755 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -157,6 +157,26 @@ pub(crate) fn sanitize_storage_error(e: extenddb_storage::error::StorageError) - DynamoDbError::InternalServerError("Internal server error".to_owned()) } +/// The null-member sentence for a `StreamSpecification` object without +/// `StreamEnabled`, on `CreateTable` and `UpdateTable` alike (measured +/// 2026-10-24). Checked before the body is deserialised, where the missing +/// field would otherwise surface as a serialization error; its order against +/// the other request checks is not measured. A `StreamSpecification` that is +/// not an object is left to deserialisation, which refuses it as a +/// `SerializationException`. +pub(crate) fn require_stream_enabled_member(body: &serde_json::Value) -> Result<(), DynamoDbError> { + if let Some(spec) = body.get("StreamSpecification").filter(|v| v.is_object()) + && spec + .get("StreamEnabled") + .is_none_or(serde_json::Value::is_null) + { + return Err(DynamoDbError::ValidationException( + "1 validation error detected: Value null at 'streamSpecification.streamEnabled' failed to satisfy constraint: Member must not be null".to_owned(), + )); + } + Ok(()) +} + /// Map a serde deserialization error to the appropriate `DynamoDB` error type. /// /// Enum validation errors (produced by custom Deserialize impls) contain diff --git a/crates/engine/src/streams.rs b/crates/engine/src/streams.rs index e57328de0..f9ac9fed6 100755 --- a/crates/engine/src/streams.rs +++ b/crates/engine/src/streams.rs @@ -27,6 +27,14 @@ pub async fn handle_describe_stream( ) -> Result { let input: DescribeStreamInput = serde_json::from_value(body) .map_err(|e| DynamoDbError::SerializationException(e.to_string()))?; + if let Some(start) = &input.exclusive_start_shard_id + && start.chars().count() < SHARD_ID_MIN_LEN + { + return Err(DynamoDbError::ValidationException(constraint( + "exclusiveStartShardId", + "have length greater than or equal to 28", + ))); + } let desc = ctx .storage @@ -51,6 +59,23 @@ pub async fn handle_list_streams( ) -> Result { let input: ListStreamsInput = serde_json::from_value(body) .map_err(|e| DynamoDbError::SerializationException(e.to_string()))?; + // Measured 2026-10-24: Limit=0 and a start ARN under 37 characters are + // refused with the model's constraint sentences (no value quoted). The + // upper bound of Limit is not measured; it is clamped to 100. + if input.limit.is_some_and(|l| l < 1) { + return Err(DynamoDbError::ValidationException(constraint( + "limit", + "have value greater than or equal to 1", + ))); + } + if let Some(start) = &input.exclusive_start_stream_arn + && start.chars().count() < 37 + { + return Err(DynamoDbError::ValidationException(constraint( + "exclusiveStartStreamArn", + "have length greater than or equal to 37", + ))); + } let limit = input.limit.unwrap_or(100).min(100); let (streams, last_arn) = ctx @@ -86,11 +111,35 @@ pub async fn handle_get_shard_iterator( let input: GetShardIteratorInput = serde_json::from_value(body) .map_err(|e| DynamoDbError::SerializationException(e.to_string()))?; + // The service's ShardId bound is 28 to 65 characters, checked before any + // lookup with the model's constraint sentences (both measured 2026-10-24). + // A SequenceNumber, when given, is at least 21 characters (measured with + // "0" on AT_SEQUENCE_NUMBER). The maximum is applied once the shard is + // not found (`shard_lookup_error`): the shards a release before v0.1.14 + // wrote are named after the table, and a table name over 40 characters + // puts them past 65. + let shard_len = input.shard_id.chars().count(); + if shard_len < SHARD_ID_MIN_LEN { + return Err(DynamoDbError::ValidationException( + SHARD_ID_TOO_SHORT.to_owned(), + )); + } + if input + .sequence_number + .as_ref() + .is_some_and(|n| n.chars().count() < 21) + { + return Err(DynamoDbError::ValidationException(constraint( + "sequenceNumber", + "have length greater than or equal to 21", + ))); + } + // Validate the stream and shard exist before issuing an iterator. ctx.storage .validate_shard(&ctx.account_id, &input.stream_arn, &input.shard_id) .await - .map_err(storage_to_dynamo)?; + .map_err(|e| shard_lookup_error(shard_len, e))?; let seq = match input.shard_iterator_type { ShardIteratorType::TrimHorizon => String::new(), @@ -106,9 +155,12 @@ pub async fn handle_get_shard_iterator( ShardIteratorType::AtSequenceNumber => { // Convert to AFTER_SEQUENCE_NUMBER by subtracting 1, so the // exclusive "after" semantics produce inclusive "at" behavior. + // Measured 2026-10-24 for AT_SEQUENCE_NUMBER; AFTER_SEQUENCE_NUMBER + // without a number is not measured and keeps its sentence below. let raw = input.sequence_number.clone().ok_or_else(|| { DynamoDbError::ValidationException( - "SequenceNumber is required for AT_SEQUENCE_NUMBER iterator type".to_owned(), + "One or more parameter values were invalid: SequenceNumber cannot be null" + .to_owned(), ) })?; let n = raw.parse::().map_err(|_| { @@ -161,9 +213,29 @@ pub async fn handle_get_shard_iterator( serialize_output(&output) } +/// The answer when the shard a `GetShardIterator` names is not found: for a +/// `ShardId` of `shard_len` characters over the service's maximum, the model's +/// constraint sentence (the service checks it before any lookup, and never has +/// a shard that long); otherwise the storage layer's not-found answer. A shard +/// that long and found passes, so a stream from before v0.1.14 on a table whose +/// name is over 40 characters stays readable. +fn shard_lookup_error(shard_len: usize, err: StorageError) -> DynamoDbError { + if shard_len > SHARD_ID_MAX_LEN { + return DynamoDbError::ValidationException(constraint( + "shardId", + "have length less than or equal to 65", + )); + } + storage_to_dynamo(err) +} + /// Shard iterator expiration: 15 minutes (900 seconds), matching real `DynamoDB`. const SHARD_ITERATOR_EXPIRY_SECS: u64 = 900; +/// The shortest and longest `ShardId` the service accepts. +const SHARD_ID_MIN_LEN: usize = 28; +const SHARD_ID_MAX_LEN: usize = 65; + /// The model's constraint sentence for a member value, as the service renders /// it for the streams API (no value quoted; measured 2026-10-24 on `shardId`, /// `exclusiveStartShardId`, `sequenceNumber`, `limit` and @@ -174,6 +246,10 @@ fn constraint(member: &str, rule: &str) -> String { ) } +/// `GetShardIterator` with a `ShardId` under 28 characters (measured 2026-10-24 +/// with `nope`); the same shape `constraint` renders. +pub const SHARD_ID_TOO_SHORT: &str = "1 validation error detected: Value at 'shardId' failed to satisfy constraint: Member must have length greater than or equal to 28"; + /// The message `DynamoDB` returns for an expired shard iterator (measured live, /// `live-r2/iter-live.json`: an iterator 990 seconds old returns /// `ExpiredIteratorException` with exactly this text). Shared so the @@ -419,180 +495,5 @@ fn storage_to_dynamo(e: StorageError) -> DynamoDbError { } #[cfg(test)] -mod tests { - /// AT_SEQUENCE_NUMBER converts to AFTER by subtracting 1 and padding to - /// the backend's stored width. Verify that an unpadded client input is - /// normalised correctly for both the 21-digit (postgres/sqlite/mongodb) - /// and 23-digit (cassandra) cases, and that the edge case n=0 maps to - /// TRIM_HORIZON (empty string). - #[test] - fn at_sequence_number_padding() { - let cases: &[(&str, usize, &str)] = &[ - ("5", 21, "000000000000000000004"), - ("000000000000000000005", 21, "000000000000000000004"), - ("5", 23, "00000000000000000000004"), - ("00000000000000000000005", 23, "00000000000000000000004"), - ("0", 21, ""), - ("1", 21, "000000000000000000000"), - ]; - for (input, width, expected) in cases { - let n: u128 = input.parse().unwrap(); - let result = if n > 0 { - format!("{:0>width$}", n - 1) - } else { - String::new() - }; - assert_eq!(&result, expected, "input={input} width={width}"); - } - } - - #[test] - fn shard_iterator_round_trip_backward_compat_and_fail_closed() { - let stream = "arn:aws:dynamodb:us-east-1:1:table/T/stream/2026-10-20T16:17:40.188"; - let enc = - super::encode_shard_iterator("shardId-1", "AFTER_SEQUENCE_NUMBER", "42", 1000, stream); - let d = super::decode_shard_iterator(&enc).expect("decode"); - assert_eq!(d.shard_id, "shardId-1"); - assert_eq!(d.sequence, "42"); - assert_eq!(d.created_at, Some(1000)); - assert_eq!(d.stream_arn.as_deref(), Some(stream)); - - // A token from an older build (four fields, no stream ARN) still - // decodes, with stream_arn None, so the authorization layer can fail - // closed on it. - let old = base64::Engine::encode( - &base64::engine::general_purpose::STANDARD, - "shardId-2|AFTER_SEQUENCE_NUMBER|7|1000", - ); - let d2 = super::decode_shard_iterator(&old).expect("decode old"); - assert_eq!(d2.shard_id, "shardId-2"); - assert_eq!(d2.created_at, Some(1000)); - assert_eq!(d2.stream_arn, None); - - // Garbage does not decode. - assert!(super::decode_shard_iterator("!!!not base64!!!").is_none()); - } - - #[test] - fn stream_not_found_maps_to_resource_not_found_with_the_arn() { - // `describe_stream` answers a stream ARN that names no stream with the - // storage not-found error that carries the ARN. - let arn = "arn:aws:dynamodb:us-east-1:1:table/T/stream/2026-10-20T16:17:40.188"; - match super::storage_to_dynamo(extenddb_storage::util::stream_not_found(arn)) { - super::DynamoDbError::ResourceNotFoundException(msg) => assert_eq!( - msg, - format!("Requested resource not found: Stream: {arn} not found") - ), - other => panic!("expected ResourceNotFoundException, got {other:?}"), - } - } - - #[test] - fn stream_does_not_exist_maps_to_resource_not_found() { - // The answer for a stream ARN that names no stream, and for a shard - // that is not one of the stream's (`get_stream_records`). - match super::storage_to_dynamo(extenddb_storage::util::stream_does_not_exist()) { - super::DynamoDbError::ResourceNotFoundException(msg) => { - assert_eq!(msg, "Requested resource not found: Stream does not exist"); - } - other => panic!("expected ResourceNotFoundException, got {other:?}"), - } - } - - #[test] - fn get_records_limit_bounds_carry_the_measured_sentences() { - let refused = |l: i64| match super::get_records_limit(Some(l)) { - Err(super::DynamoDbError::ValidationException(msg)) => msg, - other => panic!("expected ValidationException for Limit {l}, got {other:?}"), - }; - for l in [0, -1] { - assert_eq!( - refused(l), - "1 validation error detected: Value at 'limit' failed to satisfy constraint: Member must have value greater than or equal to 1" - ); - } - assert_eq!( - refused(1001), - "One or more parameter values were invalid: Limit should be less than or equal to 1000" - ); - assert_eq!(super::get_records_limit(Some(1)).unwrap(), 1); - assert_eq!(super::get_records_limit(Some(1000)).unwrap(), 1000); - assert_eq!(super::get_records_limit(None).unwrap(), 1000); - } - - fn b64(s: &str) -> String { - base64::Engine::encode(&base64::engine::general_purpose::STANDARD, s) - } - - #[test] - fn classify_current_token_is_stream() { - let stream = "arn:aws:dynamodb:us-east-1:1:table/T/stream/2026-10-20T16:17:40.188"; - let enc = - super::encode_shard_iterator("shardId-1", "AFTER_SEQUENCE_NUMBER", "42", 1000, stream); - match super::classify_shard_iterator(&enc) { - super::ShardIteratorClass::Stream(arn) => assert_eq!(arn, stream), - other => panic!("expected Stream, got {other:?}"), - } - } - - #[test] - fn classify_old_format_four_fields_is_expired() { - // Four pre-upgrade fields with a parseable timestamp, no stream ARN. - let old = b64("shardId-2|AFTER_SEQUENCE_NUMBER|7|1000"); - assert!(matches!( - super::classify_shard_iterator(&old), - super::ShardIteratorClass::Expired - )); - } - - #[test] - fn classify_undecodable_or_short_or_unparseable_is_invalid() { - // Not base64. - assert!(matches!( - super::classify_shard_iterator("!!!not base64!!!"), - super::ShardIteratorClass::Invalid - )); - // Three fields: decodes but is not the old four-field shape. - let three = b64("shardId-3|AFTER_SEQUENCE_NUMBER|7"); - assert!(matches!( - super::classify_shard_iterator(&three), - super::ShardIteratorClass::Invalid - )); - // Four fields but an unparseable timestamp. - let bad_ts = b64("shardId-4|AFTER_SEQUENCE_NUMBER|7|notanumber"); - assert!(matches!( - super::classify_shard_iterator(&bad_ts), - super::ShardIteratorClass::Invalid - )); - } - - #[test] - fn a_token_without_a_stream_arn_gets_the_answer_the_authorization_layer_gives() { - // Two and three fields have no timestamp (invalid), four fields with a - // timestamp come from before the ARN was carried (expired), and a - // timestamp that is not a number is invalid. - for token in [ - b64("shardId-5|AFTER_SEQUENCE_NUMBER"), - b64("shardId-3|AFTER_SEQUENCE_NUMBER|7"), - b64("shardId-2|AFTER_SEQUENCE_NUMBER|7|1000"), - b64("shardId-4|AFTER_SEQUENCE_NUMBER|7|notanumber"), - ] { - let it = super::decode_shard_iterator(&token).expect("the token decodes"); - let expected = match super::classify_shard_iterator(&token) { - super::ShardIteratorClass::Expired => { - super::DynamoDbError::ExpiredIteratorException( - super::SHARD_ITERATOR_EXPIRED_MESSAGE.to_owned(), - ) - } - super::ShardIteratorClass::Invalid => { - super::DynamoDbError::ValidationException("Invalid ShardIterator".to_owned()) - } - super::ShardIteratorClass::Stream(_) => panic!("the token carries no stream ARN"), - }; - assert_eq!( - format!("{:?}", super::no_stream_arn_error(&it)), - format!("{expected:?}") - ); - } - } -} +#[path = "streams_tests.rs"] +mod tests; diff --git a/crates/engine/src/streams_tests.rs b/crates/engine/src/streams_tests.rs new file mode 100644 index 000000000..1504421fd --- /dev/null +++ b/crates/engine/src/streams_tests.rs @@ -0,0 +1,217 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Unit tests for the streams request handlers: iterator tokens, the measured +//! constraint sentences and the not-found answers. + +/// AT_SEQUENCE_NUMBER converts to AFTER by subtracting 1 and padding to +/// the backend's stored width. Verify that an unpadded client input is +/// normalised correctly for both the 21-digit (postgres/sqlite/mongodb) +/// and 23-digit (cassandra) cases, and that the edge case n=0 maps to +/// TRIM_HORIZON (empty string). +#[test] +fn at_sequence_number_padding() { + let cases: &[(&str, usize, &str)] = &[ + ("5", 21, "000000000000000000004"), + ("000000000000000000005", 21, "000000000000000000004"), + ("5", 23, "00000000000000000000004"), + ("00000000000000000000005", 23, "00000000000000000000004"), + ("0", 21, ""), + ("1", 21, "000000000000000000000"), + ]; + for (input, width, expected) in cases { + let n: u128 = input.parse().unwrap(); + let result = if n > 0 { + format!("{:0>width$}", n - 1) + } else { + String::new() + }; + assert_eq!(&result, expected, "input={input} width={width}"); + } +} + +#[test] +fn shard_iterator_round_trip_backward_compat_and_fail_closed() { + let stream = "arn:aws:dynamodb:us-east-1:1:table/T/stream/2026-10-20T16:17:40.188"; + let enc = + super::encode_shard_iterator("shardId-1", "AFTER_SEQUENCE_NUMBER", "42", 1000, stream); + let d = super::decode_shard_iterator(&enc).expect("decode"); + assert_eq!(d.shard_id, "shardId-1"); + assert_eq!(d.sequence, "42"); + assert_eq!(d.created_at, Some(1000)); + assert_eq!(d.stream_arn.as_deref(), Some(stream)); + + // A token from an older build (four fields, no stream ARN) still + // decodes, with stream_arn None, so the authorization layer can fail + // closed on it. + let old = base64::Engine::encode( + &base64::engine::general_purpose::STANDARD, + "shardId-2|AFTER_SEQUENCE_NUMBER|7|1000", + ); + let d2 = super::decode_shard_iterator(&old).expect("decode old"); + assert_eq!(d2.shard_id, "shardId-2"); + assert_eq!(d2.created_at, Some(1000)); + assert_eq!(d2.stream_arn, None); + + // Garbage does not decode. + assert!(super::decode_shard_iterator("!!!not base64!!!").is_none()); +} + +#[test] +fn stream_not_found_maps_to_resource_not_found_with_the_arn() { + // `describe_stream` answers a stream ARN that names no stream with the + // storage not-found error that carries the ARN. + let arn = "arn:aws:dynamodb:us-east-1:1:table/T/stream/2026-10-20T16:17:40.188"; + match super::storage_to_dynamo(extenddb_storage::util::stream_not_found(arn)) { + super::DynamoDbError::ResourceNotFoundException(msg) => assert_eq!( + msg, + format!("Requested resource not found: Stream: {arn} not found") + ), + other => panic!("expected ResourceNotFoundException, got {other:?}"), + } +} + +#[test] +fn stream_does_not_exist_maps_to_resource_not_found() { + // The answer for a stream ARN that names no stream, and for a shard + // that is not one of the stream's (`get_stream_records`). + match super::storage_to_dynamo(extenddb_storage::util::stream_does_not_exist()) { + super::DynamoDbError::ResourceNotFoundException(msg) => { + assert_eq!(msg, "Requested resource not found: Stream does not exist"); + } + other => panic!("expected ResourceNotFoundException, got {other:?}"), + } +} + +#[test] +fn get_records_limit_bounds_carry_the_measured_sentences() { + let refused = |l: i64| match super::get_records_limit(Some(l)) { + Err(super::DynamoDbError::ValidationException(msg)) => msg, + other => panic!("expected ValidationException for Limit {l}, got {other:?}"), + }; + for l in [0, -1] { + assert_eq!( + refused(l), + "1 validation error detected: Value at 'limit' failed to satisfy constraint: Member must have value greater than or equal to 1" + ); + } + assert_eq!( + refused(1001), + "One or more parameter values were invalid: Limit should be less than or equal to 1000" + ); + assert_eq!(super::get_records_limit(Some(1)).unwrap(), 1); + assert_eq!(super::get_records_limit(Some(1000)).unwrap(), 1000); + assert_eq!(super::get_records_limit(None).unwrap(), 1000); +} + +fn b64(s: &str) -> String { + base64::Engine::encode(&base64::engine::general_purpose::STANDARD, s) +} + +#[test] +fn the_shard_id_maximum_applies_once_the_shard_is_not_found() { + // A shard of a table named with 41 characters, as a release before + // v0.1.14 wrote it: 8 + 41 + 1 + 16 = 66 characters. Found, it is not + // this function's case at all; not found, the model's sentence. + let long = format!("shardId-{}-{:016}", "t".repeat(41), 3); + assert_eq!(long.chars().count(), 66); + let sentence = "1 validation error detected: Value at 'shardId' failed to satisfy constraint: Member must have length less than or equal to 65"; + for err in [ + extenddb_storage::util::shard_does_not_exist(), + extenddb_storage::util::stream_does_not_exist(), + ] { + match super::shard_lookup_error(long.chars().count(), err) { + super::DynamoDbError::ValidationException(msg) => assert_eq!(msg, sentence), + other => panic!("expected ValidationException, got {other:?}"), + } + } + // Within the bound, the lookup's own answer stands. + match super::shard_lookup_error(65, extenddb_storage::util::shard_does_not_exist()) { + super::DynamoDbError::ResourceNotFoundException(msg) => { + assert_eq!(msg, extenddb_storage::util::SHARD_DOES_NOT_EXIST); + } + other => panic!("expected ResourceNotFoundException, got {other:?}"), + } +} + +#[test] +fn the_constraint_sentences_match_the_measured_ones() { + assert_eq!( + super::constraint("shardId", "have length greater than or equal to 28"), + super::SHARD_ID_TOO_SHORT + ); + assert_eq!( + super::constraint("limit", "have value greater than or equal to 1"), + "1 validation error detected: Value at 'limit' failed to satisfy constraint: Member must have value greater than or equal to 1" + ); +} + +#[test] +fn classify_current_token_is_stream() { + let stream = "arn:aws:dynamodb:us-east-1:1:table/T/stream/2026-10-20T16:17:40.188"; + let enc = + super::encode_shard_iterator("shardId-1", "AFTER_SEQUENCE_NUMBER", "42", 1000, stream); + match super::classify_shard_iterator(&enc) { + super::ShardIteratorClass::Stream(arn) => assert_eq!(arn, stream), + other => panic!("expected Stream, got {other:?}"), + } +} + +#[test] +fn classify_old_format_four_fields_is_expired() { + // Four pre-upgrade fields with a parseable timestamp, no stream ARN. + let old = b64("shardId-2|AFTER_SEQUENCE_NUMBER|7|1000"); + assert!(matches!( + super::classify_shard_iterator(&old), + super::ShardIteratorClass::Expired + )); +} + +#[test] +fn classify_undecodable_or_short_or_unparseable_is_invalid() { + // Not base64. + assert!(matches!( + super::classify_shard_iterator("!!!not base64!!!"), + super::ShardIteratorClass::Invalid + )); + // Three fields: decodes but is not the old four-field shape. + let three = b64("shardId-3|AFTER_SEQUENCE_NUMBER|7"); + assert!(matches!( + super::classify_shard_iterator(&three), + super::ShardIteratorClass::Invalid + )); + // Four fields but an unparseable timestamp. + let bad_ts = b64("shardId-4|AFTER_SEQUENCE_NUMBER|7|notanumber"); + assert!(matches!( + super::classify_shard_iterator(&bad_ts), + super::ShardIteratorClass::Invalid + )); +} + +#[test] +fn a_token_without_a_stream_arn_gets_the_answer_the_authorization_layer_gives() { + // Two and three fields have no timestamp (invalid), four fields with a + // timestamp come from before the ARN was carried (expired), and a + // timestamp that is not a number is invalid. + for token in [ + b64("shardId-5|AFTER_SEQUENCE_NUMBER"), + b64("shardId-3|AFTER_SEQUENCE_NUMBER|7"), + b64("shardId-2|AFTER_SEQUENCE_NUMBER|7|1000"), + b64("shardId-4|AFTER_SEQUENCE_NUMBER|7|notanumber"), + ] { + let it = super::decode_shard_iterator(&token).expect("the token decodes"); + let expected = match super::classify_shard_iterator(&token) { + super::ShardIteratorClass::Expired => super::DynamoDbError::ExpiredIteratorException( + super::SHARD_ITERATOR_EXPIRED_MESSAGE.to_owned(), + ), + super::ShardIteratorClass::Invalid => { + super::DynamoDbError::ValidationException("Invalid ShardIterator".to_owned()) + } + super::ShardIteratorClass::Stream(_) => panic!("the token carries no stream ARN"), + }; + assert_eq!( + format!("{:?}", super::no_stream_arn_error(&it)), + format!("{expected:?}") + ); + } +} diff --git a/crates/engine/src/update_table.rs b/crates/engine/src/update_table.rs index 6db763f82..ed49a6990 100755 --- a/crates/engine/src/update_table.rs +++ b/crates/engine/src/update_table.rs @@ -45,6 +45,7 @@ pub async fn handle_update_table( ], )?; + crate::require_stream_enabled_member(&body)?; let mut input: UpdateTableInput = serde_json::from_value(body).map_err(crate::deserialize_error)?; input.resolve_table_throughput_mode(); @@ -112,6 +113,16 @@ pub async fn handle_update_table( "StreamViewType must be specified when StreamEnabled is true".to_owned(), )); } + // A disable carries no view type (measured 2026-10-24 on a never-streamed + // table and on an enabled one alike; checked before the no-stream rule). + if let Some(spec) = &input.stream_specification + && !spec.stream_enabled + && spec.stream_view_type.is_some() + { + return Err(DynamoDbError::ValidationException( + "One or more parameter values were invalid: If stream is being disabled, then UpdateViewType must not be specified".to_owned(), + )); + } // Switching to PROVISIONED requires explicit throughput values. if matches!(input.billing_mode, Some(BillingMode::Provisioned)) @@ -142,6 +153,40 @@ pub async fn handle_update_table( validate_gsi_updates(&input)?; + // A stream change travels alone. Measured 2026-10-24: beside BillingMode or + // ProvisionedThroughput the service refuses with the IOPS sentence, beside + // any GlobalSecondaryIndexUpdates entry with the index sentence, and + // applies nothing. The per-index definition checks above come first, and + // the already-enabled refusal (`Table already has an enabled stream`) comes + // before the IOPS sentence, so the table's stored specification is read + // here when a companion change is present. DeletionProtectionEnabled beside + // a stream change is refused with the service's only-operation sentence + // (measured 2026-10-24); the general rule for DeletionProtection beside any + // other change is PR #474's and is not claimed here. TableClass and + // OnDemandThroughput beside a stream change are not measured and stay + // accepted. + if let Some(companion) = stream_change_companion(&input) { + let existing = ctx + .storage + .describe_table( + &ctx.account_id, + extenddb_core::types::DescribeTableInput { + table_name: input.table_name.clone(), + }, + ) + .await + .map_err(|e| update_table_err_to_dynamo(e, &input.table_name))?; + if let Some(spec) = &input.stream_specification { + extenddb_storage::stream_lineage::stream_change( + &input.table_name, + existing.stream_specification.as_ref(), + spec, + ) + .map_err(|e| update_table_err_to_dynamo(e, &input.table_name))?; + } + return Err(DynamoDbError::ValidationException(companion.to_owned())); + } + // A created GSI's projected attributes count toward the table's limit // together with the indexes it already has: the total CreateTable checks, // and so the total a restore of this table's backups checks. @@ -203,6 +248,37 @@ pub async fn handle_update_table( serialize_output(&output) } +/// The sentence the service refuses a stream change with when the same +/// `UpdateTable` also changes the table's billing or throughput, creates or +/// deletes an index, or changes deletion protection. `None` when the request +/// carries no stream change or no such companion. With both the billing and +/// the index companion present the IOPS sentence is answered (measured); the +/// deletion-protection sentence's order against the other two is not measured +/// and comes last. +fn stream_change_companion(input: &UpdateTableInput) -> Option<&'static str> { + input.stream_specification.as_ref()?; + if input.billing_mode.is_some() || input.provisioned_throughput.is_some() { + return Some( + "One or more parameter values were invalid: You cannot modify stream status while updating table IOPS", + ); + } + if input + .global_secondary_index_updates + .as_ref() + .is_some_and(|u| !u.is_empty()) + { + return Some( + "One or more parameter values were invalid: You cannot create or delete index while changing stream status", + ); + } + if input.deletion_protection_enabled.is_some() { + return Some( + "One or more parameter values were invalid: DeletionProtection modification must be the only operation in the request", + ); + } + None +} + /// The per-entry rules for `GlobalSecondaryIndexUpdates`, checked before any /// storage read: each entry has exactly one of Create, Update, or Delete, /// and a Create has a valid name, a key schema of defined attributes, and @@ -347,157 +423,5 @@ fn update_table_err_to_dynamo( } #[cfg(test)] -mod tests { - use super::*; - use extenddb_storage::error::StorageError; - - #[test] - fn a_created_gsi_throughput_below_one_is_refused() { - let input = |pt: Value| -> UpdateTableInput { - serde_json::from_value(serde_json::json!({ - "TableName": "t", - "AttributeDefinitions": [{ "AttributeName": "g", "AttributeType": "S" }], - "GlobalSecondaryIndexUpdates": [{ "Create": { - "IndexName": "gix", - "KeySchema": [{ "AttributeName": "g", "KeyType": "HASH" }], - "Projection": { "ProjectionType": "ALL" }, - "ProvisionedThroughput": pt, - }}], - })) - .unwrap() - }; - for (read, write) in [(0, 0), (0, 1), (1, 0), (-1, 5)] { - let pt = serde_json::json!({ "ReadCapacityUnits": read, "WriteCapacityUnits": write }); - match validate_gsi_updates(&input(pt)).unwrap_err() { - DynamoDbError::ValidationException(m) => { - assert!(m.ends_with("for index: gix"), "{m}"); - } - other => panic!("({read}, {write}): {other:?}"), - } - } - let ok = serde_json::json!({ "ReadCapacityUnits": 1, "WriteCapacityUnits": 1 }); - validate_gsi_updates(&input(ok)).unwrap(); - } - - #[test] - fn projected_total_counts_kept_indexes_and_created_ones() { - use extenddb_core::types::{ - GlobalSecondaryIndexUpdate, GsiDescription, LsiDescription, Projection, ProjectionType, - TableDescription, - }; - let include = |n: usize| Projection { - projection_type: ProjectionType::Include, - non_key_attributes: Some((0..n).map(|i| format!("a{i}")).collect()), - }; - let existing = TableDescription { - global_secondary_indexes: Some(vec![ - GsiDescription { - index_name: "kept".to_owned(), - key_schema: Vec::new(), - projection: include(30), - index_status: "ACTIVE".to_owned(), - provisioned_throughput: None, - index_size_bytes: 0, - item_count: 0, - index_arn: String::new(), - }, - GsiDescription { - index_name: "dropped".to_owned(), - key_schema: Vec::new(), - projection: include(50), - index_status: "ACTIVE".to_owned(), - provisioned_throughput: None, - index_size_bytes: 0, - item_count: 0, - index_arn: String::new(), - }, - ]), - local_secondary_indexes: Some(vec![LsiDescription { - index_name: "lsi".to_owned(), - key_schema: Vec::new(), - projection: include(20), - index_size_bytes: 0, - item_count: 0, - index_arn: String::new(), - }]), - ..Default::default() - }; - let create = |n: usize| -> GlobalSecondaryIndexUpdate { - serde_json::from_value(serde_json::json!({ "Create": { - "IndexName": "new", "KeySchema": [], - "Projection": serde_json::to_value(include(n)).unwrap(), - }})) - .unwrap() - }; - let delete: GlobalSecondaryIndexUpdate = - serde_json::from_value(serde_json::json!({ "Delete": { "IndexName": "dropped" } })) - .unwrap(); - - assert_eq!(projected_after_gsi_updates(&existing, &[create(1)]), 101); - assert_eq!( - projected_after_gsi_updates(&existing, &[create(1), delete]), - 51 - ); - assert!( - extenddb_core::validation::validate_projected_attribute_total( - projected_after_gsi_updates(&existing, &[create(1)]) - ) - .is_err() - ); - } - - /// A backend that cannot serve the request reports a 400, not a 500. - /// - /// This arm was missing while CreateTable had it, so the same refusal - /// answered differently depending on which operation carried it. It matters - /// for a backend whose vector capability is decided at runtime rather than at - /// compile time: its refusal arrives through this path. - #[test] - fn an_unsupported_feature_is_a_validation_exception() { - let err = update_table_err_to_dynamo( - StorageError::Unsupported("Vector indexes are not supported here".to_owned()), - "t", - ); - match err { - DynamoDbError::ValidationException(msg) => { - assert_eq!(msg, "Vector indexes are not supported here"); - } - other => panic!("expected ValidationException, got {other:?}"), - } - } - - /// The backend's whole message survives, unwrapped and unprefixed, because - /// the service's own wording for this case names both the table and the index - /// and no layer above the backend knows either. - #[test] - fn a_resource_in_use_refusal_keeps_the_backend_message() { - let measured = extenddb_core::types::vector_index_delete_in_allocation_phase("t", "vidx"); - let err = update_table_err_to_dynamo(StorageError::IndexesInUse(measured.clone()), "t"); - match err { - DynamoDbError::ResourceInUseException(msg) => assert_eq!(msg, measured), - other => panic!("expected ResourceInUseException, got {other:?}"), - } - } - - /// The measured whole string, byte for byte, from probe P2 on 2026-08-19. - #[test] - fn the_allocation_phase_refusal_is_the_measured_whole_string() { - assert_eq!( - extenddb_core::types::vector_index_delete_in_allocation_phase( - "eddbprobe-backfill", - "vidx2" - ), - "Attempt to change a resource which is still in use: Index creation is in resource \ - allocation phase. Retry deletion during backfilling phase or when the index is \ - active. Table: eddbprobe-backfill Index: vidx2" - ); - } - - /// A genuine fault still reports a 500 and is still logged as one. - #[test] - fn an_internal_failure_is_still_an_internal_server_error() { - let err = - update_table_err_to_dynamo(StorageError::Internal("disk on fire".to_owned()), "t"); - assert!(matches!(err, DynamoDbError::InternalServerError(_))); - } -} +#[path = "update_table_tests.rs"] +mod tests; diff --git a/crates/engine/src/update_table_tests.rs b/crates/engine/src/update_table_tests.rs new file mode 100644 index 000000000..9edcfd80a --- /dev/null +++ b/crates/engine/src/update_table_tests.rs @@ -0,0 +1,253 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Unit tests for the `UpdateTable` request checks. + +use super::*; +use extenddb_storage::error::StorageError; + +/// Measured 2026-10-24: a stream change beside a billing or throughput +/// change, beside an index create or delete, or beside a deletion protection +/// change is refused with its own sentence; alone it is not. +#[test] +fn a_stream_change_travels_alone() { + let with = |extra: Value| -> UpdateTableInput { + let mut body = serde_json::json!({ + "TableName": "t", + "StreamSpecification": {"StreamEnabled": false}, + }); + for (k, v) in extra.as_object().unwrap() { + body[k] = v.clone(); + } + serde_json::from_value(body).unwrap() + }; + let iops = "One or more parameter values were invalid: You cannot modify stream status while updating table IOPS"; + let index = "One or more parameter values were invalid: You cannot create or delete index while changing stream status"; + assert_eq!( + stream_change_companion(&with(serde_json::json!({"BillingMode": "PROVISIONED"}))), + Some(iops) + ); + assert_eq!( + stream_change_companion(&with(serde_json::json!({ + "ProvisionedThroughput": {"ReadCapacityUnits": 5, "WriteCapacityUnits": 5} + }))), + Some(iops) + ); + assert_eq!( + stream_change_companion(&with(serde_json::json!({ + "GlobalSecondaryIndexUpdates": [{"Delete": {"IndexName": "g"}}] + }))), + Some(index) + ); + assert_eq!( + stream_change_companion(&with(serde_json::json!({ + "GlobalSecondaryIndexUpdates": [{"Create": { + "IndexName": "g", + "KeySchema": [{"AttributeName": "g", "KeyType": "HASH"}], + "Projection": {"ProjectionType": "ALL"} + }}] + }))), + Some(index) + ); + assert_eq!(stream_change_companion(&with(serde_json::json!({}))), None); + assert_eq!( + stream_change_companion(&with( + serde_json::json!({"DeletionProtectionEnabled": true}) + )), + Some( + "One or more parameter values were invalid: DeletionProtection modification must be the only operation in the request" + ) + ); + let no_stream: UpdateTableInput = + serde_json::from_value(serde_json::json!({"TableName": "t", "BillingMode": "PROVISIONED"})) + .unwrap(); + assert_eq!(stream_change_companion(&no_stream), None); +} + +#[test] +fn a_created_gsi_throughput_below_one_is_refused() { + let input = |pt: Value| -> UpdateTableInput { + serde_json::from_value(serde_json::json!({ + "TableName": "t", + "AttributeDefinitions": [{ "AttributeName": "g", "AttributeType": "S" }], + "GlobalSecondaryIndexUpdates": [{ "Create": { + "IndexName": "gix", + "KeySchema": [{ "AttributeName": "g", "KeyType": "HASH" }], + "Projection": { "ProjectionType": "ALL" }, + "ProvisionedThroughput": pt, + }}], + })) + .unwrap() + }; + for (read, write) in [(0, 0), (0, 1), (1, 0), (-1, 5)] { + let pt = serde_json::json!({ "ReadCapacityUnits": read, "WriteCapacityUnits": write }); + match validate_gsi_updates(&input(pt)).unwrap_err() { + DynamoDbError::ValidationException(m) => { + assert!(m.ends_with("for index: gix"), "{m}"); + } + other => panic!("({read}, {write}): {other:?}"), + } + } + let ok = serde_json::json!({ "ReadCapacityUnits": 1, "WriteCapacityUnits": 1 }); + validate_gsi_updates(&input(ok)).unwrap(); +} + +#[test] +fn projected_total_counts_kept_indexes_and_created_ones() { + use extenddb_core::types::{ + GlobalSecondaryIndexUpdate, GsiDescription, LsiDescription, Projection, ProjectionType, + TableDescription, + }; + let include = |n: usize| Projection { + projection_type: ProjectionType::Include, + non_key_attributes: Some((0..n).map(|i| format!("a{i}")).collect()), + }; + let existing = TableDescription { + global_secondary_indexes: Some(vec![ + GsiDescription { + index_name: "kept".to_owned(), + key_schema: Vec::new(), + projection: include(30), + index_status: "ACTIVE".to_owned(), + provisioned_throughput: None, + index_size_bytes: 0, + item_count: 0, + index_arn: String::new(), + }, + GsiDescription { + index_name: "dropped".to_owned(), + key_schema: Vec::new(), + projection: include(50), + index_status: "ACTIVE".to_owned(), + provisioned_throughput: None, + index_size_bytes: 0, + item_count: 0, + index_arn: String::new(), + }, + ]), + local_secondary_indexes: Some(vec![LsiDescription { + index_name: "lsi".to_owned(), + key_schema: Vec::new(), + projection: include(20), + index_size_bytes: 0, + item_count: 0, + index_arn: String::new(), + }]), + ..Default::default() + }; + let create = |n: usize| -> GlobalSecondaryIndexUpdate { + serde_json::from_value(serde_json::json!({ "Create": { + "IndexName": "new", "KeySchema": [], + "Projection": serde_json::to_value(include(n)).unwrap(), + }})) + .unwrap() + }; + let delete: GlobalSecondaryIndexUpdate = + serde_json::from_value(serde_json::json!({ "Delete": { "IndexName": "dropped" } })) + .unwrap(); + + assert_eq!(projected_after_gsi_updates(&existing, &[create(1)]), 101); + assert_eq!( + projected_after_gsi_updates(&existing, &[create(1), delete]), + 51 + ); + assert!( + extenddb_core::validation::validate_projected_attribute_total(projected_after_gsi_updates( + &existing, + &[create(1)] + )) + .is_err() + ); +} + +/// A backend that cannot serve the request reports a 400, not a 500. +/// +/// This arm was missing while CreateTable had it, so the same refusal +/// answered differently depending on which operation carried it. It matters +/// for a backend whose vector capability is decided at runtime rather than at +/// compile time: its refusal arrives through this path. +#[test] +fn an_unsupported_feature_is_a_validation_exception() { + let err = update_table_err_to_dynamo( + StorageError::Unsupported("Vector indexes are not supported here".to_owned()), + "t", + ); + match err { + DynamoDbError::ValidationException(msg) => { + assert_eq!(msg, "Vector indexes are not supported here"); + } + other => panic!("expected ValidationException, got {other:?}"), + } +} + +/// The backend's whole message survives, unwrapped and unprefixed, because +/// the service's own wording for this case names both the table and the index +/// and no layer above the backend knows either. +#[test] +fn a_resource_in_use_refusal_keeps_the_backend_message() { + let measured = extenddb_core::types::vector_index_delete_in_allocation_phase("t", "vidx"); + let err = update_table_err_to_dynamo(StorageError::IndexesInUse(measured.clone()), "t"); + match err { + DynamoDbError::ResourceInUseException(msg) => assert_eq!(msg, measured), + other => panic!("expected ResourceInUseException, got {other:?}"), + } +} + +/// The measured whole string, byte for byte, from probe P2 on 2026-08-19. +#[test] +fn the_allocation_phase_refusal_is_the_measured_whole_string() { + assert_eq!( + extenddb_core::types::vector_index_delete_in_allocation_phase( + "eddbprobe-backfill", + "vidx2" + ), + "Attempt to change a resource which is still in use: Index creation is in resource \ + allocation phase. Retry deletion during backfilling phase or when the index is \ + active. Table: eddbprobe-backfill Index: vidx2" + ); +} + +/// A genuine fault still reports a 500 and is still logged as one. +#[test] +fn an_internal_failure_is_still_an_internal_server_error() { + let err = update_table_err_to_dynamo(StorageError::Internal("disk on fire".to_owned()), "t"); + assert!(matches!(err, DynamoDbError::InternalServerError(_))); +} + +/// Measured 2026-10-24: a `StreamSpecification` object without `StreamEnabled` +/// gets the null-member sentence. A member that is not an object (a string, a +/// number, an array) is not that case: it is left to deserialisation, which +/// refuses it as a `SerializationException`, as before the member check. +#[test] +fn a_stream_specification_that_is_not_an_object_is_a_serialization_error() { + let null_member = "1 validation error detected: Value null at 'streamSpecification.streamEnabled' failed to satisfy constraint: Member must not be null"; + for spec in [ + serde_json::json!({}), + serde_json::json!({"StreamViewType": "KEYS_ONLY"}), + serde_json::json!({"StreamEnabled": null}), + ] { + let body = serde_json::json!({"TableName": "t", "StreamSpecification": spec}); + match crate::require_stream_enabled_member(&body) { + Err(DynamoDbError::ValidationException(m)) => assert_eq!(m, null_member), + other => panic!("{other:?}"), + } + } + for spec in [ + serde_json::json!("x"), + serde_json::json!(1), + serde_json::json!([]), + serde_json::json!(true), + ] { + let body = serde_json::json!({"TableName": "t", "StreamSpecification": spec}); + crate::require_stream_enabled_member(&body).expect("not the member check's case"); + let err = serde_json::from_value::(body) + .map_err(crate::deserialize_error) + .expect_err("a non-object specification does not deserialise"); + assert!( + matches!(err, DynamoDbError::SerializationException(_)), + "{err:?}" + ); + } + let body = serde_json::json!({"TableName": "t", "StreamSpecification": null}); + crate::require_stream_enabled_member(&body).expect("an absent member"); +} diff --git a/crates/storage-cassandra/src/create_table.rs b/crates/storage-cassandra/src/create_table.rs index 72d436181..71430cc25 100644 --- a/crates/storage-cassandra/src/create_table.rs +++ b/crates/storage-cassandra/src/create_table.rs @@ -18,19 +18,19 @@ use crate::backup_engine::JobHeartbeat; impl CassandraEngine { /// Initialize stream shards for a table atomically. /// - /// Writes 4 shard rows into the account keyspace and updates - /// `catalog.tables.stream_label` in a single LOGGED BATCH. - /// Returns the stream label (ISO 8601 timestamp). + /// Sets `catalog.tables.stream_label` with an LWT, then writes the four + /// shard rows of the new stream into the account keyspace in one LOGGED + /// BATCH, each starting at zero. Returns the stream label. pub(crate) async fn init_stream_shards( &self, account_id: &str, table_name: &str, account_keyspace: &str, table_id: &str, + view_type: extenddb_core::types::StreamViewType, ) -> Result { - let label = extenddb_storage::util::new_stream_label(); + let label = extenddb_storage::stream_lineage::next_stream_label(None); let catalog_keyspace = self.catalog_keyspace(); - let now_ms = chrono::Utc::now().timestamp_millis(); // Update stream_label via LWT to avoid mixing plain/LWT writes on the // tables row (which was created with IF NOT EXISTS). See ADR-0021. @@ -52,24 +52,14 @@ impl CassandraEngine { return Err(StorageError::TableNotFound(table_name.to_owned())); } - // Insert 4 shard rows into account keyspace (plain inserts, separate batch) - let mut shard_statements = Vec::new(); - for i in 0..crate::stream_util::SHARDS_PER_STREAM { - let shard_id = format!("shardId-{table_id}-{i:012}"); - shard_statements.push(format!( - "INSERT INTO {account_keyspace}.stream_shards \ - (shard_id, table_id, starting_sequence_number, created_at) \ - VALUES ('{shard_id}', '{table_id}', '{}', {now_ms})", - crate::stream_util::ZERO_SEQUENCE - )); - } - - let batch = format!("BEGIN BATCH\n{}\nAPPLY BATCH", shard_statements.join(";\n")); - // Note: values are interpolated rather than bound because Cassandra LOGGED BATCH - // does not support parameterized statements spanning multiple tables. - // All interpolated values are server-generated (UUIDs, timestamps, label from chrono). - if let Err(e) = self.session.query(&batch).await { - tracing::error!("init_stream_shards batch: {e}"); + let starts = vec![ + crate::stream_util::ZERO_SEQUENCE.to_owned(); + extenddb_storage::stream_lineage::SHARDS_PER_STREAM as usize + ]; + if let Err(e) = self + .insert_stream_shards(account_keyspace, table_id, &label, view_type, &starts) + .await + { // Clear the label so ListStreams does not report a stream with no shards. let clear_cql = format!( "UPDATE {catalog_keyspace}.tables SET stream_label = null \ @@ -81,9 +71,7 @@ impl CassandraEngine { cdrs_tokio::query_values!(account_id, table_name, label.as_str()), ) .await; - return Err(StorageError::Internal(format!( - "Failed to initialize stream shards: {e}" - ))); + return Err(e); } Ok(label) @@ -131,6 +119,7 @@ impl CassandraEngine { ) -> Result<(TableDescription, Option), StorageError> { let schedule_activation = restore.is_none(); Self::validate_account_id(account_id)?; + let input = extenddb_storage::stream_lineage::without_disabled_specification(input); let table_id = restore.map_or_else(|| uuid::Uuid::new_v4().to_string(), |r| r.table_id.clone()); @@ -462,22 +451,23 @@ impl CassandraEngine { } // Initialize stream shards (if enabled) - let stream_label: Option = if input + let stream_label: Option = match input .stream_specification .as_ref() - .is_some_and(|s| s.stream_enabled) + .filter(|s| s.stream_enabled) + .and_then(|s| s.stream_view_type) { - Some( + Some(view_type) => Some( self.init_stream_shards( account_id, &input.table_name, &account_keyspace, &table_id, + view_type, ) .await?, - ) - } else { - None + ), + None => None, }; // Create data table in account keyspace diff --git a/crates/storage-cassandra/src/lib.rs b/crates/storage-cassandra/src/lib.rs index bff16c1d4..750916ac3 100644 --- a/crates/storage-cassandra/src/lib.rs +++ b/crates/storage-cassandra/src/lib.rs @@ -34,6 +34,7 @@ mod password_hash_store; pub(crate) mod propagation_hold; mod snapshot_engine; mod stream_engine; +mod stream_lifecycle; pub mod stream_util; pub mod table_engine; mod table_helpers; diff --git a/crates/storage-cassandra/src/stream_engine.rs b/crates/storage-cassandra/src/stream_engine.rs index 8c83b0927..81e0ac8de 100755 --- a/crates/storage-cassandra/src/stream_engine.rs +++ b/crates/storage-cassandra/src/stream_engine.rs @@ -4,16 +4,15 @@ //! `StreamEngine` trait implementation for `CassandraEngine`. use cdrs_tokio::types::IntoRustByName; -use extenddb_core::types::{ - SequenceNumberRange, Shard, StreamDescription, StreamRecord, StreamStatus, StreamSummary, - StreamViewType, -}; +use extenddb_core::types::{StreamDescription, StreamRecord, StreamSummary}; use extenddb_storage::StreamEngine; use extenddb_storage::error::StorageError; +use extenddb_storage::stream_lineage; use extenddb_storage::util::{parse_stream_arn, shard_in_stream, stream_arn}; use futures::future::BoxFuture; use crate::CassandraEngine; +use crate::stream_lifecycle::{physical_shard_id, shard_range_of}; impl CassandraEngine { fn account_keyspace_for(&self, account_id: &str) -> String { @@ -59,58 +58,6 @@ impl CassandraEngine { Ok(self.account_keyspace(&account_id)) } - - /// The table id and current stream label of table `table_name` within - /// `account_id` (a primary-key read of the catalog), or `None` when the - /// account has no table of that name. - async fn stream_table_row( - &self, - account_id: &str, - table_name: &str, - ) -> Result)>, StorageError> { - let catalog_keyspace = self.catalog_keyspace(); - let query = format!( - "SELECT table_id, stream_label FROM {catalog_keyspace}.tables \ - WHERE account_id = ? AND table_name = ?" - ); - let result = self - .session - .query_with_values(&query, cdrs_tokio::query_values!(account_id, table_name)) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - let rows = result - .response_body() - .map_err(|e| StorageError::Internal(e.to_string()))? - .into_rows() - .unwrap_or_default(); - rows.into_iter() - .next() - .map(|row| { - let table_id: String = row - .get_r_by_name("table_id") - .map_err(|e| StorageError::Internal(e.to_string()))?; - let label: Option = row.get_by_name("stream_label").ok().flatten(); - Ok((table_id, label)) - }) - .transpose() - } - - /// The table id of the stream `stream_arn` names within `account_id`, or - /// the "stream does not exist" error when no table of that account carries - /// that name and stream label. - async fn stream_table_id( - &self, - account_id: &str, - stream_arn: &str, - ) -> Result { - let (table_name, stream_label) = parse_stream_arn(stream_arn)?; - match self.stream_table_row(account_id, &table_name).await? { - Some((table_id, label)) if label.as_deref() == Some(stream_label.as_str()) => { - Ok(table_id) - } - _ => Err(extenddb_storage::util::stream_does_not_exist()), - } - } } impl StreamEngine for CassandraEngine { @@ -132,7 +79,7 @@ impl StreamEngine for CassandraEngine { }) } - #[allow(unused_variables)] // `after` and `start` are used in query_with_values below + #[allow(unused_variables)] // `after` is used in query_with_values below fn get_stream_records( &self, account_id: &str, @@ -152,34 +99,43 @@ impl StreamEngine for CassandraEngine { // row and the records, issued together and awaited together on the // account keyspace. The shard is the table's when its row carries // that table id, and the stream's when `shard_in_stream` says so - // from the table's label and the shard id; otherwise (another - // table, another account, or an earlier stream label on the same - // table) the answer is the "stream does not exist" error and the - // records read are discarded. The keyspace follows from the caller's - // account, so no table id is parsed out of the shard id. + // from the table's label and the shard id (with a read of the + // table's shard rows for a legacy id only); otherwise (another + // table, another account, or a stream of the same table the shard + // is not part of) the answer is the "stream does not exist" error + // and the records read are discarded. The keyspace follows from the + // caller's account, so no table id is parsed out of the shard id. + // + // A shard's records are a range of its index's partition + // (`stream_lifecycle`): after its starting number and, once closed, + // up to its ending number. The partition follows from the shard id + // alone (`physical_shard_id`), so the records read starts at the + // iterator's position together with the shard row read, and the + // range the shard row gives is applied to the rows it returns. let (table_name, stream_label) = parse_stream_arn(&stream_arn)?; - let Some((table_id, table_label)) = - self.stream_table_row(&account_id, &table_name).await? - else { + let Some((table, _)) = self.stream_table_row(&account_id, &table_name).await? else { return Err(extenddb_storage::util::stream_does_not_exist()); }; let keyspace = self.account_keyspace_for(&account_id); + let physical = physical_shard_id(&table.table_id, &shard_id); - let shard_query = - format!("SELECT table_id FROM {keyspace}.stream_shards WHERE shard_id = ?"); + let shard_query = format!( + "SELECT table_id, starting_sequence_number, ending_sequence_number \ + FROM {keyspace}.stream_shards WHERE shard_id = ?" + ); let shard_read = self .session .query_with_values(&shard_query, cdrs_tokio::query_values!(shard_id.as_str())); - let query = if let Some(ref after) = after_sequence { + let query = if after_sequence.is_some() { format!( - "SELECT record_data FROM {keyspace}.stream_records \ + "SELECT sequence_number, record_data FROM {keyspace}.stream_records \ WHERE shard_id = ? AND sequence_number > ? \ LIMIT {limit}" ) } else { format!( - "SELECT record_data FROM {keyspace}.stream_records \ + "SELECT sequence_number, record_data FROM {keyspace}.stream_records \ WHERE shard_id = ? \ LIMIT {limit}" ) @@ -189,31 +145,56 @@ impl StreamEngine for CassandraEngine { self.session .query_with_values( &query, - cdrs_tokio::query_values!(shard_id.as_str(), after.as_str()), + cdrs_tokio::query_values!(physical.as_str(), after.as_str()), ) .await } else { self.session - .query_with_values(&query, cdrs_tokio::query_values!(shard_id.as_str())) + .query_with_values(&query, cdrs_tokio::query_values!(physical.as_str())) .await } }; let (shard_result, result) = futures::join!(shard_read, records_read); - let shard_table_id: Option = shard_result + let shard_row = shard_result .map_err(|e| StorageError::Internal(e.to_string()))? .response_body() .map_err(|e| StorageError::Internal(e.to_string()))? .into_rows() .unwrap_or_default() .into_iter() - .next() + .next(); + let shard_table_id: Option = shard_row + .as_ref() .and_then(|row| row.get_r_by_name("table_id").ok()); - if shard_table_id.as_deref() != Some(table_id.as_str()) - || !shard_in_stream(&table_id, &shard_id, table_label.as_deref(), &stream_label) - { - return Err(extenddb_storage::util::stream_does_not_exist()); - } + let range = match shard_row { + Some(row) if shard_table_id.as_deref() == Some(table.table_id.as_str()) => { + // A legacy id needs the table's shard rows: it is an + // earlier, closed stream once the table has a labelled + // shard. + let owned = shard_in_stream( + &table.table_id, + &shard_id, + table.stream_label.as_deref(), + &stream_label, + || async { + let rows = self + .stream_shard_rows(&keyspace, Some(&table.table_id)) + .await?; + Ok::(stream_lineage::has_labelled_shard( + &table.table_id, + &rows, + )) + }, + ) + .await?; + if !owned { + return Err(extenddb_storage::util::stream_does_not_exist()); + } + shard_range_of(physical, &row) + } + _ => return Err(extenddb_storage::util::stream_does_not_exist()), + }; let rows = result .map_err(|e| StorageError::Internal(e.to_string()))? @@ -222,22 +203,42 @@ impl StreamEngine for CassandraEngine { .into_rows() .unwrap_or_default(); - let records: Vec = rows - .into_iter() - .map(|row| { - let data: String = row - .get_r_by_name("record_data") - .map_err(|e| StorageError::Internal(e.to_string()))?; - serde_json::from_str(&data).map_err(|e| StorageError::Internal(e.to_string())) - }) - .collect::>()?; + // The read started at the iterator's position, which is below the + // shard's start for a fresh iterator on a later stream: the rows up + // to the start are an earlier stream's and are skipped, and when no + // row of the shard follows them the next iterator starts at the + // shard's start. The rows past a closed shard's end are a later + // stream's. + let mut records: Vec = Vec::new(); + let mut skipped_to_start = false; + for row in rows { + let sequence: String = row + .get_r_by_name("sequence_number") + .map_err(|e| StorageError::Internal(e.to_string()))?; + if sequence <= range.after { + skipped_to_start = true; + continue; + } + if range.until.as_ref().is_some_and(|until| sequence > *until) { + break; + } + let data: String = row + .get_r_by_name("record_data") + .map_err(|e| StorageError::Internal(e.to_string()))?; + records.push( + serde_json::from_str(&data) + .map_err(|e| StorageError::Internal(e.to_string()))?, + ); + } - let last_seq = records.last().map(|r| r.dynamodb.sequence_number.clone()); + let last_seq = records + .last() + .map(|r| r.dynamodb.sequence_number.clone()) + .or_else(|| skipped_to_start.then(|| range.after.clone())); Ok((records, last_seq)) }) } - #[allow(unused_variables)] // `start` is used in query_with_values below fn describe_stream( &self, account_id: &str, @@ -249,155 +250,25 @@ impl StreamEngine for CassandraEngine { let exclusive_start_shard_id = input.exclusive_start_shard_id.clone(); Box::pin(async move { let (table_name, stream_label) = parse_stream_arn(&stream_arn_str)?; - let catalog_keyspace = self.catalog_keyspace(); - let account_keyspace = self.account_keyspace_for(&account_id); - - // Fetch table metadata from catalog - let query = format!( - "SELECT key_schema, stream_specification, table_status, table_id \ - FROM {catalog_keyspace}.tables \ - WHERE account_id = ? AND table_name = ? AND stream_label = ? \ - ALLOW FILTERING" - ); - let result = self - .session - .query_with_values( - &query, - cdrs_tokio::query_values!( - account_id.as_str(), - table_name.as_str(), - stream_label.as_str() - ), - ) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - - let rows = result - .response_body() - .map_err(|e| StorageError::Internal(e.to_string()))? - .into_rows() - .unwrap_or_default(); - - let row = rows - .into_iter() - .next() + let (table, ks_json) = self + .stream_table_row(&account_id, &table_name) + .await? .ok_or_else(|| extenddb_storage::util::stream_not_found(&stream_arn_str))?; - - let ks_json: String = row - .get_r_by_name("key_schema") - .map_err(|e| StorageError::Internal(e.to_string()))?; - let stream_spec_json: Option = - row.get_by_name("stream_specification").ok().flatten(); - let table_status: String = row - .get_r_by_name("table_status") - .map_err(|e| StorageError::Internal(e.to_string()))?; - let table_id: String = row - .get_r_by_name("table_id") - .map_err(|e| StorageError::Internal(e.to_string()))?; - let key_schema = serde_json::from_str(&ks_json) .map_err(|e| StorageError::Internal(e.to_string()))?; - - let stream_view_type = stream_spec_json - .and_then(|s| { - let v: serde_json::Value = serde_json::from_str(&s).ok()?; - serde_json::from_value(v["StreamViewType"].clone()).ok() - }) - .unwrap_or(StreamViewType::KeysOnly); - - // Fetch shards from account keyspace via secondary index on table_id - let limit_val = limit.unwrap_or(100); - let shard_query = if let Some(ref start) = exclusive_start_shard_id { - format!( - "SELECT shard_id, parent_shard_id, starting_sequence_number, ending_sequence_number \ - FROM {account_keyspace}.stream_shards \ - WHERE table_id = ? AND shard_id > ? \ - LIMIT {} ALLOW FILTERING", - limit_val + 1 - ) - } else { - format!( - "SELECT shard_id, parent_shard_id, starting_sequence_number, ending_sequence_number \ - FROM {account_keyspace}.stream_shards \ - WHERE table_id = ? \ - LIMIT {} ALLOW FILTERING", - limit_val + 1 - ) - }; - - let shard_result = if let Some(ref start) = exclusive_start_shard_id { - self.session - .query_with_values( - &shard_query, - cdrs_tokio::query_values!(table_id.as_str(), start.as_str()), - ) - .await - } else { - self.session - .query_with_values(&shard_query, cdrs_tokio::query_values!(table_id.as_str())) - .await - } - .map_err(|e| StorageError::Internal(e.to_string()))?; - - let shard_rows = shard_result - .response_body() - .map_err(|e| StorageError::Internal(e.to_string()))? - .into_rows() - .unwrap_or_default(); - - #[allow(clippy::cast_sign_loss)] - let limit_usize = limit_val as usize; - let last_shard = if shard_rows.len() > limit_usize { - let row = &shard_rows[limit_usize - 1]; - let id: String = row - .get_r_by_name("shard_id") - .map_err(|e| StorageError::Internal(e.to_string()))?; - Some(id) - } else { - None - }; - - let shards: Vec = shard_rows - .into_iter() - .take(limit_usize) - .map(|row| { - let shard_id: String = row - .get_r_by_name("shard_id") - .map_err(|e| StorageError::Internal(e.to_string()))?; - let parent_shard_id: Option = - row.get_by_name("parent_shard_id").ok().flatten(); - let starting: String = row - .get_r_by_name("starting_sequence_number") - .map_err(|e| StorageError::Internal(e.to_string()))?; - let ending: Option = - row.get_by_name("ending_sequence_number").ok().flatten(); - Ok(Shard { - shard_id, - parent_shard_id, - sequence_number_range: SequenceNumberRange { - starting_sequence_number: starting, - ending_sequence_number: ending, - }, - }) - }) - .collect::>()?; - - let stream_status = if table_status == "DELETING" { - StreamStatus::Disabling - } else { - StreamStatus::Enabled - }; - - Ok(StreamDescription { - stream_arn: stream_arn_str, - stream_label, - stream_status, - stream_view_type, - table_name, + let account_keyspace = self.account_keyspace_for(&account_id); + let shards = self + .stream_shard_rows(&account_keyspace, Some(&table.table_id)) + .await?; + stream_lineage::describe_stream( + stream_arn_str, + &table, key_schema, - shards, - last_evaluated_shard_id: last_shard, - }) + stream_label, + &shards, + limit, + exclusive_start_shard_id.as_deref(), + ) }) } @@ -412,75 +283,21 @@ impl StreamEngine for CassandraEngine { let table_name = table_name.map(str::to_string); let exclusive_start_stream_arn = exclusive_start_stream_arn.map(str::to_string); Box::pin(async move { - let catalog_keyspace = self.catalog_keyspace(); - - // Cassandra doesn't support IS NOT NULL filtering efficiently; fetch all and filter. - // For list_streams the result set is bounded by the number of tables per account. - let query = format!( - "SELECT table_name, stream_label FROM {catalog_keyspace}.tables \ - WHERE account_id = ? ALLOW FILTERING" - ); - let result = self - .session - .query_with_values(&query, cdrs_tokio::query_values!(account_id.as_str())) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - - use cdrs_tokio::types::IntoRustByName; - let rows = result - .response_body() - .map_err(|e| StorageError::Internal(e.to_string()))? - .into_rows() - .unwrap_or_default(); - - // Parse start cursor - let start_cursor = exclusive_start_stream_arn + let start = exclusive_start_stream_arn .as_deref() .map(parse_stream_arn) .transpose()?; - - let mut summaries: Vec = rows - .into_iter() - .filter_map(|row| { - let tn: String = row.get_r_by_name("table_name").ok()?; - let label: Option = row.get_by_name("stream_label").ok().flatten(); - let label = label?; // skip tables without streams - if let Some(ref filter_tn) = table_name - && &tn != filter_tn - { - return None; - } - Some(StreamSummary { - stream_arn: stream_arn(&self.region, &account_id, &tn, &label), - stream_label: label, - table_name: tn, - }) - }) - .collect(); - - // Sort for stable pagination (table_name, stream_label) - summaries.sort_by(|a, b| { - a.table_name - .cmp(&b.table_name) - .then(a.stream_label.cmp(&b.stream_label)) - }); - - // Apply cursor - if let Some((start_table, start_label)) = start_cursor { - summaries - .retain(|s| (&s.table_name, &s.stream_label) > (&start_table, &start_label)); - } - - #[allow(clippy::cast_sign_loss)] - let limit_usize = limit as usize; - let last_arn = if summaries.len() > limit_usize { - summaries.get(limit_usize - 1).map(|s| s.stream_arn.clone()) - } else { - None - }; - summaries.truncate(limit_usize); - - Ok((summaries, last_arn)) + let tables = self.stream_table_rows(&account_id).await?; + let account_keyspace = self.account_keyspace_for(&account_id); + let shards = self.stream_shard_rows(&account_keyspace, None).await?; + stream_lineage::list_streams( + &tables, + &shards, + table_name.as_deref(), + limit, + start, + |t, l| stream_arn(&self.region, &account_id, t, l), + ) }) } @@ -537,32 +354,22 @@ impl StreamEngine for CassandraEngine { let stream_arn = stream_arn.to_string(); let shard_id = shard_id.to_string(); Box::pin(async move { - let table_id = self.stream_table_id(&account_id, &stream_arn).await?; - let account_keyspace = self.account_keyspace_for(&account_id); - - let shard_query = format!( - "SELECT shard_id FROM {account_keyspace}.stream_shards \ - WHERE shard_id = ? AND table_id = ? ALLOW FILTERING" - ); - let shard_result = self - .session - .query_with_values( - &shard_query, - cdrs_tokio::query_values!(shard_id.as_str(), table_id.as_str()), - ) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - - let found = shard_result - .response_body() - .map_err(|e| StorageError::Internal(e.to_string()))? - .into_rows() - .is_some_and(|r| !r.is_empty()); - - if !found { - return Err(extenddb_storage::util::shard_not_in_stream(&stream_arn)); - } - Ok(()) + let (table_name, stream_label) = parse_stream_arn(&stream_arn)?; + let table = self.stream_table_row(&account_id, &table_name).await?; + let shards = match &table { + Some((t, _)) => { + let account_keyspace = self.account_keyspace_for(&account_id); + self.stream_shard_rows(&account_keyspace, Some(&t.table_id)) + .await? + } + None => Vec::new(), + }; + stream_lineage::validate_shard( + table.as_ref().map(|(t, _)| t), + &stream_label, + &shards, + &shard_id, + ) }) } @@ -573,26 +380,8 @@ impl StreamEngine for CassandraEngine { let shard_id = shard_id.to_string(); Box::pin(async move { let keyspace = self.account_keyspace_for_shard(&shard_id).await?; - let query = format!( - "SELECT sequence_number FROM {keyspace}.stream_records \ - WHERE shard_id = ? ORDER BY sequence_number DESC LIMIT 1" - ); - let result = self - .session - .query_with_values(&query, cdrs_tokio::query_values!(shard_id.as_str())) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - - let rows = result - .response_body() - .map_err(|e| StorageError::Internal(e.to_string()))? - .into_rows() - .unwrap_or_default(); - - Ok(rows - .into_iter() - .next() - .map(|row| row.get_r_by_name("sequence_number").unwrap_or_default())) + let range = self.shard_range(&keyspace, &shard_id).await?; + self.last_sequence_in(&keyspace, &range).await }) } } diff --git a/crates/storage-cassandra/src/stream_lifecycle.rs b/crates/storage-cassandra/src/stream_lifecycle.rs new file mode 100644 index 000000000..7eef053ce --- /dev/null +++ b/crates/storage-cassandra/src/stream_lifecycle.rs @@ -0,0 +1,518 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Stream lifecycle on Cassandra: closing a table's stream on +//! `UpdateTable StreamEnabled=false`, creating the next one, and reading the +//! table and shard rows the shared rules in `extenddb_storage::stream_lineage` +//! group into streams. +//! +//! The write paths compute a record's shard from the table id and the +//! partition key alone (`stream_util::assign_shard_id`), inside the LOGGED +//! BATCH of the data write, with no read of the shard rows. So records are +//! stored in one partition per shard index, `shardId-{table_id}-{index:012}`, +//! whatever stream is current, and a stream's shard is a range of that +//! partition: the records after its `StartingSequenceNumber`, up to its +//! `EndingSequenceNumber` once closed. A new stream's shards start one above +//! the larger of this server's next clock value and the last ending number +//! among the table's shards, drawn when the stream is enabled, so a record +//! belongs to at most one stream. The first stream of a table starts at zero. +//! +//! With several servers, the sequence a write draws is its own server's clock +//! (`stream_util::HybridClock`), while the start is taken on the server that +//! runs the enable. Taking the closed shards' ending numbers into the start +//! keeps a record of the old stream, written by a server whose clock is ahead, +//! from reading in both streams. A server whose clock is behind the start can +//! still write a new-stream record below it, which no stream reads; that case +//! stays with the deferred cross-instance consistency item in the design +//! document. +//! +//! `UpdateTable` writes the catalog row first and changes the shards after it +//! ([`PendingStream`]), as on PostgreSQL and MongoDB. A disable whose close +//! failed leaves a disabled stream whose shards are still open: a retried +//! disable is refused and closes them, and an enable closes them too, since an +//! enable closes the open shards of every earlier stream of the table. An +//! enable whose shard insert failed leaves an enabled stream with no shards, +//! which no record belongs to, replaced by a disable and an enable. + +use cdrs_tokio::types::IntoRustByName; +use extenddb_core::types::{StreamSpecification, StreamViewType}; +use extenddb_storage::error::StorageError; +use extenddb_storage::stream_lineage::{ + SHARDS_PER_STREAM, ShardRow, StreamChange, disabled_stream_to_close, enable_still_current, + next_stream_label, open_shard_ids_before, open_shard_ids_of, parse_shard_id, shard_id, + stored_specification, stream_change, +}; + +use crate::CassandraEngine; +use crate::cassandra_util::query_rows; +use crate::stream_util::ZERO_SEQUENCE; + +fn internal(e: impl std::fmt::Display) -> StorageError { + StorageError::Internal(e.to_string()) +} + +/// The partition holding the records of `shard_id`: the table's partition for +/// the shard's index. A shard id this table did not issue maps to itself. +#[must_use] +pub(crate) fn physical_shard_id(table_id: &str, shard_id: &str) -> String { + parse_shard_id(table_id, shard_id).map_or_else( + || shard_id.to_owned(), + |(_, index)| format!("shardId-{table_id}-{index:012}"), + ) +} + +/// The sequence number one above `sequence`, in its 23-digit shape. +fn next_sequence(sequence: &str) -> Result { + let n: u128 = sequence + .parse() + .map_err(|_| StorageError::Internal(format!("malformed sequence number {sequence}")))?; + Ok(format!("{:0width$}", n + 1, width = sequence.len())) +} + +/// The table id a shard id names (`shardId-{table_id}-{digits}`). +pub(crate) fn shard_table_id(shard_id: &str) -> Option<&str> { + shard_id + .strip_prefix("shardId-")? + .rsplit_once('-') + .map(|(table_id, _)| table_id) +} + +/// The record range of one shard, for reading its partition. +pub(crate) struct ShardRange { + /// The partition its records are in. + pub physical: String, + /// Records after this sequence number belong to the shard. + pub after: String, + /// Records up to this one, once the shard is closed. + pub until: Option, +} + +/// The record range of the shard whose `stream_shards` row is `row` (with its +/// `starting_sequence_number` and `ending_sequence_number` columns), in the +/// partition `physical`. +pub(crate) fn shard_range_of(physical: String, row: &cdrs_tokio::types::rows::Row) -> ShardRange { + let start: Option = row.get_by_name("starting_sequence_number").ok().flatten(); + ShardRange { + physical, + after: start.unwrap_or_else(|| ZERO_SEQUENCE.to_owned()), + until: row.get_by_name("ending_sequence_number").ok().flatten(), + } +} + +/// The `tables` columns an `UpdateTable` stream change sets, as text. +pub(crate) type StreamColumns = Vec<(&'static str, String)>; + +/// The shard changes of an applied `UpdateTable` stream change: on a disable, +/// close the open shards of the stream the change read (`close`, the table's +/// label at that time); on an enable, close the open shards of every earlier +/// stream, then create the new stream's shards. +#[must_use] +pub(crate) struct PendingStream { + /// The specification JSON and label the change was decided on; the + /// catalog write is conditioned on both still being these values. + pub read_specification: Option, + pub read_label: Option, + pub close: Option, + pub open: Option<(String, StreamViewType)>, +} + +impl PendingStream { + /// Run the shard changes once the catalog row is written. + /// + /// The enable's shards are inserted after its catalog row is visible, so a + /// disable that reads the row in between passes its condition, writes the + /// disabled specification and closes the shards it finds, which may be + /// none or some of the four. The enable re-reads the table's stream fields + /// after its inserts and, when the row no longer says enabled with this + /// label, closes the shards it inserted. A disable written before the + /// re-read is seen by it; one written after finds all four rows open and + /// closes them itself. A shard keeps the ending number of the first close + /// that writes one (`close_stream_shards`), so a disable whose close runs + /// after an overlapping enable started the next stream cannot move the + /// closed shard's ending number up into the new stream's records. + pub(crate) async fn apply( + self, + engine: &CassandraEngine, + account_id: &str, + account_ks: &str, + table_name: &str, + table_id: &str, + ) -> Result<(), StorageError> { + let Some((label, view_type)) = self.open else { + if let Some(label) = &self.close { + engine + .close_stream_shards(account_ks, table_id, label) + .await?; + } + return Ok(()); + }; + engine + .close_streams_before(account_ks, table_id, &label) + .await?; + // Each new shard starts one above the larger of a sequence number + // drawn now from the clock this server's writers draw from and the + // last ending number among the table's shards (the ones just closed + // included), so it is above every sequence issued so far on this + // server and every record the closed streams hold, whichever server + // wrote it. A write of the closed stream that commits after the close + // (its LOGGED BATCH drew its sequence before the ending number was + // taken) then falls between the old shard's ending number and the new + // shard's starting number: in neither stream, rather than in the new + // one under the old view type. + let local = engine + .hlc + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .generate(); + let last_end = engine + .stream_shard_rows(account_ks, Some(table_id)) + .await? + .into_iter() + .filter_map(|r| r.ending_sequence_number) + .max(); + let start = next_sequence(&last_end.map_or(local.clone(), |end| end.max(local)))?; + let starts: Vec = vec![start; SHARDS_PER_STREAM as usize]; + engine + .insert_stream_shards(account_ks, table_id, &label, view_type, &starts) + .await?; + let row = engine.stream_table_row(account_id, table_name).await?; + if !enable_still_current(row.as_ref().map(|(t, _)| t), &label) { + engine + .close_stream_shards(account_ks, table_id, &label) + .await?; + } + Ok(()) + } +} + +impl PendingStream { + /// Decide an `UpdateTable` `StreamSpecification` against the table row + /// fetched by `update_table_impl`: the catalog columns to set (as text) + /// and the shard changes to run once the row is written. + pub(crate) fn decide( + row: &cdrs_tokio::types::rows::Row, + table_name: &str, + requested: &StreamSpecification, + ) -> Result<(StreamColumns, Self), StorageError> { + let read_specification: Option = + row.get_by_name("stream_specification").ok().flatten(); + let current: Option = read_specification + .as_deref() + .map(serde_json::from_str) + .transpose() + .map_err(internal)?; + let read_label: Option = row.get_by_name("stream_label").ok().flatten(); + let change = stream_change(table_name, current.as_ref(), requested)?; + let mut cols = vec![( + "stream_specification", + serde_json::to_string(&stored_specification(change, current.as_ref(), requested)) + .map_err(internal)?, + )]; + let open = match change { + StreamChange::Disable => None, + StreamChange::Enable(view_type) => { + let label = next_stream_label(read_label.as_deref()); + cols.push(("stream_label", label.clone())); + Some((label, view_type)) + } + }; + Ok(( + cols, + Self { + read_specification, + close: read_label.clone(), + read_label, + open, + }, + )) + } +} + +impl CassandraEngine { + /// [`PendingStream::decide`], and for a disable refused because the stored + /// specification is already disabled, the close of the current label's + /// open shards first (left by a release that did not close them, or by a + /// close that failed after the catalog write); the refusal is still the + /// answer. + pub(crate) async fn decide_stream_change_or_close( + &self, + row: &cdrs_tokio::types::rows::Row, + account_ks: &str, + table_id: &str, + table_name: &str, + requested: &StreamSpecification, + ) -> Result<(StreamColumns, PendingStream), StorageError> { + match PendingStream::decide(row, table_name, requested) { + Ok(decided) => Ok(decided), + Err(refusal) => { + let stored: Option = row.get_by_name("stream_specification").ok().flatten(); + let current: Option = stored + .as_deref() + .map(serde_json::from_str) + .transpose() + .map_err(internal)?; + let label: Option = row.get_by_name("stream_label").ok().flatten(); + if let Some(label) = + disabled_stream_to_close(current.as_ref(), label.as_deref(), requested) + { + self.close_stream_shards(account_ks, table_id, label) + .await?; + } + Err(refusal) + } + } + } + + /// The refusal for a stream change whose conditional catalog write found + /// the stream fields changed: re-read the row and decide the request + /// against it, which refuses the loser of two overlapping changes with the + /// ordinary sentence (`Table already has an enabled stream` or `Table has + /// no stream to disable`). When the re-read state would accept the request + /// (a disable overlapped an enable), the table is reported as not ACTIVE, + /// the nearest of ExtendDB's answers to the service's in-progress refusal. + pub(crate) async fn refuse_overlapping_stream_change( + &self, + account_id: &str, + table_name: &str, + input: &extenddb_core::types::UpdateTableInput, + ) -> Result<(), StorageError> { + let Some(spec) = &input.stream_specification else { + return Ok(()); + }; + let catalog_ks = self.catalog_keyspace(); + let row = query_rows::( + &self.session, + &format!( + "SELECT stream_specification FROM {catalog_ks}.tables \ + WHERE account_id = ? AND table_name = ?" + ), + cdrs_tokio::query_values!(account_id, table_name), + "update_table stream re-read", + ) + .await? + .into_iter() + .next() + .ok_or_else(|| StorageError::TableNotFound(table_name.to_owned()))?; + let current: Option = row.get_by_name("stream_specification").ok().flatten(); + let current: Option = current + .as_deref() + .map(serde_json::from_str) + .transpose() + .map_err(internal)?; + stream_change(table_name, current.as_ref(), spec)?; + Err(StorageError::TableNotActive(table_name.to_owned())) + } + + /// Insert the four open shards of stream `label`, shard `i` starting at + /// `starts[i]`. The caller stores the label on the table row. + pub(crate) async fn insert_stream_shards( + &self, + account_ks: &str, + table_id: &str, + label: &str, + view_type: StreamViewType, + starts: &[String], + ) -> Result<(), StorageError> { + let now_ms = chrono::Utc::now().timestamp_millis(); + let mut statements = Vec::new(); + for (i, start) in (0..SHARDS_PER_STREAM).zip(starts) { + // Interpolated, not bound: a LOGGED BATCH here carries plain + // strings. Every value is server-generated (UUID table id, label + // digits, sequence digits). + statements.push(format!( + "INSERT INTO {account_ks}.stream_shards \ + (shard_id, table_id, starting_sequence_number, created_at) \ + VALUES ('{}', '{table_id}', '{start}', {now_ms})", + shard_id(table_id, label, view_type, i)? + )); + } + let batch = format!("BEGIN BATCH\n{}\nAPPLY BATCH", statements.join(";\n")); + self.session.query(&batch).await.map_err(|e| { + tracing::error!("insert_stream_shards batch: {e}"); + StorageError::Internal(format!("Failed to initialize stream shards: {e}")) + })?; + Ok(()) + } + + /// Close every open shard of the table: its `EndingSequenceNumber` becomes + /// the sequence number of its last record, or its starting number when it + /// holds none (both measured 2026-10-24). + /// + /// A shard's range has no upper bound while it is open, so the ending + /// number is written only if the shard has none yet: a close that read the + /// shard as open and takes its ending after another close has written one + /// (a disable applying after an overlapping enable closed the stream and + /// started the next) would otherwise take a record of the next stream from + /// the shared partition. + pub(crate) async fn close_stream_shards( + &self, + account_ks: &str, + table_id: &str, + label: &str, + ) -> Result<(), StorageError> { + self.close_selected_shards(account_ks, table_id, label, open_shard_ids_of) + .await + } + + /// Close the open shards of every stream of the table older than `label` + /// (`stream_lineage::open_shard_ids_before`), as [`Self::close_stream_shards`] + /// closes one stream's. + async fn close_streams_before( + &self, + account_ks: &str, + table_id: &str, + label: &str, + ) -> Result<(), StorageError> { + self.close_selected_shards(account_ks, table_id, label, open_shard_ids_before) + .await + } + + async fn close_selected_shards( + &self, + account_ks: &str, + table_id: &str, + label: &str, + select: fn(&str, &str, &[ShardRow]) -> Vec, + ) -> Result<(), StorageError> { + let rows = self.stream_shard_rows(account_ks, Some(table_id)).await?; + let ids = select(table_id, label, &rows); + let open = rows + .into_iter() + .filter(|r| ids.contains(&r.shard_id)) + .collect(); + self.close_shards_as_read(account_ks, table_id, open).await + } + + /// Write the ending numbers of `rows`, shard rows of table `table_id` this + /// close read as open: each one's last record after its start, or its + /// start when it holds none, unless the shard already has an ending + /// number. Public for the direct tests, which run a close that read its + /// rows before another close wrote them. + #[doc(hidden)] + pub async fn close_shards_as_read( + &self, + account_ks: &str, + table_id: &str, + rows: Vec, + ) -> Result<(), StorageError> { + for row in rows { + let range = ShardRange { + physical: physical_shard_id(table_id, &row.shard_id), + after: row.starting_sequence_number.clone(), + until: None, + }; + let ending = self + .last_sequence_in(account_ks, &range) + .await? + .unwrap_or(row.starting_sequence_number); + crate::cassandra_util::execute::( + &self.session, + &format!( + "UPDATE {account_ks}.stream_shards SET ending_sequence_number = ? \ + WHERE shard_id = ? IF ending_sequence_number = null" + ), + cdrs_tokio::query_values!(ending.as_str(), row.shard_id.as_str()), + "close_stream_shards", + ) + .await?; + } + Ok(()) + } + + /// The sequence number of the last record in a shard's range. + pub(crate) async fn last_sequence_in( + &self, + account_ks: &str, + range: &ShardRange, + ) -> Result, StorageError> { + let rows = match &range.until { + Some(until) => { + query_rows::( + &self.session, + &format!( + "SELECT sequence_number FROM {account_ks}.stream_records \ + WHERE shard_id = ? AND sequence_number > ? AND sequence_number <= ? \ + ORDER BY sequence_number DESC LIMIT 1" + ), + cdrs_tokio::query_values!( + range.physical.as_str(), + range.after.as_str(), + until.as_str() + ), + "last_sequence_in", + ) + .await? + } + None => { + query_rows::( + &self.session, + &format!( + "SELECT sequence_number FROM {account_ks}.stream_records \ + WHERE shard_id = ? AND sequence_number > ? \ + ORDER BY sequence_number DESC LIMIT 1" + ), + cdrs_tokio::query_values!(range.physical.as_str(), range.after.as_str()), + "last_sequence_in", + ) + .await? + } + }; + rows.into_iter() + .next() + .map(|r| r.get_r_by_name("sequence_number").map_err(internal)) + .transpose() + } + + /// The record range of a shard, from its row. + pub(crate) async fn shard_range( + &self, + account_ks: &str, + shard_id: &str, + ) -> Result { + let table_id = shard_table_id(shard_id).unwrap_or_default(); + let physical = physical_shard_id(table_id, shard_id); + let row = query_rows::( + &self.session, + &format!( + "SELECT starting_sequence_number, ending_sequence_number \ + FROM {account_ks}.stream_shards WHERE shard_id = ?" + ), + cdrs_tokio::query_values!(shard_id), + "shard_range", + ) + .await? + .into_iter() + .next(); + Ok(match row { + Some(r) => shard_range_of(physical, &r), + None => ShardRange { + physical, + after: ZERO_SEQUENCE.to_owned(), + until: None, + }, + }) + } +} + +#[path = "stream_lifecycle_rows.rs"] +mod rows; + +#[cfg(test)] +mod tests { + use super::*; + + const TID: &str = "550e8400-e29b-41d4-a716-446655440000"; + + #[test] + fn every_stream_of_a_table_reads_the_partition_of_its_shard_index() { + let label = "2026-10-24T08:54:32.805"; + for i in 0..SHARDS_PER_STREAM { + let labelled = shard_id(TID, label, StreamViewType::NewImage, i).unwrap(); + let legacy = format!("shardId-{TID}-{i:012}"); + assert_eq!(physical_shard_id(TID, &labelled), legacy); + assert_eq!(physical_shard_id(TID, &legacy), legacy); + assert_eq!(shard_table_id(&labelled), Some(TID)); + } + assert_eq!(physical_shard_id(TID, "shardId-other-1"), "shardId-other-1"); + } +} diff --git a/crates/storage-cassandra/src/stream_lifecycle_rows.rs b/crates/storage-cassandra/src/stream_lifecycle_rows.rs new file mode 100644 index 000000000..fe23f12c1 --- /dev/null +++ b/crates/storage-cassandra/src/stream_lifecycle_rows.rs @@ -0,0 +1,138 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The table rows and shard rows of the Cassandra stream lifecycle, read for +//! the shared rules in `extenddb_storage::stream_lineage` to group into +//! streams. + +use cdrs_tokio::types::IntoRustByName; +use extenddb_core::types::StreamSpecification; +use extenddb_storage::error::StorageError; +use extenddb_storage::stream_lineage::{ShardRow, TableStreamRow}; + +use super::internal; +use crate::CassandraEngine; +use crate::cassandra_util::query_rows; +use crate::stream_util::ZERO_SEQUENCE; + +impl CassandraEngine { + fn table_stream_row( + table_name: String, + row: &cdrs_tokio::types::rows::Row, + ) -> Result { + let spec: Option = row.get_by_name("stream_specification").ok().flatten(); + Ok(TableStreamRow { + table_name, + table_id: row.get_r_by_name("table_id").map_err(internal)?, + table_status: row.get_r_by_name("table_status").map_err(internal)?, + stream_label: row.get_by_name("stream_label").ok().flatten(), + stream_specification: spec + .as_deref() + .map(serde_json::from_str::) + .transpose() + .map_err(internal)?, + }) + } + + /// The table's stream columns and its key schema JSON, by name. + pub(crate) async fn stream_table_row( + &self, + account_id: &str, + table_name: &str, + ) -> Result, StorageError> { + let catalog_ks = self.catalog_keyspace(); + let row = query_rows::( + &self.session, + &format!( + "SELECT table_id, table_status, stream_label, stream_specification, key_schema \ + FROM {catalog_ks}.tables WHERE account_id = ? AND table_name = ?" + ), + cdrs_tokio::query_values!(account_id, table_name), + "stream_table_row", + ) + .await? + .into_iter() + .next(); + row.map(|r| { + Ok(( + Self::table_stream_row(table_name.to_owned(), &r)?, + r.get_r_by_name("key_schema").map_err(internal)?, + )) + }) + .transpose() + } + + /// Every table of the account that has had a stream. + pub(crate) async fn stream_table_rows( + &self, + account_id: &str, + ) -> Result, StorageError> { + let catalog_ks = self.catalog_keyspace(); + let rows = query_rows::( + &self.session, + &format!( + "SELECT table_name, table_id, table_status, stream_label, stream_specification \ + FROM {catalog_ks}.tables WHERE account_id = ? ALLOW FILTERING" + ), + cdrs_tokio::query_values!(account_id), + "stream_table_rows", + ) + .await?; + let mut tables = Vec::new(); + for r in rows { + let label: Option = r.get_by_name("stream_label").ok().flatten(); + if label.is_none() { + continue; + } + let name: String = r.get_r_by_name("table_name").map_err(internal)?; + tables.push(Self::table_stream_row(name, &r)?); + } + Ok(tables) + } + + /// The shard rows of one table, or of every table in the account's + /// keyspace when `table_id` is `None`. + pub(crate) async fn stream_shard_rows( + &self, + account_ks: &str, + table_id: Option<&str>, + ) -> Result, StorageError> { + let columns = "table_id, shard_id, parent_shard_id, starting_sequence_number, \ + ending_sequence_number"; + let rows = match table_id { + Some(id) => { + query_rows::( + &self.session, + &format!("SELECT {columns} FROM {account_ks}.stream_shards WHERE table_id = ?"), + cdrs_tokio::query_values!(id), + "stream_shard_rows", + ) + .await? + } + None => { + query_rows::( + &self.session, + &format!("SELECT {columns} FROM {account_ks}.stream_shards"), + cdrs_tokio::query::QueryValues::SimpleValues(Vec::new()), + "stream_shard_rows", + ) + .await? + } + }; + rows.into_iter() + .map(|r| { + Ok(ShardRow { + table_id: r.get_r_by_name("table_id").map_err(internal)?, + shard_id: r.get_r_by_name("shard_id").map_err(internal)?, + parent_shard_id: r.get_by_name("parent_shard_id").ok().flatten(), + starting_sequence_number: r + .get_by_name("starting_sequence_number") + .ok() + .flatten() + .unwrap_or_else(|| ZERO_SEQUENCE.to_owned()), + ending_sequence_number: r.get_by_name("ending_sequence_number").ok().flatten(), + }) + }) + .collect() + } +} diff --git a/crates/storage-cassandra/src/update_table.rs b/crates/storage-cassandra/src/update_table.rs index 4a20b8618..a0c438507 100644 --- a/crates/storage-cassandra/src/update_table.rs +++ b/crates/storage-cassandra/src/update_table.rs @@ -30,7 +30,7 @@ impl CassandraEngine { &self.session, &format!( "SELECT table_id, table_status, key_schema, attribute_definitions, \ - billing_mode, provisioned_throughput \ + billing_mode, provisioned_throughput, stream_specification, stream_label \ FROM {catalog_ks}.tables WHERE account_id = ? AND table_name = ?" ), cdrs_tokio::query_values!(account_id, input.table_name.as_str()), @@ -151,53 +151,24 @@ impl CassandraEngine { add_update!("on_demand_throughput", odt_json.as_str()); } - // Stream specification: add to batch, then handle shard init separately - // (init_stream_shards writes to a different keyspace and does its own batch). - let mut needs_shard_init = false; - let mut needs_label_restore = false; + // Stream specification: the catalog columns go in the tables LWT; the + // shard changes run after it (`PendingStream`), since the shards are + // in a different keyspace. + let mut pending_stream = None; if let Some(spec) = &input.stream_specification { - let spec_json = - serde_json::to_string(spec).map_err(|e| StorageError::Internal(e.to_string()))?; - add_update!("stream_specification", spec_json.as_str()); - - if spec.stream_enabled { - let account_ks = self.account_keyspace(account_id); - let existing = query_optional( - &self.session, - &format!( - "SELECT shard_id FROM {account_ks}.stream_shards \ - WHERE table_id = ? LIMIT 1 ALLOW FILTERING" - ), - cdrs_tokio::query_values!(table_id.as_str()), - "update_table stream_shards check", + let (cols, pending) = self + .decide_stream_change_or_close( + &row, + &self.account_keyspace(account_id), + &table_id, + &input.table_name, + spec, ) .await?; - - if existing.is_none() { - needs_shard_init = true; - } else { - // Re-enabling: check if stream_label needs restoring. - let label_row = query_optional( - &self.session, - &format!( - "SELECT stream_label FROM {catalog_ks}.tables \ - WHERE account_id = ? AND table_name = ?" - ), - cdrs_tokio::query_values!(account_id, input.table_name.as_str()), - "update_table stream_label check", - ) - .await?; - let has_label = label_row.is_some_and(|r| { - let v: Option = r.get_by_name("stream_label").ok().flatten(); - v.is_some() - }); - if !has_label { - needs_label_restore = true; - let label = extenddb_storage::util::new_stream_label(); - add_update!("stream_label", label.as_str()); - } - } + for (col, val) in cols { + table_cols.push((col, Value::from(val.as_str()))); } + pending_stream = Some(pending); } // GSI updates: pre-validate, then add catalog changes to batch. @@ -416,8 +387,7 @@ impl CassandraEngine { table_cols, indexes_batch, indexes_batch_has_statements, - needs_shard_init, - needs_label_restore, + pending_stream, &table_id, &gsi_creates, &gsi_deletes, @@ -451,8 +421,7 @@ impl CassandraEngine { table_cols: Vec<(&'static str, Value)>, indexes_batch: BatchQueryBuilder, indexes_batch_has_statements: bool, - needs_shard_init: bool, - needs_label_restore: bool, + pending_stream: Option, table_id: &str, gsi_creates: &[(String, String)], gsi_deletes: &[String], @@ -505,13 +474,26 @@ impl CassandraEngine { .map(|(col, _)| format!("{col} = ?")) .collect::>() .join(", "); + // A stream change is decided on the specification and label the + // row carried and written only if they still are those values: + // of two overlapping stream changes the one whose write finds them + // changed re-reads the row and is refused by `stream_change` as a + // request arriving after the other's commit would be. + let condition = match &pending_stream { + Some(_) => "IF stream_specification = ? AND stream_label = ?", + None => "IF EXISTS", + }; let cql = format!( "UPDATE {catalog_ks}.tables SET {set_clause} \ - WHERE account_id = ? AND table_name = ? IF EXISTS" + WHERE account_id = ? AND table_name = ? {condition}" ); let mut vals: Vec = table_cols.into_iter().map(|(_, v)| v).collect(); vals.push(Value::from(account_id)); vals.push(Value::from(input.table_name.as_str())); + if let Some(pending) = &pending_stream { + vals.push(Value::from(pending.read_specification.as_deref())); + vals.push(Value::from(pending.read_label.as_deref())); + } let result = crate::cassandra_util::query_lwt( &self.session, &cql, @@ -536,7 +518,10 @@ impl CassandraEngine { } })?; // IF EXISTS returns false only if the row was deleted between our - // earlier SELECT and this UPDATE — treat as TableNotFound. + // earlier SELECT and this UPDATE: treat as TableNotFound. Under a + // stream change the condition also fails when another stream + // change committed in between; `refuse_overlapping_stream_change` + // re-reads the row and answers as that later request would be. // NOTE: the tables LWT commits before the indexes batch below runs. // A failure in the indexes batch leaves the table row updated with // no corresponding index rows; the recovery worker does not cover @@ -560,6 +545,10 @@ impl CassandraEngine { .await; }); } + if pending_stream.is_some() { + self.refuse_overlapping_stream_change(account_id, &input.table_name, input) + .await?; + } return Err(StorageError::TableNotFound(input.table_name.clone())); } } @@ -591,12 +580,12 @@ impl CassandraEngine { return Err(e); } - // Post-batch: shard init (has its own internal batch across two keyspaces). - if needs_shard_init { - self.init_stream_shards(account_id, &input.table_name, &account_ks, table_id) + // Post-batch: close the stream's shards and create the new stream's. + if let Some(pending) = pending_stream { + pending + .apply(self, account_id, &account_ks, &input.table_name, table_id) .await?; } - let _ = needs_label_restore; // stream_label is set via add_update! above // Post-batch: GSI data table DDL and async backfill. if let Some(updates) = &input.global_secondary_index_updates { diff --git a/crates/storage-cassandra/tests/direct/stream_disable.rs b/crates/storage-cassandra/tests/direct/stream_disable.rs new file mode 100644 index 000000000..481e63223 --- /dev/null +++ b/crates/storage-cassandra/tests/direct/stream_disable.rs @@ -0,0 +1,424 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Disabling a table's stream closes its shards, and enabling it again +//! creates a new stream, against the rows measured on the service on +//! 2026-10-24. Records stay in one partition per shard index; each stream's +//! shard reads only its own range of it. + +use std::collections::BTreeMap; +use std::sync::Arc; +use std::time::Duration; + +use extenddb_core::types::{ + AttributeDefinition, AttributeValue, CreateTableInput, DescribeStreamInput, DescribeTableInput, + KeySchemaElement, KeyType, ScalarAttributeType, StreamDescription, StreamSpecification, + StreamStatus, StreamViewType, TableDescription, +}; +use extenddb_storage::error::StorageError; +use extenddb_storage::{DataEngine, StreamCapture, StreamEngine, TableEngine}; +use extenddb_storage_cassandra::CassandraEngine; + +use crate::helpers::setup_engine; + +pub(crate) async fn activate(engine: &CassandraEngine) { + tokio::time::sleep(Duration::from_millis(350)).await; + engine + .process_control_plane_transitions() + .await + .expect("process table transitions"); +} + +pub(crate) async fn describe_table( + engine: &CassandraEngine, + account: &str, + name: &str, +) -> TableDescription { + engine + .describe_table( + account, + DescribeTableInput { + table_name: name.to_owned(), + }, + ) + .await + .unwrap() +} + +pub(crate) async fn describe_stream( + engine: &CassandraEngine, + account: &str, + arn: &str, +) -> StreamDescription { + engine + .describe_stream( + account, + &DescribeStreamInput { + stream_arn: arn.to_owned(), + limit: None, + exclusive_start_shard_id: None, + }, + ) + .await + .unwrap() +} + +pub(crate) async fn set_stream( + engine: &CassandraEngine, + account: &str, + name: &str, + spec: serde_json::Value, +) -> Result { + let input = serde_json::json!({"TableName": name, "StreamSpecification": spec}); + let out = engine + .update_table(account, serde_json::from_value(input).unwrap()) + .await; + activate(engine).await; + out +} + +pub(crate) async fn put( + engine: &CassandraEngine, + account: &str, + name: &str, + pk: &str, + view: StreamViewType, +) { + let key_info = engine.table_key_info(account, name).await.unwrap(); + let mut item = BTreeMap::new(); + item.insert("id".to_owned(), AttributeValue::S(pk.to_owned())); + let capture = StreamCapture { + view_type: view, + user_identity: None, + region: Arc::from("us-east-1"), + }; + engine + .put_item( + &key_info, + item, + false, + None, + &Default::default(), + Some(&capture), + ) + .await + .unwrap(); +} + +/// Every record a shard of the stream `arn` yields, from its start. +pub(crate) async fn read_all( + engine: &CassandraEngine, + account: &str, + arn: &str, + shard: &str, +) -> Vec { + let (records, _) = engine + .get_stream_records(account, arn, shard, None, 1000) + .await + .unwrap(); + records + .into_iter() + .map(|r| r.dynamodb.sequence_number) + .collect() +} + +#[tokio::test] +async fn disable_closes_every_shard_and_re_enable_creates_a_new_stream() { + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = crate::helpers::unique_test_account(); + crate::helpers::ensure_test_account(&engine, &account) + .await + .unwrap(); + let name = "StreamDisable"; + engine + .create_table( + &account, + CreateTableInput { + table_name: name.to_owned(), + key_schema: vec![KeySchemaElement { + attribute_name: "id".to_owned(), + key_type: KeyType::Hash, + }], + attribute_definitions: vec![AttributeDefinition { + attribute_name: "id".to_owned(), + attribute_type: ScalarAttributeType::S, + }], + stream_specification: Some(StreamSpecification { + stream_enabled: true, + stream_view_type: Some(StreamViewType::NewAndOldImages), + }), + ..Default::default() + }, + ) + .await + .unwrap(); + activate(&engine).await; + for pk in ["k0", "k1", "k2"] { + put(&engine, &account, name, pk, StreamViewType::NewAndOldImages).await; + } + let enabled = describe_table(&engine, &account, name).await; + let old_arn = enabled.latest_stream_arn.clone().unwrap(); + + let disabled = set_stream( + &engine, + &account, + name, + serde_json::json!({"StreamEnabled": false}), + ) + .await + .unwrap(); + let mut rendered = disabled.clone(); + rendered.prepare_for_response(); + assert!(rendered.stream_specification.is_none()); + assert_eq!( + disabled.latest_stream_arn.as_deref(), + Some(old_arn.as_str()) + ); + let mut table = describe_table(&engine, &account, name).await; + table.prepare_for_response(); + assert!(table.stream_specification.is_none()); + + let closed = describe_stream(&engine, &account, &old_arn).await; + assert!(matches!(closed.stream_status, StreamStatus::Disabled)); + assert_eq!(closed.stream_view_type, StreamViewType::NewAndOldImages); + let mut before = 0; + for shard in &closed.shards { + let range = &shard.sequence_number_range; + let records = read_all(&engine, &account, &old_arn, &shard.shard_id).await; + before += records.len(); + let expected = records + .last() + .cloned() + .unwrap_or_else(|| range.starting_sequence_number.clone()); + assert_eq!( + range.ending_sequence_number.as_deref(), + Some(expected.as_str()) + ); + } + assert_eq!(before, 3); + + match set_stream( + &engine, + &account, + name, + serde_json::json!({"StreamEnabled": false}), + ) + .await + { + Err(StorageError::Validation(m)) => { + assert_eq!( + m, + format!("Table has no stream to disable: TableName: {name}") + ); + } + other => panic!("{other:?}"), + } + + let reenabled = set_stream( + &engine, + &account, + name, + serde_json::json!({"StreamEnabled": true, "StreamViewType": "KEYS_ONLY"}), + ) + .await + .unwrap(); + let new_arn = reenabled.latest_stream_arn.clone().unwrap(); + assert_ne!(new_arn, old_arn); + let new = describe_stream(&engine, &account, &new_arn).await; + assert!(matches!(new.stream_status, StreamStatus::Enabled)); + assert_eq!(new.stream_view_type, StreamViewType::KeysOnly); + assert!( + new.shards + .iter() + .all(|n| !closed.shards.iter().any(|o| o.shard_id == n.shard_id)) + ); + match set_stream( + &engine, + &account, + name, + serde_json::json!({"StreamEnabled": true, "StreamViewType": "NEW_IMAGE"}), + ) + .await + { + Err(StorageError::Validation(m)) => { + assert_eq!( + m, + format!("Table already has an enabled stream: TableName: {name}") + ); + } + other => panic!("{other:?}"), + } + + // A write after the re-enable lands in the shared partition, and only the + // new stream's shard yields it. + put(&engine, &account, name, "after", StreamViewType::KeysOnly).await; + let mut old_after = 0; + for shard in &closed.shards { + old_after += read_all(&engine, &account, &old_arn, &shard.shard_id) + .await + .len(); + } + assert_eq!(old_after, 3); + let mut on_new = 0; + for shard in &new.shards { + on_new += read_all(&engine, &account, &new_arn, &shard.shard_id) + .await + .len(); + } + assert_eq!(on_new, 1); + // A token pairing one stream's ARN with the other stream's shard id names + // a shard the stream does not have. + for (arn, shard) in [ + (&new_arn, &closed.shards[0].shard_id), + (&old_arn, &new.shards[0].shard_id), + ] { + match engine + .get_stream_records(&account, arn, shard, None, 1000) + .await + { + Err(StorageError::TableNotFound(m)) => { + assert_eq!(m, extenddb_storage::util::STREAM_DOES_NOT_EXIST); + } + other => panic!("{other:?}"), + } + } + // LATEST on a closed shard resolves to its last record, not to the new + // stream's. + for shard in &closed.shards { + let latest = engine + .latest_sequence_number(&shard.shard_id) + .await + .unwrap(); + let end = shard.sequence_number_range.ending_sequence_number.clone(); + if latest.is_some() { + assert_eq!(latest, end); + } + } + + let (listed, _) = engine + .list_streams(&account, Some(name), 100, None) + .await + .unwrap(); + let arns: Vec<&str> = listed.iter().map(|s| s.stream_arn.as_str()).collect(); + assert_eq!(arns, [new_arn.as_str(), old_arn.as_str()]); + match engine + .validate_shard(&account, &old_arn, &new.shards[0].shard_id) + .await + { + Err(StorageError::TableNotFound(m)) => { + assert_eq!(m, "Requested resource not found: Shard does not exist"); + } + other => panic!("{other:?}"), + } +} + +/// Of two stream changes fired at the same instant, one is applied and the +/// other refused (the conditional catalog write); the table ends with one +/// enabled stream of four open shards and every other shard closed. +#[tokio::test] +async fn overlapping_stream_changes_apply_exactly_one() { + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = crate::helpers::unique_test_account(); + crate::helpers::ensure_test_account(&engine, &account) + .await + .unwrap(); + let name = "StreamOverlap"; + engine + .create_table( + &account, + CreateTableInput { + table_name: name.to_owned(), + key_schema: vec![KeySchemaElement { + attribute_name: "id".to_owned(), + key_type: KeyType::Hash, + }], + attribute_definitions: vec![AttributeDefinition { + attribute_name: "id".to_owned(), + attribute_type: ScalarAttributeType::S, + }], + stream_specification: Some(StreamSpecification { + stream_enabled: true, + stream_view_type: Some(StreamViewType::KeysOnly), + }), + ..Default::default() + }, + ) + .await + .unwrap(); + activate(&engine).await; + let input = |spec: serde_json::Value| -> extenddb_core::types::UpdateTableInput { + serde_json::from_value(serde_json::json!({"TableName": name, "StreamSpecification": spec})) + .unwrap() + }; + let disable = serde_json::json!({"StreamEnabled": false}); + let enable = serde_json::json!({"StreamEnabled": true, "StreamViewType": "NEW_IMAGE"}); + for round in 0..6 { + let (a, b) = match round % 3 { + 0 => (disable.clone(), disable.clone()), + 1 => (enable.clone(), disable.clone()), + _ => (disable.clone(), enable.clone()), + }; + let (ra, rb) = tokio::join!( + engine.update_table(&account, input(a)), + engine.update_table(&account, input(b)), + ); + activate(&engine).await; + let ok = [&ra, &rb].iter().filter(|r| r.is_ok()).count(); + for r in [&ra, &rb] { + if let Err(e) = r { + assert!( + matches!( + e, + StorageError::Validation(_) | StorageError::TableNotActive(_) + ), + "{e:?}" + ); + } + } + assert_eq!( + ok, 1, + "round {round}: exactly one change applies: {ra:?} {rb:?}" + ); + + // Audit: one enabled stream with four open shards at most, every + // other shard closed. + let table = describe_table(&engine, &account, name).await; + let (listed, _) = engine + .list_streams(&account, Some(name), 100, None) + .await + .unwrap(); + let mut enabled = 0; + for s in &listed { + let d = describe_stream(&engine, &account, &s.stream_arn).await; + let open = d + .shards + .iter() + .filter(|x| x.sequence_number_range.ending_sequence_number.is_none()) + .count(); + if matches!(d.stream_status, StreamStatus::Enabled) { + enabled += 1; + assert_eq!((d.shards.len(), open), (4, 4), "{}", s.stream_arn); + assert_eq!( + table.latest_stream_arn.as_deref(), + Some(s.stream_arn.as_str()) + ); + } else { + assert_eq!(open, 0, "a disabled stream has every shard closed"); + } + } + let spec_enabled = table + .stream_specification + .as_ref() + .is_some_and(|s| s.stream_enabled); + assert_eq!(enabled, usize::from(spec_enabled)); + // Back to enabled for the next round. + let _ = engine.update_table(&account, input(enable.clone())).await; + activate(&engine).await; + } +} diff --git a/crates/storage-cassandra/tests/direct/stream_endings.rs b/crates/storage-cassandra/tests/direct/stream_endings.rs new file mode 100644 index 000000000..cd8a316ac --- /dev/null +++ b/crates/storage-cassandra/tests/direct/stream_endings.rs @@ -0,0 +1,244 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! A closed stream's ending numbers against the next stream's start on +//! Cassandra, where every stream of a table keeps its records in the same +//! partition per shard index. + +use extenddb_core::types::{ + AttributeDefinition, CreateTableInput, KeySchemaElement, KeyType, ScalarAttributeType, + StreamSpecification, StreamViewType, +}; +use extenddb_storage::TableEngine; + +use crate::helpers::setup_engine; +use crate::stream_disable::{activate, describe_stream, describe_table, put, read_all, set_stream}; + +/// The closed stream's last record came from a server whose clock is ahead of +/// this one: the new stream starts above that ending number, not at this +/// server's clock, so the record stays in one stream. +#[tokio::test] +async fn a_new_stream_starts_above_the_closed_streams_ending_numbers() { + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = crate::helpers::unique_test_account(); + crate::helpers::ensure_test_account(&engine, &account) + .await + .unwrap(); + let name = "StreamSkew"; + engine + .create_table( + &account, + CreateTableInput { + table_name: name.to_owned(), + key_schema: vec![KeySchemaElement { + attribute_name: "id".to_owned(), + key_type: KeyType::Hash, + }], + attribute_definitions: vec![AttributeDefinition { + attribute_name: "id".to_owned(), + attribute_type: ScalarAttributeType::S, + }], + stream_specification: Some(StreamSpecification { + stream_enabled: true, + stream_view_type: Some(StreamViewType::KeysOnly), + }), + ..Default::default() + }, + ) + .await + .unwrap(); + activate(&engine).await; + let old_arn = describe_table(&engine, &account, name) + .await + .latest_stream_arn + .unwrap(); + set_stream( + &engine, + &account, + name, + serde_json::json!({"StreamEnabled": false}), + ) + .await + .unwrap(); + let closed = describe_stream(&engine, &account, &old_arn).await; + // An ending number an hour ahead of this server's clock, as a record from + // a server that far ahead would leave it. + let ahead = format!( + "{:013}{:06}{:04}", + chrono::Utc::now().timestamp_millis() + 3_600_000, + 0, + 1 + ); + let account_ks = engine.account_keyspace(&account); + let update = format!( + "UPDATE {account_ks}.stream_shards SET ending_sequence_number = ? WHERE shard_id = ?" + ); + for shard in &closed.shards { + engine + .session_arc() + .query_with_values( + &update, + cdrs_tokio::query_values!(ahead.as_str(), shard.shard_id.as_str()), + ) + .await + .unwrap(); + } + + let reenabled = set_stream( + &engine, + &account, + name, + serde_json::json!({"StreamEnabled": true, "StreamViewType": "KEYS_ONLY"}), + ) + .await + .unwrap(); + let new = describe_stream(&engine, &account, &reenabled.latest_stream_arn.unwrap()).await; + assert_eq!(new.shards.len(), 4); + for shard in &new.shards { + let start = &shard.sequence_number_range.starting_sequence_number; + assert_eq!(start.len(), ahead.len()); + assert!( + start.as_str() > ahead.as_str(), + "start {start} is not above the closed stream's ending number {ahead}" + ); + } +} + +/// A disable whose close read the old stream's shards as open and writes their +/// ending numbers only after an overlapping enable closed them, started the +/// new stream and the new stream took records: the old shards keep the ending +/// numbers the first close wrote, below the new stream's start. +#[tokio::test] +async fn a_late_close_keeps_the_ending_numbers_the_first_close_wrote() { + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = crate::helpers::unique_test_account(); + crate::helpers::ensure_test_account(&engine, &account) + .await + .unwrap(); + let name = "StreamLateClose"; + engine + .create_table( + &account, + CreateTableInput { + table_name: name.to_owned(), + key_schema: vec![KeySchemaElement { + attribute_name: "id".to_owned(), + key_type: KeyType::Hash, + }], + attribute_definitions: vec![AttributeDefinition { + attribute_name: "id".to_owned(), + attribute_type: ScalarAttributeType::S, + }], + stream_specification: Some(StreamSpecification { + stream_enabled: true, + stream_view_type: Some(StreamViewType::KeysOnly), + }), + ..Default::default() + }, + ) + .await + .unwrap(); + activate(&engine).await; + let table = describe_table(&engine, &account, name).await; + let old_arn = table.latest_stream_arn.clone().unwrap(); + for i in 0..8 { + put( + &engine, + &account, + name, + &format!("a{i}"), + StreamViewType::KeysOnly, + ) + .await; + } + // The rows a disable reads before it closes: every shard open. + let read_open: Vec = + describe_stream(&engine, &account, &old_arn) + .await + .shards + .into_iter() + .map(|s| extenddb_storage::stream_lineage::ShardRow { + table_id: table.table_id.clone(), + shard_id: s.shard_id, + parent_shard_id: s.parent_shard_id, + starting_sequence_number: s.sequence_number_range.starting_sequence_number, + ending_sequence_number: None, + }) + .collect(); + assert_eq!(read_open.len(), 4); + + set_stream( + &engine, + &account, + name, + serde_json::json!({"StreamEnabled": false}), + ) + .await + .unwrap(); + let closed = describe_stream(&engine, &account, &old_arn).await; + let reenabled = set_stream( + &engine, + &account, + name, + serde_json::json!({"StreamEnabled": true, "StreamViewType": "KEYS_ONLY"}), + ) + .await + .unwrap(); + let new_arn = reenabled.latest_stream_arn.unwrap(); + for i in 0..8 { + put( + &engine, + &account, + name, + &format!("b{i}"), + StreamViewType::KeysOnly, + ) + .await; + } + let new = describe_stream(&engine, &account, &new_arn).await; + let mut new_records = 0; + for shard in &new.shards { + new_records += read_all(&engine, &account, &new_arn, &shard.shard_id) + .await + .len(); + } + assert_eq!(new_records, 8, "the new stream holds the eight new records"); + + // The late close of the old stream's label. + engine + .close_shards_as_read( + &engine.account_keyspace(&account), + &table.table_id, + read_open, + ) + .await + .unwrap(); + let after = describe_stream(&engine, &account, &old_arn).await; + for (a, b) in after.shards.iter().zip(&closed.shards) { + assert_eq!(a.shard_id, b.shard_id); + assert_eq!( + a.sequence_number_range.ending_sequence_number, + b.sequence_number_range.ending_sequence_number, + "the late close moved shard {}'s ending number", + a.shard_id + ); + } + let start = &new.shards[0].sequence_number_range.starting_sequence_number; + for shard in &after.shards { + let end = shard + .sequence_number_range + .ending_sequence_number + .as_deref() + .unwrap(); + assert!( + end < start.as_str(), + "ending {end} is not below the new start {start}" + ); + } +} diff --git a/crates/storage-cassandra/tests/direct/stream_ownership.rs b/crates/storage-cassandra/tests/direct/stream_ownership.rs index fa1170c59..00af0aa4c 100644 --- a/crates/storage-cassandra/tests/direct/stream_ownership.rs +++ b/crates/storage-cassandra/tests/direct/stream_ownership.rs @@ -5,36 +5,91 @@ //! //! The catalog read is keyed on the caller's account and the table name the //! ARN carries, the shard read on that table's id, and the stream label is -//! decided in Rust from the table's current label. Each test here removes one -//! of those from the picture (same label on two tables, same name in two -//! accounts, a new label on the same shards) so that one predicate alone +//! decided in Rust from the label the shard id carries. Each test here removes +//! one of those from the picture (same label on two tables, same name in two +//! accounts, a new label on the same partitions) so that one predicate alone //! decides the answer. use std::collections::BTreeMap; use std::sync::Arc; +use cdrs_tokio::types::IntoRustByName; use extenddb_core::types::{AttributeValue, StreamViewType, TableKeyInfo}; use extenddb_storage::error::StorageError; +use extenddb_storage::stream_lineage::{ShardStream, parse_shard_id, shard_id}; use extenddb_storage::util::stream_arn; use extenddb_storage::{DataEngine, StreamCapture, StreamEngine, StreamRecordsResult}; use extenddb_storage_cassandra::CassandraEngine; use crate::helpers::setup_engine; -use crate::streams::{setup_stream_table, shard_for}; +use crate::streams::{setup_stream_table, stream_shard_for}; const REGION: &str = "us-east-1"; const LABEL: &str = "2026-01-01T00:00:00.000"; const LATER_LABEL: &str = "2026-02-02T00:00:00.000"; -/// Pin a table's stream label so two tables can share one. +/// Pin a table's stream label so two tables can share one. A shard id carries +/// its stream's label, so each shard row of the table moves to the id the +/// stream `label` gives its index, with the row's sequence range kept. async fn relabel(engine: &CassandraEngine, key_info: &TableKeyInfo, label: &str) { + let session = engine.session_arc(); + let account_ks = engine.account_keyspace(&key_info.account_id); + let rows = session + .query_with_values( + &format!( + "SELECT shard_id, starting_sequence_number, ending_sequence_number \ + FROM {account_ks}.stream_shards WHERE table_id = ?" + ), + cdrs_tokio::query_values!(key_info.table_id.as_str()), + ) + .await + .expect("shard rows") + .response_body() + .expect("body") + .into_rows() + .unwrap_or_default(); + for row in rows { + let old: String = row.get_r_by_name("shard_id").expect("shard_id"); + let start: String = row + .get_r_by_name("starting_sequence_number") + .expect("starting_sequence_number"); + let end: Option = row.get_by_name("ending_sequence_number").ok().flatten(); + let Some((ShardStream::Labelled { view_type, .. }, index)) = + parse_shard_id(&key_info.table_id, &old) + else { + panic!("{old} is not a labelled shard of {}", key_info.table_id); + }; + let new = shard_id(&key_info.table_id, label, view_type, index).expect("label shape"); + session + .query_with_values( + &format!("DELETE FROM {account_ks}.stream_shards WHERE shard_id = ?"), + cdrs_tokio::query_values!(old.as_str()), + ) + .await + .expect("delete the old shard row"); + session + .query_with_values( + &format!( + "INSERT INTO {account_ks}.stream_shards \ + (shard_id, table_id, starting_sequence_number, ending_sequence_number) \ + VALUES (?, ?, ?, ?)" + ), + cdrs_tokio::query_values!( + new.as_str(), + key_info.table_id.as_str(), + start.as_str(), + end + ), + ) + .await + .expect("insert the relabelled shard row"); + } let catalog_keyspace = engine.catalog_keyspace(); let query = format!( "UPDATE {catalog_keyspace}.tables SET stream_label = ? \ WHERE account_id = ? AND table_name = ?" ); - engine - .session_arc() + session .query_with_values( &query, cdrs_tokio::query_values!( @@ -97,7 +152,8 @@ async fn stream_table_in( } } -/// One item written under `pk`, which lands in `shard_for(pk, table_id)`. +/// One item written under `pk`, which lands in the partition `shard_for(pk, +/// table_id)` names; the stream's shard for it is `shard_of(pk, table, label)`. async fn put(engine: &CassandraEngine, key_info: &TableKeyInfo, pk: &str, val: u32) { let mut item = BTreeMap::new(); item.insert("id".to_string(), AttributeValue::S(pk.to_string())); @@ -124,6 +180,17 @@ fn arn_of(key_info: &TableKeyInfo, label: &str) -> String { stream_arn(REGION, &key_info.account_id, &key_info.table_name, label) } +/// The shard of the stream `label` of `key_info`'s table that a record written +/// under `pk` lands in. +fn shard_of(pk: &str, key_info: &TableKeyInfo, label: &str) -> String { + let view_type = key_info + .stream_specification + .as_ref() + .and_then(|s| s.stream_view_type) + .expect("a stream-enabled table"); + stream_shard_for(pk, &key_info.table_id, label, view_type) +} + fn assert_stream_not_found(result: StreamRecordsResult) { match result { Err(StorageError::TableNotFound(msg)) => { @@ -147,8 +214,8 @@ async fn test_another_tables_shard_with_the_same_label_is_not_found() { relabel(&engine, &b, LABEL).await; put(&engine, &a, "pk-a", 1).await; put(&engine, &b, "pk-b", 1).await; - let shard_a = shard_for("pk-a", &a.table_id); - let shard_b = shard_for("pk-b", &b.table_id); + let shard_a = shard_of("pk-a", &a, LABEL); + let shard_b = shard_of("pk-b", &b, LABEL); let arn_a = arn_of(&a, LABEL); let arn_b = arn_of(&b, LABEL); @@ -169,7 +236,7 @@ async fn test_another_tables_shard_with_the_same_label_is_not_found() { .get_stream_records( &a.account_id, &arn_a, - "shardId-forged-000000000000", + "shardId-forged-0000000000000000", None, 10, ) @@ -201,8 +268,8 @@ async fn test_another_accounts_same_named_stream_is_not_found() { relabel(&engine, &other, LABEL).await; put(&engine, &own, "pk-o", 1).await; put(&engine, &other, "pk-o", 1).await; - let own_shard = shard_for("pk-o", &own.table_id); - let other_shard = shard_for("pk-o", &other.table_id); + let own_shard = shard_of("pk-o", &own, LABEL); + let other_shard = shard_of("pk-o", &other, LABEL); // From both callers, so whichever table a lookup without the account // would pick, one caller reads the other account's shard. @@ -228,8 +295,9 @@ async fn test_another_accounts_same_named_stream_is_not_found() { } } -/// The table takes a new stream label while keeping its shard rows: the old -/// ARN's iterator is refused and the new ARN reads every record. +/// The table's stream takes a new label while keeping its partitions: the old +/// ARN's iterator, and the old shard id under the new ARN, are refused, and +/// the new stream's shard reads every record of its partition. #[tokio::test] async fn test_an_earlier_label_on_the_same_shards_is_not_found() { if crate::helpers::skip_without_cassandra() { @@ -239,7 +307,7 @@ async fn test_an_earlier_label_on_the_same_shards_is_not_found() { let (t, _) = setup_stream_table(&engine, "Relabel").await; relabel(&engine, &t, LABEL).await; put(&engine, &t, "pk-r", 1).await; - let shard = shard_for("pk-r", &t.table_id); + let shard = shard_of("pk-r", &t, LABEL); let old_arn = arn_of(&t, LABEL); engine .get_stream_records(&t.account_id, &old_arn, &shard, None, 10) @@ -248,13 +316,20 @@ async fn test_an_earlier_label_on_the_same_shards_is_not_found() { relabel(&engine, &t, LATER_LABEL).await; put(&engine, &t, "pk-r", 2).await; + let new_arn = arn_of(&t, LATER_LABEL); + let new_shard = shard_of("pk-r", &t, LATER_LABEL); assert_stream_not_found( engine .get_stream_records(&t.account_id, &old_arn, &shard, None, 10) .await, ); + assert_stream_not_found( + engine + .get_stream_records(&t.account_id, &new_arn, &shard, None, 10) + .await, + ); let (records, _) = engine - .get_stream_records(&t.account_id, &arn_of(&t, LATER_LABEL), &shard, None, 10) + .get_stream_records(&t.account_id, &new_arn, &new_shard, None, 10) .await .expect("new label reads"); assert_eq!(records.len(), 2); diff --git a/crates/storage-cassandra/tests/direct/streams.rs b/crates/storage-cassandra/tests/direct/streams.rs index bc0efa206..e0a39ac7c 100644 --- a/crates/storage-cassandra/tests/direct/streams.rs +++ b/crates/storage-cassandra/tests/direct/streams.rs @@ -143,6 +143,22 @@ pub(crate) fn shard_for(pk: &str, table_id: &str) -> String { extenddb_storage_cassandra::stream_util::assign_shard_id(pk, table_id) } +/// The shard of the stream `label` (view type `view_type`) of `table_id` that +/// records written under `pk` land in: the stream's shard for the partition +/// `shard_for` names, which is the id `get_stream_records` reads it by. +pub(crate) fn stream_shard_for( + pk: &str, + table_id: &str, + label: &str, + view_type: StreamViewType, +) -> String { + let (_, index) = + extenddb_storage::stream_lineage::parse_shard_id(table_id, &shard_for(pk, table_id)) + .expect("the partition id is the table's"); + extenddb_storage::stream_lineage::shard_id(table_id, label, view_type, index) + .expect("label shape") +} + #[tokio::test] async fn test_put_item_insert_writes_stream_record() { if crate::helpers::skip_without_cassandra() { @@ -406,7 +422,15 @@ async fn test_validate_shard_ok() { &key_info.table_name, &stream_label, ); - let shard_id = shard_for("any-pk", &key_info.table_id); + // The stream's own shard id for the partition the record lands in. + let index = crc32fast::hash(b"any-pk") % 4; + let shard_id = extenddb_storage::stream_lineage::shard_id( + &key_info.table_id, + &stream_label, + StreamViewType::NewAndOldImages, + index, + ) + .unwrap(); engine .validate_shard(&key_info.account_id, &arn, &shard_id) @@ -517,7 +541,12 @@ async fn test_get_stream_records_returns_written_records() { .unwrap(); } - let shard_id = shard_for("pk-read", &key_info.table_id); + let shard_id = stream_shard_for( + "pk-read", + &key_info.table_id, + &stream_label, + StreamViewType::NewAndOldImages, + ); let (records, last_seq) = engine .get_stream_records(&key_info.account_id, &arn, &shard_id, None, 10) .await @@ -564,7 +593,12 @@ async fn test_get_stream_records_after_sequence_paginates() { .unwrap(); } - let shard_id = shard_for("pk-page", &key_info.table_id); + let shard_id = stream_shard_for( + "pk-page", + &key_info.table_id, + &stream_label, + StreamViewType::NewAndOldImages, + ); let (first_page, last_seq) = engine .get_stream_records(&key_info.account_id, &arn, &shard_id, None, 2) .await diff --git a/crates/storage-cassandra/tests/direct_integration.rs b/crates/storage-cassandra/tests/direct_integration.rs index 863bbda69..7942a31af 100644 --- a/crates/storage-cassandra/tests/direct_integration.rs +++ b/crates/storage-cassandra/tests/direct_integration.rs @@ -53,6 +53,10 @@ mod scan; mod settings_store; #[path = "direct/snapshot_engine.rs"] mod snapshot_engine; +#[path = "direct/stream_disable.rs"] +mod stream_disable; +#[path = "direct/stream_endings.rs"] +mod stream_endings; #[path = "direct/stream_ownership.rs"] mod stream_ownership; #[path = "direct/streams.rs"] diff --git a/crates/storage-mongodb/migrations/data/0001_baseline.expected.json b/crates/storage-mongodb/migrations/data/0001_baseline.expected.json index e4c708147..a8cf1c1ff 100644 --- a/crates/storage-mongodb/migrations/data/0001_baseline.expected.json +++ b/crates/storage-mongodb/migrations/data/0001_baseline.expected.json @@ -102,6 +102,19 @@ "options": { "unique": true } + }, + "table_id_1_shard_id_1": { + "key": [ + [ + "table_id", + 1 + ], + [ + "shard_id", + 1 + ] + ], + "options": {} } } } diff --git a/crates/storage-mongodb/src/data_engine.rs b/crates/storage-mongodb/src/data_engine.rs index be8d9de03..10aca1d2b 100644 --- a/crates/storage-mongodb/src/data_engine.rs +++ b/crates/storage-mongodb/src/data_engine.rs @@ -2107,14 +2107,17 @@ impl MongoEngine { // ordering safe under contention — see // stream_engine::next_sequence_number_in_session for the full // rationale. - let shard_id = self + let Some(shard_id) = self .assign_shard_in_session( &key_info.account_id, &key_info.table_name, &pk_str, &mut *session, ) - .await?; + .await? + else { + return Ok(()); + }; let seq = self .next_sequence_number_in_session(&shard_id, &mut *session) .await?; diff --git a/crates/storage-mongodb/src/lib.rs b/crates/storage-mongodb/src/lib.rs index 14c735a72..50881030d 100644 --- a/crates/storage-mongodb/src/lib.rs +++ b/crates/storage-mongodb/src/lib.rs @@ -25,6 +25,7 @@ mod password_hash_store; pub mod pushdown; mod snapshot_engine; mod stream_engine; +mod stream_lifecycle; mod table_engine; mod ttl_worker; mod worker_store; diff --git a/crates/storage-mongodb/src/migrations/m0001_baseline.rs b/crates/storage-mongodb/src/migrations/m0001_baseline.rs index 3ff88aab7..30eb325d3 100644 --- a/crates/storage-mongodb/src/migrations/m0001_baseline.rs +++ b/crates/storage-mongodb/src/migrations/m0001_baseline.rs @@ -13,7 +13,9 @@ //! after the post-check: a body or post-check that fails leaves the row, so //! the 0.1.14 server still serves the catalog. The data body creates the //! data database's collections and indexes, which before 1.0 only `init` -//! created. Every request is safe to +//! created, and the `{table_id, shard_id}` index on `stream_shards`, which +//! no release before 1.0 created: a released deployment gets it from the +//! run that adopts it. Every request is safe to //! send again: creating a collection that exists with the same options //! succeeds, one that exists with other options fails with NamespaceExists //! (code 48), which is accepted by its code and left to the post-check, and @@ -112,7 +114,9 @@ fn catalog_indexes() -> Vec { ] } -/// The data indexes, as every released `init` created them. +/// The data indexes: those every released `init` created, and the +/// `{table_id, shard_id}` index on `stream_shards`, which no release before +/// 1.0 created. fn data_indexes() -> Vec { vec![ // 540 s, so a token is kept for at most 10 minutes with the TTL @@ -122,6 +126,13 @@ fn data_indexes() -> Vec { unique("idempotency_tokens", doc! { "account_id": 1, "token": 1 }), // Two stream enables cannot insert the same shard. unique("stream_shards", doc! { "shard_id": 1 }), + // A table's shards in shard id order. Every write to a streamed table + // selects the table's open shards by `table_id` and sorts them by + // `shard_id`, and DescribeStream, GetShardIterator and ListStreams read + // a table's shard documents the same way. Without it each of those + // reads scans the whole collection, which holds the shards of every + // table of every account, and each enable adds four documents to it. + plain("stream_shards", doc! { "table_id": 1, "shard_id": 1 }), // DynamoDB's 24-hour stream retention. expiring("stream_records", doc! { "created_at": 1 }, Duration::from_secs(24 * 3600)), plain("stream_records", doc! { "shard_id": 1, "sequence_number": 1 }), diff --git a/crates/storage-mongodb/src/migrations/postcheck.rs b/crates/storage-mongodb/src/migrations/postcheck.rs index 70ff7746b..8e9785909 100644 --- a/crates/storage-mongodb/src/migrations/postcheck.rs +++ b/crates/storage-mongodb/src/migrations/postcheck.rs @@ -343,5 +343,17 @@ mod tests { ); let unique = &data["idempotency_tokens"].indexes["account_id_1_token_1"]; assert_eq!(unique.options.get("unique"), Some(&json!(true))); + // A table's stream shards in shard id order. No release before 1.0 + // created this index, so migration 1 adds it to a released data + // database before the check. + let by_table = &data["stream_shards"].indexes["table_id_1_shard_id_1"]; + assert_eq!( + by_table.key, + [ + ("table_id".to_owned(), json!(1)), + ("shard_id".to_owned(), json!(1)) + ] + ); + assert!(by_table.options.is_empty()); } } diff --git a/crates/storage-mongodb/src/stream_engine.rs b/crates/storage-mongodb/src/stream_engine.rs index 097b3bb2a..4d5d7b826 100644 --- a/crates/storage-mongodb/src/stream_engine.rs +++ b/crates/storage-mongodb/src/stream_engine.rs @@ -13,22 +13,18 @@ use futures::TryStreamExt; use futures::future::BoxFuture; use mongodb::bson::DateTime as BsonDateTime; -use mongodb::bson::{self, Document, doc}; +use mongodb::bson::{Document, doc}; use mongodb::options::FindOptions; -use extenddb_core::types::{ - DescribeStreamInput, SequenceNumberRange, Shard, StreamDescription, StreamEventName, - StreamRecord, StreamStatus, StreamSummary, StreamViewType, -}; +use extenddb_core::types::{DescribeStreamInput, StreamDescription, StreamEventName, StreamRecord}; use extenddb_storage::StreamEngine; use extenddb_storage::error::StorageError; +use extenddb_storage::stream_lineage; use extenddb_storage::util::{parse_stream_arn, shard_in_stream, stream_arn}; use extenddb_storage::{StreamListResult, StreamRecordsResult}; use crate::MongoEngine; -const SHARDS_PER_STREAM: u32 = 4; - /// Map a `StreamEventName` to its DynamoDB wire-format string. /// /// DynamoDB Streams records use uppercase event names (`INSERT`, `MODIFY`, `REMOVE`). @@ -41,69 +37,7 @@ pub(crate) fn event_name_ddb_str(name: StreamEventName) -> &'static str { } } -/// Build the mongo `stream_shards.shard_id` for a given table_id + shard index. -/// -/// **Security invariant (RFC-0003 §5.3, §8.2):** shard_id must incorporate -/// the table's globally-unique `table_id` (a UUID), not the caller-visible -/// `table_name`. Table names are only unique per-account, so two accounts -/// creating "orders" tables would generate colliding shard_ids under a -/// name-derived scheme, letting one account's `GetRecords(shard_id)` read -/// the other's stream records. `table_id` is a per-table-instance UUID that -/// resets on `DeleteTable + CreateTable` — the recreated table gets fresh -/// shard_ids, so leftover stream records from the deleted table never -/// resurface either. -/// -/// UUIDs are not guessable, so an attacker cannot synthesize a shard_id -/// belonging to another tenant without first observing it (which itself -/// requires an authenticated path scoped to that tenant's account). -pub(crate) fn build_shard_id(table_id: &str, shard_index: u32) -> String { - format!("shardId-{table_id}-{shard_index:012}") -} - impl MongoEngine { - /// Initialize stream shards for a table. Only creates shard documents; - /// the caller is responsible for setting `stream_label` on the table doc. - /// - /// Uses the table's UUID (`table_id`), not `table_name`, in the shard_id - /// — see `build_shard_id` for the security rationale. - pub(crate) async fn init_stream_shards(&self, table_id: &str) -> Result<(), StorageError> { - let shards_coll = self.data_db.collection::("stream_shards"); - for i in 0..SHARDS_PER_STREAM { - let shard_id = build_shard_id(table_id, i); - let start_seq = format!("{:021}", 0); - let insert_res = shards_coll - .insert_one(doc! { - "shard_id": &shard_id, - "table_id": table_id, - "starting_sequence_number": &start_seq, - "created_at": BsonDateTime::now(), - }) - .await; - if let Err(e) = insert_res { - // Treat a duplicate `shard_id` as idempotent no-op: two - // concurrent `UpdateTable(stream_enabled=true)` calls - // can both observe "no shards yet" under snapshot - // isolation and both reach this insert; the unique - // index on `stream_shards.shard_id` (`bootstrapper.rs`) - // means one of them lands E11000. RFC-0003 §10.3 - // requires repeated `UpdateTable` calls with the same - // specification to not corrupt state — swallowing - // this E11000 makes the redundant call a no-op rather - // than a wire-visible 500. - let is_dup = matches!( - *e.kind, - mongodb::error::ErrorKind::Write(mongodb::error::WriteFailure::WriteError( - mongodb::error::WriteError { code: 11000, .. } - )) - ); - if !is_dup { - return Err(StorageError::Internal(e.to_string())); - } - } - } - Ok(()) - } - /// Draw the next sequence number for a shard *inside* the given /// transaction session. /// @@ -153,13 +87,18 @@ impl MongoEngine { /// *inside* the given transaction session. Pairs with /// `next_sequence_number_in_session` so the shard set the write is /// routed to is read at the same snapshot as the sequence draw. + /// + /// Only the open shards count: they belong to the table's enabled + /// stream. `None` when there are none, because the stream was disabled + /// after the caller read the table's key info; the write then records + /// nothing. pub(crate) async fn assign_shard_in_session( &self, account_id: &str, table_name: &str, partition_key: &str, session: &mut mongodb::ClientSession, - ) -> Result { + ) -> Result, StorageError> { let tables_coll = self.catalog_db.collection::("tables"); let table_doc = tables_coll .find_one(doc! { "_id": { "account_id": account_id, "table_name": table_name } }) @@ -172,7 +111,9 @@ impl MongoEngine { let shards_coll = self.data_db.collection::("stream_shards"); let opts = FindOptions::builder().sort(doc! { "shard_id": 1 }).build(); let mut cursor = shards_coll - .find(doc! { "table_id": table_id }) + .find( + doc! { "table_id": table_id, "ending_sequence_number": mongodb::bson::Bson::Null }, + ) .with_options(opts) .session(&mut *session) .await @@ -183,21 +124,18 @@ impl MongoEngine { .await .map_err(|e| StorageError::Internal(e.to_string()))?; - if shard_docs.is_empty() { - return Err(StorageError::Internal(format!( - "No stream shards for table {table_name}" - ))); - } - let shard_ids: Vec<&str> = shard_docs .iter() .filter_map(|d| d.get_str("shard_id").ok()) .collect(); + if shard_ids.is_empty() { + return Ok(None); + } let hash = crc32fast::hash(partition_key.as_bytes()); #[allow(clippy::cast_possible_truncation)] let idx = (hash as usize) % shard_ids.len(); - Ok(shard_ids[idx].to_owned()) + Ok(Some(shard_ids[idx].to_owned())) } /// Delete every stream_shards document for a given table_id, and every @@ -310,7 +248,9 @@ impl StreamEngine for MongoEngine { // shard that is not that table's (another table, or another // account) returns no document. Whether the shard belongs to the // stream the ARN's label names is decided in Rust from the table's - // label and the shard id (`shard_in_stream`). + // label and the shard id (`shard_in_stream`), with a third round + // trip for a legacy id only: whether the table has a labelled + // shard. let table_doc = self .catalog_db .collection::("tables") @@ -370,7 +310,11 @@ impl StreamEngine for MongoEngine { let Some(row_shard_id) = docs.first().and_then(|d| d.get_str("shard_id").ok()) else { return Err(extenddb_storage::util::stream_does_not_exist()); }; - if !shard_in_stream(&table_id, row_shard_id, table_label, &stream_label) { + if !shard_in_stream(&table_id, row_shard_id, table_label, &stream_label, || { + self.table_has_labelled_shard(&table_id) + }) + .await? + { return Err(extenddb_storage::util::stream_does_not_exist()); } @@ -392,114 +336,26 @@ impl StreamEngine for MongoEngine { let exclusive_start_shard_id = input.exclusive_start_shard_id.clone(); Box::pin(async move { let (table_name, stream_label) = parse_stream_arn(&stream_arn_val)?; - - let tables_coll = self.catalog_db.collection::("tables"); - let table_doc = tables_coll - .find_one(doc! { - "_id": { "account_id": &account_id, "table_name": &table_name }, - "stream_label": &stream_label, - }) - .await - .map_err(|e| StorageError::Internal(e.to_string()))? + let (table, table_doc) = self + .stream_table_row(&account_id, &table_name) + .await? .ok_or_else(|| extenddb_storage::util::stream_not_found(&stream_arn_val))?; - let key_schema = table_doc .get("key_schema") - .and_then(|b| bson::from_bson(b.clone()).ok()) + .and_then(|b| mongodb::bson::from_bson(b.clone()).ok()) .ok_or_else(|| StorageError::Internal("Missing key_schema".to_owned()))?; - - let stream_view_type = table_doc - .get("stream_specification") - .and_then(|b| { - let json: serde_json::Value = bson::from_bson(b.clone()).ok()?; - json.get("StreamViewType") - .and_then(|sv| serde_json::from_value::(sv.clone()).ok()) - }) - .unwrap_or(StreamViewType::KeysOnly); - - let table_status = table_doc.get_str("table_status").unwrap_or("ACTIVE"); - let table_id = table_doc.get_str("table_id").unwrap_or_default(); - - let limit = limit.unwrap_or(100); - let shards_coll = self.data_db.collection::("stream_shards"); - - let filter = if let Some(ref start) = exclusive_start_shard_id { - doc! { - "table_id": table_id, - "shard_id": { "$gt": start }, - } - } else { - doc! { "table_id": table_id } - }; - - let opts = FindOptions::builder() - .sort(doc! { "shard_id": 1 }) - .limit(limit + 1) - .build(); - - let cursor = shards_coll - .find(filter) - .with_options(opts) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - - let shard_docs: Vec = cursor - .try_collect() - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - - #[allow(clippy::cast_sign_loss)] - let limit_usize = limit as usize; - let last_shard = if shard_docs.len() > limit_usize { - shard_docs.get(limit_usize - 1).and_then(|d| { - d.get_str("shard_id") - .ok() - .map(std::borrow::ToOwned::to_owned) - }) - } else { - None - }; - - let shards: Vec = shard_docs - .into_iter() - .take(limit_usize) - .filter_map(|d| { - Some(Shard { - shard_id: d.get_str("shard_id").ok()?.to_owned(), - parent_shard_id: d - .get_str("parent_shard_id") - .ok() - .map(std::borrow::ToOwned::to_owned), - sequence_number_range: SequenceNumberRange { - starting_sequence_number: d - .get_str("starting_sequence_number") - .ok()? - .to_owned(), - ending_sequence_number: d - .get_str("ending_sequence_number") - .ok() - .map(std::borrow::ToOwned::to_owned), - }, - }) - }) - .collect(); - - let stream_status = if table_status == "DELETING" { - StreamStatus::Disabling - } else { - StreamStatus::Enabled - }; - - Ok(StreamDescription { - stream_arn: stream_arn_val, - stream_label, - stream_status, - stream_view_type, - table_name, + let shards = self + .stream_shard_rows(std::slice::from_ref(&table.table_id)) + .await?; + stream_lineage::describe_stream( + stream_arn_val, + &table, key_schema, - shards, - last_evaluated_shard_id: last_shard, - }) + stream_label, + &shards, + limit, + exclusive_start_shard_id.as_deref(), + ) }) } @@ -515,73 +371,21 @@ impl StreamEngine for MongoEngine { let exclusive_start_stream_arn = exclusive_start_stream_arn.map(std::borrow::ToOwned::to_owned); Box::pin(async move { - let tables_coll = self.catalog_db.collection::("tables"); - - let mut filter = doc! { - "_id.account_id": &account_id, - "stream_label": { "$ne": null }, - }; - - if let Some(ref tn) = table_name { - filter.insert("_id.table_name", tn.as_str()); - } - - if let Some(ref start_arn) = exclusive_start_stream_arn { - let (start_table, start_label) = parse_stream_arn(start_arn)?; - if table_name.is_some() { - filter.insert("stream_label", doc! { "$gt": &start_label }); - } else { - filter.insert( - "$or", - bson::bson!([ - { "_id.table_name": { "$gt": &start_table } }, - { "_id.table_name": &start_table, "stream_label": { "$gt": &start_label } } - ]), - ); - } - } - - let opts = FindOptions::builder() - .sort(doc! { "_id.table_name": 1, "stream_label": 1 }) - .limit(limit + 1) - .build(); - - let cursor = tables_coll - .find(filter) - .with_options(opts) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - - let docs: Vec = cursor - .try_collect() - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - - #[allow(clippy::cast_sign_loss)] - let limit_usize = limit as usize; - - let summaries: Vec = docs - .iter() - .take(limit_usize) - .filter_map(|d| { - let id = d.get_document("_id").ok()?; - let tn = id.get_str("table_name").ok()?; - let label = d.get_str("stream_label").ok()?; - Some(StreamSummary { - stream_arn: stream_arn(&self.region, &account_id, tn, label), - stream_label: label.to_owned(), - table_name: tn.to_owned(), - }) - }) - .collect(); - - let last_arn = if docs.len() > limit_usize { - summaries.last().map(|s| s.stream_arn.clone()) - } else { - None - }; - - Ok((summaries, last_arn)) + let start = exclusive_start_stream_arn + .as_deref() + .map(parse_stream_arn) + .transpose()?; + let tables = self.stream_table_rows(&account_id).await?; + let ids: Vec = tables.iter().map(|t| t.table_id.clone()).collect(); + let shards = self.stream_shard_rows(&ids).await?; + stream_lineage::list_streams( + &tables, + &shards, + table_name.as_deref(), + limit, + start, + |t, l| stream_arn(&self.region, &account_id, t, l), + ) }) } @@ -622,8 +426,9 @@ impl StreamEngine for MongoEngine { let shards_coll = self.data_db.collection::("stream_shards"); let opts = FindOptions::builder().sort(doc! { "shard_id": 1 }).build(); + // Only the open shards: they belong to the table's enabled stream. let cursor = shards_coll - .find(doc! { "table_id": table_id }) + .find(doc! { "table_id": table_id, "ending_sequence_number": mongodb::bson::Bson::Null }) .with_options(opts) .await .map_err(|e| StorageError::Internal(e.to_string()))?; @@ -695,32 +500,20 @@ impl StreamEngine for MongoEngine { let shard_id = shard_id.to_owned(); Box::pin(async move { let (table_name, stream_label) = parse_stream_arn(&stream_arn_val)?; - - let tables_coll = self.catalog_db.collection::("tables"); - let table_doc = tables_coll - .find_one(doc! { - "_id": { "account_id": &account_id, "table_name": &table_name }, - "stream_label": &stream_label, - }) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - - let Some(table_doc) = table_doc else { - return Err(extenddb_storage::util::stream_does_not_exist()); + let table = self.stream_table_row(&account_id, &table_name).await?; + let shards = match &table { + Some((t, _)) => { + self.stream_shard_rows(std::slice::from_ref(&t.table_id)) + .await? + } + None => Vec::new(), }; - - let table_id = table_doc.get_str("table_id").unwrap_or_default(); - - let shards_coll = self.data_db.collection::("stream_shards"); - let exists = shards_coll - .find_one(doc! { "shard_id": &shard_id, "table_id": table_id }) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - - if exists.is_none() { - return Err(extenddb_storage::util::shard_not_in_stream(&stream_arn_val)); - } - Ok(()) + stream_lineage::validate_shard( + table.as_ref().map(|(t, _)| t), + &stream_label, + &shards, + &shard_id, + ) }) } @@ -764,16 +557,28 @@ mod tests { assert_eq!(event_name_ddb_str(StreamEventName::Remove), "REMOVE"); } + const LABEL: &str = "2026-10-24T08:54:14.037"; + + fn shard(table_id: &str, index: u32) -> String { + stream_lineage::shard_id( + table_id, + LABEL, + extenddb_core::types::StreamViewType::KeysOnly, + index, + ) + .unwrap() + } + #[test] fn shard_id_embeds_table_id_not_table_name() { let table_id = "550e8400-e29b-41d4-a716-446655440000"; assert_eq!( - build_shard_id(table_id, 0), - "shardId-550e8400-e29b-41d4-a716-446655440000-000000000000" + shard(table_id, 0), + "shardId-550e8400-e29b-41d4-a716-446655440000-1792832054037100" ); assert_eq!( - build_shard_id(table_id, 3), - "shardId-550e8400-e29b-41d4-a716-446655440000-000000000003" + shard(table_id, 3), + "shardId-550e8400-e29b-41d4-a716-446655440000-1792832054037103" ); } @@ -785,25 +590,40 @@ mod tests { // shard. let table_a = "550e8400-e29b-41d4-a716-446655440000"; let table_b = "6ba7b810-9dad-11d1-80b4-00c04fd430c8"; - assert_ne!(build_shard_id(table_a, 0), build_shard_id(table_b, 0)); + assert_ne!(shard(table_a, 0), shard(table_b, 0)); } #[test] fn shard_id_format_is_stable_across_shards_of_same_table() { // Same table_id, different shard index → deterministic ordering. let table_id = "550e8400-e29b-41d4-a716-446655440000"; - let s0 = build_shard_id(table_id, 0); - let s1 = build_shard_id(table_id, 1); - let s2 = build_shard_id(table_id, 2); - let s3 = build_shard_id(table_id, 3); - assert!(s0 < s1); - assert!(s1 < s2); - assert!(s2 < s3); + let ids: Vec = (0..4).map(|i| shard(table_id, i)).collect(); + assert!(ids.windows(2).all(|w| w[0] < w[1])); } } /// The records in the result of the `get_stream_records` aggregation; see the /// module documentation. +impl MongoEngine { + /// Whether table `table_id` has a labelled shard document + /// (`stream_lineage::labelled_shard_id_pattern`, on the + /// `{table_id, shard_id}` index), which makes its legacy shards an + /// earlier, closed stream (`util::shard_in_stream`). + async fn table_has_labelled_shard(&self, table_id: &str) -> Result { + let found = self + .data_db + .collection::("stream_shards") + .find_one(doc! { + "table_id": table_id, + "shard_id": { "$regex": stream_lineage::labelled_shard_id_pattern(table_id) }, + }) + .projection(doc! { "_id": 0, "shard_id": 1 }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(found.is_some()) + } +} + #[path = "stream_record_docs.rs"] mod record_docs; diff --git a/crates/storage-mongodb/src/stream_lifecycle.rs b/crates/storage-mongodb/src/stream_lifecycle.rs new file mode 100644 index 000000000..272dc954e --- /dev/null +++ b/crates/storage-mongodb/src/stream_lifecycle.rs @@ -0,0 +1,370 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Stream lifecycle on MongoDB: creating a table's stream, closing it on +//! `UpdateTable StreamEnabled=false`, and reading the table and shard +//! documents the shared rules in `extenddb_storage::stream_lineage` group into +//! streams. +//! +//! The table document is in the catalog database and the shards and records +//! in the data database. `UpdateTable` sets the specification and label on the +//! table document first and changes the shards after that write +//! ([`PendingStream`]), the order the PostgreSQL backend uses for the same +//! two-database split. A failure between the two leaves the table document's +//! answer in force. A disable whose close failed leaves a disabled stream whose +//! shards are still open: a retried disable is refused and closes them, and an +//! enable closes them too, since an enable closes the open shards of every +//! earlier stream of the table. An enable whose shard insert failed leaves an +//! enabled stream with no shards, which records nothing and is replaced by a +//! disable and an enable. + +use futures::TryStreamExt; +use mongodb::bson::DateTime as BsonDateTime; +use mongodb::bson::{self, Bson, Document, doc}; +use mongodb::options::FindOptions; + +use extenddb_core::types::{StreamSpecification, StreamViewType}; +use extenddb_storage::error::StorageError; +use extenddb_storage::stream_lineage::{ + SHARDS_PER_STREAM, ShardRow, StreamChange, TableStreamRow, disabled_stream_to_close, + enable_still_current, next_stream_label, open_shard_ids_before, open_shard_ids_of, shard_id, + stored_specification, stream_change, +}; + +use crate::MongoEngine; + +fn internal(e: impl std::fmt::Display) -> StorageError { + StorageError::Internal(e.to_string()) +} + +fn spec_of(doc: &Document) -> Option { + doc.get("stream_specification") + .and_then(|b| bson::from_bson(b.clone()).ok()) +} + +fn table_row(table_name: &str, doc: &Document) -> TableStreamRow { + TableStreamRow { + table_name: table_name.to_owned(), + table_id: doc.get_str("table_id").unwrap_or_default().to_owned(), + table_status: doc.get_str("table_status").unwrap_or("ACTIVE").to_owned(), + stream_label: doc + .get_str("stream_label") + .ok() + .filter(|s| !s.is_empty()) + .map(str::to_owned), + stream_specification: spec_of(doc), + } +} + +/// The shard changes of an applied `UpdateTable` stream change: on a disable, +/// close the open shards of the stream the change read (`close`, the table's +/// label at that time); on an enable, close the open shards of every earlier +/// stream, then create the new stream's shards. +#[must_use] +pub(crate) struct PendingStream { + table_id: String, + close: Option, + open: Option<(String, StreamViewType)>, +} + +impl PendingStream { + /// Run the shard changes once the table document is written. + /// + /// The enable's shards are inserted after its table document is visible, + /// so a disable that reads the document in between passes its check, + /// writes the disabled specification and closes the shards it finds, which + /// may be none or some of the four. The enable re-reads the table's stream + /// fields after its inserts and, when the document no longer says enabled + /// with this label, closes the shards it inserted. A disable written + /// before the re-read is seen by it; one written after finds all four + /// shards open and closes them itself; both closing is idempotent through + /// the `ending_sequence_number: null` filter. A disable closes only the + /// stream it read, so one applying after an overlapping enable leaves the + /// new stream open. + pub(crate) async fn apply( + self, + engine: &MongoEngine, + account_id: &str, + table_name: &str, + ) -> Result<(), StorageError> { + let Some((label, view_type)) = self.open else { + if let Some(label) = &self.close { + engine.close_stream_shards(&self.table_id, label).await?; + } + return Ok(()); + }; + engine.close_streams_before(&self.table_id, &label).await?; + engine + .init_stream_shards(&self.table_id, &label, view_type) + .await?; + let row = engine.stream_table_row(account_id, table_name).await?; + if !enable_still_current(row.as_ref().map(|(t, _)| t), &label) { + engine.close_stream_shards(&self.table_id, &label).await?; + } + Ok(()) + } +} + +impl MongoEngine { + /// Insert the four open shards of stream `label`. The caller stores the + /// label on the table document. + pub(crate) async fn init_stream_shards( + &self, + table_id: &str, + label: &str, + view_type: StreamViewType, + ) -> Result<(), StorageError> { + let shards_coll = self.data_db.collection::("stream_shards"); + for i in 0..SHARDS_PER_STREAM { + let insert_res = shards_coll + .insert_one(doc! { + "shard_id": shard_id(table_id, label, view_type, i)?, + "table_id": table_id, + "starting_sequence_number": format!("{:021}", 0), + "created_at": BsonDateTime::now(), + }) + .await; + if let Err(e) = insert_res { + // A duplicate `shard_id` (E11000 on the unique index the + // bootstrapper creates) is a repeat of the same insert, from + // a retried request: keep the shard that is there. + let is_dup = matches!( + *e.kind, + mongodb::error::ErrorKind::Write(mongodb::error::WriteFailure::WriteError( + mongodb::error::WriteError { code: 11000, .. } + )) + ); + if !is_dup { + return Err(internal(e)); + } + } + } + Ok(()) + } + + /// Close every open shard of the table: its `EndingSequenceNumber` becomes + /// the sequence number of its last record, or its starting number when it + /// holds none (both measured 2026-10-24). + pub(crate) async fn close_stream_shards( + &self, + table_id: &str, + label: &str, + ) -> Result<(), StorageError> { + self.close_selected_shards(table_id, label, open_shard_ids_of) + .await + } + + /// Close the open shards of every stream of the table older than `label` + /// (`stream_lineage::open_shard_ids_before`), as [`Self::close_stream_shards`] + /// closes one stream's. + async fn close_streams_before(&self, table_id: &str, label: &str) -> Result<(), StorageError> { + self.close_selected_shards(table_id, label, open_shard_ids_before) + .await + } + + async fn close_selected_shards( + &self, + table_id: &str, + label: &str, + select: fn(&str, &str, &[ShardRow]) -> Vec, + ) -> Result<(), StorageError> { + let shards_coll = self.data_db.collection::("stream_shards"); + let records_coll = self.data_db.collection::("stream_records"); + let rows = self + .stream_shard_rows(std::slice::from_ref(&table_id.to_owned())) + .await?; + let ids = select(table_id, label, &rows); + if ids.is_empty() { + return Ok(()); + } + let open: Vec = shards_coll + .find(doc! { "shard_id": { "$in": &ids }, "ending_sequence_number": Bson::Null }) + .await + .map_err(internal)? + .try_collect() + .await + .map_err(internal)?; + for shard in open { + let id = shard.get_str("shard_id").map_err(internal)?; + let last: Option = records_coll + .find(doc! { "shard_id": id }) + .with_options( + FindOptions::builder() + .sort(doc! { "sequence_number": -1 }) + .limit(1) + .build(), + ) + .await + .map_err(internal)? + .try_next() + .await + .map_err(internal)?; + let ending = match last + .as_ref() + .and_then(|d| d.get_str("sequence_number").ok()) + { + Some(seq) => seq.to_owned(), + None => shard + .get_str("starting_sequence_number") + .map_err(internal)? + .to_owned(), + }; + shards_coll + .update_one( + doc! { "shard_id": id, "ending_sequence_number": Bson::Null }, + doc! { "$set": { "ending_sequence_number": ending } }, + ) + .await + .map_err(internal)?; + } + Ok(()) + } + + /// Decide an `UpdateTable` `StreamSpecification` against the table + /// document and add its catalog fields to `update_doc`. + /// + /// Disabling stores `{StreamEnabled: false}` with the stream's view type, + /// which DescribeTable omits, and keeps `stream_label`, so + /// `LatestStreamArn` keeps naming the disabled stream. Enabling stores the specification and + /// a new label. The returned shard changes run once `update_doc` is + /// written. This backend has no transaction around UpdateTable, so two + /// concurrent stream changes can both pass the check. + pub(crate) fn prepare_stream_update( + update_doc: &mut Document, + table_doc: &Document, + table_name: &str, + requested: &StreamSpecification, + ) -> Result { + let table = table_row(table_name, table_doc); + let change = stream_change(table_name, table.stream_specification.as_ref(), requested)?; + update_doc.insert( + "stream_specification", + bson::to_bson(&stored_specification( + change, + table.stream_specification.as_ref(), + requested, + )) + .map_err(internal)?, + ); + let open = match change { + StreamChange::Disable => None, + StreamChange::Enable(view_type) => { + let label = next_stream_label(table.stream_label.as_deref()); + update_doc.insert("stream_label", &label); + Some((label, view_type)) + } + }; + Ok(PendingStream { + table_id: table.table_id, + close: table.stream_label, + open, + }) + } + + /// [`Self::prepare_stream_update`], and for a disable refused because the + /// stored specification is already disabled, the close of the current + /// label's open shards first (left by a release that did not close them, + /// or by a close that failed after the catalog write); the refusal is + /// still the answer. + pub(crate) async fn prepare_stream_update_or_close( + &self, + update_doc: &mut Document, + table_doc: &Document, + table_name: &str, + requested: &StreamSpecification, + ) -> Result { + match Self::prepare_stream_update(update_doc, table_doc, table_name, requested) { + Ok(pending) => Ok(pending), + Err(refusal) => { + let table = table_row(table_name, table_doc); + if let Some(label) = disabled_stream_to_close( + table.stream_specification.as_ref(), + table.stream_label.as_deref(), + requested, + ) { + self.close_stream_shards(&table.table_id, label).await?; + } + Err(refusal) + } + } + } + + /// The table's stream fields and its document, by name. + pub(crate) async fn stream_table_row( + &self, + account_id: &str, + table_name: &str, + ) -> Result, StorageError> { + let doc = self + .catalog_db + .collection::("tables") + .find_one(doc! { "_id": { "account_id": account_id, "table_name": table_name } }) + .await + .map_err(internal)?; + Ok(doc.map(|d| (table_row(table_name, &d), d))) + } + + /// Every table of the account that has had a stream. + pub(crate) async fn stream_table_rows( + &self, + account_id: &str, + ) -> Result, StorageError> { + let docs: Vec = self + .catalog_db + .collection::("tables") + .find(doc! { "_id.account_id": account_id, "stream_label": { "$ne": Bson::Null } }) + .await + .map_err(internal)? + .try_collect() + .await + .map_err(internal)?; + Ok(docs + .iter() + .filter_map(|d| { + let name = d.get_document("_id").ok()?.get_str("table_name").ok()?; + Some(table_row(name, d)) + }) + .collect()) + } + + /// The shard documents of the given tables. + pub(crate) async fn stream_shard_rows( + &self, + table_ids: &[String], + ) -> Result, StorageError> { + if table_ids.is_empty() { + return Ok(Vec::new()); + } + let docs: Vec = self + .data_db + .collection::("stream_shards") + .find(doc! { "table_id": { "$in": table_ids } }) + .await + .map_err(internal)? + .try_collect() + .await + .map_err(internal)?; + Ok(docs + .iter() + .filter_map(|d| { + Some(ShardRow { + table_id: d.get_str("table_id").ok()?.to_owned(), + shard_id: d.get_str("shard_id").ok()?.to_owned(), + parent_shard_id: d.get_str("parent_shard_id").ok().map(str::to_owned), + starting_sequence_number: d + .get_str("starting_sequence_number") + .ok()? + .to_owned(), + ending_sequence_number: d + .get_str("ending_sequence_number") + .ok() + .map(str::to_owned), + }) + }) + .collect()) + } +} + +#[cfg(test)] +#[path = "stream_live_tests.rs"] +mod live_tests; diff --git a/crates/storage-mongodb/src/stream_live_tests.rs b/crates/storage-mongodb/src/stream_live_tests.rs new file mode 100644 index 000000000..67c9ff6da --- /dev/null +++ b/crates/storage-mongodb/src/stream_live_tests.rs @@ -0,0 +1,266 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Live MongoDB tests for the stream lifecycle: a disable closes the stream, +//! an enable creates a new one, and of two overlapping stream changes exactly +//! one is applied. +//! +//! They need a replica set and read its URI from `EXTENDDB_TEST_MONGODB_URI` +//! (for example `mongodb://127.0.0.1:27482/?directConnection=true`). They are +//! `#[ignore]`d so `cargo test --lib` passes without one; run them with +//! `cargo test -p extenddb-storage-mongodb --lib -- --ignored live_stream`. +//! Each test uses its own catalog and data databases and drops them. + +use std::sync::Arc; + +use mongodb::bson::{Document, doc}; + +use extenddb_core::types::{ + AttributeDefinition, BillingMode, CreateTableInput, DescribeStreamInput, DescribeTableInput, + KeySchemaElement, KeyType, ScalarAttributeType, StreamSpecification, StreamStatus, + StreamViewType, TableDescription, +}; +use extenddb_storage::error::StorageError; +use extenddb_storage::{StreamEngine, TableEngine}; + +use crate::MongoEngine; + +const ACCOUNT: &str = "123456789012"; + +fn uri() -> String { + std::env::var("EXTENDDB_TEST_MONGODB_URI").expect( + "set EXTENDDB_TEST_MONGODB_URI to a MongoDB replica set, e.g. \ + mongodb://127.0.0.1:27482/?directConnection=true", + ) +} + +async fn engine() -> MongoEngine { + let tag = uuid::Uuid::new_v4().simple().to_string(); + let client = mongodb::Client::with_uri_str(uri()).await.expect("client"); + let engine = MongoEngine { + client: client.clone(), + catalog_db: client.database(&format!("s{}_cat", &tag[..12])), + data_db: client.database(&format!("s{}_data", &tag[..12])), + region: "us-east-1".to_owned(), + gsi_cache: Arc::default(), + tx_read_concern: mongodb::options::ReadConcern::snapshot(), + backup_liveness: crate::backup_engine::BackupLiveness::default(), + }; + engine + .catalog_db + .collection::("settings") + .insert_one(doc! { "_id": "control_plane_delay_seconds", "value": "0" }) + .await + .expect("settings"); + engine +} + +async fn finish(engine: MongoEngine) { + engine.catalog_db.drop().await.expect("drop catalog"); + engine.data_db.drop().await.expect("drop data"); +} + +async fn stream_table(engine: &MongoEngine, name: &str) -> TableDescription { + engine + .create_table_impl( + ACCOUNT, + CreateTableInput { + table_name: name.to_owned(), + key_schema: vec![KeySchemaElement { + attribute_name: "pk".to_owned(), + key_type: KeyType::Hash, + }], + attribute_definitions: vec![AttributeDefinition { + attribute_name: "pk".to_owned(), + attribute_type: ScalarAttributeType::S, + }], + billing_mode: Some(BillingMode::PayPerRequest), + stream_specification: Some(StreamSpecification { + stream_enabled: true, + stream_view_type: Some(StreamViewType::KeysOnly), + }), + ..Default::default() + }, + false, + ) + .await + .expect("create") +} + +fn set_stream(name: &str, spec: serde_json::Value) -> extenddb_core::types::UpdateTableInput { + serde_json::from_value(serde_json::json!({"TableName": name, "StreamSpecification": spec})) + .expect("input") +} + +async fn describe_stream( + engine: &MongoEngine, + arn: &str, +) -> extenddb_core::types::StreamDescription { + engine + .describe_stream( + ACCOUNT, + &DescribeStreamInput { + stream_arn: arn.to_owned(), + limit: None, + exclusive_start_shard_id: None, + }, + ) + .await + .expect("describe stream") +} + +/// Of two stream changes fired at the same instant, one is applied and the +/// other refused; the table ends with one stream enabled and four open +/// shards, and every other shard closed. +async fn overlapping_pair( + engine: &MongoEngine, + name: &str, + a: serde_json::Value, + b: serde_json::Value, +) -> (usize, usize) { + let (ra, rb) = tokio::join!( + engine.update_table(ACCOUNT, set_stream(name, a)), + engine.update_table(ACCOUNT, set_stream(name, b)), + ); + let ok = [&ra, &rb].iter().filter(|r| r.is_ok()).count(); + let refused = [&ra, &rb] + .iter() + .filter(|r| { + matches!( + r, + Err(StorageError::Validation(_)) | Err(StorageError::TableNotActive(_)) + ) + }) + .count(); + for r in [&ra, &rb] { + if let Err(e) = r { + assert!( + matches!( + e, + StorageError::Validation(_) | StorageError::TableNotActive(_) + ), + "{e:?}" + ); + } + } + (ok, refused) +} + +async fn audit(engine: &MongoEngine, name: &str) { + let table = engine + .describe_table( + ACCOUNT, + DescribeTableInput { + table_name: name.to_owned(), + }, + ) + .await + .expect("describe table"); + let (listed, _) = engine + .list_streams(ACCOUNT, Some(name), 100, None) + .await + .expect("list"); + let mut enabled = 0; + for s in &listed { + let d = describe_stream(engine, &s.stream_arn).await; + let open = d + .shards + .iter() + .filter(|x| x.sequence_number_range.ending_sequence_number.is_none()) + .count(); + match d.stream_status { + StreamStatus::Enabled => { + enabled += 1; + assert_eq!(d.shards.len(), 4, "{}", s.stream_arn); + assert_eq!(open, 4, "an enabled stream has every shard open"); + assert_eq!( + table.latest_stream_arn.as_deref(), + Some(s.stream_arn.as_str()) + ); + } + _ => assert_eq!(open, 0, "a disabled stream has every shard closed"), + } + } + let spec_enabled = table + .stream_specification + .as_ref() + .is_some_and(|s| s.stream_enabled); + assert_eq!(enabled, usize::from(spec_enabled)); +} + +#[tokio::test] +#[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] +async fn live_stream_overlapping_changes_apply_exactly_one() { + let engine = engine().await; + let name = "overlap"; + stream_table(&engine, name).await; + let disable = serde_json::json!({"StreamEnabled": false}); + let enable = serde_json::json!({"StreamEnabled": true, "StreamViewType": "NEW_IMAGE"}); + let mut applied = 0; + let mut refused = 0; + for round in 0..10 { + // Alternate the pairs so both orders and both mixes are exercised. + let (a, b) = match round % 3 { + 0 => (disable.clone(), disable.clone()), + 1 => (enable.clone(), disable.clone()), + _ => (disable.clone(), enable.clone()), + }; + let (ok, no) = overlapping_pair(&engine, name, a, b).await; + assert_eq!( + (ok, no), + (1, 1), + "round {round}: exactly one change applies" + ); + applied += ok; + refused += no; + audit(&engine, name).await; + // Bring the table back to enabled for the next round. + let _ = engine + .update_table(ACCOUNT, set_stream(name, enable.clone())) + .await; + audit(&engine, name).await; + } + assert_eq!((applied, refused), (10, 10)); + finish(engine).await; +} + +#[tokio::test] +#[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] +async fn live_stream_disable_closes_and_enable_replaces() { + let engine = engine().await; + let desc = stream_table(&engine, "lifecycle").await; + let old = desc.latest_stream_arn.expect("arn"); + engine + .update_table( + ACCOUNT, + set_stream("lifecycle", serde_json::json!({"StreamEnabled": false})), + ) + .await + .expect("disable"); + let closed = describe_stream(&engine, &old).await; + assert!(matches!(closed.stream_status, StreamStatus::Disabled)); + assert_eq!(closed.stream_view_type, StreamViewType::KeysOnly); + assert!( + closed + .shards + .iter() + .all(|x| x.sequence_number_range.ending_sequence_number.is_some()) + ); + let re = engine + .update_table( + ACCOUNT, + set_stream( + "lifecycle", + serde_json::json!({"StreamEnabled": true, "StreamViewType": "NEW_IMAGE"}), + ), + ) + .await + .expect("enable"); + let new = re.latest_stream_arn.expect("new arn"); + assert_ne!(new, old); + let d = describe_stream(&engine, &new).await; + assert!(matches!(d.stream_status, StreamStatus::Enabled)); + assert_eq!(d.stream_view_type, StreamViewType::NewImage); + audit(&engine, "lifecycle").await; + finish(engine).await; +} diff --git a/crates/storage-mongodb/src/stream_records_live_tests.rs b/crates/storage-mongodb/src/stream_records_live_tests.rs index ec5956c7a..8c346d088 100644 --- a/crates/storage-mongodb/src/stream_records_live_tests.rs +++ b/crates/storage-mongodb/src/stream_records_live_tests.rs @@ -34,6 +34,14 @@ fn uri() -> String { ) } +/// A shard id in the shape releases before stream labels were part of the id +/// wrote: shard `index` of table `table_id`. Such an id belongs to the +/// table's current stream, so these fixtures read as a table whose stream was +/// enabled by an earlier release. +fn legacy_shard_id(table_id: &str, index: u32) -> String { + format!("shardId-{table_id}-{index:012}") +} + struct Fixture { engine: MongoEngine, } @@ -73,7 +81,7 @@ impl Fixture { .data_db .collection::("stream_shards") .insert_one(doc! { - "shard_id": super::build_shard_id(&table_id, i), + "shard_id": legacy_shard_id(&table_id, i), "table_id": &table_id, "starting_sequence_number": format!("{:021}", 0), }) @@ -160,7 +168,7 @@ async fn records_live_a_page_over_16_mb_is_read_in_full() { let f = Fixture::new().await; let table_id = f.stream_table(ACCOUNT, "big", LABEL).await; let arn = stream_arn(REGION, ACCOUNT, "big", LABEL); - let shard_id = super::build_shard_id(&table_id, 0); + let shard_id = legacy_shard_id(&table_id, 0); // Inserted in descending order so the read's sort, not insertion order, // produces the sequence order the page is asserted in. let seqs: Vec = (1..=40).rev().collect(); @@ -205,7 +213,7 @@ async fn records_live_a_page_over_16_mb_is_read_in_full() { async fn records_live_an_earlier_label_on_the_same_shards_is_not_found() { let f = Fixture::new().await; let table_id = f.stream_table(ACCOUNT, "t", LABEL).await; - let shard_id = super::build_shard_id(&table_id, 0); + let shard_id = legacy_shard_id(&table_id, 0); f.write_records(&shard_id, &table_id, &[1], 10).await; let old_arn = stream_arn(REGION, ACCOUNT, "t", LABEL); f.engine @@ -239,8 +247,8 @@ async fn records_live_another_accounts_same_named_stream_is_not_found() { let f = Fixture::new().await; let own_id = f.stream_table(ACCOUNT, "t", LABEL).await; let other_id = f.stream_table(OTHER_ACCOUNT, "t", LABEL).await; - let own_shard = super::build_shard_id(&own_id, 0); - let other_shard = super::build_shard_id(&other_id, 0); + let own_shard = legacy_shard_id(&own_id, 0); + let other_shard = legacy_shard_id(&other_id, 0); f.write_records(&own_shard, &own_id, &[1], 10).await; f.write_records(&other_shard, &other_id, &[1], 10).await; @@ -281,9 +289,9 @@ async fn records_live_another_tables_shard_with_the_same_label_is_not_found() { let f = Fixture::new().await; let a_id = f.stream_table(ACCOUNT, "a", LABEL).await; let b_id = f.stream_table(ACCOUNT, "b", LABEL).await; - f.write_records(&super::build_shard_id(&a_id, 0), &a_id, &[1], 10) + f.write_records(&legacy_shard_id(&a_id, 0), &a_id, &[1], 10) .await; - f.write_records(&super::build_shard_id(&b_id, 0), &b_id, &[1, 2], 10) + f.write_records(&legacy_shard_id(&b_id, 0), &b_id, &[1, 2], 10) .await; let arn_a = stream_arn(REGION, ACCOUNT, "a", LABEL); let arn_b = stream_arn(REGION, ACCOUNT, "b", LABEL); @@ -292,12 +300,12 @@ async fn records_live_another_tables_shard_with_the_same_label_is_not_found() { // pick, one of them reads a shard that is not its own. assert_stream_not_found( f.engine - .get_stream_records(ACCOUNT, &arn_a, &super::build_shard_id(&b_id, 0), None, 10) + .get_stream_records(ACCOUNT, &arn_a, &legacy_shard_id(&b_id, 0), None, 10) .await, ); assert_stream_not_found( f.engine - .get_stream_records(ACCOUNT, &arn_b, &super::build_shard_id(&a_id, 0), None, 10) + .get_stream_records(ACCOUNT, &arn_b, &legacy_shard_id(&a_id, 0), None, 10) .await, ); assert_stream_not_found( @@ -307,7 +315,7 @@ async fn records_live_another_tables_shard_with_the_same_label_is_not_found() { ); let (records, _) = f .engine - .get_stream_records(ACCOUNT, &arn_a, &super::build_shard_id(&a_id, 0), None, 10) + .get_stream_records(ACCOUNT, &arn_a, &legacy_shard_id(&a_id, 0), None, 10) .await .expect("own shard reads"); assert_eq!(records.len(), 1); @@ -323,7 +331,7 @@ async fn records_live_a_record_document_without_record_data_is_an_internal_error let f = Fixture::new().await; let table_id = f.stream_table(ACCOUNT, "t", LABEL).await; let arn = stream_arn(REGION, ACCOUNT, "t", LABEL); - let shard_id = super::build_shard_id(&table_id, 0); + let shard_id = legacy_shard_id(&table_id, 0); f.write_records(&shard_id, &table_id, &[1], 10).await; f.engine .data_db @@ -365,8 +373,8 @@ async fn records_live_an_idle_shard_reads_an_empty_page() { let f = Fixture::new().await; let table_id = f.stream_table(ACCOUNT, "t", LABEL).await; let arn = stream_arn(REGION, ACCOUNT, "t", LABEL); - let written = super::build_shard_id(&table_id, 0); - let idle = super::build_shard_id(&table_id, 1); + let written = legacy_shard_id(&table_id, 0); + let idle = legacy_shard_id(&table_id, 1); f.write_records(&written, &table_id, &[1, 2], 10).await; let last_written = format!("{:021}", 2); diff --git a/crates/storage-mongodb/src/table_engine.rs b/crates/storage-mongodb/src/table_engine.rs index 32b61ef7d..33e7e20c5 100644 --- a/crates/storage-mongodb/src/table_engine.rs +++ b/crates/storage-mongodb/src/table_engine.rs @@ -155,6 +155,7 @@ impl MongoEngine { restore: Option<&mut crate::backup_engine::RestoreRegistration<'_>>, ) -> Result { Self::validate_account_id(account_id)?; + let input = extenddb_storage::stream_lineage::without_disabled_specification(input); let table_id = uuid::Uuid::new_v4().to_string(); let table_arn_val = table_arn(&self.region, account_id, &input.table_name); @@ -384,10 +385,16 @@ impl MongoEngine { } // Initialize stream shards if streaming is enabled. shard_id is - // derived from table_id (UUID), never table_name — see - // stream_engine::build_shard_id for the security rationale. - if stream_label_opt.is_some() { - self.init_stream_shards(&table_id).await?; + // derived from table_id (UUID), never table_name, and names the + // stream's label and view type (`extenddb_storage::stream_lineage`). + if let (Some(label), Some(view_type)) = ( + stream_label_opt.as_deref(), + input + .stream_specification + .as_ref() + .and_then(|s| s.stream_view_type), + ) { + self.init_stream_shards(&table_id, label, view_type).await?; } // Handle GSI creation @@ -877,68 +884,75 @@ impl MongoEngine { update_doc.insert("on_demand_throughput", odt_bson); } + // A stream change is decided on the specification and label the table + // document carries, and written only if they are still those values: + // this backend has no transaction around UpdateTable, so of two + // overlapping stream changes the one whose write finds the document + // changed re-reads it and is refused by `stream_change` as a request + // arriving after the other's commit would be. + let mut pending_stream = None; + let mut stream_guard = Document::new(); if let Some(ss) = &input.stream_specification { - let ss_bson = bson::to_bson(ss).map_err(|e| StorageError::Internal(e.to_string()))?; - update_doc.insert("stream_specification", ss_bson); - if ss.stream_enabled { - let table_doc = tables_coll - .find_one(doc! { "_id": { "account_id": account_id, "table_name": &input.table_name } }) - .await - .map_err(|e| StorageError::Internal(e.to_string()))? - .ok_or_else(|| StorageError::TableNotFound(input.table_name.clone()))?; - let table_id = table_doc - .get_str("table_id") - .map_err(|_| StorageError::Internal("missing table_id".to_string()))?; - - // Idempotent re-enable: if shards already exist for this - // table, reuse them and preserve the existing - // stream_label. Otherwise a repeat UpdateTable would - // insert duplicate shards (DescribeStream would then - // report N × k) and rotate stream_label, invalidating - // stream ARNs previously handed out to consumers. - let shards_coll = self.data_db.collection::("stream_shards"); - let existing_shard = shards_coll - .find_one(doc! { "table_id": table_id }) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - if existing_shard.is_none() { - let label = extenddb_storage::util::format_stream_label( - time::OffsetDateTime::now_utc(), - ); - update_doc.insert("stream_label", &label); - self.init_stream_shards(table_id).await?; - } else if table_doc - .get_str("stream_label") - .ok() - .filter(|s| !s.is_empty()) - .is_none() - { - // Shards exist but the label was cleared by a - // previous disable — restore a fresh label so the - // ARN resolves again. - let label = extenddb_storage::util::format_stream_label( - time::OffsetDateTime::now_utc(), - ); - update_doc.insert("stream_label", &label); - } + let table_doc = tables_coll + .find_one( + doc! { "_id": { "account_id": account_id, "table_name": &input.table_name } }, + ) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + .ok_or_else(|| StorageError::TableNotFound(input.table_name.clone()))?; + pending_stream = Some( + self.prepare_stream_update_or_close( + &mut update_doc, + &table_doc, + &input.table_name, + ss, + ) + .await?, + ); + for field in ["stream_specification", "stream_label"] { + stream_guard.insert( + field, + table_doc.get(field).cloned().unwrap_or(bson::Bson::Null), + ); } } if !update_doc.is_empty() { + let mut filter = active_table_filter(&table_filter); + filter.extend(stream_guard); let result = tables_coll - .update_one( - active_table_filter(&table_filter), - doc! { "$set": &update_doc }, - ) + .update_one(filter, doc! { "$set": &update_doc }) .await .map_err(|e| StorageError::Internal(e.to_string()))?; if result.matched_count == 0 { - // Deleted or left ACTIVE since the check above; report which. + // Deleted or left ACTIVE since the check above, or the stream + // fields changed under a stream change; report which. A stream + // change the re-read state would still accept (a disable that + // overlapped an enable) is reported as not ACTIVE, the nearest + // of ExtendDB's answers to the service's in-progress refusal. self.require_active_table(&tables_coll, &table_filter, &input.table_name) .await?; + if let Some(ss) = &input.stream_specification { + let table_doc = tables_coll + .find_one(doc! { + "_id": { "account_id": account_id, "table_name": &input.table_name } + }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + .ok_or_else(|| StorageError::TableNotFound(input.table_name.clone()))?; + let _would_apply = Self::prepare_stream_update( + &mut Document::new(), + &table_doc, + &input.table_name, + ss, + )?; + } return Err(StorageError::TableNotActive(input.table_name.clone())); } } + if let Some(pending) = pending_stream { + pending.apply(self, account_id, &input.table_name).await?; + } if clear_provisioned_throughput { let table_doc = tables_coll diff --git a/crates/storage-mongodb/tests/fixtures/golden/0001_baseline.txt b/crates/storage-mongodb/tests/fixtures/golden/0001_baseline.txt index 01656c16b..b357ba9f5 100644 --- a/crates/storage-mongodb/tests/fixtures/golden/0001_baseline.txt +++ b/crates/storage-mongodb/tests/fixtures/golden/0001_baseline.txt @@ -541,6 +541,19 @@ "options": { "unique": true } + }, + "table_id_1_shard_id_1": { + "key": [ + [ + "table_id", + 1 + ], + [ + "shard_id", + 1 + ] + ], + "options": {} } } } diff --git a/crates/storage-mongodb/tests/migrations_fresh.rs b/crates/storage-mongodb/tests/migrations_fresh.rs index 9b5270035..0d4bc911a 100644 --- a/crates/storage-mongodb/tests/migrations_fresh.rs +++ b/crates/storage-mongodb/tests/migrations_fresh.rs @@ -482,6 +482,66 @@ async fn a_second_run_applies_nothing_and_reconverge_keeps_the_rows() { d.drop_databases().await; } +/// A data database migrated by a build whose migration 1 did not create the +/// `{table_id, shard_id}` index on `stream_shards`: its ledgers record every +/// migration, so `migrate --yes` applies nothing, and `migrate --yes +/// --reconverge` runs migration 1's data step again, which creates the index +/// and leaves the ledger rows as they are. +#[tokio::test(flavor = "multi_thread")] +async fn reconverge_adds_the_stream_shards_table_index_to_a_migrated_data_database() { + let Some(uri) = server() else { return }; + let d = Deployment::new(&uri, "shardrerun").await; + d.create().await; + let shards = d.data().collection::("stream_shards"); + let names = || async { + shards + .list_index_names() + .await + .expect("stream_shards index names") + }; + assert!(names().await.iter().any(|n| n == "table_id_1_shard_id_1")); + shards + .drop_index("table_id_1_shard_id_1") + .await + .expect("drop the index, as a build without it left the store"); + let before = d.catalog_ledger().await; + assert_eq!( + d.status().await, + CatalogStatus::Current { + version: MigrationId::LATEST.number() + } + ); + let again = d.migrate().await.expect("migrate"); + assert!(again.applied.is_empty(), "{again:?}"); + assert!(!names().await.iter().any(|n| n == "table_id_1_shard_id_1")); + + let reconverged = d.reconverge().await.expect("reconverge"); + assert!( + reconverged + .applied + .iter() + .any(|step| step.id == 1 && step.store_name == d.target.data), + "{reconverged:?}" + ); + let indexes: Vec = { + use futures::TryStreamExt; + shards + .list_indexes() + .await + .expect("list the stream_shards indexes") + .map_ok(|m| m.keys) + .try_collect() + .await + .expect("read the stream_shards indexes") + }; + assert!( + indexes.contains(&doc! { "table_id": 1, "shard_id": 1 }), + "{indexes:?}" + ); + assert_eq!(d.catalog_ledger().await, before); + d.drop_databases().await; +} + #[tokio::test(flavor = "multi_thread")] async fn forget_clears_an_unreleased_id_and_migrate_applies_it_again() { let Some(uri) = server() else { return }; diff --git a/crates/storage-mongodb/tests/migrations_released.rs b/crates/storage-mongodb/tests/migrations_released.rs index d85f0342e..d031cfb31 100644 --- a/crates/storage-mongodb/tests/migrations_released.rs +++ b/crates/storage-mongodb/tests/migrations_released.rs @@ -222,6 +222,55 @@ async fn every_released_catalog_upgrades_to_the_definitions_of_a_fresh_install() } } +/// The indexes of the data database's `stream_shards`, by name, with their +/// keys. +async fn stream_shards_indexes(d: &Deployment) -> BTreeMap { + use futures::TryStreamExt; + let models: Vec = d + .data() + .collection::("stream_shards") + .list_indexes() + .await + .expect("list the stream_shards indexes") + .try_collect() + .await + .expect("read the stream_shards indexes"); + models + .into_iter() + .map(|m| (m.options.and_then(|o| o.name).unwrap_or_default(), m.keys)) + .collect() +} + +/// The `{table_id, shard_id}` index on `stream_shards`, which no release +/// before 1.0 created: every released data database has only the unique +/// `shard_id` index, and the `migrate --yes` that adopts it adds the index +/// with migration 1's data step. A second run applies nothing. +#[tokio::test(flavor = "multi_thread")] +async fn migrate_adds_the_stream_shards_table_index_to_every_released_data_database() { + let Some(uri) = server() else { return }; + for state in STATES { + let d = Deployment::new(&uri, "shardindex").await; + fixture::load(&d, &fixture::read(state)).await; + let before = stream_shards_indexes(&d).await; + assert_eq!( + before.keys().map(String::as_str).collect::>(), + ["_id_", "shard_id_1"], + "{state}" + ); + d.migrate().await.expect("migrate"); + let after = stream_shards_indexes(&d).await; + assert_eq!( + after.get("table_id_1_shard_id_1"), + Some(&doc! { "table_id": 1, "shard_id": 1 }), + "{state}: {after:?}" + ); + let again = d.migrate().await.expect("second migrate"); + assert!(again.applied.is_empty(), "{state}: {again:?}"); + assert_eq!(stream_shards_indexes(&d).await, after, "{state}"); + d.drop_databases().await; + } +} + /// Migration 2's body on the oldest released catalog: every stored policy /// after the upgrade, which the committed golden file pins. #[tokio::test(flavor = "multi_thread")] diff --git a/crates/storage-postgres/src/create_table.rs b/crates/storage-postgres/src/create_table.rs index 7e4098a82..9587adfd0 100755 --- a/crates/storage-postgres/src/create_table.rs +++ b/crates/storage-postgres/src/create_table.rs @@ -110,6 +110,7 @@ impl PostgresEngine { restore: Option<&RestoreRegistration<'_>>, ) -> Result { Self::validate_account_id(account_id)?; + let input = extenddb_storage::stream_lineage::without_disabled_specification(input); let table_id = uuid::Uuid::new_v4().to_string(); let table_arn = table_arn(&self.region, account_id, &input.table_name); let billing_mode = input.billing_mode.unwrap_or(BillingMode::Provisioned); @@ -438,22 +439,24 @@ impl PostgresEngine { // If data DDL fails, the catalog entry is cleaned up (see below). // Initialize stream shards and label if streams are enabled on this table. - let stream_label = if input + let stream_label = match input .stream_specification .as_ref() - .is_some_and(|s| s.stream_enabled) + .filter(|s| s.stream_enabled) + .and_then(|s| s.stream_view_type) { - let label = Self::init_stream_shards( - &mut tx, - &self.data_pool, - account_id, - &input.table_name, - &table_id, - ) - .await?; - Some(label) - } else { - None + Some(view_type) => Some( + Self::init_stream_shards( + &mut tx, + &self.data_pool, + account_id, + &input.table_name, + &table_id, + view_type, + ) + .await?, + ), + None => None, }; tx.commit() diff --git a/crates/storage-postgres/src/data/transactions.rs b/crates/storage-postgres/src/data/transactions.rs index 6748ed00d..480847fbf 100644 --- a/crates/storage-postgres/src/data/transactions.rs +++ b/crates/storage-postgres/src/data/transactions.rs @@ -13,14 +13,15 @@ use extenddb_core::types::{ use extenddb_core::validation; use extenddb_storage::error::StorageError; use extenddb_storage::util::pk_to_text; -use extenddb_storage::{IdempotencyKey, TransactGetOp, TransactWriteOp}; +use extenddb_storage::{IdempotencyKey, StreamCapture, TransactGetOp, TransactWriteOp}; use super::index::{ IndexMeta, db_error, enqueue_async_indexes, fetch_write_path_indexes, sync_indexes, }; use super::tx_helpers::{ check_idempotency_token_in_tx, delete_item_in_tx, fetch_item_for_update, fetch_item_in_tx, - insert_item_if_absent_in_tx, upsert_item_in_tx, write_stream_record_in_tx, + insert_item_if_absent_in_tx, stream_record_shard_in_tx, upsert_item_in_tx, + write_stream_record_to_shard_in_tx, }; use crate::PostgresEngine; use crate::pg_util::is_conflict_abort; @@ -230,26 +231,44 @@ impl PostgresEngine { }); } - // Write stream records atomically within the transaction (BLOCKER #1 fix). - for (op, (old_item, new_item)) in ops.iter().zip(op_items.iter()) { - let capture = match op { - TransactWriteOp::Put { stream, .. } - | TransactWriteOp::Delete { stream, .. } - | TransactWriteOp::Update { stream, .. } => stream.as_ref(), - TransactWriteOp::ConditionCheck { .. } => None, - }; - if let Some(capture) = capture { - write_stream_record_in_tx( + // Write stream records atomically within the transaction (BLOCKER #1 fix), + // in shard id order. Each record locks its shard's row FOR KEY SHARE, + // and a disable locks the stream's open rows FOR UPDATE in shard id + // order (`close_stream_shards`). In request order, a transaction with + // items on shard 2 and then shard 0 would hold shard 2 and wait for + // shard 0. A close that held shard 0 would wait for shard 2. This code + // chooses the shard of every record first. It then sorts the records + // by shard id, byte by byte, as the close sorts the rows. Two records + // on one shard keep their request order. + let mut records: Vec<(String, usize)> = Vec::new(); + for (i, (op, (old_item, new_item))) in ops.iter().zip(op_items.iter()).enumerate() { + if transact_op_stream(op).is_none() { + continue; + } + let shard = stream_record_shard_in_tx( + &mut tx, + transact_op_key_info(op), + old_item.as_ref(), + new_item.as_ref(), + ) + .await + .map_err(cancel_all)?; + if let Some(shard) = shard { + records.push((shard, i)); + } + } + records.sort(); + for (shard, i) in &records { + let op = &ops[*i]; + let (old_item, new_item) = &op_items[*i]; + if let Some(capture) = transact_op_stream(op) { + write_stream_record_to_shard_in_tx( &mut tx, - match op { - TransactWriteOp::Put { key_info, .. } - | TransactWriteOp::Delete { key_info, .. } - | TransactWriteOp::Update { key_info, .. } - | TransactWriteOp::ConditionCheck { key_info, .. } => key_info, - }, + transact_op_key_info(op), capture, old_item.as_ref(), new_item.as_ref(), + shard, ) .await .map_err(cancel_all)?; @@ -419,6 +438,27 @@ fn transact_op_table_id<'a>(op: &'a TransactWriteOp<'_>) -> &'a str { } } +/// The table metadata of a transactional write operation. +fn transact_op_key_info<'a>(op: &'a TransactWriteOp<'_>) -> &'a TableKeyInfo { + match op { + TransactWriteOp::Put { key_info, .. } + | TransactWriteOp::Delete { key_info, .. } + | TransactWriteOp::Update { key_info, .. } + | TransactWriteOp::ConditionCheck { key_info, .. } => key_info, + } +} + +/// The stream capture of a transactional write operation, if its table has an +/// enabled stream. A `ConditionCheck` writes nothing and records nothing. +fn transact_op_stream<'a>(op: &'a TransactWriteOp<'_>) -> Option<&'a StreamCapture> { + match op { + TransactWriteOp::Put { stream, .. } + | TransactWriteOp::Delete { stream, .. } + | TransactWriteOp::Update { stream, .. } => stream.as_ref(), + TransactWriteOp::ConditionCheck { .. } => None, + } +} + /// Error type for individual transactional write operations. /// /// Separates user-driven cancellations (condition failures, validation errors) diff --git a/crates/storage-postgres/src/data/tx_helpers.rs b/crates/storage-postgres/src/data/tx_helpers.rs index 69db42b1d..0ed177767 100755 --- a/crates/storage-postgres/src/data/tx_helpers.rs +++ b/crates/storage-postgres/src/data/tx_helpers.rs @@ -250,8 +250,9 @@ pub(super) async fn delete_item_in_tx( /// Write a stream record within an existing transaction. /// /// Builds the stream record from the old/new items and the `StreamCapture` -/// parameters, assigns a shard, and stores the record in `stream_pending`, -/// all within the caller's transaction. The record gets its sequence number +/// parameters, assigns a shard ([`stream_record_shard_in_tx`]), and stores the +/// record in `stream_pending` ([`write_stream_record_to_shard_in_tx`]), all +/// within the caller's transaction. The record gets its sequence number /// after the transaction commits (`number_shard_batch` in `stream_engine.rs`). /// /// The event type is determined from the old/new items: @@ -260,13 +261,98 @@ pub(super) async fn delete_item_in_tx( /// - old=Some, new=None → Remove /// /// For Delete operations where the item didn't exist, no stream record is written. -#[allow(clippy::too_many_arguments)] pub(super) async fn write_stream_record_in_tx( tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, key_info: &TableKeyInfo, capture: &StreamCapture, old_item: Option<&Item>, new_item: Option<&Item>, +) -> Result<(), StorageError> { + let Some(shard_id) = stream_record_shard_in_tx(tx, key_info, old_item, new_item).await? else { + return Ok(()); + }; + write_stream_record_to_shard_in_tx(tx, key_info, capture, old_item, new_item, &shard_id).await +} + +/// The open shard that receives the stream record of a write. `None` when +/// the write records nothing: no item changed (a delete of an item that did +/// not exist), or the table has no open shard. This reads the shard rows in +/// the caller's transaction without a lock. +/// [`write_stream_record_to_shard_in_tx`] locks the chosen shard and checks +/// that it is still open. +pub(super) async fn stream_record_shard_in_tx( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + key_info: &TableKeyInfo, + old_item: Option<&Item>, + new_item: Option<&Item>, +) -> Result, StorageError> { + // No stream record if nothing changed (e.g., delete of non-existent item). + let Some(source) = new_item.or(old_item) else { + return Ok(None); + }; + + // Assign shard within the transaction. + let pk_name = &key_info.key_schema[0].attribute_name; + let pk_str = source + .get(pk_name) + .map(|v| match v { + AttributeValue::S(s) => s.clone(), + AttributeValue::N(n) => n.clone(), + AttributeValue::B(b) => BASE64.encode(b), + _ => String::new(), + }) + .unwrap_or_default(); + + // Only the open shards: they belong to the table's enabled stream. A + // disabled stream's shards are closed and take no new records, so a write + // that read the table's key info before a disable records nothing. + // + // The open set is read without a lock, and only the shard this record + // goes to is locked, FOR KEY SHARE, re-checking that it is still open + // (`write_stream_record_to_shard_in_tx`). Locking every open row + // FOR SHARE made concurrent writers to one table extend the same + // MultiXacts on all four rows for a record that goes into one. + let mut shards: Vec<(String,)> = sqlx::query_as( + "SELECT shard_id FROM stream_shards \ + WHERE table_id = $1 AND ending_sequence_number IS NULL \ + ORDER BY shard_id", + ) + .bind(&key_info.table_id) + .fetch_all(&mut **tx) + .await + .map_err(db_error)?; + + if shards.is_empty() { + // No shards — streams may not be fully set up yet. Skip silently. + return Ok(None); + } + + let hash = crc32fast::hash(pk_str.as_bytes()); + #[allow(clippy::cast_possible_truncation)] + let idx = (hash as usize) % shards.len(); + Ok(Some(shards.swap_remove(idx).0)) +} + +/// Store the stream record of a write in `stream_pending`, in shard +/// `shard_id` ([`stream_record_shard_in_tx`]), within the caller's +/// transaction. Records nothing when no item changed, or when a disable +/// closed the shard after [`stream_record_shard_in_tx`] chose it. +/// +/// This locks the shard's row FOR KEY SHARE and checks again that the shard +/// is open. A disable locks the open rows FOR UPDATE before it takes the +/// ending numbers (`close_stream_shards`), and that mode conflicts with this +/// lock. The write therefore commits before the close takes the ending +/// number, or it waits here, finds the row closed and records nothing. A +/// transaction that stores several records stores them in shard id order, +/// the order in which the disable locks the rows +/// (`transact_write_items_impl`). +pub(super) async fn write_stream_record_to_shard_in_tx( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + key_info: &TableKeyInfo, + capture: &StreamCapture, + old_item: Option<&Item>, + new_item: Option<&Item>, + shard_id: &str, ) -> Result<(), StorageError> { // No stream record if nothing changed (e.g., delete of non-existent item). let source_item = new_item.or(old_item); @@ -306,38 +392,19 @@ pub(super) async fn write_stream_record_in_tx( let size = source_item.map_or(0, |i| i64::try_from(item_size_bytes(i)).unwrap_or(i64::MAX)); - // Assign shard within the transaction. - let pk_name = &key_info.key_schema[0].attribute_name; - let pk_str = source - .get(pk_name) - .map(|v| match v { - AttributeValue::S(s) => s.clone(), - AttributeValue::N(n) => n.clone(), - AttributeValue::B(b) => BASE64.encode(b), - _ => String::new(), - }) - .unwrap_or_default(); - - let shards: Vec<(String,)> = sqlx::query_as( + let still_open: Option<(String,)> = sqlx::query_as( "SELECT shard_id FROM stream_shards \ - WHERE table_id = $1 \ - ORDER BY shard_id", + WHERE shard_id = $1 AND ending_sequence_number IS NULL FOR KEY SHARE", ) - .bind(&key_info.table_id) - .fetch_all(&mut **tx) + .bind(shard_id) + .fetch_optional(&mut **tx) .await .map_err(db_error)?; - - if shards.is_empty() { - // No shards — streams may not be fully set up yet. Skip silently. + if still_open.is_none() { + // Closed between the read and the lock: the stream is disabled. return Ok(()); } - let hash = crc32fast::hash(pk_str.as_bytes()); - #[allow(clippy::cast_possible_truncation)] - let idx = (hash as usize) % shards.len(); - let shard_id = &shards[idx].0; - // The record's one clock read: clock_timestamp() is the database clock // when this statement runs, after the write's locking reads, while the // column default (NOW()) is the transaction start, which in a long diff --git a/crates/storage-postgres/src/lib.rs b/crates/storage-postgres/src/lib.rs index 3724a9415..b28ebf715 100755 --- a/crates/storage-postgres/src/lib.rs +++ b/crates/storage-postgres/src/lib.rs @@ -30,6 +30,7 @@ mod snapshot_engine; #[doc(hidden)] pub use snapshot_engine::map_fetch_error; mod stream_engine; +mod stream_lifecycle; #[doc(hidden)] pub mod stream_records_sql; mod table_engine; diff --git a/crates/storage-postgres/src/stream_engine.rs b/crates/storage-postgres/src/stream_engine.rs index 285d31b32..6cc51e52b 100755 --- a/crates/storage-postgres/src/stream_engine.rs +++ b/crates/storage-postgres/src/stream_engine.rs @@ -3,23 +3,17 @@ //! `StreamEngine` trait implementation for `PostgresEngine`. -use extenddb_core::types::{ - SequenceNumberRange, Shard, StreamDescription, StreamRecord, StreamStatus, StreamSummary, - StreamViewType, -}; +use extenddb_core::types::{StreamDescription, StreamRecord, StreamSummary}; use extenddb_storage::StreamEngine; use extenddb_storage::error::StorageError; -use extenddb_storage::util::{new_stream_label, parse_stream_arn, shard_in_stream, stream_arn}; +use extenddb_storage::stream_lineage; +use extenddb_storage::util::{parse_stream_arn, shard_in_stream, stream_arn}; use futures::future::BoxFuture; -use sqlx::PgPool; use std::time::Duration; use crate::PostgresEngine; use crate::stream_records_sql::{STREAM_RECORDS_AFTER, STREAM_RECORDS_FROM_START}; -/// Number of fixed shards per stream (hash-based assignment). -const SHARDS_PER_STREAM: u32 = 4; - /// The most pending records one numbering transaction moves. const NUMBER_BATCH: i64 = 1_000; @@ -263,7 +257,7 @@ impl PostgresEngine { /// Number up to [`NUMBER_LIMIT`] of one shard's committed pending /// records, in batches, and stop at a shard that another transaction /// numbers. Returns how many records moved. - async fn number_shard(&self, shard_id: &str) -> Result { + pub(crate) async fn number_shard(&self, shard_id: &str) -> Result { let mut numbered = 0; while numbered < NUMBER_LIMIT { match self.number_shard_batch(shard_id, None).await? { @@ -315,15 +309,39 @@ impl PostgresEngine { self.number_pending_stream_records().await } - /// Initialize stream shards for a table and set the `stream_label`. - /// - /// Updates the stream label in the catalog (via the provided catalog - /// transaction) and creates shard rows in the data database within a - /// data transaction for atomicity. - /// - /// # Errors - /// - /// Returns [`StorageError::Internal`] if any query fails. + /// Number every committed pending record of shard `shard_id` in `tx`, a + /// READ COMMITTED transaction that holds the shard's row `FOR UPDATE`: + /// the close of a shard (`close_stream_shards` in `stream_lifecycle.rs`), + /// so the ending number it takes in `tx` covers every record whose write + /// committed before it. The lock conflicts with the `FOR KEY SHARE` lock + /// a write takes on its shard's row, so no record of the shard is stored + /// while `tx` runs, and with [`LOCK_SHARD`], so no other transaction + /// numbers the shard. The close numbers the shard's backlog with + /// [`Self::number_shard`] before it takes the lock, so this moves the + /// records committed after that. Returns how many records moved. + pub(crate) async fn number_locked_shard( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + shard_id: &str, + ) -> Result { + let mut numbered = 0; + loop { + let (moved, looked): (i64, i64) = sqlx::query_as(NUMBER_PENDING) + .bind(shard_id) + .bind(NUMBER_BATCH) + .bind(NUMBER_BATCH_BYTES) + .fetch_one(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + numbered += moved.unsigned_abs(); + // A batch takes at least one record when one is pending, so the + // loop ends once a batch takes every record it looked at and + // looked at fewer than a full batch. + if moved == looked && looked < NUMBER_BATCH { + return Ok(numbered); + } + } + } + /// Remove the shards of deleted tables once their stream has aged out. /// /// DeleteTable leaves a table's shards and records in place, as the service @@ -392,64 +410,6 @@ impl PostgresEngine { } Ok(result.rows_affected()) } - - pub(crate) async fn init_stream_shards( - tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, - data_pool: &PgPool, - account_id: &str, - table_name: &str, - table_id: &str, - ) -> Result { - // Formatted here, not in SQL, so every backend issues the same label - // shape from one function (`extenddb_storage::util::format_stream_label`). - let label = new_stream_label(); - sqlx::query( - "UPDATE tables SET stream_label = $3 WHERE account_id = $1 AND table_name = $2", - ) - .bind(account_id) - .bind(table_name) - .bind(&label) - .execute(&mut **tx) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - - // P54 Bug 1: Stream shards live in the data database for atomic - // writes with stream records and item data. Use a transaction so - // all shards are created atomically. - let mut data_tx = data_pool - .begin() - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - - for i in 0..SHARDS_PER_STREAM { - // Derived from the table id, not its name. A name is reused by - // delete-and-recreate and by every account that picks it, while - // `stream_shards.shard_id` is unique across the data database - // and PostgreSQL keeps a deleted table's shards. A name of - // up to 255 bytes also overflowed the 65-character ShardId limit - // the AWS SDKs enforce. The table id is a UUID, giving a - // 61-character id. - let shard_id = format!("shardId-{table_id}-{i:016}"); - let start_seq = format!("{:021}", 0); - sqlx::query( - "INSERT INTO stream_shards (shard_id, table_id, starting_sequence_number) \ - VALUES ($1, $2, $3)", - ) - .bind(&shard_id) - .bind(table_id) - .bind(&start_seq) - .execute(&mut *data_tx) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - } - - data_tx - .commit() - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - - Ok(label) - } } impl StreamEngine for PostgresEngine { @@ -500,7 +460,8 @@ impl StreamEngine for PostgresEngine { // shard that is not that table's (another table, or another // account) returns no rows. Whether the shard belongs to the stream // the ARN's label names is decided in Rust from the table's label - // and the shard id (`shard_in_stream`). + // and the shard id (`shard_in_stream`), with a third round trip for + // a legacy id only: whether the table has a labelled shard. let table: Option<(String, Option)> = sqlx::query_as( "SELECT table_id, stream_label FROM tables \ WHERE account_id = $1 AND table_name = $2", @@ -549,7 +510,10 @@ impl StreamEngine for PostgresEngine { row_shard_id, table_label.as_deref(), &stream_label, - ) { + || self.table_has_labelled_shard(&table_id), + ) + .await? + { return Err(extenddb_storage::util::stream_does_not_exist()); } @@ -577,99 +541,24 @@ impl StreamEngine for PostgresEngine { let exclusive_start_shard_id = input.exclusive_start_shard_id.clone(); Box::pin(async move { let (table_name, stream_label) = parse_stream_arn(&stream_arn)?; - - let row: Option<(serde_json::Value, serde_json::Value, Option, String, String)> = - sqlx::query_as( - "SELECT key_schema, attribute_definitions, stream_specification, table_status, table_id \ - FROM tables WHERE account_id = $1 AND table_name = $2 AND stream_label = $3", - ) - .bind(&account_id) - .bind(&table_name) - .bind(&stream_label) - .fetch_optional(&self.pool) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - - let (ks_json, _ad_json, stream_spec_json, table_status, table_id) = - row.ok_or_else(|| extenddb_storage::util::stream_not_found(&stream_arn))?; - + let (table, ks_json) = self + .stream_table_row(&account_id, &table_name) + .await? + .ok_or_else(|| extenddb_storage::util::stream_not_found(&stream_arn))?; let key_schema = serde_json::from_value(ks_json) .map_err(|e| StorageError::Internal(e.to_string()))?; - - let stream_view_type = stream_spec_json - .and_then(|v| { - v.get("StreamViewType") - .and_then(|sv| serde_json::from_value::(sv.clone()).ok()) - }) - .unwrap_or(StreamViewType::KeysOnly); - - let limit = limit.unwrap_or(100); - let shard_rows: Vec<(String, Option, String, Option)> = if let Some( - ref start, - ) = - exclusive_start_shard_id - { - sqlx::query_as( - "SELECT shard_id, parent_shard_id, starting_sequence_number, ending_sequence_number \ - FROM stream_shards WHERE table_id = $1 AND shard_id > $2 \ - ORDER BY shard_id LIMIT $3", - ) - .bind(&table_id) - .bind(start) - .bind(limit + 1) - .fetch_all(&self.data_pool) - .await - .map_err(|e| StorageError::Internal(e.to_string()))? - } else { - sqlx::query_as( - "SELECT shard_id, parent_shard_id, starting_sequence_number, ending_sequence_number \ - FROM stream_shards WHERE table_id = $1 \ - ORDER BY shard_id LIMIT $2", - ) - .bind(&table_id) - .bind(limit + 1) - .fetch_all(&self.data_pool) - .await - .map_err(|e| StorageError::Internal(e.to_string()))? - }; - - #[allow(clippy::cast_sign_loss)] - let limit_usize = limit as usize; - let last_shard = if shard_rows.len() > limit_usize { - Some(shard_rows[limit_usize - 1].0.clone()) - } else { - None - }; - - let shards: Vec = shard_rows - .into_iter() - .take(limit_usize) - .map(|(id, parent, start, end)| Shard { - shard_id: id, - parent_shard_id: parent, - sequence_number_range: SequenceNumberRange { - starting_sequence_number: start, - ending_sequence_number: end, - }, - }) - .collect(); - - let stream_status = if table_status == "DELETING" { - StreamStatus::Disabling - } else { - StreamStatus::Enabled - }; - - Ok(StreamDescription { + let shards = self + .stream_shard_rows(std::slice::from_ref(&table.table_id)) + .await?; + stream_lineage::describe_stream( stream_arn, - stream_label, - stream_status, - stream_view_type, - table_name, + &table, key_schema, - shards, - last_evaluated_shard_id: last_shard, - }) + stream_label, + &shards, + limit, + exclusive_start_shard_id.as_deref(), + ) }) } @@ -685,84 +574,21 @@ impl StreamEngine for PostgresEngine { let exclusive_start_stream_arn = exclusive_start_stream_arn.map(std::string::ToString::to_string); Box::pin(async move { - let rows: Vec<(String, String, String)> = match ( + let start = exclusive_start_stream_arn + .as_deref() + .map(parse_stream_arn) + .transpose()?; + let tables = self.stream_table_rows(&account_id).await?; + let ids: Vec = tables.iter().map(|t| t.table_id.clone()).collect(); + let shards = self.stream_shard_rows(&ids).await?; + stream_lineage::list_streams( + &tables, + &shards, table_name.as_deref(), - exclusive_start_stream_arn.as_deref(), - ) { - (Some(tn), Some(start_arn)) => { - let (_, start_label) = parse_stream_arn(start_arn)?; - sqlx::query_as( - "SELECT table_name, table_arn, stream_label FROM tables \ - WHERE account_id = $1 AND stream_label IS NOT NULL AND table_name = $2 AND stream_label > $3 \ - ORDER BY stream_label LIMIT $4", - ) - .bind(&account_id) - .bind(tn) - .bind(&start_label) - .bind(limit + 1) - .fetch_all(&self.pool) - .await - .map_err(|e| StorageError::Internal(e.to_string()))? - } - (Some(tn), None) => sqlx::query_as( - "SELECT table_name, table_arn, stream_label FROM tables \ - WHERE account_id = $1 AND stream_label IS NOT NULL AND table_name = $2 \ - ORDER BY stream_label LIMIT $3", - ) - .bind(&account_id) - .bind(tn) - .bind(limit + 1) - .fetch_all(&self.pool) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?, - (None, Some(start_arn)) => { - let (start_table, start_label) = parse_stream_arn(start_arn)?; - sqlx::query_as( - "SELECT table_name, table_arn, stream_label FROM tables \ - WHERE account_id = $1 AND stream_label IS NOT NULL \ - AND (table_name, stream_label) > ($2, $3) \ - ORDER BY table_name, stream_label LIMIT $4", - ) - .bind(&account_id) - .bind(&start_table) - .bind(&start_label) - .bind(limit + 1) - .fetch_all(&self.pool) - .await - .map_err(|e| StorageError::Internal(e.to_string()))? - } - (None, None) => sqlx::query_as( - "SELECT table_name, table_arn, stream_label FROM tables \ - WHERE account_id = $1 AND stream_label IS NOT NULL \ - ORDER BY table_name, stream_label LIMIT $2", - ) - .bind(&account_id) - .bind(limit + 1) - .fetch_all(&self.pool) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?, - }; - - #[allow(clippy::cast_sign_loss)] - let limit_usize = limit as usize; - - let summaries: Vec = rows - .iter() - .take(limit_usize) - .map(|(tn, _table_arn, label)| StreamSummary { - stream_arn: stream_arn(&self.region, &account_id, tn, label), - stream_label: label.clone(), - table_name: tn.clone(), - }) - .collect(); - - let last_arn = if rows.len() > limit_usize { - summaries.last().map(|s| s.stream_arn.clone()) - } else { - None - }; - - Ok((summaries, last_arn)) + limit, + start, + |t, l| stream_arn(&self.region, &account_id, t, l), + ) }) } @@ -803,9 +629,11 @@ impl StreamEngine for PostgresEngine { .await .map_err(|e| StorageError::Internal(e.to_string()))?; + // Only the open shards: they belong to the table's enabled stream. + // A disabled stream's closed shards take no new records. let shards: Vec<(String,)> = sqlx::query_as( "SELECT shard_id FROM stream_shards \ - WHERE table_id = $1 \ + WHERE table_id = $1 AND ending_sequence_number IS NULL \ ORDER BY shard_id", ) .bind(&table_id) @@ -850,34 +678,20 @@ impl StreamEngine for PostgresEngine { let shard_id = shard_id.to_string(); Box::pin(async move { let (table_name, stream_label) = parse_stream_arn(&stream_arn)?; - - let table_id: Option = sqlx::query_scalar( - "SELECT table_id FROM tables \ - WHERE account_id = $1 AND table_name = $2 AND stream_label = $3", - ) - .bind(&account_id) - .bind(&table_name) - .bind(&stream_label) - .fetch_optional(&self.pool) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - - let Some(table_id) = table_id else { - return Err(extenddb_storage::util::stream_does_not_exist()); + let table = self.stream_table_row(&account_id, &table_name).await?; + let shards = match &table { + Some((t, _)) => { + self.stream_shard_rows(std::slice::from_ref(&t.table_id)) + .await? + } + None => Vec::new(), }; - - let exists: Option<(i32,)> = - sqlx::query_as("SELECT 1 FROM stream_shards WHERE shard_id = $1 AND table_id = $2") - .bind(&shard_id) - .bind(&table_id) - .fetch_optional(&self.data_pool) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - - if exists.is_none() { - return Err(extenddb_storage::util::shard_not_in_stream(&stream_arn)); - } - Ok(()) + stream_lineage::validate_shard( + table.as_ref().map(|(t, _)| t), + &stream_label, + &shards, + &shard_id, + ) }) } @@ -908,3 +722,23 @@ impl StreamEngine for PostgresEngine { }) } } + +impl PostgresEngine { + /// Whether table `table_id` has a labelled shard row + /// (`stream_lineage::labelled_shard_id_range`, compared in byte order + /// whatever the database's collation), which makes its legacy rows an + /// earlier, closed stream (`util::shard_in_stream`). + async fn table_has_labelled_shard(&self, table_id: &str) -> Result { + let (from, to) = extenddb_storage::stream_lineage::labelled_shard_id_range(table_id); + sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM stream_shards WHERE table_id = $1 \ + AND shard_id COLLATE \"C\" >= $2 AND shard_id COLLATE \"C\" < $3)", + ) + .bind(table_id) + .bind(&from) + .bind(&to) + .fetch_one(&self.data_pool) + .await + .map_err(|e| StorageError::Internal(e.to_string())) + } +} diff --git a/crates/storage-postgres/src/stream_lifecycle.rs b/crates/storage-postgres/src/stream_lifecycle.rs new file mode 100644 index 000000000..6854a95e5 --- /dev/null +++ b/crates/storage-postgres/src/stream_lifecycle.rs @@ -0,0 +1,498 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Stream lifecycle on PostgreSQL: creating a table's stream, closing it on +//! `UpdateTable StreamEnabled=false`, and reading the table and shard rows the +//! shared rules in `extenddb_storage::stream_lineage` group into streams. +//! +//! The table row is in the catalog database and the shards and records in the +//! data database, so a stream change is two commits. `UpdateTable` changes the +//! catalog row (specification and label) in its transaction and changes the +//! shards after that transaction commits ([`PendingStream`]). A failure between +//! the two leaves the catalog's answer in force. A disable whose close failed +//! leaves a disabled stream whose shards are still open: a retried disable is +//! refused and closes them, and an enable closes them too, since an enable +//! closes the open shards of every earlier stream of the table. An enable whose +//! shard insert failed leaves an enabled stream with no shards, which records +//! nothing and is replaced by a disable and an enable. + +use extenddb_core::types::{StreamSpecification, StreamViewType}; +use extenddb_storage::error::StorageError; +use extenddb_storage::stream_lineage::{ + SHARDS_PER_STREAM, ShardRow, StreamChange, TableStreamRow, disabled_stream_to_close, + enable_still_current, next_stream_label, open_shard_ids_before, open_shard_ids_of, shard_id, + stored_specification, stream_change, +}; +use sqlx::PgPool; + +use crate::PostgresEngine; + +type ShardTuple = (String, String, Option, String, Option); + +fn internal(e: impl std::fmt::Display) -> StorageError { + StorageError::Internal(e.to_string()) +} + +fn spec_of(value: Option) -> Result, StorageError> { + value + .map(serde_json::from_value) + .transpose() + .map_err(internal) +} + +/// The shard changes of a committed `UpdateTable` stream change: on a +/// disable, close the open shards of the stream the change read (`close`, the +/// table's label at that time); on an enable, close the open shards of every +/// earlier stream, then create the new stream's shards. +#[must_use] +pub(crate) struct PendingStream { + table_id: String, + close: Option, + open: Option<(String, StreamViewType)>, +} + +impl PendingStream { + /// The shard changes of a disable that read stream `label`, for tests that + /// apply one after an overlapping enable's. + #[cfg(feature = "test-hooks")] + pub(crate) fn disable_of(table_id: &str, label: &str) -> Self { + Self { + table_id: table_id.to_owned(), + close: Some(label.to_owned()), + open: None, + } + } + + /// Run the shard changes once the catalog transaction has committed. + /// + /// The enable's shards are inserted after its catalog row is visible, so a + /// disable that reads the row in between passes its check, commits the + /// disabled specification and closes the shards it finds, which may be + /// none or some of the four. The enable re-reads the table's stream fields + /// after its inserts and, when the row no longer says enabled with this + /// label, closes the shards it inserted. A disable committed before the + /// re-read is seen by it; one committed after finds all four rows open + /// and closes them itself; both closing is idempotent through the + /// `ending_sequence_number IS NULL` guard. A disable closes only the + /// stream it read, so one applying after an overlapping enable leaves the + /// new stream open. + pub(crate) async fn apply( + self, + engine: &PostgresEngine, + account_id: &str, + table_name: &str, + ) -> Result<(), StorageError> { + let Some((label, view_type)) = self.open else { + if let Some(label) = &self.close { + engine.close_stream_shards(&self.table_id, label).await?; + } + return Ok(()); + }; + engine.close_streams_before(&self.table_id, &label).await?; + PostgresEngine::insert_stream_shards(&engine.data_pool, &self.table_id, &label, view_type) + .await?; + let row = engine.stream_table_row(account_id, table_name).await?; + if !enable_still_current(row.as_ref().map(|(t, _)| t), &label) { + engine.close_stream_shards(&self.table_id, &label).await?; + } + Ok(()) + } +} + +impl PostgresEngine { + /// Create a new stream on the table: `tables.stream_label` set to a new + /// label in the caller's catalog transaction, and four open shards named + /// for it committed in the data database. Returns the label. + pub(crate) async fn init_stream_shards( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + data_pool: &PgPool, + account_id: &str, + table_name: &str, + table_id: &str, + view_type: StreamViewType, + ) -> Result { + let label = Self::set_new_stream_label(tx, account_id, table_name).await?; + Self::insert_stream_shards(data_pool, table_id, &label, view_type).await?; + Ok(label) + } + + /// Set `tables.stream_label` to a label later than the table's previous + /// one, in the caller's catalog transaction. + async fn set_new_stream_label( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + account_id: &str, + table_name: &str, + ) -> Result { + let previous: Option = sqlx::query_scalar( + "SELECT stream_label FROM tables WHERE account_id = $1 AND table_name = $2", + ) + .bind(account_id) + .bind(table_name) + .fetch_optional(&mut **tx) + .await + .map_err(internal)? + .flatten(); + let label = next_stream_label(previous.as_deref()); + sqlx::query( + "UPDATE tables SET stream_label = $3 WHERE account_id = $1 AND table_name = $2", + ) + .bind(account_id) + .bind(table_name) + .bind(&label) + .execute(&mut **tx) + .await + .map_err(internal)?; + Ok(label) + } + + /// Insert the four open shards of stream `label`. Stream shards live in + /// the data database for atomic writes with stream records and item data; + /// one transaction creates all four. + async fn insert_stream_shards( + data_pool: &PgPool, + table_id: &str, + label: &str, + view_type: StreamViewType, + ) -> Result<(), StorageError> { + let mut data_tx = data_pool.begin().await.map_err(internal)?; + for i in 0..SHARDS_PER_STREAM { + sqlx::query( + "INSERT INTO stream_shards (shard_id, table_id, starting_sequence_number) \ + VALUES ($1, $2, $3)", + ) + .bind(shard_id(table_id, label, view_type, i)?) + .bind(table_id) + .bind(format!("{:021}", 0)) + .execute(&mut *data_tx) + .await + .map_err(internal)?; + } + data_tx.commit().await.map_err(internal)?; + Ok(()) + } + + /// Close every open shard of the table: its `EndingSequenceNumber` becomes + /// the sequence number of its last record, or its starting number when it + /// holds none (both measured 2026-10-24). + /// + /// The service's rule, measured 2026-10-24 with 569 writes racing a + /// disable: every write accepted before the stream reports `DISABLED` + /// lands in the closed stream before its shard's `EndingSequenceNumber`, + /// and a write after that records nothing. Held here by a lock in the data + /// database: the write path locks its record's shard row `FOR KEY SHARE` + /// in its data transaction, and this close locks the open rows + /// `FOR UPDATE` first. A write stores its record in `stream_pending` with + /// no number, and the record is numbered after the write commits (#808), + /// so the close numbers each shard's pending records under its lock + /// (`PostgresEngine::number_locked_shard`) and then takes the ending + /// numbers, all in one READ COMMITTED transaction. A write that locked its + /// shard before the close commits first and its record is numbered at or + /// below the ending number, and a write that arrives after the lock + /// re-evaluates the closed row and records nothing. What + /// remains is the gap every backend has: a write acknowledged after the + /// disable response that recorded into no stream, because its key info + /// pre-dated the disable and it found no open shard. + /// + /// A write that waits for the lock holds its item locks and a pool + /// connection. The close therefore numbers each shard's backlog first, in + /// the numbering worker's batches (`PostgresEngine::number_shard`). Those + /// transactions lock the shard's row `FOR NO KEY UPDATE`, and a writer's + /// `FOR KEY SHARE` does not wait for that mode. The numbering under the + /// lock then moves only the records committed in between. + /// + /// The close locks the rows in shard id order, and `TransactWriteItems` + /// stores its records, and locks their shards, in the same order + /// (`transact_write_items_impl`). Both compare the ids byte by byte. Each + /// then waits only for a shard with a higher id than every shard it holds, + /// so the two cannot wait for each other. In request order, a transaction + /// with items on shard 2 and then shard 0 would hold shard 2 and wait for + /// shard 0. A close that held shard 0 would wait for shard 2, and + /// PostgreSQL would cancel one of the two after `deadlock_timeout`. + pub(crate) async fn close_stream_shards( + &self, + table_id: &str, + label: &str, + ) -> Result<(), StorageError> { + self.close_selected_shards(table_id, label, open_shard_ids_of) + .await + } + + /// Close the open shards of every stream of the table older than `label` + /// (`stream_lineage::open_shard_ids_before`), as [`Self::close_stream_shards`] + /// closes one stream's. + async fn close_streams_before(&self, table_id: &str, label: &str) -> Result<(), StorageError> { + self.close_selected_shards(table_id, label, open_shard_ids_before) + .await + } + + async fn close_selected_shards( + &self, + table_id: &str, + label: &str, + select: fn(&str, &str, &[ShardRow]) -> Vec, + ) -> Result<(), StorageError> { + let data_pool = &self.data_pool; + let rows: Vec = sqlx::query_as( + "SELECT table_id, shard_id, parent_shard_id, starting_sequence_number, \ + ending_sequence_number \ + FROM stream_shards WHERE table_id = $1", + ) + .bind(table_id) + .fetch_all(data_pool) + .await + .map_err(internal)?; + let ids = select(table_id, label, &shard_rows(rows)); + if ids.is_empty() { + return Ok(()); + } + // The backlog first, without blocking writers. The close logs a + // failure here, and the numbering under the lock below numbers what + // this leaves. + for shard_id in &ids { + if let Err(e) = self.number_shard(shard_id).await { + tracing::warn!( + "Stream record numbering before the close of shard {shard_id} failed: {e}" + ); + } + } + // READ COMMITTED whatever the database's default: each statement after + // the lock then reads the records of the writes the lock waited for. + // Under REPEATABLE READ the snapshot would date from the start of the + // lock statement, before those writes committed. + let mut data_tx = data_pool + .begin_with("BEGIN ISOLATION LEVEL READ COMMITTED") + .await + .map_err(internal)?; + // PostgreSQL applies ORDER BY before the lock, so the statement locks + // the rows in shard id order. COLLATE "C" compares the ids byte by + // byte, as `TransactWriteItems` sorts its records, whatever the + // database's collation. + let locked: Vec = sqlx::query_scalar( + "SELECT shard_id FROM stream_shards \ + WHERE shard_id = ANY($1) AND ending_sequence_number IS NULL \ + ORDER BY shard_id COLLATE \"C\" FOR UPDATE", + ) + .bind(&ids) + .fetch_all(&mut *data_tx) + .await + .map_err(internal)?; + // Only the rows this transaction locked: a shard another close has + // closed was numbered by that close. + for shard_id in &locked { + Self::number_locked_shard(&mut data_tx, shard_id).await?; + } + sqlx::query( + "UPDATE stream_shards s SET ending_sequence_number = COALESCE(\ + (SELECT MAX(r.sequence_number) FROM stream_records r \ + WHERE r.shard_id = s.shard_id), \ + s.starting_sequence_number) \ + WHERE s.shard_id = ANY($1) AND s.ending_sequence_number IS NULL", + ) + .bind(&ids) + .execute(&mut *data_tx) + .await + .map_err(internal)?; + data_tx.commit().await.map_err(internal)?; + Ok(()) + } + + /// Apply an `UpdateTable` `StreamSpecification` in the caller's catalog + /// transaction, which holds the table row `FOR UPDATE`. + /// + /// Disabling stores `{StreamEnabled: false}` with the stream's view type, + /// which DescribeTable omits, and keeps `stream_label`, so + /// `LatestStreamArn` keeps naming the disabled stream. Enabling stores the specification and + /// a new label. The shard changes are returned, for the caller to apply + /// with [`PendingStream::apply`] once the catalog has committed: a request + /// refused after this point (an invalid index update in the same call) + /// then leaves no shards behind. + /// + /// A disable refused because the stored specification is already disabled + /// first closes the current label's open shards in the data database + /// (left by a release that did not close them, or by a close that failed + /// after its catalog write), then returns the refusal; the catalog + /// transaction has nothing to commit. + pub(crate) async fn apply_stream_specification( + &self, + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + account_id: &str, + table_name: &str, + table_id: &str, + requested: &StreamSpecification, + ) -> Result { + let current: Option = sqlx::query_scalar( + "SELECT stream_specification FROM tables WHERE account_id = $1 AND table_name = $2", + ) + .bind(account_id) + .bind(table_name) + .fetch_optional(&mut **tx) + .await + .map_err(internal)? + .flatten(); + let current = spec_of(current)?; + let current_label: Option = sqlx::query_scalar( + "SELECT stream_label FROM tables WHERE account_id = $1 AND table_name = $2", + ) + .bind(account_id) + .bind(table_name) + .fetch_optional(&mut **tx) + .await + .map_err(internal)? + .flatten(); + let change = match stream_change(table_name, current.as_ref(), requested) { + Ok(change) => change, + Err(refusal) => { + if let Some(label) = + disabled_stream_to_close(current.as_ref(), current_label.as_deref(), requested) + { + self.close_stream_shards(table_id, label).await?; + } + return Err(refusal); + } + }; + let stored = + serde_json::to_value(stored_specification(change, current.as_ref(), requested)) + .map_err(internal)?; + sqlx::query( + "UPDATE tables SET stream_specification = $1 \ + WHERE account_id = $2 AND table_name = $3", + ) + .bind(stored) + .bind(account_id) + .bind(table_name) + .execute(&mut **tx) + .await + .map_err(internal)?; + let open = match change { + StreamChange::Disable => None, + StreamChange::Enable(view_type) => Some(( + Self::set_new_stream_label(tx, account_id, table_name).await?, + view_type, + )), + }; + Ok(PendingStream { + table_id: table_id.to_owned(), + close: current_label, + open, + }) + } + + /// The table's stream columns and its key schema, by name. + pub(crate) async fn stream_table_row( + &self, + account_id: &str, + table_name: &str, + ) -> Result, StorageError> { + type Row = ( + String, + String, + Option, + Option, + serde_json::Value, + ); + let row: Option = sqlx::query_as( + "SELECT table_id, table_status, stream_label, stream_specification, key_schema \ + FROM tables WHERE account_id = $1 AND table_name = $2", + ) + .bind(account_id) + .bind(table_name) + .fetch_optional(&self.pool) + .await + .map_err(internal)?; + row.map(|(table_id, table_status, stream_label, spec, key_schema)| { + Ok(( + TableStreamRow { + table_name: table_name.to_owned(), + table_id, + table_status, + stream_label, + stream_specification: spec_of(spec)?, + }, + key_schema, + )) + }) + .transpose() + } + + /// Every table of the account that has had a stream. + pub(crate) async fn stream_table_rows( + &self, + account_id: &str, + ) -> Result, StorageError> { + type Row = ( + String, + String, + String, + Option, + Option, + ); + let rows: Vec = sqlx::query_as( + "SELECT table_name, table_id, table_status, stream_label, stream_specification \ + FROM tables WHERE account_id = $1 AND stream_label IS NOT NULL", + ) + .bind(account_id) + .fetch_all(&self.pool) + .await + .map_err(internal)?; + rows.into_iter() + .map(|(table_name, table_id, table_status, stream_label, spec)| { + Ok(TableStreamRow { + table_name, + table_id, + table_status, + stream_label, + stream_specification: spec_of(spec)?, + }) + }) + .collect() + } + + /// The shard rows of the given tables. + pub(crate) async fn stream_shard_rows( + &self, + table_ids: &[String], + ) -> Result, StorageError> { + if table_ids.is_empty() { + return Ok(Vec::new()); + } + let rows: Vec = sqlx::query_as( + "SELECT table_id, shard_id, parent_shard_id, starting_sequence_number, \ + ending_sequence_number \ + FROM stream_shards WHERE table_id = ANY($1)", + ) + .bind(table_ids) + .fetch_all(&self.data_pool) + .await + .map_err(internal)?; + Ok(shard_rows(rows)) + } + + /// Apply the shard changes of a disable that read stream `label` of table + /// `table_id`, as a disable whose apply runs after an overlapping enable's + /// would. + #[cfg(feature = "test-hooks")] + #[doc(hidden)] + pub async fn apply_disable_shard_changes_for_tests( + &self, + account_id: &str, + table_name: &str, + table_id: &str, + label: &str, + ) -> Result<(), StorageError> { + PendingStream::disable_of(table_id, label) + .apply(self, account_id, table_name) + .await + } +} + +fn shard_rows(rows: Vec) -> Vec { + rows.into_iter() + .map(|(table_id, shard_id, parent, start, end)| ShardRow { + table_id, + shard_id, + parent_shard_id: parent, + starting_sequence_number: start, + ending_sequence_number: end, + }) + .collect() +} diff --git a/crates/storage-postgres/src/update_table.rs b/crates/storage-postgres/src/update_table.rs index dec99a93b..ced9ca105 100755 --- a/crates/storage-postgres/src/update_table.rs +++ b/crates/storage-postgres/src/update_table.rs @@ -331,75 +331,21 @@ impl PostgresEngine { .map_err(|e| StorageError::Internal(e.to_string()))?; } - // Apply stream specification change (enable/disable streams). - if let Some(spec) = &input.stream_specification { - let spec_json = - serde_json::to_value(spec).map_err(|e| StorageError::Internal(e.to_string()))?; - sqlx::query( - "UPDATE tables SET stream_specification = $1 \ - WHERE account_id = $2 AND table_name = $3", - ) - .bind(&spec_json) - .bind(account_id) - .bind(&input.table_name) - .execute(&mut *tx) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - - if spec.stream_enabled { - // Check if shards already exist (re-enabling streams on a table - // that previously had them). Query the data pool since - // stream_shards lives in the data database. - let existing: Option<(String,)> = sqlx::query_as( - "SELECT shard_id FROM stream_shards \ - WHERE table_id = $1 \ - LIMIT 1", + // Apply stream specification change (enable/disable streams). The + // shard changes run once the catalog has committed. + let pending_stream = match &input.stream_specification { + Some(spec) => Some( + self.apply_stream_specification( + &mut tx, + account_id, + &input.table_name, + &table_id, + spec, ) - .bind(&table_id) - .fetch_optional(&self.data_pool) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - - if existing.is_none() { - Self::init_stream_shards( - &mut tx, - &self.data_pool, - account_id, - &input.table_name, - &table_id, - ) - .await?; - } else { - // Shards exist but stream_label may be NULL if streams were - // previously disabled and the disable path cleared the label. - // This is a defensive check — init_stream_shards sets the - // label on first enable, but re-enable after disable needs - // to restore it. - let current_label: Option = sqlx::query_scalar( - "SELECT stream_label FROM tables \ - WHERE account_id = $1 AND table_name = $2", - ) - .bind(account_id) - .bind(&input.table_name) - .fetch_one(&mut *tx) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - - if current_label.is_none() { - sqlx::query( - "UPDATE tables SET stream_label = $3 \ - WHERE account_id = $1 AND table_name = $2", - ) - .bind(account_id) - .bind(&input.table_name) - .bind(extenddb_storage::util::new_stream_label()) - .execute(&mut *tx) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - } - } - } - } + .await?, + ), + None => None, + }; // Apply table class change. if let Some(tc) = &input.table_class { @@ -901,6 +847,10 @@ impl PostgresEngine { return Err(e); } + if let Some(pending) = pending_stream { + pending.apply(self, account_id, &input.table_name).await?; + } + // Drop the data tables of deleted GSIs, after the catalog commit. if let Some(updates) = &input.global_secondary_index_updates { let mut delete_idx = 0usize; diff --git a/crates/storage-postgres/tests/common/stream_disable_fixture.rs b/crates/storage-postgres/tests/common/stream_disable_fixture.rs new file mode 100644 index 000000000..3eddc4124 --- /dev/null +++ b/crates/storage-postgres/tests/common/stream_disable_fixture.rs @@ -0,0 +1,290 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +#![allow(dead_code)] + +//! The scratch database and request helpers the stream disable tests share. +//! +//! Needs `EXTENDDB_TEST_PG_CONNECTION_STRING` (host-only, e.g. +//! `postgres://user:pass@127.0.0.1:5432`); the tests skip otherwise. Each +//! test builds its own scratch database and drops it afterwards. + +pub use extenddb_core::expression::ExpressionMaps; +pub use extenddb_core::types::{ + AttributeDefinition, BillingMode, CreateTableInput, DescribeStreamInput, DescribeTableInput, + KeySchemaElement, KeyType, ScalarAttributeType, StreamDescription, StreamSpecification, + StreamStatus, StreamViewType, TableDescription, +}; +pub use extenddb_storage::error::StorageError; +pub use extenddb_storage::{DataEngine, StreamCapture, StreamEngine, TableEngine}; +pub use extenddb_storage_postgres::{PostgresConfig, PostgresEngine}; +pub use sqlx::PgPool; +pub use sqlx::postgres::PgPoolOptions; + +pub const ACCOUNT: &str = "123456789012"; +pub const REGION: &str = "us-east-1"; + +pub struct Scratch { + pub engine: PostgresEngine, + pub db: PgPool, + pub admin: PgPool, + pub db_name: String, +} + +impl Scratch { + pub async fn cleanup(self) { + let Scratch { + engine, + db, + admin, + db_name, + } = self; + drop(engine); + db.close().await; + sqlx::query(&format!( + "DROP DATABASE IF EXISTS \"{db_name}\" WITH (FORCE)" + )) + .execute(&admin) + .await + .expect("drop the scratch database"); + admin.close().await; + } +} + +/// The server's base URL, or `None` to skip. With `EXTENDDB_TEST_PG_REQUIRED` +/// set to any value but `0`, a job that must reach a server fails here +/// instead. +pub fn base_conn() -> Option { + let conn = std::env::var("EXTENDDB_TEST_PG_CONNECTION_STRING").unwrap_or_default(); + if conn.trim().is_empty() { + let required = + std::env::var("EXTENDDB_TEST_PG_REQUIRED").is_ok_and(|v| !v.is_empty() && v != "0"); + assert!( + !required, + "EXTENDDB_TEST_PG_CONNECTION_STRING is not set, and EXTENDDB_TEST_PG_REQUIRED says \ + this job must reach a PostgreSQL server" + ); + return None; + } + Some(conn.trim_end_matches('/').to_owned()) +} + +/// Every `.sql` file under `migrations/` and `data_migrations/`, each +/// directory in filename order. +pub fn shipped_migrations() -> Vec { + let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")); + let mut out = Vec::new(); + for dir in ["migrations", "data_migrations"] { + let mut files: Vec<_> = std::fs::read_dir(root.join(dir)) + .expect("read the migrations directory") + .map(|e| e.expect("directory entry").path()) + .filter(|p| p.extension().is_some_and(|e| e == "sql")) + .collect(); + files.sort(); + for f in files { + out.push(std::fs::read_to_string(&f).expect("read a migration file")); + } + } + out +} + +pub async fn scratch() -> Scratch { + let base = base_conn().expect("caller checks base_conn() first"); + let db_name = format!("eddb_sdis_{}", uuid::Uuid::new_v4().simple())[..24].to_owned(); + let admin = PgPoolOptions::new() + .max_connections(1) + .connect(&format!("{base}/postgres")) + .await + .expect("connect to the postgres maintenance database"); + sqlx::query(&format!("CREATE DATABASE \"{db_name}\"")) + .execute(&admin) + .await + .expect("create the scratch database"); + let url = format!("{base}/{db_name}"); + let db = PgPoolOptions::new() + .max_connections(2) + .connect(&url) + .await + .expect("connect to the scratch database"); + for sql in shipped_migrations() { + sqlx::raw_sql(&sql) + .execute(&db) + .await + .expect("apply a shipped migration"); + } + // One scratch database stands in for both the catalog and the data + // database; drop the foreign key that cannot exist across the two. + sqlx::query("ALTER TABLE stream_shards DROP CONSTRAINT IF EXISTS stream_shards_table_id_fkey") + .execute(&db) + .await + .expect("match the two-database layout"); + sqlx::query( + "INSERT INTO settings (key, value) VALUES ('control_plane_delay_seconds', '0') \ + ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value", + ) + .execute(&db) + .await + .expect("pin the control-plane delay to zero"); + sqlx::query("INSERT INTO accounts (account_id, account_name) VALUES ($1, $2)") + .bind(ACCOUNT) + .bind(format!("acct-{db_name}")) + .execute(&db) + .await + .expect("seed the account row"); + let engine = PostgresEngine::new( + &PostgresConfig { + connection_string: url, + pool_size: 10, + max_item_size_bytes: 400_000, + }, + REGION, + ) + .await + .expect("open a PostgresEngine on the scratch database"); + Scratch { + engine, + db, + admin, + db_name, + } +} + +pub fn table_input(name: &str, stream: bool) -> CreateTableInput { + CreateTableInput { + table_name: name.to_owned(), + key_schema: vec![KeySchemaElement { + attribute_name: "pk".to_owned(), + key_type: KeyType::Hash, + }], + attribute_definitions: vec![AttributeDefinition { + attribute_name: "pk".to_owned(), + attribute_type: ScalarAttributeType::S, + }], + billing_mode: Some(BillingMode::PayPerRequest), + stream_specification: stream.then_some(StreamSpecification { + stream_enabled: true, + stream_view_type: Some(StreamViewType::NewAndOldImages), + }), + ..Default::default() + } +} + +/// Finish the control-plane transition the worker would run. +pub async fn activate(s: &Scratch, name: &str) { + sqlx::query("UPDATE tables SET table_status = 'ACTIVE' WHERE table_name = $1") + .bind(name) + .execute(&s.db) + .await + .expect("activate"); +} + +pub async fn set_stream( + s: &Scratch, + name: &str, + spec: serde_json::Value, +) -> Result { + let input = serde_json::json!({"TableName": name, "StreamSpecification": spec}); + let out = s + .engine + .update_table(ACCOUNT, serde_json::from_value(input).expect("input")) + .await; + activate(s, name).await; + out +} + +pub async fn describe_table(s: &Scratch, name: &str) -> TableDescription { + s.engine + .describe_table( + ACCOUNT, + DescribeTableInput { + table_name: name.to_owned(), + }, + ) + .await + .expect("describe table") +} + +pub async fn describe_stream(s: &Scratch, arn: &str) -> StreamDescription { + s.engine + .describe_stream( + ACCOUNT, + &DescribeStreamInput { + stream_arn: arn.to_owned(), + limit: None, + exclusive_start_shard_id: None, + }, + ) + .await + .expect("describe stream") +} + +pub async fn put(s: &Scratch, name: &str, pk: &str, view_type: StreamViewType) { + let key_info = s + .engine + .table_key_info(ACCOUNT, name) + .await + .expect("key info"); + let item = serde_json::from_value(serde_json::json!({"pk": {"S": pk}})).expect("item"); + let capture = StreamCapture { + view_type, + user_identity: None, + region: REGION.into(), + }; + s.engine + .put_item( + &key_info, + item, + false, + None, + &ExpressionMaps::default(), + Some(&capture), + ) + .await + .expect("put"); +} + +/// Number every committed record still in `stream_pending`, as the numbering +/// worker would: a write's record gets its sequence number after the write +/// commits. +pub async fn number_pending(s: &Scratch) { + s.engine + .number_pending_stream_records_for_tests() + .await + .expect("number the pending stream records"); +} + +/// The records still waiting in `stream_pending` for their numbers. +pub async fn pending_count(s: &Scratch) -> i64 { + sqlx::query_scalar("SELECT COUNT(*) FROM stream_pending") + .fetch_one(&s.db) + .await + .expect("count pending records") +} + +pub async fn last_record(s: &Scratch, shard: &str) -> Option { + number_pending(s).await; + sqlx::query_scalar("SELECT MAX(sequence_number) FROM stream_records WHERE shard_id = $1") + .bind(shard) + .fetch_one(&s.db) + .await + .expect("last record") +} + +pub async fn record_count(s: &Scratch) -> i64 { + number_pending(s).await; + sqlx::query_scalar("SELECT COUNT(*) FROM stream_records") + .fetch_one(&s.db) + .await + .expect("count records") +} + +pub async fn open_shard_ids(s: &Scratch, table_id: &str) -> Vec { + sqlx::query_scalar( + "SELECT shard_id FROM stream_shards \ + WHERE table_id = $1 AND ending_sequence_number IS NULL ORDER BY shard_id", + ) + .bind(table_id) + .fetch_all(&s.db) + .await + .expect("open shard ids") +} diff --git a/crates/storage-postgres/tests/stream_commit_order.rs b/crates/storage-postgres/tests/stream_commit_order.rs index 72f8acd89..4f34f756d 100644 --- a/crates/storage-postgres/tests/stream_commit_order.rs +++ b/crates/storage-postgres/tests/stream_commit_order.rs @@ -42,7 +42,6 @@ use sqlx::{Connection, PgConnection, PgPool}; const ACCOUNT: &str = "123456789012"; const REGION: &str = "us-east-1"; const TABLE: &str = "t_stream_order"; -const LABEL: &str = "2026-01-01T00:00:00.000"; /// The advisory lock key of the gate. const GATE: i64 = 0x5374_4f72; @@ -77,8 +76,7 @@ impl Scratch { } /// The test table, keyed on `pk` and `sk`, with a NEW_AND_OLD_IMAGES - /// stream whose label is pinned to [`LABEL`]. Returns its key info, its - /// table id and its stream ARN. + /// stream. Returns its key info, its table id and its stream ARN. async fn stream_table(&self) -> (TableKeyInfo, String, String) { let key = |name: &str, key_type| { ( @@ -114,12 +112,12 @@ impl Scratch { ) .await .expect("create the table"); - sqlx::query("UPDATE tables SET stream_label = $1 WHERE table_id = $2") - .bind(LABEL) - .bind(&desc.table_id) - .execute(&self.db) - .await - .expect("pin the stream label"); + // The stream's label is carried in its shard ids, so the ARN takes the + // label the table was created with rather than pinning one. + let label = desc + .latest_stream_label + .clone() + .expect("a stream-enabled table has a label"); let key_info = self .engine .table_key_info(ACCOUNT, TABLE) @@ -128,7 +126,7 @@ impl Scratch { ( key_info, desc.table_id, - stream_arn(REGION, ACCOUNT, TABLE, LABEL), + stream_arn(REGION, ACCOUNT, TABLE, &label), ) } diff --git a/crates/storage-postgres/tests/stream_disable.rs b/crates/storage-postgres/tests/stream_disable.rs new file mode 100644 index 000000000..047c7d1d3 --- /dev/null +++ b/crates/storage-postgres/tests/stream_disable.rs @@ -0,0 +1,445 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Disabling a table's stream closes its shards, and enabling it again +//! creates a new stream, against the rows measured on the service on +//! 2026-10-24. +//! +//! Needs `EXTENDDB_TEST_PG_CONNECTION_STRING` (host-only, e.g. +//! `postgres://user:pass@127.0.0.1:5432`); skips otherwise. Each test builds +//! its own scratch database and drops it afterwards. + +#[path = "common/stream_disable_fixture.rs"] +mod fixture; + +use fixture::*; + +#[tokio::test] +async fn disable_closes_every_shard_and_re_enable_creates_a_new_stream() { + if base_conn().is_none() { + eprintln!("SKIP: EXTENDDB_TEST_PG_CONNECTION_STRING is not set"); + return; + } + let s = scratch().await; + s.engine + .create_table(ACCOUNT, table_input("t482", true)) + .await + .expect("create"); + activate(&s, "t482").await; + for pk in ["k0", "k1", "k2"] { + put(&s, "t482", pk, StreamViewType::NewAndOldImages).await; + } + let enabled = describe_table(&s, "t482").await; + let old_arn = enabled.latest_stream_arn.clone().expect("stream arn"); + + let disabled = set_stream(&s, "t482", serde_json::json!({"StreamEnabled": false})) + .await + .expect("disable"); + let mut disabled = disabled; + disabled.prepare_for_response(); + assert!(disabled.stream_specification.is_none()); + assert_eq!( + disabled.latest_stream_arn.as_deref(), + Some(old_arn.as_str()) + ); + let mut table = describe_table(&s, "t482").await; + table.prepare_for_response(); + assert!(table.stream_specification.is_none()); + assert_eq!(table.latest_stream_label, enabled.latest_stream_label); + + let closed = describe_stream(&s, &old_arn).await; + assert!(matches!(closed.stream_status, StreamStatus::Disabled)); + assert_eq!(closed.stream_view_type, StreamViewType::NewAndOldImages); + assert_eq!(closed.shards.len(), 4); + for shard in &closed.shards { + let range = &shard.sequence_number_range; + let expected = last_record(&s, &shard.shard_id) + .await + .unwrap_or_else(|| range.starting_sequence_number.clone()); + assert_eq!( + range.ending_sequence_number.as_deref(), + Some(expected.as_str()) + ); + } + + // A write still carrying the enabled view type records nothing. + put(&s, "t482", "stale", StreamViewType::NewAndOldImages).await; + assert_eq!(record_count(&s).await, 3); + + match set_stream(&s, "t482", serde_json::json!({"StreamEnabled": false})).await { + Err(StorageError::Validation(m)) => { + assert_eq!(m, "Table has no stream to disable: TableName: t482"); + } + other => panic!("{other:?}"), + } + + let reenabled = set_stream( + &s, + "t482", + serde_json::json!({"StreamEnabled": true, "StreamViewType": "KEYS_ONLY"}), + ) + .await + .expect("re-enable"); + let new_arn = reenabled.latest_stream_arn.clone().expect("new arn"); + assert_ne!(new_arn, old_arn); + let new = describe_stream(&s, &new_arn).await; + assert!(matches!(new.stream_status, StreamStatus::Enabled)); + assert_eq!(new.stream_view_type, StreamViewType::KeysOnly); + assert!( + new.shards + .iter() + .all(|n| !closed.shards.iter().any(|o| o.shard_id == n.shard_id)) + ); + match set_stream( + &s, + "t482", + serde_json::json!({"StreamEnabled": true, "StreamViewType": "NEW_IMAGE"}), + ) + .await + { + Err(StorageError::Validation(m)) => { + assert_eq!(m, "Table already has an enabled stream: TableName: t482"); + } + other => panic!("{other:?}"), + } + + put(&s, "t482", "after", StreamViewType::KeysOnly).await; + assert_eq!(record_count(&s).await, 4); + let old_again = describe_stream(&s, &old_arn).await; + assert!(matches!(old_again.stream_status, StreamStatus::Disabled)); + for (a, b) in old_again.shards.iter().zip(&closed.shards) { + assert_eq!(a.shard_id, b.shard_id); + assert_eq!( + a.sequence_number_range.ending_sequence_number, + b.sequence_number_range.ending_sequence_number + ); + } + + let (listed, _) = s + .engine + .list_streams(ACCOUNT, Some("t482"), 100, None) + .await + .expect("list"); + let arns: Vec<&str> = listed.iter().map(|x| x.stream_arn.as_str()).collect(); + assert_eq!(arns, [new_arn.as_str(), old_arn.as_str()]); + match s + .engine + .validate_shard(ACCOUNT, &old_arn, &new.shards[0].shard_id) + .await + { + Err(StorageError::TableNotFound(m)) => { + assert_eq!(m, "Requested resource not found: Shard does not exist"); + } + other => panic!("{other:?}"), + } + s.cleanup().await; +} + +#[tokio::test] +async fn disable_on_a_table_that_never_had_a_stream_is_refused() { + if base_conn().is_none() { + eprintln!("SKIP: EXTENDDB_TEST_PG_CONNECTION_STRING is not set"); + return; + } + let s = scratch().await; + s.engine + .create_table(ACCOUNT, table_input("nostream", false)) + .await + .expect("create"); + activate(&s, "nostream").await; + match set_stream(&s, "nostream", serde_json::json!({"StreamEnabled": false})).await { + Err(StorageError::Validation(m)) => { + assert_eq!(m, "Table has no stream to disable: TableName: nostream"); + } + other => panic!("{other:?}"), + } + // An UpdateTable that enables a stream and is then refused for an index + // change in the same call leaves no shards and no label behind. + let input = serde_json::json!({ + "TableName": "nostream", + "StreamSpecification": {"StreamEnabled": true, "StreamViewType": "KEYS_ONLY"}, + "GlobalSecondaryIndexUpdates": [{"Delete": {"IndexName": "missing"}}], + }); + let refused = s + .engine + .update_table(ACCOUNT, serde_json::from_value(input).expect("input")) + .await; + assert!(refused.is_err(), "{refused:?}"); + let shards: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM stream_shards") + .fetch_one(&s.db) + .await + .expect("count shards"); + assert_eq!(shards, 0); + assert!( + describe_table(&s, "nostream") + .await + .latest_stream_arn + .is_none() + ); + s.cleanup().await; +} + +/// Writers racing a disable: every record a write lands is in the closed +/// stream at or below its shard's `EndingSequenceNumber`, or the write records +/// nothing. The write path locks only its own shard row, `FOR KEY SHARE`, and +/// the close locks the open rows `FOR UPDATE` and numbers their pending +/// records before taking the ending numbers, so a record is never numbered +/// past them. +#[tokio::test] +async fn a_write_racing_a_disable_lands_before_the_ending_number_or_not_at_all() { + if base_conn().is_none() { + eprintln!("SKIP: EXTENDDB_TEST_PG_CONNECTION_STRING is not set"); + return; + } + let s = scratch().await; + s.engine + .create_table(ACCOUNT, table_input("race", true)) + .await + .expect("create"); + activate(&s, "race").await; + let key_info = s + .engine + .table_key_info(ACCOUNT, "race") + .await + .expect("key info"); + let arn = describe_table(&s, "race") + .await + .latest_stream_arn + .expect("stream arn"); + + let mut writers = tokio::task::JoinSet::new(); + for w in 0..8u32 { + let engine = s.engine.clone(); + let key_info = key_info.clone(); + writers.spawn(async move { + let mut acked = 0u32; + let mut failed = 0u32; + for i in 0..40u32 { + let item = + serde_json::from_value(serde_json::json!({"pk": {"S": format!("w{w}-{i}")}})) + .expect("item"); + let capture = StreamCapture { + view_type: StreamViewType::NewAndOldImages, + user_identity: None, + region: REGION.into(), + }; + match engine + .put_item( + &key_info, + item, + false, + None, + &ExpressionMaps::default(), + Some(&capture), + ) + .await + { + Ok(_) => acked += 1, + Err(_) => failed += 1, + } + } + (acked, failed) + }); + } + tokio::time::sleep(std::time::Duration::from_millis(40)).await; + set_stream(&s, "race", serde_json::json!({"StreamEnabled": false})) + .await + .expect("disable"); + let (mut acked, mut failed) = (0u32, 0u32); + while let Some(done) = writers.join_next().await { + let (a, f) = done.expect("writer task"); + acked += a; + failed += f; + } + assert_eq!(acked + failed, 320); + // The close numbered the records of every write that locked its shard + // before the close did, and no write stored a record after the close. A + // record left pending would get a number above its shard's ending number + // once the numbering worker numbered it, so the records are numbered here + // before the check below. + assert_eq!(pending_count(&s).await, 0, "{acked} acked, {failed} failed"); + number_pending(&s).await; + + let closed = describe_stream(&s, &arn).await; + assert!(matches!(closed.stream_status, StreamStatus::Disabled)); + assert!( + closed + .shards + .iter() + .all(|x| x.sequence_number_range.ending_sequence_number.is_some()) + ); + let past_the_end: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM stream_records r JOIN stream_shards s ON s.shard_id = r.shard_id \ + WHERE s.ending_sequence_number IS NULL OR r.sequence_number > s.ending_sequence_number", + ) + .fetch_one(&s.db) + .await + .expect("count records past an ending number"); + assert_eq!(past_the_end, 0, "{acked} acked, {failed} failed"); + let recorded = record_count(&s).await; + assert!( + recorded >= 1 && recorded <= i64::from(acked), + "{recorded} records for {acked} acked writes" + ); + s.cleanup().await; +} + +/// A disable numbers the records waiting in `stream_pending` before it locks +/// the shard rows `FOR UPDATE`, in the numbering worker's transactions, which +/// writers do not wait for. The transaction that closes the shards, whose id +/// is the `xmin` of each closed row, then numbers none of the records. +#[tokio::test] +async fn a_disable_numbers_the_backlog_before_it_locks_the_shards() { + if base_conn().is_none() { + eprintln!("SKIP: EXTENDDB_TEST_PG_CONNECTION_STRING is not set"); + return; + } + let s = scratch().await; + s.engine + .create_table(ACCOUNT, table_input("backlog", true)) + .await + .expect("create"); + activate(&s, "backlog").await; + for pk in ["k0", "k1", "k2", "k3", "k4", "k5"] { + put(&s, "backlog", pk, StreamViewType::NewAndOldImages).await; + } + assert_eq!(pending_count(&s).await, 6); + + set_stream(&s, "backlog", serde_json::json!({"StreamEnabled": false})) + .await + .expect("disable"); + assert_eq!(pending_count(&s).await, 0); + let numbered_by_the_close: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM stream_records r JOIN stream_shards s ON s.shard_id = r.shard_id \ + WHERE r.xmin = s.xmin", + ) + .fetch_one(&s.db) + .await + .expect("count the records the closing transaction numbered"); + assert_eq!(numbered_by_the_close, 0); + let past_the_end: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM stream_records r JOIN stream_shards s ON s.shard_id = r.shard_id \ + WHERE s.ending_sequence_number IS NULL OR r.sequence_number > s.ending_sequence_number", + ) + .fetch_one(&s.db) + .await + .expect("count records past an ending number"); + assert_eq!(past_the_end, 0); + assert_eq!(record_count(&s).await, 6); + s.cleanup().await; +} + +/// A disable whose close failed after its catalog write, then an enable whose +/// close of the label it read failed too: the table is enabled under a later +/// label with no shards of its own, and the earlier stream's shards are still +/// open. The next enable closes every earlier stream's open shards, whatever +/// label it read. +#[tokio::test] +async fn an_enable_closes_an_earlier_stream_left_open() { + if base_conn().is_none() { + eprintln!("SKIP: EXTENDDB_TEST_PG_CONNECTION_STRING is not set"); + return; + } + let s = scratch().await; + s.engine + .create_table(ACCOUNT, table_input("leftopen", true)) + .await + .expect("create"); + activate(&s, "leftopen").await; + put(&s, "leftopen", "k0", StreamViewType::NewAndOldImages).await; + let table = describe_table(&s, "leftopen").await; + let first_arn = table.latest_stream_arn.clone().expect("stream arn"); + let first = describe_stream(&s, &first_arn).await; + let first_ids: Vec = first.shards.iter().map(|x| x.shard_id.clone()).collect(); + let later = + extenddb_storage::stream_lineage::next_stream_label(Some(first.stream_label.as_str())); + sqlx::query("UPDATE tables SET stream_label = $1 WHERE table_name = 'leftopen'") + .bind(&later) + .execute(&s.db) + .await + .expect("move the label on, as the enable's catalog write did"); + assert_eq!(open_shard_ids(&s, &table.table_id).await, first_ids); + + set_stream(&s, "leftopen", serde_json::json!({"StreamEnabled": false})) + .await + .expect("disable"); + let reenabled = set_stream( + &s, + "leftopen", + serde_json::json!({"StreamEnabled": true, "StreamViewType": "KEYS_ONLY"}), + ) + .await + .expect("enable"); + let new = describe_stream(&s, &reenabled.latest_stream_arn.expect("new arn")).await; + let new_ids: Vec = new.shards.iter().map(|x| x.shard_id.clone()).collect(); + assert_eq!( + open_shard_ids(&s, &table.table_id).await, + new_ids, + "only the new stream's shards are open" + ); + for id in &first_ids { + let (start, end): (String, Option) = sqlx::query_as( + "SELECT starting_sequence_number, ending_sequence_number \ + FROM stream_shards WHERE shard_id = $1", + ) + .bind(id) + .fetch_one(&s.db) + .await + .expect("shard row"); + let expected = last_record(&s, id).await.unwrap_or(start); + assert_eq!(end.as_deref(), Some(expected.as_str())); + } + s.cleanup().await; +} + +/// A disable that read stream A and applies its shard changes only after an +/// overlapping enable has closed A and created stream B: it closes A's +/// shards, which are closed already, and leaves B's open. +#[tokio::test] +async fn a_disable_applying_after_an_overlapping_enable_leaves_the_new_stream_open() { + if base_conn().is_none() { + eprintln!("SKIP: EXTENDDB_TEST_PG_CONNECTION_STRING is not set"); + return; + } + let s = scratch().await; + s.engine + .create_table(ACCOUNT, table_input("overlap", true)) + .await + .expect("create"); + activate(&s, "overlap").await; + put(&s, "overlap", "k0", StreamViewType::NewAndOldImages).await; + let table = describe_table(&s, "overlap").await; + let old_label = table.latest_stream_label.clone().expect("label"); + set_stream(&s, "overlap", serde_json::json!({"StreamEnabled": false})) + .await + .expect("disable"); + let closed = describe_stream(&s, &table.latest_stream_arn.clone().expect("arn")).await; + let reenabled = set_stream( + &s, + "overlap", + serde_json::json!({"StreamEnabled": true, "StreamViewType": "KEYS_ONLY"}), + ) + .await + .expect("enable"); + let new = describe_stream(&s, &reenabled.latest_stream_arn.expect("new arn")).await; + let new_ids: Vec = new.shards.iter().map(|x| x.shard_id.clone()).collect(); + put(&s, "overlap", "k1", StreamViewType::KeysOnly).await; + + s.engine + .apply_disable_shard_changes_for_tests(ACCOUNT, "overlap", &table.table_id, &old_label) + .await + .expect("the late disable's shard changes"); + assert_eq!( + open_shard_ids(&s, &table.table_id).await, + new_ids, + "the new stream's four shards stay open" + ); + let old_again = describe_stream(&s, &closed.stream_arn).await; + for (a, b) in old_again.shards.iter().zip(&closed.shards) { + assert_eq!( + a.sequence_number_range.ending_sequence_number, + b.sequence_number_range.ending_sequence_number + ); + } + s.cleanup().await; +} diff --git a/crates/storage-postgres/tests/stream_disable_legacy.rs b/crates/storage-postgres/tests/stream_disable_legacy.rs new file mode 100644 index 000000000..fc07646c2 --- /dev/null +++ b/crates/storage-postgres/tests/stream_disable_legacy.rs @@ -0,0 +1,261 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Streams an earlier release left behind: the name-based shards of a +//! stream created before v0.1.14, and a disabled stream whose shards stayed +//! open. +//! +//! Needs `EXTENDDB_TEST_PG_CONNECTION_STRING` (host-only, e.g. +//! `postgres://user:pass@127.0.0.1:5432`); skips otherwise. Each test builds +//! its own scratch database and drops it afterwards. + +#[path = "common/stream_disable_fixture.rs"] +mod fixture; + +use fixture::*; + +/// A stream-enabled table created under v0.1.13 or earlier has four shard +/// rows named `shardId-{table_name}-{index:016}`. Its stream stays +/// describable, readable and listed after the upgrade, keeps its view type +/// once disabled, and is replaced by a labelled stream on re-enable. +#[tokio::test] +async fn a_stream_from_before_v0_1_14_survives_the_upgrade() { + if base_conn().is_none() { + eprintln!("SKIP: EXTENDDB_TEST_PG_CONNECTION_STRING is not set"); + return; + } + let s = scratch().await; + s.engine + .create_table(ACCOUNT, table_input("legacy", false)) + .await + .expect("create"); + activate(&s, "legacy").await; + let table_id = describe_table(&s, "legacy").await.table_id; + // What a pre-v0.1.14 release wrote: the label and specification on the + // catalog row, four name-based open shards in the data database. + let label = "2026-10-20T10:00:00.000"; + sqlx::query( + "UPDATE tables SET stream_label = $1, stream_specification = $2 \ + WHERE account_id = $3 AND table_name = 'legacy'", + ) + .bind(label) + .bind(serde_json::json!({"StreamEnabled": true, "StreamViewType": "NEW_IMAGE"})) + .bind(ACCOUNT) + .execute(&s.db) + .await + .expect("seed the catalog row"); + for i in 0..4 { + sqlx::query( + "INSERT INTO stream_shards (shard_id, table_id, starting_sequence_number) \ + VALUES ($1, $2, $3)", + ) + .bind(format!("shardId-legacy-{i:016}")) + .bind(&table_id) + .bind(format!("{:021}", 0)) + .execute(&s.db) + .await + .expect("seed a legacy shard"); + } + let arn = describe_table(&s, "legacy") + .await + .latest_stream_arn + .expect("the seeded stream arn"); + assert!(arn.ends_with(label)); + + put(&s, "legacy", "k", StreamViewType::NewImage).await; + let open = describe_stream(&s, &arn).await; + assert!(matches!(open.stream_status, StreamStatus::Enabled)); + assert_eq!(open.stream_view_type, StreamViewType::NewImage); + let ids: Vec<&str> = open.shards.iter().map(|x| x.shard_id.as_str()).collect(); + assert_eq!( + ids, + (0..4) + .map(|i| format!("shardId-legacy-{i:016}")) + .collect::>() + ); + s.engine + .validate_shard(ACCOUNT, &arn, "shardId-legacy-0000000000000001") + .await + .expect("a legacy shard belongs to the stream"); + assert_eq!(record_count(&s).await, 1); + let (listed, _) = s + .engine + .list_streams(ACCOUNT, Some("legacy"), 100, None) + .await + .expect("list"); + assert_eq!(listed.len(), 1); + assert_eq!(listed[0].stream_arn, arn); + + set_stream(&s, "legacy", serde_json::json!({"StreamEnabled": false})) + .await + .expect("disable"); + let closed = describe_stream(&s, &arn).await; + assert!(matches!(closed.stream_status, StreamStatus::Disabled)); + assert_eq!( + closed.stream_view_type, + StreamViewType::NewImage, + "a disabled legacy stream keeps its view type" + ); + assert!( + closed + .shards + .iter() + .all(|x| x.sequence_number_range.ending_sequence_number.is_some()) + ); + // The backend keeps the disabled specification (with the view type) and + // the engine omits it from every response. + let stored = describe_table(&s, "legacy").await.stream_specification; + assert_eq!( + stored, + Some(StreamSpecification { + stream_enabled: false, + stream_view_type: Some(StreamViewType::NewImage), + }) + ); + let mut rendered = describe_table(&s, "legacy").await; + rendered.prepare_for_response(); + assert!(rendered.stream_specification.is_none()); + put(&s, "legacy", "dark", StreamViewType::NewImage).await; + assert_eq!( + record_count(&s).await, + 1, + "a write after the disable records nothing" + ); + + let re = set_stream( + &s, + "legacy", + serde_json::json!({"StreamEnabled": true, "StreamViewType": "KEYS_ONLY"}), + ) + .await + .expect("re-enable"); + let new_arn = re.latest_stream_arn.expect("new arn"); + assert_ne!(new_arn, arn); + let new = describe_stream(&s, &new_arn).await; + assert!(matches!(new.stream_status, StreamStatus::Enabled)); + assert_eq!(new.shards.len(), 4); + assert!(new.shards.iter().all(|x| x.shard_id.len() == 61)); + // The enable closed the four legacy rows: none is still open. + let open_legacy: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM stream_shards \ + WHERE shard_id LIKE 'shardId-legacy-%' AND ending_sequence_number IS NULL", + ) + .fetch_one(&s.db) + .await + .expect("count open legacy shards"); + assert_eq!( + open_legacy, 0, + "the enable closes the legacy stream's shards" + ); + s.cleanup().await; +} + +/// A stream disabled by a release that stored `{StreamEnabled: false}` and +/// left its four name-based shards open (the same state a close that failed +/// after its catalog write leaves): writes still record into it. A disable is +/// refused, as on the service, and closes the open shards first, so the next +/// write records nothing. +#[tokio::test] +async fn a_disable_refused_on_a_disabled_stream_closes_the_shards_it_left_open() { + if base_conn().is_none() { + eprintln!("SKIP: EXTENDDB_TEST_PG_CONNECTION_STRING is not set"); + return; + } + let s = scratch().await; + s.engine + .create_table(ACCOUNT, table_input("base", false)) + .await + .expect("create"); + activate(&s, "base").await; + let table_id = describe_table(&s, "base").await.table_id; + let label = "2026-10-20T10:00:00.000"; + sqlx::query( + "UPDATE tables SET stream_label = $1, stream_specification = $2 \ + WHERE account_id = $3 AND table_name = 'base'", + ) + .bind(label) + .bind(serde_json::json!({"StreamEnabled": false})) + .bind(ACCOUNT) + .execute(&s.db) + .await + .expect("seed the catalog row"); + for i in 0..4 { + sqlx::query( + "INSERT INTO stream_shards (shard_id, table_id, starting_sequence_number) \ + VALUES ($1, $2, $3)", + ) + .bind(format!("shardId-base-{i:016}")) + .bind(&table_id) + .bind(format!("{:021}", 0)) + .execute(&s.db) + .await + .expect("seed an open shard"); + } + let open_shards = || async { + sqlx::query_scalar::<_, i64>( + "SELECT COUNT(*) FROM stream_shards WHERE ending_sequence_number IS NULL", + ) + .fetch_one(&s.db) + .await + .expect("count open shards") + }; + let arn = describe_table(&s, "base") + .await + .latest_stream_arn + .expect("the seeded stream arn"); + assert!(matches!( + describe_stream(&s, &arn).await.stream_status, + StreamStatus::Disabled + )); + put(&s, "base", "dark", StreamViewType::KeysOnly).await; + assert_eq!(record_count(&s).await, 1, "the open shards still record"); + + match set_stream(&s, "base", serde_json::json!({"StreamEnabled": false})).await { + Err(StorageError::Validation(m)) => { + assert_eq!(m, "Table has no stream to disable: TableName: base"); + } + other => panic!("{other:?}"), + } + assert_eq!( + open_shards().await, + 0, + "the refused disable closed the shards" + ); + let closed = describe_stream(&s, &arn).await; + assert!(matches!(closed.stream_status, StreamStatus::Disabled)); + for shard in &closed.shards { + let expected = last_record(&s, &shard.shard_id) + .await + .unwrap_or_else(|| shard.sequence_number_range.starting_sequence_number.clone()); + assert_eq!( + shard + .sequence_number_range + .ending_sequence_number + .as_deref(), + Some(expected.as_str()) + ); + } + put(&s, "base", "after", StreamViewType::KeysOnly).await; + assert_eq!( + record_count(&s).await, + 1, + "a write after the close records nothing" + ); + // The stored specification is untouched, and a second refusal has nothing + // left to close. + assert_eq!( + describe_table(&s, "base").await.stream_specification, + Some(StreamSpecification { + stream_enabled: false, + stream_view_type: None, + }) + ); + match set_stream(&s, "base", serde_json::json!({"StreamEnabled": false})).await { + Err(StorageError::Validation(m)) => { + assert_eq!(m, "Table has no stream to disable: TableName: base"); + } + other => panic!("{other:?}"), + } + assert_eq!(open_shards().await, 0); + s.cleanup().await; +} diff --git a/crates/storage-postgres/tests/stream_disable_lock_order.rs b/crates/storage-postgres/tests/stream_disable_lock_order.rs new file mode 100644 index 000000000..26e809adc --- /dev/null +++ b/crates/storage-postgres/tests/stream_disable_lock_order.rs @@ -0,0 +1,341 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! A disable and a `TransactWriteItems` lock the rows of a stream's shards in +//! one order, shard id order. A transaction that stores records on two shards +//! of a stream therefore cannot deadlock with the close of that stream. +//! +//! A trigger on `stream_pending` waits on an advisory lock, the gate. Each +//! test uses it to hold the transaction at its first stream record, after the +//! transaction locked that record's shard row. The test then starts the +//! disable. When the close waits for the transaction, the test opens the gate. +//! If the two lock in different orders, the transaction then waits for a row +//! that the close holds. PostgreSQL then cancels one of them after +//! `deadlock_timeout`. +//! +//! Needs `EXTENDDB_TEST_PG_CONNECTION_STRING` (host-only, e.g. +//! `postgres://user:pass@127.0.0.1:5432`), and skips without it. Each test +//! builds its own scratch database and drops it when it passes. A failing +//! test leaves its database, named `eddb_sdis_*`, behind. + +#[path = "common/stream_disable_fixture.rs"] +mod fixture; + +use std::time::Duration; + +use extenddb_core::types::{Item, ReturnValuesOnConditionCheckFailure, TableKeyInfo}; +use extenddb_storage::TransactWriteOp; +use fixture::*; +use sqlx::{Connection, PgConnection}; + +/// The advisory lock key of the gate. +const GATE: i64 = 0x4c6b_4f72; + +/// What a raced transaction and disable returned. +type Raced = ( + Result<(), StorageError>, + Result, +); + +/// Open a connection to the scratch database outside its pools. Returns it and +/// its backend pid. +async fn connect(s: &Scratch) -> (PgConnection, i32) { + let base = base_conn().expect("caller checks base_conn() first"); + let mut conn = PgConnection::connect(&format!("{base}/{}", s.db_name)) + .await + .expect("connect to the scratch database"); + let pid: i32 = sqlx::query_scalar("SELECT pg_backend_pid()") + .fetch_one(&mut conn) + .await + .expect("read the backend pid"); + (conn, pid) +} + +/// Make a write wait at the end of each stream record it stores while the gate +/// is closed. The write has locked the record's shard row by then. +async fn install_record_gate(s: &Scratch) { + sqlx::raw_sql(&format!( + "CREATE FUNCTION hold_record() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN \ + PERFORM pg_advisory_xact_lock_shared({GATE}); \ + RETURN NULL; \ + END $$; \ + CREATE TRIGGER hold_record AFTER INSERT ON stream_pending \ + FOR EACH ROW EXECUTE FUNCTION hold_record();" + )) + .execute(&s.db) + .await + .expect("install the gate trigger"); +} + +/// Close the gate on a session of its own. Returns the session and its pid. +async fn close_gate(s: &Scratch) -> (PgConnection, i32) { + let (mut gate, pid) = connect(s).await; + sqlx::query("SELECT pg_advisory_lock($1)") + .bind(GATE) + .execute(&mut gate) + .await + .expect("close the gate"); + (gate, pid) +} + +async fn open_gate(gate: &mut PgConnection) { + sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(GATE) + .execute(gate) + .await + .expect("open the gate"); +} + +/// The pid of a backend that waits for a lock of kind `event` that the backend +/// `blocker` holds. Panics after 10 s. +async fn blocked_by(db: &PgPool, blocker: i32, event: &str) -> i32 { + for _ in 0..1_000 { + let pid: Option = sqlx::query_scalar( + "SELECT pid FROM pg_stat_activity \ + WHERE datname = current_database() AND wait_event_type = 'Lock' \ + AND wait_event = $2 AND $1 = ANY(pg_blocking_pids(pid)) LIMIT 1", + ) + .bind(blocker) + .bind(event) + .fetch_optional(db) + .await + .expect("read pg_stat_activity"); + if let Some(pid) = pid { + return pid; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + panic!("no backend waited on the {event} lock of backend {blocker}"); +} + +/// The open shards of the table, in byte order. +async fn shards_in_order(s: &Scratch, table_id: &str) -> Vec { + let mut ids = open_shard_ids(s, table_id).await; + ids.sort(); + ids +} + +/// A partition key whose records go to `shard` of table `name`. +async fn key_on(s: &Scratch, name: &str, shard: &str) -> String { + for i in 0..1_000 { + let pk = format!("k{i}"); + let assigned = s + .engine + .assign_shard(ACCOUNT, name, &pk) + .await + .expect("assign a shard"); + if assigned == shard { + return pk; + } + } + panic!("none of 1,000 keys goes to shard {shard}"); +} + +fn put_op<'a>( + key_info: &'a TableKeyInfo, + item: &'a Item, + maps: &'a ExpressionMaps, +) -> TransactWriteOp<'a> { + TransactWriteOp::Put { + key_info, + item, + condition: None, + maps, + return_values_on_ccf: ReturnValuesOnConditionCheckFailure::None, + stream: Some(StreamCapture { + view_type: StreamViewType::NewAndOldImages, + user_identity: None, + region: REGION.into(), + }), + } +} + +/// Race a `TransactWriteItems` that puts the items `pks`, in that order, +/// against a disable of the table's stream. The gate holds the transaction at +/// its first stream record until the close waits for it. +async fn race_a_disable(s: &Scratch, name: &str, pks: [&str; 2]) -> Raced { + let key_info = s + .engine + .table_key_info(ACCOUNT, name) + .await + .expect("key info"); + install_record_gate(s).await; + let (mut gate, gate_pid) = close_gate(s).await; + + let items: Vec = pks + .iter() + .map(|pk| serde_json::from_value(serde_json::json!({"pk": {"S": pk}})).expect("item")) + .collect(); + let engine = s.engine.clone(); + let transaction = tokio::spawn(async move { + let maps = ExpressionMaps::default(); + let ops: Vec> = items + .iter() + .map(|item| put_op(&key_info, item, &maps)) + .collect(); + engine.transact_write_items(&ops, None).await + }); + let writer = blocked_by(&s.db, gate_pid, "advisory").await; + + let input = serde_json::json!({ + "TableName": name, + "StreamSpecification": {"StreamEnabled": false}, + }); + let engine = s.engine.clone(); + let disable = tokio::spawn(async move { + engine + .update_table(ACCOUNT, serde_json::from_value(input).expect("input")) + .await + }); + // The close waits for the shard row the transaction has locked. + blocked_by(&s.db, writer, "transactionid").await; + open_gate(&mut gate).await; + + let written = transaction.await.expect("the transaction task"); + let disabled = disable.await.expect("the disable task"); + activate(s, name).await; + (written, disabled) +} + +/// Both calls succeeded, the close numbered the transaction's two records, and +/// each is in the closed stream at or below its shard's ending number. +async fn assert_closed_with_both_records(s: &Scratch, raced: Raced, arn: &str, shards: [&str; 2]) { + let (written, disabled) = raced; + written.expect("the transaction commits"); + disabled.expect("the disable succeeds"); + // The close waited for the transaction to commit, so its numbering under + // the lock numbered both records. + assert_eq!(pending_count(s).await, 0); + let closed = describe_stream(s, arn).await; + assert!(matches!(closed.stream_status, StreamStatus::Disabled)); + assert!( + closed + .shards + .iter() + .all(|x| x.sequence_number_range.ending_sequence_number.is_some()) + ); + let past_the_end: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM stream_records r JOIN stream_shards s ON s.shard_id = r.shard_id \ + WHERE s.ending_sequence_number IS NULL OR r.sequence_number > s.ending_sequence_number", + ) + .fetch_one(&s.db) + .await + .expect("count records past an ending number"); + assert_eq!(past_the_end, 0); + let mut recorded: Vec = sqlx::query_scalar("SELECT shard_id FROM stream_records") + .fetch_all(&s.db) + .await + .expect("the shards of the records"); + recorded.sort(); + let mut expected = shards.map(str::to_owned).to_vec(); + expected.sort(); + assert_eq!(recorded, expected); +} + +/// The items are in descending shard order. With the records in request +/// order, the transaction would hold the higher shard. The close would hold +/// the lower shard and wait for the higher one. The transaction would then +/// wait for the lower shard. +#[tokio::test] +async fn a_transaction_with_records_on_two_shards_in_descending_order_does_not_deadlock_a_disable() +{ + if base_conn().is_none() { + eprintln!("SKIP: EXTENDDB_TEST_PG_CONNECTION_STRING is not set"); + return; + } + let s = scratch().await; + s.engine + .create_table(ACCOUNT, table_input("lockdesc", true)) + .await + .expect("create"); + activate(&s, "lockdesc").await; + let table = describe_table(&s, "lockdesc").await; + let arn = table.latest_stream_arn.clone().expect("stream arn"); + let shards = shards_in_order(&s, &table.table_id).await; + let (low, high) = (shards[0].as_str(), shards[3].as_str()); + let (pk_low, pk_high) = ( + key_on(&s, "lockdesc", low).await, + key_on(&s, "lockdesc", high).await, + ); + + let raced = race_a_disable(&s, "lockdesc", [&pk_high, &pk_low]).await; + assert_closed_with_both_records(&s, raced, &arn, [low, high]).await; + s.cleanup().await; +} + +/// With index scans off, the close's statement reads the rows in the order +/// the table stores them. An update stores a new version of the lowest shard's +/// row after the other three, so the statement reads that row last. Without +/// its ORDER BY, the close would lock the other three rows and wait for the +/// lowest. The transaction would hold the lowest and wait for the second. +#[tokio::test] +async fn a_disable_locks_its_shard_rows_in_shard_id_order_whatever_order_they_are_stored_in() { + if base_conn().is_none() { + eprintln!("SKIP: EXTENDDB_TEST_PG_CONNECTION_STRING is not set"); + return; + } + let mut s = scratch().await; + s.engine + .create_table(ACCOUNT, table_input("lockscan", true)) + .await + .expect("create"); + activate(&s, "lockscan").await; + let table = describe_table(&s, "lockscan").await; + let arn = table.latest_stream_arn.clone().expect("stream arn"); + let shards = shards_in_order(&s, &table.table_id).await; + let (first, second) = (shards[0].as_str(), shards[1].as_str()); + let (pk_first, pk_second) = ( + key_on(&s, "lockscan", first).await, + key_on(&s, "lockscan", second).await, + ); + + sqlx::query( + "UPDATE stream_shards SET starting_sequence_number = starting_sequence_number \ + WHERE shard_id = $1", + ) + .bind(first) + .execute(&s.db) + .await + .expect("store a new version of the lowest shard's row"); + for setting in [ + "enable_indexscan", + "enable_indexonlyscan", + "enable_bitmapscan", + ] { + sqlx::query(&format!( + "ALTER DATABASE \"{}\" SET {setting} = off", + s.db_name + )) + .execute(&s.admin) + .await + .expect("turn index scans off"); + } + // The engine's pooled sessions predate the settings. An engine opened now + // takes them. + s.engine = PostgresEngine::new( + &PostgresConfig { + connection_string: format!("{}/{}", base_conn().expect("checked above"), s.db_name), + pool_size: 10, + max_item_size_bytes: 400_000, + }, + REGION, + ) + .await + .expect("open a PostgresEngine with index scans off"); + let (mut scan, _) = connect(&s).await; + let read: Vec = sqlx::query_scalar( + "SELECT shard_id FROM stream_shards \ + WHERE shard_id = ANY($1) AND ending_sequence_number IS NULL", + ) + .bind(&shards) + .fetch_all(&mut scan) + .await + .expect("read the shard rows as the close's statement reads them"); + assert_eq!(read.last().map(String::as_str), Some(first), "{read:?}"); + + // Request order is shard id order, the order the transaction stores its + // records in. + let raced = race_a_disable(&s, "lockscan", [&pk_first, &pk_second]).await; + assert_closed_with_both_records(&s, raced, &arn, [first, second]).await; + s.cleanup().await; +} diff --git a/crates/storage-postgres/tests/stream_records_page_order.rs b/crates/storage-postgres/tests/stream_records_page_order.rs index 06ba9e33d..38fad92d0 100644 --- a/crates/storage-postgres/tests/stream_records_page_order.rs +++ b/crates/storage-postgres/tests/stream_records_page_order.rs @@ -23,7 +23,6 @@ use sqlx::postgres::PgPoolOptions; const ACCOUNT: &str = "111111111111"; const REGION: &str = "us-east-1"; -const LABEL: &str = "2026-01-01T00:00:00.000"; /// The server's base URL, or `None` to skip. With `EXTENDDB_TEST_PG_REQUIRED` /// set to any value but `0`, a job that must reach a server fails here @@ -159,12 +158,12 @@ async fn a_page_is_in_sequence_order_when_the_records_were_written_in_reverse() .await .expect("create a stream-enabled table"); let table_id = desc.table_id.clone(); - sqlx::query("UPDATE tables SET stream_label = $1 WHERE table_id = $2") - .bind(LABEL) - .bind(&table_id) - .execute(&db) - .await - .expect("pin the stream label"); + // The stream's label is carried in its shard ids, so the ARN takes the + // label the table was created with rather than pinning one. + let label = desc + .latest_stream_label + .clone() + .expect("a stream-enabled table has a label"); let shard: String = sqlx::query_scalar( "SELECT shard_id FROM stream_shards WHERE table_id = $1 ORDER BY shard_id LIMIT 1", ) @@ -193,7 +192,7 @@ async fn a_page_is_in_sequence_order_when_the_records_were_written_in_reverse() .await .expect("analyze the records"); - let arn = stream_arn(REGION, ACCOUNT, "t", LABEL); + let arn = stream_arn(REGION, ACCOUNT, "t", &label); let want = |from: u64| -> Vec { (from..from + 3).map(|n| format!("{n:021}")).collect() }; diff --git a/crates/storage-postgres/tests/stream_shard_ownership.rs b/crates/storage-postgres/tests/stream_shard_ownership.rs index 6ff100327..b5d579a08 100644 --- a/crates/storage-postgres/tests/stream_shard_ownership.rs +++ b/crates/storage-postgres/tests/stream_shard_ownership.rs @@ -24,6 +24,7 @@ use extenddb_core::types::{ ScalarAttributeType, StreamSpecification, StreamViewType, }; use extenddb_storage::error::StorageError; +use extenddb_storage::stream_lineage::{ShardStream, parse_shard_id, shard_id}; use extenddb_storage::util::stream_arn; use extenddb_storage::{StreamEngine, StreamRecordsResult, TableEngine}; use extenddb_storage_postgres::stream_records_sql::{ @@ -73,20 +74,19 @@ impl Scratch { .create_table(account, stream_table(name)) .await .expect("create a stream-enabled table"); - sqlx::query("UPDATE tables SET stream_label = $1 WHERE table_id = $2") - .bind(label) - .bind(&desc.table_id) - .execute(&self.db) - .await - .expect("pin the stream label"); - let shard: String = sqlx::query_scalar( + self.relabel(&desc.table_id, label).await; + let shard = self.first_shard(&desc.table_id).await; + (desc.table_id, shard) + } + + async fn first_shard(&self, table_id: &str) -> String { + sqlx::query_scalar( "SELECT shard_id FROM stream_shards WHERE table_id = $1 ORDER BY shard_id LIMIT 1", ) - .bind(&desc.table_id) + .bind(table_id) .fetch_one(&self.db) .await - .expect("one shard id"); - (desc.table_id, shard) + .expect("one shard id") } async fn write_records(&self, shard_id: &str, table_id: &str, seqs: &[u64]) { @@ -130,7 +130,48 @@ impl Scratch { .expect("analyze the records"); } + /// Pin the table's stream label to `label`. A shard id carries its + /// stream's label, so each shard row of the table, and the records keyed + /// on it, move to the id the stream `label` gives its index. async fn relabel(&self, table_id: &str, label: &str) { + let ids: Vec = + sqlx::query_scalar("SELECT shard_id FROM stream_shards WHERE table_id = $1") + .bind(table_id) + .fetch_all(&self.db) + .await + .expect("shard ids"); + for old in ids { + let Some((ShardStream::Labelled { view_type, .. }, index)) = + parse_shard_id(table_id, &old) + else { + panic!("{old} is not a labelled shard of {table_id}"); + }; + let new = shard_id(table_id, label, view_type, index).expect("label shape"); + // The records reference the shard row, so the new row comes first + // and the old one goes last. + sqlx::query( + "INSERT INTO stream_shards (shard_id, table_id, parent_shard_id, \ + starting_sequence_number, ending_sequence_number) \ + SELECT $1, table_id, parent_shard_id, starting_sequence_number, \ + ending_sequence_number FROM stream_shards WHERE shard_id = $2", + ) + .bind(&new) + .bind(&old) + .execute(&self.db) + .await + .expect("insert the relabelled shard row"); + sqlx::query("UPDATE stream_records SET shard_id = $1 WHERE shard_id = $2") + .bind(&new) + .bind(&old) + .execute(&self.db) + .await + .expect("move the shard's records"); + sqlx::query("DELETE FROM stream_shards WHERE shard_id = $1") + .bind(&old) + .execute(&self.db) + .await + .expect("remove the old shard row"); + } sqlx::query("UPDATE tables SET stream_label = $1 WHERE table_id = $2") .bind(label) .bind(table_id) @@ -420,8 +461,9 @@ async fn another_accounts_same_named_stream_is_not_found() { s.cleanup().await; } -/// The table takes a new stream label while keeping its shard rows: the old -/// ARN's iterator is refused and the new ARN reads every record. +/// The table's stream takes a new label while keeping its records: the old +/// ARN's iterator, and the old shard id under the new ARN, are refused, and +/// the new stream's shard reads every record. #[tokio::test] async fn an_earlier_label_on_the_same_shards_is_not_found() { if base_conn().is_none() { @@ -438,16 +480,23 @@ async fn an_earlier_label_on_the_same_shards_is_not_found() { .expect("current label reads"); s.relabel(&table_id, LATER_LABEL).await; - s.write_records(&shard, &table_id, &[2]).await; + let new_shard = s.first_shard(&table_id).await; + assert_ne!(new_shard, shard); + s.write_records(&new_shard, &table_id, &[2]).await; + let new_arn = stream_arn(REGION, ACCOUNT, "t", LATER_LABEL); assert_stream_not_found( s.engine .get_stream_records(ACCOUNT, &old_arn, &shard, None, 10) .await, ); - let new_arn = stream_arn(REGION, ACCOUNT, "t", LATER_LABEL); + assert_stream_not_found( + s.engine + .get_stream_records(ACCOUNT, &new_arn, &shard, None, 10) + .await, + ); let (records, _) = s .engine - .get_stream_records(ACCOUNT, &new_arn, &shard, None, 10) + .get_stream_records(ACCOUNT, &new_arn, &new_shard, None, 10) .await .expect("new label reads"); assert_eq!(records.len(), 2); diff --git a/crates/storage-sqlite/src/create_table.rs b/crates/storage-sqlite/src/create_table.rs index 509e73086..d936472a4 100644 --- a/crates/storage-sqlite/src/create_table.rs +++ b/crates/storage-sqlite/src/create_table.rs @@ -73,6 +73,7 @@ impl SqliteEngine { restore: Option<&RestoreRegistration<'_>>, ) -> Result { Self::validate_account_id(account_id)?; + let input = extenddb_storage::stream_lineage::without_disabled_specification(input); // D1: every writer holds the engine write lock. This method runs two // write transactions (catalog rows, then data-table DDL); without the // lock they contend with data-plane writers at the SQLite level, and a @@ -421,14 +422,23 @@ impl SqliteEngine { } // Stream shards (same transaction — single file). - let stream_label = if input + let stream_label = match input .stream_specification .as_ref() - .is_some_and(|s| s.stream_enabled) + .filter(|s| s.stream_enabled) + .and_then(|s| s.stream_view_type) { - Some(Self::init_stream_shards(&mut tx, account_id, &input.table_name, &table_id).await?) - } else { - None + Some(view_type) => Some( + Self::init_stream_shards( + &mut tx, + account_id, + &input.table_name, + &table_id, + view_type, + ) + .await?, + ), + None => None, }; tx.commit() diff --git a/crates/storage-sqlite/src/data/tx_helpers.rs b/crates/storage-sqlite/src/data/tx_helpers.rs index 0ee6ba208..0af8b654e 100644 --- a/crates/storage-sqlite/src/data/tx_helpers.rs +++ b/crates/storage-sqlite/src/data/tx_helpers.rs @@ -217,12 +217,17 @@ pub(super) async fn write_stream_record_in_tx( }) .unwrap_or_default(); - let shards: Vec<(String,)> = - sqlx::query_as("SELECT shard_id FROM stream_shards WHERE table_id = ? ORDER BY shard_id") - .bind(&key_info.table_id) - .fetch_all(&mut **tx) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; + // Only the open shards: they belong to the table's enabled stream. A + // disabled stream's shards are closed and take no new records, so a write + // that read the table's key info before a disable records nothing. + let shards: Vec<(String,)> = sqlx::query_as( + "SELECT shard_id FROM stream_shards \ + WHERE table_id = ? AND ending_sequence_number IS NULL ORDER BY shard_id", + ) + .bind(&key_info.table_id) + .fetch_all(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; if shards.is_empty() { return Ok(()); // Streams not enabled for this table. } diff --git a/crates/storage-sqlite/src/lib.rs b/crates/storage-sqlite/src/lib.rs index 7c2188096..3fa65f642 100644 --- a/crates/storage-sqlite/src/lib.rs +++ b/crates/storage-sqlite/src/lib.rs @@ -44,6 +44,7 @@ mod snapshot_engine; mod sqlite_util; mod store; mod stream; +mod stream_lifecycle; #[cfg(test)] mod stream_tests; mod table_engine; diff --git a/crates/storage-sqlite/src/stream.rs b/crates/storage-sqlite/src/stream.rs index 5b00eabfb..c273d9a6f 100644 --- a/crates/storage-sqlite/src/stream.rs +++ b/crates/storage-sqlite/src/stream.rs @@ -4,16 +4,14 @@ //! `StreamEngine` trait implementation for `SqliteEngine`. //! //! Shards, records, and the sequence counter all live in the one SQLite file, -//! so `init_stream_shards` runs in the caller's catalog transaction. -//! Monotonic sequence numbers come from the `seq_counters` table (there is no -//! PostgreSQL sequence). Retention cleanup compares an RFC 3339 cutoff computed -//! in Rust. +//! so `init_stream_shards` (`stream_lifecycle.rs`) runs in the caller's +//! catalog transaction. Monotonic sequence numbers come from the +//! `seq_counters` table (there is no PostgreSQL sequence). Retention cleanup +//! compares an RFC 3339 cutoff computed in Rust. -use extenddb_core::types::{ - DescribeStreamInput, SequenceNumberRange, Shard, StreamDescription, StreamRecord, StreamStatus, - StreamSummary, StreamViewType, -}; +use extenddb_core::types::{DescribeStreamInput, StreamDescription, StreamRecord}; use extenddb_storage::error::StorageError; +use extenddb_storage::stream_lineage; use extenddb_storage::util::{parse_stream_arn, shard_in_stream, stream_arn}; use extenddb_storage::{StreamEngine, StreamListResult, StreamRecordsResult}; use futures::future::BoxFuture; @@ -21,53 +19,6 @@ use futures::future::BoxFuture; use crate::sqlite_util::format_timestamp; use crate::store::SqliteEngine; -/// Fixed shards per stream (hash-based partition-key assignment). -const SHARDS_PER_STREAM: u32 = 4; - -impl SqliteEngine { - /// Initialize stream shards and set `stream_label`, within the caller's - /// transaction. Returns the assigned stream label. - pub(crate) async fn init_stream_shards( - tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, - account_id: &str, - table_name: &str, - table_id: &str, - ) -> Result { - // Formatted here, not in SQL, so every backend issues the same label - // shape from one function (`extenddb_storage::util::format_stream_label`). - let label = extenddb_storage::util::new_stream_label(); - sqlx::query("UPDATE tables SET stream_label = ? WHERE account_id = ? AND table_name = ?") - .bind(&label) - .bind(account_id) - .bind(table_name) - .execute(&mut **tx) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - - for i in 0..SHARDS_PER_STREAM { - // Derived from the table id, not its name. A name is shared by - // every account that picks it, while `stream_shards.shard_id` is - // unique across the database. A name of up to 255 bytes also - // overflowed the 65-character ShardId limit - // the AWS SDKs enforce. The table id is a UUID, giving a - // 61-character id. - let shard_id = format!("shardId-{table_id}-{i:016}"); - let start_seq = format!("{:021}", 0); - sqlx::query( - "INSERT INTO stream_shards (shard_id, table_id, starting_sequence_number) \ - VALUES (?, ?, ?)", - ) - .bind(&shard_id) - .bind(table_id) - .bind(&start_seq) - .execute(&mut **tx) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - } - Ok(label) - } -} - impl StreamEngine for SqliteEngine { fn write_stream_record( &self, @@ -141,7 +92,9 @@ impl StreamEngine for SqliteEngine { // all. The sequence bound sits in the join condition so the LEFT // JOIN keeps its one row for an idle shard. Whether the shard // belongs to the stream the ARN's label names is decided in Rust - // from the table's label and the shard id (`shard_in_stream`). + // from the table's label and the shard id (`shard_in_stream`), + // with a second query for a legacy id only: whether the table has + // a labelled shard. let rows: Vec<(String, Option, String, Option)> = if let Some(after) = after { sqlx::query_as( @@ -185,7 +138,10 @@ impl StreamEngine for SqliteEngine { row_shard_id, table_label.as_deref(), &stream_label, - ) { + || self.table_has_labelled_shard(table_id), + ) + .await? + { return Err(extenddb_storage::util::stream_does_not_exist()); } @@ -208,92 +164,28 @@ impl StreamEngine for SqliteEngine { ) -> BoxFuture<'_, Result> { let account_id = account_id.to_owned(); let stream_arn = input.stream_arn.clone(); - let limit = input.limit.unwrap_or(100); + let limit = input.limit; let start = input.exclusive_start_shard_id.clone(); Box::pin(async move { let (table_name, stream_label) = parse_stream_arn(&stream_arn)?; - let row: Option<(String, Option, String, String)> = sqlx::query_as( - "SELECT key_schema, stream_specification, table_status, table_id \ - FROM tables WHERE account_id = ? AND table_name = ? AND stream_label = ?", - ) - .bind(&account_id) - .bind(&table_name) - .bind(&stream_label) - .fetch_optional(&self.pool) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - - let (ks_json, stream_spec_json, table_status, table_id) = - row.ok_or_else(|| extenddb_storage::util::stream_not_found(&stream_arn))?; - + let (table, ks_json) = self + .stream_table_row(&account_id, &table_name) + .await? + .ok_or_else(|| extenddb_storage::util::stream_not_found(&stream_arn))?; let key_schema = serde_json::from_str(&ks_json) .map_err(|e| StorageError::Internal(e.to_string()))?; - let stream_view_type = stream_spec_json - .as_deref() - .and_then(|s| serde_json::from_str::(s).ok()) - .and_then(|v| { - v.get("StreamViewType") - .and_then(|sv| serde_json::from_value::(sv.clone()).ok()) - }) - .unwrap_or(StreamViewType::KeysOnly); - - let shard_rows: Vec<(String, Option, String, Option)> = - if let Some(ref s) = start { - sqlx::query_as( - "SELECT shard_id, parent_shard_id, starting_sequence_number, ending_sequence_number \ - FROM stream_shards WHERE table_id = ? AND shard_id > ? \ - ORDER BY shard_id LIMIT ?", - ) - .bind(&table_id) - .bind(s) - .bind(limit + 1) - .fetch_all(&self.pool) - .await - } else { - sqlx::query_as( - "SELECT shard_id, parent_shard_id, starting_sequence_number, ending_sequence_number \ - FROM stream_shards WHERE table_id = ? ORDER BY shard_id LIMIT ?", - ) - .bind(&table_id) - .bind(limit + 1) - .fetch_all(&self.pool) - .await - } - .map_err(|e| StorageError::Internal(e.to_string()))?; - - #[allow(clippy::cast_sign_loss)] - let limit_usize = limit.max(0) as usize; - let last_shard = - (shard_rows.len() > limit_usize).then(|| shard_rows[limit_usize - 1].0.clone()); - let shards: Vec = shard_rows - .into_iter() - .take(limit_usize) - .map(|(id, parent, start, end)| Shard { - shard_id: id, - parent_shard_id: parent, - sequence_number_range: SequenceNumberRange { - starting_sequence_number: start, - ending_sequence_number: end, - }, - }) - .collect(); - - let stream_status = if table_status == "DELETING" { - StreamStatus::Disabling - } else { - StreamStatus::Enabled - }; - - Ok(StreamDescription { + let shards = self + .stream_shard_rows(&account_id, Some(&table.table_id)) + .await?; + stream_lineage::describe_stream( stream_arn, - stream_label, - stream_status, - stream_view_type, - table_name, + &table, key_schema, - shards, - last_evaluated_shard_id: last_shard, - }) + stream_label, + &shards, + limit, + start.as_deref(), + ) }) } @@ -308,77 +200,17 @@ impl StreamEngine for SqliteEngine { let table_name = table_name.map(str::to_owned); let start_arn = exclusive_start_stream_arn.map(str::to_owned); Box::pin(async move { - let rows: Vec<(String, String)> = match (table_name.as_deref(), start_arn.as_deref()) { - (Some(tn), Some(arn)) => { - let (_, start_label) = parse_stream_arn(arn)?; - sqlx::query_as( - "SELECT table_name, stream_label FROM tables \ - WHERE account_id = ? AND stream_label IS NOT NULL AND table_name = ? \ - AND stream_label > ? ORDER BY stream_label LIMIT ?", - ) - .bind(&account_id) - .bind(tn) - .bind(&start_label) - .bind(limit + 1) - .fetch_all(&self.pool) - .await - } - (Some(tn), None) => { - sqlx::query_as( - "SELECT table_name, stream_label FROM tables \ - WHERE account_id = ? AND stream_label IS NOT NULL AND table_name = ? \ - ORDER BY stream_label LIMIT ?", - ) - .bind(&account_id) - .bind(tn) - .bind(limit + 1) - .fetch_all(&self.pool) - .await - } - (None, Some(arn)) => { - let (start_table, start_label) = parse_stream_arn(arn)?; - sqlx::query_as( - "SELECT table_name, stream_label FROM tables \ - WHERE account_id = ? AND stream_label IS NOT NULL \ - AND (table_name, stream_label) > (?, ?) \ - ORDER BY table_name, stream_label LIMIT ?", - ) - .bind(&account_id) - .bind(&start_table) - .bind(&start_label) - .bind(limit + 1) - .fetch_all(&self.pool) - .await - } - (None, None) => { - sqlx::query_as( - "SELECT table_name, stream_label FROM tables \ - WHERE account_id = ? AND stream_label IS NOT NULL \ - ORDER BY table_name, stream_label LIMIT ?", - ) - .bind(&account_id) - .bind(limit + 1) - .fetch_all(&self.pool) - .await - } - } - .map_err(|e| StorageError::Internal(e.to_string()))?; - - #[allow(clippy::cast_sign_loss)] - let limit_usize = limit.max(0) as usize; - let summaries: Vec = rows - .iter() - .take(limit_usize) - .map(|(tn, label)| StreamSummary { - stream_arn: stream_arn(&self.region, &account_id, tn, label), - stream_label: label.clone(), - table_name: tn.clone(), - }) - .collect(); - let last = (rows.len() > limit_usize) - .then(|| summaries.last().map(|s| s.stream_arn.clone())) - .flatten(); - Ok((summaries, last)) + let start = start_arn.as_deref().map(parse_stream_arn).transpose()?; + let tables = self.stream_table_rows(&account_id).await?; + let shards = self.stream_shard_rows(&account_id, None).await?; + stream_lineage::list_streams( + &tables, + &shards, + table_name.as_deref(), + limit, + start, + |t, l| stream_arn(&self.region, &account_id, t, l), + ) }) } @@ -420,8 +252,11 @@ impl StreamEngine for SqliteEngine { .await .map_err(|e| StorageError::Internal(e.to_string()))?; + // Only the open shards: they belong to the table's enabled stream. + // A disabled stream's closed shards take no new records. let shards: Vec<(String,)> = sqlx::query_as( - "SELECT shard_id FROM stream_shards WHERE table_id = ? ORDER BY shard_id", + "SELECT shard_id FROM stream_shards \ + WHERE table_id = ? AND ending_sequence_number IS NULL ORDER BY shard_id", ) .bind(&table_id) .fetch_all(&self.pool) @@ -472,31 +307,20 @@ impl StreamEngine for SqliteEngine { let shard_id = shard_id.to_owned(); Box::pin(async move { let (table_name, stream_label) = parse_stream_arn(&stream_arn)?; - let table_id: Option = sqlx::query_scalar( - "SELECT table_id FROM tables \ - WHERE account_id = ? AND table_name = ? AND stream_label = ?", - ) - .bind(&account_id) - .bind(&table_name) - .bind(&stream_label) - .fetch_optional(&self.pool) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - - let Some(table_id) = table_id else { - return Err(extenddb_storage::util::stream_does_not_exist()); + let table = self.stream_table_row(&account_id, &table_name).await?; + let shards = match &table { + Some((t, _)) => { + self.stream_shard_rows(&account_id, Some(&t.table_id)) + .await? + } + None => Vec::new(), }; - let exists: Option<(i64,)> = - sqlx::query_as("SELECT 1 FROM stream_shards WHERE shard_id = ? AND table_id = ?") - .bind(&shard_id) - .bind(&table_id) - .fetch_optional(&self.pool) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - if exists.is_none() { - return Err(extenddb_storage::util::shard_not_in_stream(&stream_arn)); - } - Ok(()) + stream_lineage::validate_shard( + table.as_ref().map(|(t, _)| t), + &stream_label, + &shards, + &shard_id, + ) }) } @@ -518,3 +342,23 @@ impl StreamEngine for SqliteEngine { }) } } + +impl SqliteEngine { + /// Whether table `table_id` has a labelled shard row + /// (`stream_lineage::labelled_shard_id_range`), which makes its legacy + /// rows an earlier, closed stream (`util::shard_in_stream`). + async fn table_has_labelled_shard(&self, table_id: &str) -> Result { + let (from, to) = stream_lineage::labelled_shard_id_range(table_id); + let found: i64 = sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM stream_shards \ + WHERE table_id = ? AND shard_id >= ? AND shard_id < ?)", + ) + .bind(table_id) + .bind(&from) + .bind(&to) + .fetch_one(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(found != 0) + } +} diff --git a/crates/storage-sqlite/src/stream_lifecycle.rs b/crates/storage-sqlite/src/stream_lifecycle.rs new file mode 100644 index 000000000..155acad1b --- /dev/null +++ b/crates/storage-sqlite/src/stream_lifecycle.rs @@ -0,0 +1,261 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Stream lifecycle on SQLite: creating a table's stream, closing it on +//! `UpdateTable StreamEnabled=false`, and reading the table and shard rows the +//! shared rules in `extenddb_storage::stream_lineage` group into streams. +//! +//! Shards and records live in the catalog file, so every step runs in the +//! caller's transaction under the engine write lock, and a data write cannot +//! interleave with a close. + +use extenddb_core::types::{StreamSpecification, StreamViewType}; +use extenddb_storage::error::StorageError; +use extenddb_storage::stream_lineage::{ + SHARDS_PER_STREAM, ShardRow, StreamChange, TableStreamRow, disabled_stream_to_close, + next_stream_label, shard_id, stored_specification, stream_change, +}; + +use crate::store::SqliteEngine; + +type ShardTuple = (String, String, Option, String, Option); +/// `(table_id, table_status, stream_label, stream_specification, key_schema)`. +type TableTuple = (String, String, Option, Option, String); +/// `(table_name, table_id, table_status, stream_label, stream_specification)`. +type TableListTuple = (String, String, String, Option, Option); + +fn shard_row((table_id, shard_id, parent, start, end): ShardTuple) -> ShardRow { + ShardRow { + table_id, + shard_id, + parent_shard_id: parent, + starting_sequence_number: start, + ending_sequence_number: end, + } +} + +fn internal(e: impl std::fmt::Display) -> StorageError { + StorageError::Internal(e.to_string()) +} + +impl SqliteEngine { + /// Create a new stream on the table within the caller's transaction: four + /// open shards named for a new label, and `tables.stream_label` set to that + /// label. Returns the label. + pub(crate) async fn init_stream_shards( + tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, + account_id: &str, + table_name: &str, + table_id: &str, + view_type: StreamViewType, + ) -> Result { + let previous: Option = sqlx::query_scalar( + "SELECT stream_label FROM tables WHERE account_id = ? AND table_name = ?", + ) + .bind(account_id) + .bind(table_name) + .fetch_optional(&mut **tx) + .await + .map_err(internal)? + .flatten(); + let label = next_stream_label(previous.as_deref()); + sqlx::query("UPDATE tables SET stream_label = ? WHERE account_id = ? AND table_name = ?") + .bind(&label) + .bind(account_id) + .bind(table_name) + .execute(&mut **tx) + .await + .map_err(internal)?; + for i in 0..SHARDS_PER_STREAM { + sqlx::query( + "INSERT INTO stream_shards (shard_id, table_id, starting_sequence_number) \ + VALUES (?, ?, ?)", + ) + .bind(shard_id(table_id, &label, view_type, i)?) + .bind(table_id) + .bind(format!("{:021}", 0)) + .execute(&mut **tx) + .await + .map_err(internal)?; + } + Ok(label) + } + + /// Close every open shard of the table: its `EndingSequenceNumber` becomes + /// the sequence number of its last record, or its starting number when it + /// holds none (both measured 2026-10-24). + async fn close_stream_shards( + tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, + table_id: &str, + ) -> Result<(), StorageError> { + sqlx::query( + "UPDATE stream_shards SET ending_sequence_number = COALESCE(\ + (SELECT MAX(r.sequence_number) FROM stream_records r \ + WHERE r.shard_id = stream_shards.shard_id), \ + starting_sequence_number) \ + WHERE table_id = ? AND ending_sequence_number IS NULL", + ) + .bind(table_id) + .execute(&mut **tx) + .await + .map_err(internal)?; + Ok(()) + } + + /// Apply an `UpdateTable` `StreamSpecification` within the caller's + /// transaction. + /// + /// Disabling stores `{StreamEnabled: false}` with the stream's view type, + /// which DescribeTable omits, keeps `stream_label` (`LatestStreamArn` + /// keeps naming the disabled stream) and closes the stream's shards. + /// Enabling closes any shard still open (a stream disabled before shards + /// were closed on disable) and creates a new stream. + /// + /// `Ok(Some(refusal))` is a disable refused because the stored + /// specification is already disabled, whose open shards (left by a release + /// that did not close them, or by a close that failed after its catalog + /// write) this call closed in the transaction: the caller commits it and + /// answers the refusal. Any other refusal is `Err`, with nothing to commit. + pub(crate) async fn apply_stream_specification( + tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, + account_id: &str, + table_name: &str, + table_id: &str, + requested: &StreamSpecification, + ) -> Result, StorageError> { + let row: Option<(Option, Option)> = sqlx::query_as( + "SELECT stream_specification, stream_label FROM tables \ + WHERE account_id = ? AND table_name = ?", + ) + .bind(account_id) + .bind(table_name) + .fetch_optional(&mut **tx) + .await + .map_err(internal)?; + let (current, current_label) = row.unwrap_or_default(); + let current: Option = current + .as_deref() + .map(serde_json::from_str) + .transpose() + .map_err(internal)?; + let change = match stream_change(table_name, current.as_ref(), requested) { + Ok(change) => change, + Err(refusal) => { + if disabled_stream_to_close(current.as_ref(), current_label.as_deref(), requested) + .is_some() + { + Self::close_stream_shards(tx, table_id).await?; + return Ok(Some(refusal)); + } + return Err(refusal); + } + }; + let stored = + serde_json::to_string(&stored_specification(change, current.as_ref(), requested)) + .map_err(internal)?; + sqlx::query( + "UPDATE tables SET stream_specification = ? WHERE account_id = ? AND table_name = ?", + ) + .bind(stored) + .bind(account_id) + .bind(table_name) + .execute(&mut **tx) + .await + .map_err(internal)?; + Self::close_stream_shards(tx, table_id).await?; + if let StreamChange::Enable(view_type) = change { + Self::init_stream_shards(tx, account_id, table_name, table_id, view_type).await?; + } + Ok(None) + } + + /// The table's stream columns and its key schema JSON, by name. + pub(crate) async fn stream_table_row( + &self, + account_id: &str, + table_name: &str, + ) -> Result, StorageError> { + let row: Option = sqlx::query_as( + "SELECT table_id, table_status, stream_label, stream_specification, key_schema \ + FROM tables WHERE account_id = ? AND table_name = ?", + ) + .bind(account_id) + .bind(table_name) + .fetch_optional(&self.pool) + .await + .map_err(internal)?; + row.map(|(table_id, table_status, stream_label, spec, key_schema)| { + Ok(( + TableStreamRow { + table_name: table_name.to_owned(), + table_id, + table_status, + stream_label, + stream_specification: spec + .as_deref() + .map(serde_json::from_str) + .transpose() + .map_err(internal)?, + }, + key_schema, + )) + }) + .transpose() + } + + /// Every table of the account that has had a stream. + pub(crate) async fn stream_table_rows( + &self, + account_id: &str, + ) -> Result, StorageError> { + let rows: Vec = sqlx::query_as( + "SELECT table_name, table_id, table_status, stream_label, stream_specification \ + FROM tables WHERE account_id = ? AND stream_label IS NOT NULL", + ) + .bind(account_id) + .fetch_all(&self.pool) + .await + .map_err(internal)?; + rows.into_iter() + .map(|(table_name, table_id, table_status, stream_label, spec)| { + Ok(TableStreamRow { + table_name, + table_id, + table_status, + stream_label, + stream_specification: spec + .as_deref() + .map(serde_json::from_str) + .transpose() + .map_err(internal)?, + }) + }) + .collect() + } + + /// The shard rows of one table, or of every table of the account when + /// `table_id` is `None`. + pub(crate) async fn stream_shard_rows( + &self, + account_id: &str, + table_id: Option<&str>, + ) -> Result, StorageError> { + let rows: Vec = sqlx::query_as( + "SELECT s.table_id, s.shard_id, s.parent_shard_id, s.starting_sequence_number, \ + s.ending_sequence_number \ + FROM stream_shards s JOIN tables t ON t.table_id = s.table_id \ + WHERE t.account_id = ? AND (? IS NULL OR s.table_id = ?)", + ) + .bind(account_id) + .bind(table_id) + .bind(table_id) + .fetch_all(&self.pool) + .await + .map_err(internal)?; + Ok(rows.into_iter().map(shard_row).collect()) + } +} + +#[cfg(test)] +#[path = "stream_lifecycle_tests.rs"] +mod tests; diff --git a/crates/storage-sqlite/src/stream_lifecycle_tests.rs b/crates/storage-sqlite/src/stream_lifecycle_tests.rs new file mode 100644 index 000000000..d35b88da5 --- /dev/null +++ b/crates/storage-sqlite/src/stream_lifecycle_tests.rs @@ -0,0 +1,482 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Disabling and re-enabling a table's stream on SQLite, against the rows +//! measured on the service on 2026-10-24. + +use extenddb_core::expression::ExpressionMaps; +use extenddb_core::types::{ + DescribeStreamInput, DescribeTableInput, StreamDescription, StreamSpecification, StreamStatus, + StreamViewType, TableDescription, +}; +use extenddb_storage::error::StorageError; +use extenddb_storage::{DataEngine, StreamCapture, StreamEngine, TableEngine}; + +use crate::store::SqliteEngine; + +const ACCOUNT: &str = "123456789012"; + +async fn engine() -> SqliteEngine { + let engine = SqliteEngine::new(":memory:", 1, "us-east-1", 400 * 1024) + .await + .unwrap(); + crate::migrations::apply_for_tests(&engine.pool).await; + sqlx::query("INSERT INTO accounts (account_id, account_name) VALUES (?, 'stream-test')") + .bind(ACCOUNT) + .execute(&engine.pool) + .await + .unwrap(); + engine +} + +async fn create(engine: &SqliteEngine, name: &str, stream: bool) { + let mut input = serde_json::json!({ + "TableName": name, + "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}], + "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}], + "BillingMode": "PAY_PER_REQUEST", + }); + if stream { + input["StreamSpecification"] = + serde_json::json!({"StreamEnabled": true, "StreamViewType": "NEW_AND_OLD_IMAGES"}); + } + engine + .create_table(ACCOUNT, serde_json::from_value(input).unwrap()) + .await + .unwrap(); + activate(engine, name).await; +} + +/// Finish the control-plane transition the worker would run. +async fn activate(engine: &SqliteEngine, name: &str) { + sqlx::query("UPDATE tables SET table_status = 'ACTIVE' WHERE table_name = ?") + .bind(name) + .execute(&engine.pool) + .await + .unwrap(); +} + +async fn set_stream( + engine: &SqliteEngine, + name: &str, + spec: serde_json::Value, +) -> Result { + let input = serde_json::json!({"TableName": name, "StreamSpecification": spec}); + let out = engine + .update_table(ACCOUNT, serde_json::from_value(input).unwrap()) + .await; + activate(engine, name).await; + out +} + +async fn describe_table(engine: &SqliteEngine, name: &str) -> TableDescription { + engine + .describe_table( + ACCOUNT, + DescribeTableInput { + table_name: name.to_owned(), + }, + ) + .await + .unwrap() +} + +async fn describe_stream(engine: &SqliteEngine, arn: &str) -> StreamDescription { + engine + .describe_stream( + ACCOUNT, + &DescribeStreamInput { + stream_arn: arn.to_owned(), + limit: None, + exclusive_start_shard_id: None, + }, + ) + .await + .unwrap() +} + +/// Put an item through the data path with the stream capture the engine +/// builds from `view_type`, as a caller holding stale key info would. +async fn put(engine: &SqliteEngine, name: &str, pk: &str, view_type: StreamViewType) { + let key_info = engine.table_key_info(ACCOUNT, name).await.unwrap(); + let item = serde_json::from_value(serde_json::json!({"pk": {"S": pk}})).unwrap(); + let capture = StreamCapture { + view_type, + user_identity: None, + region: "us-east-1".into(), + }; + engine + .put_item( + &key_info, + item, + false, + None, + &ExpressionMaps::default(), + Some(&capture), + ) + .await + .unwrap(); +} + +async fn records_in(engine: &SqliteEngine, shard: &str) -> Vec { + sqlx::query_scalar( + "SELECT sequence_number FROM stream_records WHERE shard_id = ? ORDER BY sequence_number", + ) + .bind(shard) + .fetch_all(&engine.pool) + .await + .unwrap() +} + +#[tokio::test] +async fn disable_closes_every_shard_and_re_enable_creates_a_new_stream() { + let engine = engine().await; + create(&engine, "t482", true).await; + for pk in ["k0", "k1", "k2", "k3", "k4", "k5"] { + put(&engine, "t482", pk, StreamViewType::NewAndOldImages).await; + } + let enabled = describe_table(&engine, "t482").await; + let old_arn = enabled.latest_stream_arn.clone().unwrap(); + let before = describe_stream(&engine, &old_arn).await; + assert!(matches!(before.stream_status, StreamStatus::Enabled)); + assert!( + before + .shards + .iter() + .all(|s| s.sequence_number_range.ending_sequence_number.is_none()) + ); + + let disabled = set_stream(&engine, "t482", serde_json::json!({"StreamEnabled": false})) + .await + .unwrap(); + // The backend keeps `{StreamEnabled: false}` with the closed stream's + // view type; the engine omits it from every response. + assert_eq!( + disabled.stream_specification, + Some(StreamSpecification { + stream_enabled: false, + stream_view_type: Some(StreamViewType::NewAndOldImages), + }) + ); + let mut rendered = disabled.clone(); + rendered.prepare_for_response(); + assert!(rendered.stream_specification.is_none()); + assert_eq!( + disabled.latest_stream_arn.as_deref(), + Some(old_arn.as_str()) + ); + let mut table = describe_table(&engine, "t482").await; + table.prepare_for_response(); + assert!(table.stream_specification.is_none()); + assert_eq!(table.latest_stream_label, enabled.latest_stream_label); + + let closed = describe_stream(&engine, &old_arn).await; + assert!(matches!(closed.stream_status, StreamStatus::Disabled)); + assert_eq!(closed.stream_view_type, StreamViewType::NewAndOldImages); + assert_eq!(closed.shards.len(), 4); + for shard in &closed.shards { + let range = &shard.sequence_number_range; + let last = records_in(&engine, &shard.shard_id).await.pop(); + let expected = last.unwrap_or_else(|| range.starting_sequence_number.clone()); + assert_eq!( + range.ending_sequence_number.as_deref(), + Some(expected.as_str()) + ); + } + + // A write still carrying the enabled view type records nothing. + put(&engine, "t482", "stale", StreamViewType::NewAndOldImages).await; + let total: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM stream_records") + .fetch_one(&engine.pool) + .await + .unwrap(); + assert_eq!(total, 6); + + let reenabled = set_stream( + &engine, + "t482", + serde_json::json!({"StreamEnabled": true, "StreamViewType": "KEYS_ONLY"}), + ) + .await + .unwrap(); + let new_arn = reenabled.latest_stream_arn.clone().unwrap(); + assert_ne!(new_arn, old_arn); + assert!(reenabled.latest_stream_label > enabled.latest_stream_label); + let new = describe_stream(&engine, &new_arn).await; + assert!(matches!(new.stream_status, StreamStatus::Enabled)); + assert_eq!(new.stream_view_type, StreamViewType::KeysOnly); + assert!( + new.shards + .iter() + .all(|s| !closed.shards.iter().any(|o| o.shard_id == s.shard_id)) + ); + + put(&engine, "t482", "after", StreamViewType::KeysOnly).await; + let old_again = describe_stream(&engine, &old_arn).await; + assert!(matches!(old_again.stream_status, StreamStatus::Disabled)); + assert_eq!(old_again.stream_view_type, StreamViewType::NewAndOldImages); + for (a, b) in old_again.shards.iter().zip(&closed.shards) { + assert_eq!(a.shard_id, b.shard_id); + assert_eq!( + a.sequence_number_range.ending_sequence_number, + b.sequence_number_range.ending_sequence_number + ); + assert_eq!(records_in(&engine, &a.shard_id).await.len(), { + records_in(&engine, &b.shard_id).await.len() + }); + } + let mut on_new = 0; + for s in &new.shards { + on_new += records_in(&engine, &s.shard_id).await.len(); + } + assert_eq!(on_new, 1); + + let (listed, _) = engine + .list_streams(ACCOUNT, Some("t482"), 100, None) + .await + .unwrap(); + let arns: Vec<&str> = listed.iter().map(|s| s.stream_arn.as_str()).collect(); + assert_eq!(arns, [new_arn.as_str(), old_arn.as_str()]); + + // Each ARN accepts only its own shards. + engine + .validate_shard(ACCOUNT, &old_arn, &closed.shards[0].shard_id) + .await + .unwrap(); + for (arn, shard) in [ + (&old_arn, &new.shards[0].shard_id), + (&new_arn, &closed.shards[0].shard_id), + ] { + match engine.validate_shard(ACCOUNT, arn, shard).await { + Err(StorageError::TableNotFound(m)) => { + assert_eq!(m, "Requested resource not found: Shard does not exist"); + } + other => panic!("{other:?}"), + } + } + engine.pool.close().await; +} + +#[tokio::test] +async fn enable_while_enabled_and_disable_without_a_stream_are_refused() { + let engine = engine().await; + create(&engine, "t482", true).await; + create(&engine, "nostream", false).await; + for view in ["NEW_AND_OLD_IMAGES", "KEYS_ONLY"] { + let spec = serde_json::json!({"StreamEnabled": true, "StreamViewType": view}); + match set_stream(&engine, "t482", spec).await { + Err(StorageError::Validation(m)) => { + assert_eq!(m, "Table already has an enabled stream: TableName: t482"); + } + other => panic!("{other:?}"), + } + } + match set_stream( + &engine, + "nostream", + serde_json::json!({"StreamEnabled": false}), + ) + .await + { + Err(StorageError::Validation(m)) => { + assert_eq!(m, "Table has no stream to disable: TableName: nostream"); + } + other => panic!("{other:?}"), + } + set_stream(&engine, "t482", serde_json::json!({"StreamEnabled": false})) + .await + .unwrap(); + match set_stream(&engine, "t482", serde_json::json!({"StreamEnabled": false})).await { + Err(StorageError::Validation(m)) => { + assert_eq!(m, "Table has no stream to disable: TableName: t482"); + } + other => panic!("{other:?}"), + } + // A refused request changes nothing: one stream, four shards. + let shards: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM stream_shards") + .fetch_one(&engine.pool) + .await + .unwrap(); + assert_eq!(shards, 4); + engine.pool.close().await; +} + +#[tokio::test] +async fn a_stream_from_before_labelled_shard_ids_closes_and_is_replaced() { + let engine = engine().await; + create(&engine, "legacy", true).await; + // Rewrite the shards in the shape earlier builds created. + sqlx::query( + "UPDATE stream_shards SET shard_id = 'shardId-' || table_id || '-' || \ + printf('%016d', CAST(substr(shard_id, -2) AS INTEGER))", + ) + .execute(&engine.pool) + .await + .unwrap(); + let arn = describe_table(&engine, "legacy") + .await + .latest_stream_arn + .unwrap(); + put(&engine, "legacy", "k", StreamViewType::NewAndOldImages).await; + let open = describe_stream(&engine, &arn).await; + assert!(matches!(open.stream_status, StreamStatus::Enabled)); + assert_eq!(open.stream_view_type, StreamViewType::NewAndOldImages); + assert!(open.shards.iter().all(|s| { + s.shard_id + .rsplit('-') + .next() + .is_some_and(|suffix| suffix.starts_with("0000000000000")) + })); + + set_stream( + &engine, + "legacy", + serde_json::json!({"StreamEnabled": false}), + ) + .await + .unwrap(); + let closed = describe_stream(&engine, &arn).await; + assert!(matches!(closed.stream_status, StreamStatus::Disabled)); + assert!( + closed + .shards + .iter() + .all(|s| s.sequence_number_range.ending_sequence_number.is_some()) + ); + let re = set_stream( + &engine, + "legacy", + serde_json::json!({"StreamEnabled": true, "StreamViewType": "NEW_IMAGE"}), + ) + .await + .unwrap(); + let new = describe_stream(&engine, &re.latest_stream_arn.unwrap()).await; + assert!(matches!(new.stream_status, StreamStatus::Enabled)); + assert_eq!(new.shards.len(), 4); + engine.pool.close().await; +} + +/// A stream disabled by a release that stored `{StreamEnabled: false}` and +/// left its four shards open (the same state a close that failed after its +/// catalog write leaves): writes still record into it. A disable is refused, +/// as on the service, and closes the open shards first, so the next write +/// records nothing; an enable on that state closes them too. +#[tokio::test] +async fn a_disable_refused_on_a_disabled_stream_closes_the_shards_it_left_open() { + let engine = engine().await; + create(&engine, "base", true).await; + let arn = describe_table(&engine, "base") + .await + .latest_stream_arn + .unwrap(); + // The shards in the shape the earlier release created them, and what its + // disable wrote: the specification alone. + sqlx::query( + "UPDATE stream_shards SET shard_id = 'shardId-' || table_id || '-' || \ + printf('%016d', CAST(substr(shard_id, -2) AS INTEGER)) \ + WHERE table_id = (SELECT table_id FROM tables WHERE table_name = 'base')", + ) + .execute(&engine.pool) + .await + .unwrap(); + put(&engine, "base", "before", StreamViewType::NewAndOldImages).await; + sqlx::query( + "UPDATE tables SET stream_specification = '{\"StreamEnabled\":false}' \ + WHERE table_name = 'base'", + ) + .execute(&engine.pool) + .await + .unwrap(); + let open_shards = || async { + sqlx::query_scalar::<_, i64>( + "SELECT COUNT(*) FROM stream_shards WHERE ending_sequence_number IS NULL", + ) + .fetch_one(&engine.pool) + .await + .unwrap() + }; + assert_eq!(open_shards().await, 4); + let described = describe_stream(&engine, &arn).await; + assert!(matches!(described.stream_status, StreamStatus::Disabled)); + put(&engine, "base", "dark", StreamViewType::NewAndOldImages).await; + let recorded = || async { + sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM stream_records") + .fetch_one(&engine.pool) + .await + .unwrap() + }; + assert_eq!(recorded().await, 2, "the open shards still record"); + + match set_stream(&engine, "base", serde_json::json!({"StreamEnabled": false})).await { + Err(StorageError::Validation(m)) => { + assert_eq!(m, "Table has no stream to disable: TableName: base"); + } + other => panic!("{other:?}"), + } + assert_eq!( + open_shards().await, + 0, + "the refused disable closed the shards" + ); + let closed = describe_stream(&engine, &arn).await; + assert!(matches!(closed.stream_status, StreamStatus::Disabled)); + assert_eq!(closed.stream_view_type, StreamViewType::KeysOnly); + for shard in &closed.shards { + let last = records_in(&engine, &shard.shard_id).await.pop(); + let expected = + last.unwrap_or_else(|| shard.sequence_number_range.starting_sequence_number.clone()); + assert_eq!( + shard + .sequence_number_range + .ending_sequence_number + .as_deref(), + Some(expected.as_str()) + ); + } + put(&engine, "base", "after", StreamViewType::NewAndOldImages).await; + assert_eq!( + recorded().await, + 2, + "a write after the close records nothing" + ); + // A second refusal has nothing left to close and changes nothing. + match set_stream(&engine, "base", serde_json::json!({"StreamEnabled": false})).await { + Err(StorageError::Validation(m)) => { + assert_eq!(m, "Table has no stream to disable: TableName: base"); + } + other => panic!("{other:?}"), + } + assert_eq!(open_shards().await, 0); + + // The same state, met by an enable: the enable closes the open shards. + create(&engine, "base2", true).await; + sqlx::query( + "UPDATE tables SET stream_specification = '{\"StreamEnabled\":false}' \ + WHERE table_name = 'base2'", + ) + .execute(&engine.pool) + .await + .unwrap(); + assert_eq!(open_shards().await, 4); + set_stream( + &engine, + "base2", + serde_json::json!({"StreamEnabled": true, "StreamViewType": "KEYS_ONLY"}), + ) + .await + .unwrap(); + assert_eq!( + open_shards().await, + 4, + "only the new stream's shards are open" + ); + let closed_by_enable: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM stream_shards s JOIN tables t ON t.table_id = s.table_id \ + WHERE t.table_name = 'base2' AND s.ending_sequence_number IS NOT NULL", + ) + .fetch_one(&engine.pool) + .await + .unwrap(); + assert_eq!(closed_by_enable, 4); + engine.pool.close().await; +} diff --git a/crates/storage-sqlite/src/stream_tests.rs b/crates/storage-sqlite/src/stream_tests.rs index c474318e8..3630e48e7 100644 --- a/crates/storage-sqlite/src/stream_tests.rs +++ b/crates/storage-sqlite/src/stream_tests.rs @@ -289,3 +289,176 @@ async fn a_malformed_stream_arn_is_a_validation_error() { .expect_err("malformed ARN"); assert!(matches!(err, StorageError::Validation(_)), "{err:?}"); } + +/// Shard `i` of the stream `label` on `table_id`, in the shape +/// `init_stream_shards` writes since streams were named in shard ids. +fn labelled_shard(table_id: &str, label: &str, i: u32) -> String { + extenddb_storage::stream_lineage::shard_id( + table_id, + label, + extenddb_core::types::StreamViewType::KeysOnly, + i, + ) + .expect("label shape") +} + +/// Four shard rows of the stream `label` on `table_id`, closed at +/// `ending_sequence_number` when it is given. +async fn labelled_shards(engine: &SqliteEngine, table_id: &str, label: &str, end: Option) { + for i in 0..4u32 { + sqlx::query( + "INSERT INTO stream_shards (shard_id, table_id, starting_sequence_number, \ + ending_sequence_number) VALUES (?, ?, ?, ?)", + ) + .bind(labelled_shard(table_id, label, i)) + .bind(table_id) + .bind(format!("{:021}", 0)) + .bind(end.map(|e| format!("{e:021}"))) + .execute(&engine.pool) + .await + .expect("shard"); + } +} + +/// A table whose stream was disabled and enabled again: the closed stream's +/// labelled shards, then the enabled stream's, with the table's label on the +/// new stream. Returns the table id. +async fn re_enabled_table(engine: &SqliteEngine, old: &str, new: &str) -> String { + let table_id = stream_table(engine, ACCOUNT, "t", new).await; + sqlx::query("DELETE FROM stream_shards WHERE table_id = ?") + .bind(&table_id) + .execute(&engine.pool) + .await + .expect("drop the legacy rows the fixture wrote"); + labelled_shards(engine, &table_id, old, Some(2)).await; + labelled_shards(engine, &table_id, new, None).await; + write_records( + engine, + &labelled_shard(&table_id, old, 0), + &table_id, + &[1, 2], + ) + .await; + write_records(engine, &labelled_shard(&table_id, new, 0), &table_id, &[3]).await; + table_id +} + +#[tokio::test] +async fn a_closed_stream_stays_readable_after_the_table_has_a_newer_one() { + let engine = engine().await; + let (old, new) = ("2026-01-01T00:00:00.000", "2026-02-02T00:00:00.000"); + let table_id = re_enabled_table(&engine, old, new).await; + let old_arn = stream_arn(REGION, ACCOUNT, "t", old); + let new_arn = stream_arn(REGION, ACCOUNT, "t", new); + + let (records, last) = engine + .get_stream_records( + ACCOUNT, + &old_arn, + &labelled_shard(&table_id, old, 0), + None, + 10, + ) + .await + .expect("the closed stream reads under its own ARN"); + assert_eq!(records.len(), 2); + assert_eq!(last.as_deref(), Some(format!("{:021}", 2).as_str())); + + let (records, _) = engine + .get_stream_records( + ACCOUNT, + &new_arn, + &labelled_shard(&table_id, new, 0), + None, + 10, + ) + .await + .expect("the new stream reads under its own ARN"); + assert_eq!(records.len(), 1); + assert_eq!(records[0].dynamodb.sequence_number, format!("{:021}", 3)); +} + +#[tokio::test] +async fn the_newer_arn_with_an_older_streams_shard_id_is_not_found() { + let engine = engine().await; + let (old, new) = ("2026-01-01T00:00:00.000", "2026-02-02T00:00:00.000"); + let table_id = re_enabled_table(&engine, old, new).await; + let old_arn = stream_arn(REGION, ACCOUNT, "t", old); + let new_arn = stream_arn(REGION, ACCOUNT, "t", new); + + // An iterator pairing the new ARN with a closed shard, and one pairing the + // closed stream's ARN with a new shard: neither shard is the stream's. + assert_stream_not_found( + engine + .get_stream_records( + ACCOUNT, + &new_arn, + &labelled_shard(&table_id, old, 0), + None, + 10, + ) + .await, + ); + assert_stream_not_found( + engine + .get_stream_records( + ACCOUNT, + &old_arn, + &labelled_shard(&table_id, new, 0), + None, + 10, + ) + .await, + ); +} + +#[tokio::test] +async fn the_newer_arn_with_a_legacy_shard_of_the_closed_stream_is_not_found() { + // A stream created before shard ids named their stream, disabled and + // enabled again under the labelled scheme: the legacy rows are closed, and + // the table's label is the new labelled stream's. + let engine = engine().await; + let new = "2026-02-02T00:00:00.000"; + let table_id = stream_table(&engine, ACCOUNT, "t", new).await; + write_records(&engine, &shard(&table_id, 0), &table_id, &[1, 2]).await; + sqlx::query( + "UPDATE stream_shards SET ending_sequence_number = ? WHERE table_id = ? AND shard_id = ?", + ) + .bind(format!("{:021}", 2)) + .bind(&table_id) + .bind(shard(&table_id, 0)) + .execute(&engine.pool) + .await + .expect("close shard 0"); + sqlx::query( + "UPDATE stream_shards SET ending_sequence_number = starting_sequence_number \ + WHERE table_id = ? AND ending_sequence_number IS NULL", + ) + .bind(&table_id) + .execute(&engine.pool) + .await + .expect("close the other legacy shards"); + labelled_shards(&engine, &table_id, new, None).await; + write_records(&engine, &labelled_shard(&table_id, new, 0), &table_id, &[3]).await; + let new_arn = stream_arn(REGION, ACCOUNT, "t", new); + + for i in 0..4u32 { + assert_stream_not_found( + engine + .get_stream_records(ACCOUNT, &new_arn, &shard(&table_id, i), None, 10) + .await, + ); + } + // The control: the new stream's own shard reads under the new ARN. + let (records, _) = engine + .get_stream_records( + ACCOUNT, + &new_arn, + &labelled_shard(&table_id, new, 0), + None, + 10, + ) + .await + .expect("the new stream reads under its own ARN"); + assert_eq!(records.len(), 1); +} diff --git a/crates/storage-sqlite/src/update_table.rs b/crates/storage-sqlite/src/update_table.rs index d1e0e5769..8e70f9c57 100644 --- a/crates/storage-sqlite/src/update_table.rs +++ b/crates/storage-sqlite/src/update_table.rs @@ -311,50 +311,23 @@ impl SqliteEngine { ) .await?; } - if let Some(spec) = &input.stream_specification { - let j = - serde_json::to_string(spec).map_err(|e| StorageError::Internal(e.to_string()))?; - update_col( + if let Some(spec) = &input.stream_specification + && let Some(refusal) = Self::apply_stream_specification( &mut tx, account_id, &input.table_name, - "stream_specification", - &j, + &table_id, + spec, ) - .await?; - if spec.stream_enabled { - let existing: Option<(String,)> = - sqlx::query_as("SELECT shard_id FROM stream_shards WHERE table_id = ? LIMIT 1") - .bind(&table_id) - .fetch_optional(&mut *tx) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - if existing.is_none() { - Self::init_stream_shards(&mut tx, account_id, &input.table_name, &table_id) - .await?; - } else { - let label: Option = sqlx::query_scalar( - "SELECT stream_label FROM tables WHERE account_id = ? AND table_name = ?", - ) - .bind(account_id) - .bind(&input.table_name) - .fetch_one(&mut *tx) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - if label.is_none() { - sqlx::query( - "UPDATE tables SET stream_label = ? \ - WHERE account_id = ? AND table_name = ?", - ) - .bind(extenddb_storage::util::new_stream_label()) - .bind(account_id) - .bind(&input.table_name) - .execute(&mut *tx) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - } - } - } + .await? + { + // A disable of an already disabled stream: the request is refused, + // and the transaction holds only the close of the shards the first + // disable left open (a stream change travels alone in the request). + tx.commit() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + return Err(refusal); } // GSI create/delete. diff --git a/crates/storage/src/lib.rs b/crates/storage/src/lib.rs index 79f8665a0..8aecff154 100755 --- a/crates/storage/src/lib.rs +++ b/crates/storage/src/lib.rs @@ -30,6 +30,7 @@ pub mod snapshot; #[doc(hidden)] #[cfg(feature = "test-hooks")] pub mod snapshot_conformance; +pub mod stream_lineage; #[doc(hidden)] #[cfg(feature = "test-hooks")] pub mod test_pause; @@ -629,8 +630,10 @@ pub trait StreamEngine: Send + Sync { /// Cassandra after reading the table), and return `StorageError::TableNotFound` /// with the message `Requested resource not found: Stream does not exist` when the /// stream does not exist for that account or the shard is not one of that stream's - /// shards. A shard of another table, another account, or an earlier stream label on - /// the same table therefore yields no records. + /// shards (`util::shard_in_stream`). A shard of another table, another account, or + /// another stream of the same table therefore yields no records; a closed stream's + /// shards keep reading under the closed stream's ARN after the table has a newer + /// stream. fn get_stream_records( &self, account_id: &str, diff --git a/crates/storage/src/stream_lineage.rs b/crates/storage/src/stream_lineage.rs new file mode 100644 index 000000000..ad1f09fd6 --- /dev/null +++ b/crates/storage/src/stream_lineage.rs @@ -0,0 +1,571 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The streams a table has had, and which of them a shard belongs to. +//! +//! Disabling a table's stream closes it: the stream reports `DISABLED`, every +//! shard gets an `EndingSequenceNumber`, and the stream stays describable and +//! listed. Enabling it again creates a new stream with a new label and new +//! shards. A table therefore has a sequence of streams, and only the newest +//! one can be enabled. +//! +//! Each stream's identity is carried in its shard ids rather than in a catalog +//! row, so no backend needs a schema change: a shard id is +//! `shardId-{table_id}-{label_millis:013}{view_type:1}{index:02}`, where +//! `label_millis` is the stream label as epoch milliseconds and `view_type` +//! one digit for the stream's `StreamViewType`. A shard id written before +//! this scheme (`shardId-{table_id}-{index}`, all zeros but the index) +//! belongs to the table's current stream, as it always did. +//! +//! Every backend reads its table row and shard rows and hands them to the +//! functions here, so the grouping, the status rule, the paging order and the +//! refusal messages are written once. + +use extenddb_core::types::{ + KeySchemaElement, SequenceNumberRange, Shard, StreamDescription, StreamSpecification, + StreamStatus, StreamViewType, +}; + +use crate::error::StorageError; +use crate::util::{ + format_stream_label, shard_does_not_exist, stream_does_not_exist, stream_not_found, +}; + +/// Fixed shards per stream (hash-based partition-key assignment). +pub const SHARDS_PER_STREAM: u32 = 4; + +/// A `CreateTable` request with a disabled `StreamSpecification` dropped. +/// +/// The service does not store `{StreamEnabled: false}`: `DescribeTable` on +/// such a table has no `StreamSpecification`, `LatestStreamArn` or +/// `LatestStreamLabel`, and the table answers `UpdateTable` as one created +/// without the member (measured 2026-10-24). Every backend passes its request +/// through here before storing it. +#[must_use] +pub fn without_disabled_specification( + mut input: extenddb_core::types::CreateTableInput, +) -> extenddb_core::types::CreateTableInput { + if input + .stream_specification + .as_ref() + .is_some_and(|s| !s.stream_enabled) + { + input.stream_specification = None; + } + input +} + +/// What an `UpdateTable` `StreamSpecification` asks of the table's stream. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum StreamChange { + /// Create a new stream with this view type. + Enable(StreamViewType), + /// Close the enabled stream. + Disable, +} + +/// Decide a `StreamSpecification` change against the table's stored one. +/// +/// # Errors +/// +/// `Validation` with the service's sentence when the request enables a stream +/// on a table whose stream is enabled (whatever the view type), or disables +/// one on a table with no enabled stream (never had one, or already +/// disabled). Both measured 2026-10-24. +pub fn stream_change( + table_name: &str, + current: Option<&StreamSpecification>, + requested: &StreamSpecification, +) -> Result { + let enabled = current.is_some_and(|s| s.stream_enabled); + match (requested.stream_enabled, enabled) { + (true, true) => Err(StorageError::Validation(format!( + "Table already has an enabled stream: TableName: {table_name}" + ))), + (false, false) => Err(StorageError::Validation(format!( + "Table has no stream to disable: TableName: {table_name}" + ))), + // Unreachable through the engine, which refuses an enable without a + // view type before storage with this same sentence. The service's + // sentence for that case, measured 2026-10-24, is "One or more + // parameter values were invalid: If stream is being enabled then + // UpdateViewType is required", and it precedes the already-enabled + // check; the engine's sentence pre-dates this change and is not + // changed here. + (true, false) => requested + .stream_view_type + .map(StreamChange::Enable) + .ok_or_else(|| { + StorageError::Validation( + "StreamViewType must be specified when StreamEnabled is true".to_owned(), + ) + }), + (false, true) => Ok(StreamChange::Disable), + } +} + +/// The stream a refused disable closes before it is answered: the table's +/// current label, when `requested` disables a table whose stored specification +/// is already disabled. +/// +/// A release before shards were closed on disable left a disabled stream's +/// four shards open, and a close that fails after its catalog write leaves the +/// same state until the request is retried; either way `DescribeStream` +/// reports `DISABLED` while writes keep recording into the open shards. The +/// retry is refused with [`stream_change`]'s sentence, as on the service, and +/// the backend closes the current label's open shards first, which is what the +/// first disable would have done. `None` when the request enables, when the +/// table never had a stream (no stored specification), or when it has no +/// label. +#[must_use] +pub fn disabled_stream_to_close<'a>( + current: Option<&StreamSpecification>, + current_label: Option<&'a str>, + requested: &StreamSpecification, +) -> Option<&'a str> { + (!requested.stream_enabled && current.is_some_and(|c| !c.stream_enabled)) + .then_some(current_label) + .flatten() +} + +/// The specification to store once a stream change is decided: the request's +/// own on an enable, and on a disable `{StreamEnabled: false}` carrying the +/// closed stream's view type, so a legacy stream (whose view type lives only +/// here) still reports it once disabled. Responses omit a disabled +/// specification (`TableDescription::prepare_for_response`). +#[must_use] +pub fn stored_specification( + change: StreamChange, + current: Option<&StreamSpecification>, + requested: &StreamSpecification, +) -> StreamSpecification { + match change { + StreamChange::Enable(_) => requested.clone(), + StreamChange::Disable => StreamSpecification { + stream_enabled: false, + stream_view_type: current.and_then(|c| c.stream_view_type), + }, + } +} + +/// The label for a new stream: now, or one millisecond after the table's +/// previous label when the clock has not moved past it, so two streams of one +/// table never share an ARN. +#[must_use] +pub fn next_stream_label(previous: Option<&str>) -> String { + let now = crate::util::new_stream_label(); + match previous { + Some(prev) if now.as_str() <= prev => label_millis(prev) + .and_then(|m| millis_label(m + 1)) + .unwrap_or(now), + _ => now, + } +} + +/// The epoch milliseconds a stream label names, or `None` for a string that is +/// not in the label shape. +#[must_use] +pub fn label_millis(label: &str) -> Option { + let b = label.as_bytes(); + let seps = [ + (4, b'-'), + (7, b'-'), + (10, b'T'), + (13, b':'), + (16, b':'), + (19, b'.'), + ]; + if b.len() != 23 || seps.iter().any(|&(i, c)| b[i] != c) { + return None; + } + let num = |from: usize, to: usize| -> Option { + let s = label.get(from..to)?; + s.bytes() + .all(|c| c.is_ascii_digit()) + .then(|| s.parse().ok())? + }; + let month = time::Month::try_from(u8::try_from(num(5, 7)?).ok()?).ok()?; + let date = time::Date::from_calendar_date( + i32::from(num(0, 4)?), + month, + u8::try_from(num(8, 10)?).ok()?, + ) + .ok()?; + let at = time::Time::from_hms_milli( + u8::try_from(num(11, 13)?).ok()?, + u8::try_from(num(14, 16)?).ok()?, + u8::try_from(num(17, 19)?).ok()?, + num(20, 23)?, + ) + .ok()?; + i64::try_from(date.with_time(at).assume_utc().unix_timestamp_nanos() / 1_000_000).ok() +} + +fn millis_label(millis: i64) -> Option { + time::OffsetDateTime::from_unix_timestamp_nanos(i128::from(millis) * 1_000_000) + .ok() + .map(format_stream_label) +} + +fn view_digit(view_type: StreamViewType) -> char { + match view_type { + StreamViewType::KeysOnly => '1', + StreamViewType::NewImage => '2', + StreamViewType::OldImage => '3', + StreamViewType::NewAndOldImages => '4', + } +} + +fn digit_view(digit: u8) -> Option { + match digit { + b'1' => Some(StreamViewType::KeysOnly), + b'2' => Some(StreamViewType::NewImage), + b'3' => Some(StreamViewType::OldImage), + b'4' => Some(StreamViewType::NewAndOldImages), + _ => None, + } +} + +/// The id of shard `index` of the stream `label` on table `table_id`. +/// +/// Derived from the table id, not its name: a name is shared by every account +/// that picks it and reused by delete-and-recreate, while a shard id must be +/// unique across a backend's shard store. 8 + 36 + 1 + 16 = 61 characters for +/// a UUID table id, inside the 65-character limit the AWS SDKs enforce on +/// `ShardId`. +/// +/// # Errors +/// +/// `Internal` when `label` is not in the label shape; every label comes from +/// [`next_stream_label`] or an earlier stored label. +pub fn shard_id( + table_id: &str, + label: &str, + view_type: StreamViewType, + index: u32, +) -> Result { + let millis = label_millis(label) + .ok_or_else(|| StorageError::Internal(format!("malformed stream label {label}")))?; + Ok(format!( + "shardId-{table_id}-{millis:013}{}{index:02}", + view_digit(view_type) + )) +} + +/// Which stream of its table a shard id belongs to. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ShardStream { + /// An id from before streams were named in shard ids: the table's current + /// stream. + Legacy, + /// An id naming its stream's label and view type. + Labelled { + label: String, + view_type: StreamViewType, + }, +} + +/// Parse a shard id of table `table_id`: its stream and its index within the +/// stream. `None` when the id is not one of this table's. +#[must_use] +pub fn parse_shard_id(table_id: &str, shard_id: &str) -> Option<(ShardStream, u32)> { + let suffix = shard_id + .strip_prefix("shardId-")? + .strip_prefix(table_id)? + .strip_prefix('-')?; + if suffix.is_empty() || !suffix.bytes().all(|c| c.is_ascii_digit()) { + return None; + } + if suffix.len() == 16 && !suffix.starts_with("0000000000000") { + let millis: i64 = suffix[..13].parse().ok()?; + let view_type = digit_view(suffix.as_bytes()[13])?; + let index: u32 = suffix[14..].parse().ok()?; + return Some(( + ShardStream::Labelled { + label: millis_label(millis)?, + view_type, + }, + index, + )); + } + Some((ShardStream::Legacy, suffix.parse().ok()?)) +} + +/// One shard row as a backend stores it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ShardRow { + pub table_id: String, + pub shard_id: String, + pub parent_shard_id: Option, + pub starting_sequence_number: String, + pub ending_sequence_number: Option, +} + +/// The stream-relevant columns of one table row. +#[derive(Debug, Clone)] +pub struct TableStreamRow { + pub table_name: String, + pub table_id: String, + pub table_status: String, + /// `LatestStreamLabel`: the newest stream, enabled or not. + pub stream_label: Option, + pub stream_specification: Option, +} + +/// One stream of a table and its shards, sorted by shard id. +#[derive(Debug, Clone)] +pub struct TableStream { + pub label: String, + pub view_type: StreamViewType, + pub shards: Vec, +} + +/// Group a table's shards into its streams, oldest first. +/// +/// Legacy shard ids (any id of this table that is not in the labelled shape) +/// take the table's current label and the view type the stored specification +/// carries, enabled or disabled (`KEYS_ONLY` when it carries none, which is +/// what DescribeStream reported for them before), unless a labelled stream +/// already holds that label. +#[must_use] +pub fn table_streams(table: &TableStreamRow, shards: &[ShardRow]) -> Vec { + let mut streams: Vec = Vec::new(); + let mut legacy: Vec = Vec::new(); + for row in shards.iter().filter(|r| r.table_id == table.table_id) { + match parse_shard_id(&table.table_id, &row.shard_id) { + Some((ShardStream::Labelled { label, view_type }, _)) => { + match streams.iter_mut().find(|s| s.label == label) { + Some(s) => s.shards.push(row.clone()), + None => streams.push(TableStream { + label, + view_type, + shards: vec![row.clone()], + }), + } + } + // Ids from before streams were named in shard ids: the + // `shardId-{table_id}-{index}` form, and the + // `shardId-{table_name}-{index}` form SQLite and PostgreSQL wrote + // before v0.1.14. The rows are already this table's (filtered by + // table id above), so any id that does not parse is a legacy one. + Some((ShardStream::Legacy, _)) | None => legacy.push(row.clone()), + } + } + if let Some(current) = table.stream_label.as_deref() + && !legacy.is_empty() + && !streams.iter().any(|s| s.label == current) + { + let view_type = table + .stream_specification + .as_ref() + .and_then(|s| s.stream_view_type) + .unwrap_or(StreamViewType::KeysOnly); + streams.push(TableStream { + label: current.to_owned(), + view_type, + shards: legacy, + }); + } + for s in &mut streams { + s.shards.sort_by(|a, b| a.shard_id.cmp(&b.shard_id)); + } + streams.sort_by(|a, b| a.label.cmp(&b.label)); + streams +} + +/// The ids of the open shards of the stream `label` of table `table_id`: the +/// labelled rows carrying that label, or the legacy rows when no labelled row +/// carries it (a legacy stream's label is the table's label at the time it was +/// read). A disable closes exactly these, so a disable that applies after an +/// overlapping enable has committed does not close the stream the enable just +/// opened. +#[must_use] +pub fn open_shard_ids_of(table_id: &str, label: &str, shards: &[ShardRow]) -> Vec { + let rows: Vec<&ShardRow> = shards.iter().filter(|r| r.table_id == table_id).collect(); + let labelled_exists = rows.iter().any(|r| { + matches!( + parse_shard_id(table_id, &r.shard_id), + Some((ShardStream::Labelled { label: l, .. }, _)) if l == label + ) + }); + let mut ids: Vec = rows + .into_iter() + .filter(|r| r.ending_sequence_number.is_none()) + .filter(|r| match parse_shard_id(table_id, &r.shard_id) { + Some((ShardStream::Labelled { label: l, .. }, _)) => l == label, + Some((ShardStream::Legacy, _)) | None => !labelled_exists, + }) + .map(|r| r.shard_id.clone()) + .collect(); + ids.sort(); + ids +} + +/// The ids of the open shards of every stream of table `table_id` older than +/// `label`: the labelled rows carrying an earlier label, and the legacy rows. +/// An enable creating stream `label` closes these, so a stream left open by a +/// close that failed after its catalog write is closed by the next enable, +/// whatever label that enable read. +#[must_use] +pub fn open_shard_ids_before(table_id: &str, label: &str, shards: &[ShardRow]) -> Vec { + let mut ids: Vec = shards + .iter() + .filter(|r| r.table_id == table_id && r.ending_sequence_number.is_none()) + .filter(|r| match parse_shard_id(table_id, &r.shard_id) { + Some((ShardStream::Labelled { label: l, .. }, _)) => l.as_str() < label, + Some((ShardStream::Legacy, _)) | None => true, + }) + .map(|r| r.shard_id.clone()) + .collect(); + ids.sort(); + ids +} + +/// A stream's status: only the table's newest stream, while its specification +/// is enabled, is `ENABLED` (`DISABLING` once the table is being deleted); +/// every other stream of the table is `DISABLED`. +#[must_use] +pub fn stream_status(table: &TableStreamRow, label: &str) -> StreamStatus { + let current = table.stream_label.as_deref() == Some(label) + && table + .stream_specification + .as_ref() + .is_some_and(|s| s.stream_enabled); + if !current { + StreamStatus::Disabled + } else if table.table_status == "DELETING" { + StreamStatus::Disabling + } else { + StreamStatus::Enabled + } +} + +/// Whether the stream `label` an enable just created is still the table's +/// enabled stream, read from the catalog after its shards were inserted. On +/// the backends whose catalog and shards are two commits, a disable that reads +/// the row between the enable's commit and its shard inserts closes the shards +/// it finds at that instant, so the enable re-reads the row afterwards and +/// closes what it inserted when the answer is `false` (the label moved on, the +/// specification is no longer enabled, or the table is gone). +#[must_use] +pub fn enable_still_current(table: Option<&TableStreamRow>, label: &str) -> bool { + table.is_some_and(|t| { + t.stream_label.as_deref() == Some(label) + && t.stream_specification + .as_ref() + .is_some_and(|s| s.stream_enabled) + }) +} + +/// The shard ids of the table's open shards: the ones its enabled stream +/// writes to. Empty when the stream is disabled. +#[must_use] +pub fn open_shard_ids(shards: &[ShardRow]) -> Vec { + let mut ids: Vec = shards + .iter() + .filter(|r| r.ending_sequence_number.is_none()) + .map(|r| r.shard_id.clone()) + .collect(); + ids.sort(); + ids +} + +/// Build a `DescribeStream` answer from the table row and all its shard rows. +/// +/// # Errors +/// +/// `TableNotFound` with the stream-not-found message when the table has no +/// stream with this label. +#[allow(clippy::too_many_arguments)] +pub fn describe_stream( + stream_arn: String, + table: &TableStreamRow, + key_schema: Vec, + label: String, + shards: &[ShardRow], + limit: Option, + exclusive_start_shard_id: Option<&str>, +) -> Result { + let stream = table_streams(table, shards) + .into_iter() + .find(|s| s.label == label) + .ok_or_else(|| stream_not_found(&stream_arn))?; + let limit = usize::try_from(limit.unwrap_or(100)).unwrap_or(0).max(1); + let mut remaining: Vec<&ShardRow> = stream + .shards + .iter() + .filter(|r| exclusive_start_shard_id.is_none_or(|s| r.shard_id.as_str() > s)) + .collect(); + let last_evaluated_shard_id = + (remaining.len() > limit).then(|| remaining[limit - 1].shard_id.clone()); + remaining.truncate(limit); + Ok(StreamDescription { + stream_status: stream_status(table, &label), + stream_view_type: stream.view_type, + stream_arn, + stream_label: label, + table_name: table.table_name.clone(), + key_schema, + shards: remaining + .into_iter() + .map(|r| Shard { + shard_id: r.shard_id.clone(), + parent_shard_id: r.parent_shard_id.clone(), + sequence_number_range: SequenceNumberRange { + starting_sequence_number: r.starting_sequence_number.clone(), + ending_sequence_number: r.ending_sequence_number.clone(), + }, + }) + .collect(), + last_evaluated_shard_id, + }) +} + +/// Check that `shard_id` is a shard of the stream `label` of `table`. +/// +/// # Errors +/// +/// [`crate::util::STREAM_DOES_NOT_EXIST`] when the table has no such stream, +/// and [`crate::util::SHARD_DOES_NOT_EXIST`] when the stream exists but the +/// shard is not one of its own (measured 2026-10-24 for a shard of the table's +/// other stream and for a well-formed shard id the account never had). +pub fn validate_shard( + table: Option<&TableStreamRow>, + label: &str, + shards: &[ShardRow], + shard_id: &str, +) -> Result<(), StorageError> { + let stream = table + .and_then(|t| { + table_streams(t, shards) + .into_iter() + .find(|s| s.label == label) + }) + .ok_or_else(stream_does_not_exist)?; + if stream.shards.iter().any(|r| r.shard_id == shard_id) { + Ok(()) + } else { + Err(shard_does_not_exist()) + } +} + +/// The sentence for an `ExclusiveStartStreamArn` that names a stream of a +/// table other than `TableName` (measured 2026-10-24). +#[must_use] +pub fn invalid_exclusive_start_stream_arn(arn: &str) -> StorageError { + StorageError::Validation(format!( + "One or more parameter values were invalid: ExclusiveStartStreamArn provided is invalid: {arn}" + )) +} + +#[path = "stream_lineage_labelled.rs"] +mod labelled; +pub use labelled::{has_labelled_shard, labelled_shard_id_pattern, labelled_shard_id_range}; + +#[path = "stream_lineage_list.rs"] +mod list; +pub use list::list_streams; + +#[cfg(test)] +#[path = "stream_lineage_tests.rs"] +mod tests; diff --git a/crates/storage/src/stream_lineage_labelled.rs b/crates/storage/src/stream_lineage_labelled.rs new file mode 100644 index 000000000..d357e58e8 --- /dev/null +++ b/crates/storage/src/stream_lineage_labelled.rs @@ -0,0 +1,43 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Which of a table's shard rows are labelled, for the backends that check it +//! before serving a legacy shard (`util::shard_in_stream`). + +use super::{ShardRow, ShardStream, parse_shard_id}; + +/// The byte-order range `[from, to)` that holds the labelled shard ids of +/// table `table_id` and none of its legacy ones, for the backends that write +/// 16-digit suffixes (SQLite and PostgreSQL). A labelled suffix starts with +/// the label's 13 milliseconds digits, not all zeros, so it sorts at or above +/// `0000000000001`; a legacy `{index:016}` suffix starts with 13 zeros and +/// sorts below. The upper end keeps the `shardId-{table_name}-{index:016}` +/// form out, which sorts anywhere relative to the table id. +#[must_use] +pub fn labelled_shard_id_range(table_id: &str) -> (String, String) { + ( + format!("shardId-{table_id}-0000000000001"), + format!("shardId-{table_id}."), + ) +} + +/// A regular expression matching the labelled shard ids of table `table_id` +/// and none of its legacy ones, for MongoDB, whose legacy ids have 12-digit +/// suffixes that no byte-order bound separates from a labelled one. Table ids +/// are UUIDs, so the id needs no escaping. +#[must_use] +pub fn labelled_shard_id_pattern(table_id: &str) -> String { + format!("^shardId-{table_id}-(?!0{{13}})[0-9]{{16}}$") +} + +/// Whether any of `shards` is a labelled shard of table `table_id`. +#[must_use] +pub fn has_labelled_shard(table_id: &str, shards: &[ShardRow]) -> bool { + shards.iter().any(|r| { + r.table_id == table_id + && matches!( + parse_shard_id(table_id, &r.shard_id), + Some((ShardStream::Labelled { .. }, _)) + ) + }) +} diff --git a/crates/storage/src/stream_lineage_list.rs b/crates/storage/src/stream_lineage_list.rs new file mode 100644 index 000000000..a25d813b8 --- /dev/null +++ b/crates/storage/src/stream_lineage_list.rs @@ -0,0 +1,68 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The `ListStreams` page over a table's streams; see the parent module. + +use extenddb_core::types::StreamSummary; + +use super::{ShardRow, TableStreamRow, invalid_exclusive_start_stream_arn, table_streams}; +use crate::error::StorageError; + +/// Build a `ListStreams` page from the account's tables and their shards. +/// +/// Every stream of every table is listed, disabled ones included, in the +/// order the service answered on 2026-10-24: table names descending, and a +/// table's streams newest first. `exclusive_start` is the (table name, label) +/// of the previous page's last stream; `arn` formats a stream ARN. +/// `LastEvaluatedStreamArn` is present whenever `limit` cut the page, even +/// when no stream follows: with two streams and `Limit=1`, page two carries it +/// and page three is empty (measured 2026-10-24). +/// +/// # Errors +/// +/// `Validation` when `exclusive_start` names a table other than `table_name`. +pub fn list_streams( + tables: &[TableStreamRow], + shards: &[ShardRow], + table_name: Option<&str>, + limit: i64, + exclusive_start: Option<(String, String)>, + arn: impl Fn(&str, &str) -> String, +) -> Result<(Vec, Option), StorageError> { + if let (Some(name), Some((start_table, start_label))) = (table_name, &exclusive_start) + && start_table != name + { + return Err(invalid_exclusive_start_stream_arn(&arn( + start_table, + start_label, + ))); + } + let mut listed: Vec<(String, String)> = tables + .iter() + .filter(|t| table_name.is_none_or(|n| t.table_name == n)) + .flat_map(|t| { + table_streams(t, shards) + .into_iter() + .map(|s| (t.table_name.clone(), s.label)) + }) + .collect(); + listed.sort_by(|a, b| b.cmp(a)); + if let Some(start) = exclusive_start { + listed.retain(|entry| *entry < start); + } + let limit = usize::try_from(limit).unwrap_or(0).max(1); + let last = (listed.len() >= limit).then(|| { + let (t, l) = &listed[limit - 1]; + arn(t, l) + }); + listed.truncate(limit); + let summaries = listed + .into_iter() + .map(|(table_name, label)| StreamSummary { + stream_arn: arn(&table_name, &label), + stream_label: label, + table_name, + }) + .collect(); + Ok((summaries, last)) +} diff --git a/crates/storage/src/stream_lineage_tests.rs b/crates/storage/src/stream_lineage_tests.rs new file mode 100644 index 000000000..94da3f2df --- /dev/null +++ b/crates/storage/src/stream_lineage_tests.rs @@ -0,0 +1,568 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +use super::*; + +const TID: &str = "0865b556-c8dd-433c-bb2a-f6a8fb6b6e65"; +const OLD: &str = "2026-10-24T08:54:14.037"; +const NEW: &str = "2026-10-24T08:54:32.805"; + +fn spec(enabled: bool, view: Option) -> StreamSpecification { + StreamSpecification { + stream_enabled: enabled, + stream_view_type: view, + } +} + +fn table(label: Option<&str>, spec: Option) -> TableStreamRow { + TableStreamRow { + table_name: "t482".to_owned(), + table_id: TID.to_owned(), + table_status: "ACTIVE".to_owned(), + stream_label: label.map(str::to_owned), + stream_specification: spec, + } +} + +fn row(shard_id: String, end: Option<&str>) -> ShardRow { + ShardRow { + table_id: TID.to_owned(), + shard_id, + parent_shard_id: None, + starting_sequence_number: "0".repeat(21), + ending_sequence_number: end.map(str::to_owned), + } +} + +fn stream_rows(label: &str, view: StreamViewType, end: Option<&str>) -> Vec { + (0..SHARDS_PER_STREAM) + .map(|i| row(shard_id(TID, label, view, i).unwrap(), end)) + .collect() +} + +#[test] +fn the_two_refusals_are_the_measured_sentences() { + let enabled = spec(true, Some(StreamViewType::NewAndOldImages)); + for requested in [ + spec(true, Some(StreamViewType::NewAndOldImages)), + spec(true, Some(StreamViewType::KeysOnly)), + ] { + match stream_change("t482-1792832053", Some(&enabled), &requested) { + Err(StorageError::Validation(m)) => assert_eq!( + m, + "Table already has an enabled stream: TableName: t482-1792832053" + ), + other => panic!("{other:?}"), + } + } + let disabled = spec(false, None); + for current in [None, Some(&disabled)] { + match stream_change("x482-nostream-1792832053", current, &disabled) { + Err(StorageError::Validation(m)) => assert_eq!( + m, + "Table has no stream to disable: TableName: x482-nostream-1792832053" + ), + other => panic!("{other:?}"), + } + } + assert_eq!( + stream_change("t", Some(&enabled), &disabled).unwrap(), + StreamChange::Disable + ); + assert_eq!( + stream_change("t", Some(&disabled), &enabled).unwrap(), + StreamChange::Enable(StreamViewType::NewAndOldImages) + ); + assert_eq!( + stream_change("t", None, &enabled).unwrap(), + StreamChange::Enable(StreamViewType::NewAndOldImages) + ); +} + +#[test] +fn a_refused_disable_of_a_disabled_stream_closes_the_current_label_first() { + let label = "2026-10-20T10:00:00.000"; + let disabled = spec(false, Some(StreamViewType::NewImage)); + let disable = spec(false, None); + let enable = spec(true, Some(StreamViewType::KeysOnly)); + // The stored specification says disabled: the first disable (an earlier + // release's, or one whose close failed) may have left the shards open. + assert_eq!( + disabled_stream_to_close(Some(&disabled), Some(label), &disable), + Some(label) + ); + assert_eq!( + disabled_stream_to_close(Some(&spec(false, None)), Some(label), &disable), + Some(label) + ); + // A table that never had a stream, or has no label, has nothing to close; + // an enabled stream is closed by the disable itself; an enable closes + // through its own change. + assert_eq!(disabled_stream_to_close(None, Some(label), &disable), None); + assert_eq!( + disabled_stream_to_close(Some(&disabled), None, &disable), + None + ); + assert_eq!( + disabled_stream_to_close( + Some(&spec(true, Some(StreamViewType::KeysOnly))), + Some(label), + &disable + ), + None + ); + assert_eq!( + disabled_stream_to_close(Some(&disabled), Some(label), &enable), + None + ); +} + +#[test] +fn labels_round_trip_through_shard_ids() { + assert_eq!(label_millis("1970-01-01T00:00:00.001"), Some(1)); + assert_eq!(label_millis("2026-10-24T08:54:14.03"), None); + assert_eq!(label_millis("2026-10-24 08:54:14.037"), None); + assert_eq!(label_millis("2026-13-24T08:54:14.037"), None); + for view in [ + StreamViewType::KeysOnly, + StreamViewType::NewImage, + StreamViewType::OldImage, + StreamViewType::NewAndOldImages, + ] { + let id = shard_id(TID, OLD, view, 3).unwrap(); + assert!(id.len() <= 65, "{id}"); + assert_eq!( + parse_shard_id(TID, &id), + Some(( + ShardStream::Labelled { + label: OLD.to_owned(), + view_type: view + }, + 3 + )) + ); + } + // Ids written before streams were named in shard ids: 16 digits on SQLite + // and PostgreSQL, 12 on MongoDB and Cassandra. + for legacy in [ + format!("shardId-{TID}-{:016}", 2), + format!("shardId-{TID}-{:012}", 2), + ] { + assert_eq!(parse_shard_id(TID, &legacy), Some((ShardStream::Legacy, 2))); + } + assert_eq!( + parse_shard_id( + "other", + &shard_id(TID, OLD, StreamViewType::KeysOnly, 0).unwrap() + ), + None + ); + assert!(shard_id(TID, "not a label", StreamViewType::KeysOnly, 0).is_err()); +} + +#[test] +fn a_new_label_is_later_than_the_previous_one() { + let far_future = "2999-01-01T00:00:00.000"; + assert_eq!( + next_stream_label(Some(far_future)), + "2999-01-01T00:00:00.001" + ); + let fresh = next_stream_label(Some(OLD)); + assert!(fresh.as_str() > OLD, "{fresh}"); + assert!(label_millis(&next_stream_label(None)).is_some()); +} + +#[test] +fn a_disabled_stream_stays_describable_with_its_ending_numbers() { + let mut shards = stream_rows(OLD, StreamViewType::NewAndOldImages, Some("1")); + shards[3].ending_sequence_number = Some(shards[3].starting_sequence_number.clone()); + let t = table(Some(OLD), None); + let d = describe_stream( + "arn".into(), + &t, + Vec::new(), + OLD.into(), + &shards, + None, + None, + ) + .unwrap(); + assert!(matches!(d.stream_status, StreamStatus::Disabled)); + assert_eq!(d.stream_view_type, StreamViewType::NewAndOldImages); + assert_eq!(d.shards.len(), 4); + assert!(d.last_evaluated_shard_id.is_none()); + assert!( + d.shards + .iter() + .all(|s| s.sequence_number_range.ending_sequence_number.is_some()) + ); + + // Limit=2 pages through the disabled stream as through an enabled one. + let p1 = describe_stream( + "arn".into(), + &t, + Vec::new(), + OLD.into(), + &shards, + Some(2), + None, + ) + .unwrap(); + assert_eq!(p1.shards.len(), 2); + let start = p1.last_evaluated_shard_id.clone().unwrap(); + assert_eq!(start, p1.shards[1].shard_id); + let p2 = describe_stream( + "arn".into(), + &t, + Vec::new(), + OLD.into(), + &shards, + Some(2), + Some(&start), + ) + .unwrap(); + assert_eq!(p2.shards.len(), 2); + assert!(p2.last_evaluated_shard_id.is_none()); +} + +#[test] +fn a_re_enabled_table_has_two_streams_and_only_the_new_one_is_enabled() { + let mut shards = stream_rows(OLD, StreamViewType::NewAndOldImages, Some("5")); + shards.extend(stream_rows(NEW, StreamViewType::KeysOnly, None)); + let t = table(Some(NEW), Some(spec(true, Some(StreamViewType::KeysOnly)))); + let streams = table_streams(&t, &shards); + assert_eq!(streams.len(), 2); + assert_eq!(streams[0].label, OLD); + assert!(matches!(stream_status(&t, OLD), StreamStatus::Disabled)); + assert!(matches!(stream_status(&t, NEW), StreamStatus::Enabled)); + let old = describe_stream("a".into(), &t, Vec::new(), OLD.into(), &shards, None, None).unwrap(); + assert_eq!(old.stream_view_type, StreamViewType::NewAndOldImages); + let new = describe_stream("a".into(), &t, Vec::new(), NEW.into(), &shards, None, None).unwrap(); + assert!( + new.shards + .iter() + .all(|s| !old.shards.iter().any(|o| o.shard_id == s.shard_id)) + ); + assert_eq!(open_shard_ids(&shards).len(), 4); + + // Each ARN accepts only its own stream's shards. + validate_shard(Some(&t), OLD, &shards, &old.shards[0].shard_id).unwrap(); + for (label, shard) in [ + (OLD, &new.shards[0].shard_id), + (NEW, &old.shards[0].shard_id), + ] { + match validate_shard(Some(&t), label, &shards, shard) { + Err(StorageError::TableNotFound(m)) => { + assert_eq!(m, "Requested resource not found: Shard does not exist"); + } + other => panic!("{other:?}"), + } + } + match validate_shard(Some(&t), "2020-01-01T00:00:00.000", &shards, "s") { + Err(StorageError::TableNotFound(m)) => { + assert_eq!(m, "Requested resource not found: Stream does not exist"); + } + other => panic!("{other:?}"), + } +} + +/// SQLite and PostgreSQL wrote `shardId-{table_name}-{index:016}` before +/// v0.1.14; those rows are this table's shards and its stream must stay +/// describable after an upgrade. +#[test] +fn name_based_shard_ids_from_before_v0_1_14_are_the_current_stream() { + let rows: Vec = (0..4) + .map(|i| row(format!("shardId-t482-{i:016}"), None)) + .collect(); + let t = table(Some(OLD), Some(spec(true, Some(StreamViewType::NewImage)))); + let streams = table_streams(&t, &rows); + assert_eq!(streams.len(), 1); + assert_eq!(streams[0].label, OLD); + assert_eq!(streams[0].view_type, StreamViewType::NewImage); + assert_eq!(streams[0].shards.len(), 4); + validate_shard(Some(&t), OLD, &rows, "shardId-t482-0000000000000002").unwrap(); + let d = describe_stream("arn".into(), &t, Vec::new(), OLD.into(), &rows, None, None).unwrap(); + assert_eq!(d.shards.len(), 4); + assert!(matches!(d.stream_status, StreamStatus::Enabled)); + // A shard row of another table never reaches the grouping. + let foreign = ShardRow { + table_id: "other".to_owned(), + ..row("shardId-other-0000000000000000".to_owned(), None) + }; + assert_eq!(table_streams(&t, &[foreign]).len(), 0); +} + +#[test] +fn legacy_shards_belong_to_the_current_label_until_a_new_stream_takes_it() { + let legacy: Vec = (0..4) + .map(|i| row(format!("shardId-{TID}-{i:016}"), None)) + .collect(); + let t = table(Some(OLD), Some(spec(true, Some(StreamViewType::NewImage)))); + let streams = table_streams(&t, &legacy); + assert_eq!(streams.len(), 1); + assert_eq!(streams[0].label, OLD); + assert_eq!(streams[0].view_type, StreamViewType::NewImage); + + let mut both = legacy; + both.extend(stream_rows(NEW, StreamViewType::NewImage, None)); + let t = table(Some(NEW), Some(spec(true, Some(StreamViewType::NewImage)))); + let streams = table_streams(&t, &both); + assert_eq!( + streams.len(), + 1, + "the legacy stream's label is no longer known" + ); + assert_eq!(streams[0].label, NEW); + assert!(streams[0].shards.iter().all(|s| s.shard_id.len() == 61)); +} + +#[test] +fn list_streams_orders_tables_descending_and_streams_newest_first() { + let mut shards = stream_rows(OLD, StreamViewType::NewAndOldImages, Some("5")); + shards.extend(stream_rows(NEW, StreamViewType::NewAndOldImages, None)); + let other_id = "11111111-2222-3333-4444-555555555555"; + shards.extend((0..4).map(|i| ShardRow { + table_id: other_id.to_owned(), + shard_id: shard_id(other_id, OLD, StreamViewType::KeysOnly, i).unwrap(), + ..row(String::new(), None) + })); + let tables = vec![ + table( + Some(NEW), + Some(spec(true, Some(StreamViewType::NewAndOldImages))), + ), + TableStreamRow { + table_name: "a-table".to_owned(), + table_id: other_id.to_owned(), + ..table(Some(OLD), Some(spec(true, Some(StreamViewType::KeysOnly)))) + }, + ]; + let arn = |t: &str, l: &str| format!("{t}/{l}"); + let (all, last) = list_streams(&tables, &shards, None, 100, None, arn).unwrap(); + let got: Vec = all.iter().map(|s| s.stream_arn.clone()).collect(); + assert_eq!( + got, + [ + format!("t482/{NEW}"), + format!("t482/{OLD}"), + format!("a-table/{OLD}") + ] + ); + assert!(last.is_none()); + + let (only, last) = list_streams(&tables, &shards, Some("t482"), 100, None, arn).unwrap(); + assert_eq!(only.len(), 2); + assert_eq!(only[0].stream_label, NEW); + assert!(last.is_none()); + + // Limit=1 over the two streams of one table: three pages, the last ARN + // present on every page the limit cut, the third page empty. + let page = |start: Option<&str>| { + list_streams( + &tables, + &shards, + Some("t482"), + 1, + start.map(|l| ("t482".to_owned(), l.to_owned())), + arn, + ) + .unwrap() + }; + let (p1, last) = page(None); + assert_eq!(p1[0].stream_label, NEW); + assert_eq!(last.as_deref(), Some(format!("t482/{NEW}").as_str())); + let (p2, last) = page(Some(NEW)); + assert_eq!(p2[0].stream_label, OLD); + assert_eq!(last.as_deref(), Some(format!("t482/{OLD}").as_str())); + let (p3, last) = page(Some(OLD)); + assert!(p3.is_empty()); + assert!(last.is_none()); + + // No Limit: after the older stream nothing, after the newest the older. + let from = |l: &str| { + list_streams( + &tables, + &shards, + Some("t482"), + 100, + Some(("t482".to_owned(), l.to_owned())), + arn, + ) + .unwrap() + }; + let (after_old, last) = from(OLD); + assert!(after_old.is_empty()); + assert!(last.is_none()); + let (after_new, last) = from(NEW); + assert_eq!(after_new.len(), 1); + assert_eq!(after_new[0].stream_label, OLD); + assert!(last.is_none()); + + // A start ARN of another table, with TableName, is refused. + match list_streams( + &tables, + &shards, + Some("t482"), + 100, + Some(("a-table".to_owned(), OLD.to_owned())), + arn, + ) { + Err(StorageError::Validation(m)) => assert_eq!( + m, + format!( + "One or more parameter values were invalid: ExclusiveStartStreamArn provided is invalid: a-table/{OLD}" + ) + ), + other => panic!("{other:?}"), + } +} + +#[test] +fn a_disabled_create_table_specification_is_not_stored() { + let make = |spec: Option| extenddb_core::types::CreateTableInput { + table_name: "t".to_owned(), + stream_specification: spec, + ..Default::default() + }; + assert_eq!( + without_disabled_specification(make(Some(spec(false, None)))).stream_specification, + None + ); + let enabled = spec(true, Some(StreamViewType::KeysOnly)); + assert_eq!( + without_disabled_specification(make(Some(enabled.clone()))).stream_specification, + Some(enabled) + ); + assert_eq!( + without_disabled_specification(make(None)).stream_specification, + None + ); +} + +/// A disable closes the shards of the stream it decided on and no other, +/// including when an overlapping enable has already opened the next stream. +#[test] +fn a_close_selects_only_the_shards_of_its_label() { + let mut shards = stream_rows(OLD, StreamViewType::NewAndOldImages, None); + shards.extend(stream_rows(NEW, StreamViewType::KeysOnly, None)); + let old_ids = open_shard_ids_of(TID, OLD, &shards); + assert_eq!(old_ids.len(), 4); + assert!( + old_ids + .iter() + .all(|id| shards[..4].iter().any(|r| &r.shard_id == id)) + ); + let new_ids = open_shard_ids_of(TID, NEW, &shards); + assert_eq!(new_ids.len(), 4); + assert!(old_ids.iter().all(|id| !new_ids.contains(id))); + // A label no shard carries selects nothing. + assert!(open_shard_ids_of(TID, "2020-01-01T00:00:00.000", &shards).is_empty()); + // Closed shards are never selected again. + shards[0].ending_sequence_number = Some("5".to_owned()); + assert_eq!(open_shard_ids_of(TID, OLD, &shards).len(), 3); + + // Legacy rows belong to whatever label the change read, until a labelled + // stream carries that label. + let legacy: Vec = (0..4) + .map(|i| row(format!("shardId-t482-{i:016}"), None)) + .collect(); + assert_eq!(open_shard_ids_of(TID, OLD, &legacy).len(), 4); + let mut both = legacy.clone(); + both.extend(stream_rows(OLD, StreamViewType::KeysOnly, None)); + assert_eq!(open_shard_ids_of(TID, OLD, &both).len(), 4); + assert!( + open_shard_ids_of(TID, OLD, &both) + .iter() + .all(|id| id.len() == 61) + ); + // Another table's rows are never selected. + let foreign = vec![ShardRow { + table_id: "other".to_owned(), + ..row( + shard_id("other", OLD, StreamViewType::KeysOnly, 0).unwrap(), + None, + ) + }]; + assert!(open_shard_ids_of(TID, OLD, &foreign).is_empty()); +} + +#[test] +fn an_enable_re_read_sees_whether_its_stream_is_still_current() { + let enabled = table(Some(NEW), Some(spec(true, Some(StreamViewType::NewImage)))); + assert!(enable_still_current(Some(&enabled), NEW)); + // A disable committed between the enable's catalog write and its shard + // inserts: the label is still the enable's, the specification is not + // enabled. + let disabled = table(Some(NEW), Some(spec(false, Some(StreamViewType::NewImage)))); + assert!(!enable_still_current(Some(&disabled), NEW)); + // A later enable replaced the label. + assert!(!enable_still_current(Some(&enabled), OLD)); + // No specification at all, or no table row. + assert!(!enable_still_current(Some(&table(Some(NEW), None)), NEW)); + assert!(!enable_still_current(None, NEW)); +} + +#[test] +fn the_labelled_range_holds_labelled_ids_and_no_legacy_ones() { + let (from, to) = labelled_shard_id_range(TID); + let inside = |id: &str| id >= from.as_str() && id < to.as_str(); + for view in [StreamViewType::KeysOnly, StreamViewType::NewAndOldImages] { + for i in 0..SHARDS_PER_STREAM { + assert!(inside(&shard_id(TID, OLD, view, i).unwrap())); + } + } + for i in 0..SHARDS_PER_STREAM { + assert!(!inside(&format!("shardId-{TID}-{i:016}"))); + } + // The name forms sort on either side of the table id. + assert!(!inside("shardId-Orders-0000000000000002")); + assert!(!inside("shardId-00-orders-0000000000000002")); + assert!(!inside("shardId-zz-orders-0000000000000002")); +} + +#[test] +fn a_table_has_a_labelled_shard_once_an_enable_wrote_one() { + let legacy: Vec = (0..SHARDS_PER_STREAM) + .map(|i| row(format!("shardId-{TID}-{i:012}"), Some("1"))) + .collect(); + assert!(!has_labelled_shard(TID, &legacy)); + let mut both = legacy; + both.extend(stream_rows(NEW, StreamViewType::KeysOnly, None)); + assert!(has_labelled_shard(TID, &both)); + // Another table's labelled rows do not count. + let foreign = vec![ShardRow { + table_id: "other".to_owned(), + ..row( + shard_id("other", OLD, StreamViewType::KeysOnly, 0).unwrap(), + None, + ) + }]; + assert!(!has_labelled_shard(TID, &foreign)); +} + +#[test] +fn an_enable_closes_the_open_shards_of_every_earlier_stream() { + const LATEST: &str = "2026-10-24T08:55:00.000"; + let mut rows: Vec = (0..SHARDS_PER_STREAM) + .map(|i| row(format!("shardId-{TID}-{i:016}"), None)) + .collect(); + rows.extend(stream_rows(OLD, StreamViewType::KeysOnly, None)); + rows.extend(stream_rows(NEW, StreamViewType::NewImage, Some("5"))); + rows.extend(stream_rows(LATEST, StreamViewType::NewImage, None)); + let mut expected: Vec = (0..SHARDS_PER_STREAM) + .map(|i| format!("shardId-{TID}-{i:016}")) + .collect(); + expected.extend( + stream_rows(OLD, StreamViewType::KeysOnly, None) + .into_iter() + .map(|r| r.shard_id), + ); + expected.sort(); + // The legacy rows and the open OLD rows; NEW's rows are closed and + // LATEST's are not earlier. + assert_eq!(open_shard_ids_before(TID, LATEST, &rows), expected); + assert!( + open_shard_ids_before(TID, OLD, &stream_rows(OLD, StreamViewType::KeysOnly, None)) + .is_empty() + ); +} diff --git a/crates/storage/src/util/arn.rs b/crates/storage/src/util/arn.rs index f463ee1f0..e9a6f4c5c 100755 --- a/crates/storage/src/util/arn.rs +++ b/crates/storage/src/util/arn.rs @@ -119,20 +119,23 @@ pub fn stream_does_not_exist() -> StorageError { } /// What `GetShardIterator` answers for a shard that is not one of the named -/// stream's shards: the stream-not-found sentence with a trailing period. The -/// service answers "Requested resource not found: Shard does not exist" -/// (measured 2026-10-24), so this text differs from it. +/// stream's shards. Measured 2026-10-24 for a shard id of the table's other +/// stream, on the old and on the new stream ARN, and for a well-formed shard id +/// the account never had. +pub const SHARD_DOES_NOT_EXIST: &str = "Requested resource not found: Shard does not exist"; + +/// [`SHARD_DOES_NOT_EXIST`] as the storage error the engine maps to a +/// `ResourceNotFoundException`. #[must_use] -pub fn shard_not_in_stream(stream_arn: &str) -> StorageError { - StorageError::TableNotFound(format!( - "Requested resource not found: Stream: {stream_arn} not found." - )) +pub fn shard_does_not_exist() -> StorageError { + StorageError::TableNotFound(SHARD_DOES_NOT_EXIST.to_owned()) } #[cfg(test)] mod stream_not_found_tests { use super::{ - STREAM_DOES_NOT_EXIST, shard_not_in_stream, stream_does_not_exist, stream_not_found, + SHARD_DOES_NOT_EXIST, STREAM_DOES_NOT_EXIST, shard_does_not_exist, stream_does_not_exist, + stream_not_found, }; use crate::error::StorageError; @@ -164,11 +167,12 @@ mod stream_not_found_tests { } #[test] - fn a_shard_of_no_stream_of_the_table_keeps_the_sentence_with_the_period() { + fn get_shard_iterator_says_a_shard_of_no_stream_of_the_table_does_not_exist() { assert_eq!( - message(shard_not_in_stream(ARN)), - format!("Requested resource not found: Stream: {ARN} not found.") + SHARD_DOES_NOT_EXIST, + "Requested resource not found: Shard does not exist" ); + assert_eq!(message(shard_does_not_exist()), SHARD_DOES_NOT_EXIST); } } diff --git a/crates/storage/src/util/mod.rs b/crates/storage/src/util/mod.rs index bb79c2103..d93e5177b 100755 --- a/crates/storage/src/util/mod.rs +++ b/crates/storage/src/util/mod.rs @@ -13,9 +13,9 @@ mod key; mod shard_stream; pub use arn::{ - ARN_PARTITION, STREAM_DOES_NOT_EXIST, format_stream_label, index_arn, new_stream_label, - parse_stream_arn, shard_not_in_stream, stream_arn, stream_does_not_exist, stream_not_found, - table_arn, + ARN_PARTITION, SHARD_DOES_NOT_EXIST, STREAM_DOES_NOT_EXIST, format_stream_label, index_arn, + new_stream_label, parse_stream_arn, shard_does_not_exist, stream_arn, stream_does_not_exist, + stream_not_found, table_arn, }; pub use backup_id::{ BackupArnError, BackupArnParts, MAX_BACKUP_ARN_LEN, backup_arn, backup_creation_date_time, diff --git a/crates/storage/src/util/shard_stream.rs b/crates/storage/src/util/shard_stream.rs index 93982c78b..113198737 100644 --- a/crates/storage/src/util/shard_stream.rs +++ b/crates/storage/src/util/shard_stream.rs @@ -10,54 +10,198 @@ //! label names. The rule lives in one place so a change to how streams are //! identified touches one function and its test. +use std::future::Future; + +use crate::stream_lineage::{ShardStream, parse_shard_id}; + /// Whether shard `shard_id`, a row of table `table_id` whose current stream /// label is `table_label` (`None` when the table never had a stream), belongs /// to the stream labelled `arn_label`. /// -/// Today a table has one stream and every shard row of the table belongs to -/// it, so the shard is the stream's exactly when the ARN's label is the -/// table's current label. `table_id` and `shard_id` are part of the contract -/// for a scheme that names the stream inside the shard id; this rule does not -/// read them. -#[must_use] -pub fn shard_in_stream( +/// A table has a sequence of streams, one per enable, and each stream's +/// identity is in its shard ids ([`crate::stream_lineage::shard_id`]). A +/// labelled id belongs to the label it carries, so a closed stream's shards +/// stay readable under the closed stream's ARN after the table has a newer +/// stream, and an iterator pairing the newer ARN with one of them is refused. +/// An id from before streams were named in shard ids (the +/// `shardId-{table_id}-{index}` and `shardId-{table_name}-{index}` shapes) +/// belongs to the table's current stream while the table has no labelled +/// shard: it is the ARN's when the ARN's label is the table's current label +/// and `table_has_labelled_shard` answers `false`. From a table's first enable +/// under the labelled scheme every later stream is labelled, so legacy rows +/// beside a labelled row are an earlier, closed stream, which the newer ARN +/// does not name. The row is already this table's (the callers key the shard +/// read on the table id), so any id that does not parse is a legacy one. +/// +/// `table_has_labelled_shard` reads the table's shard rows; it runs only for a +/// legacy id whose table label matches the ARN's, so a labelled id is decided +/// from the id alone. +/// +/// # Errors +/// +/// Whatever `table_has_labelled_shard` returns. +pub async fn shard_in_stream( table_id: &str, shard_id: &str, table_label: Option<&str>, arn_label: &str, -) -> bool { - let _ = (table_id, shard_id); - table_label == Some(arn_label) + table_has_labelled_shard: impl FnOnce() -> F, +) -> Result +where + F: Future>, +{ + match parse_shard_id(table_id, shard_id) { + Some((ShardStream::Labelled { label, .. }, _)) => Ok(label == arn_label), + Some((ShardStream::Legacy, _)) | None => { + if table_label != Some(arn_label) { + return Ok(false); + } + Ok(!table_has_labelled_shard().await?) + } + } } #[cfg(test)] mod tests { + use std::cell::Cell; + use std::convert::Infallible; + + use extenddb_core::types::StreamViewType; + use super::shard_in_stream; + use crate::stream_lineage::shard_id; const TABLE: &str = "0f3c1c1e-7b1d-4e3b-9a6e-4a2d3e5f6a7b"; const LABEL: &str = "2026-01-01T00:00:00.000"; const LATER: &str = "2026-02-02T00:00:00.000"; - fn shard(i: u32) -> String { + fn legacy_shard(i: u32) -> String { format!("shardId-{TABLE}-{i:016}") } + fn labelled_shard(label: &str, i: u32) -> String { + shard_id(TABLE, label, StreamViewType::KeysOnly, i).expect("label shape") + } + + /// `shard_in_stream` on a table whose shard rows include a labelled one + /// when `has_labelled` is set. + fn in_stream( + shard: &str, + table_label: Option<&str>, + arn_label: &str, + has_labelled: bool, + ) -> bool { + let answer = futures::executor::block_on(shard_in_stream( + TABLE, + shard, + table_label, + arn_label, + || async move { Ok::(has_labelled) }, + )); + match answer { + Ok(v) => v, + Err(never) => match never {}, + } + } + + #[test] + fn a_labelled_shard_belongs_to_the_stream_its_id_names() { + assert!(in_stream( + &labelled_shard(LABEL, 0), + Some(LABEL), + LABEL, + true + )); + assert!(in_stream( + &labelled_shard(LABEL, 3), + Some(LABEL), + LABEL, + true + )); + } + + #[test] + fn a_closed_stream_stays_readable_after_the_table_has_a_newer_one() { + // The table was disabled and enabled again: its label moved on to the + // new stream, and the closed stream's shards still answer to the + // closed stream's ARN. + assert!(in_stream( + &labelled_shard(LABEL, 0), + Some(LATER), + LABEL, + true + )); + assert!(in_stream( + &labelled_shard(LATER, 0), + Some(LATER), + LATER, + true + )); + } + + #[test] + fn the_newer_arn_with_an_older_streams_shard_id_is_not_its_stream() { + // An iterator forged from the new ARN and an old shard id, or the + // other way round, names a shard the stream does not have. + assert!(!in_stream( + &labelled_shard(LABEL, 0), + Some(LATER), + LATER, + true + )); + assert!(!in_stream( + &labelled_shard(LATER, 0), + Some(LATER), + LABEL, + true + )); + } + + #[test] + fn a_legacy_shard_belongs_to_its_tables_current_stream() { + assert!(in_stream(&legacy_shard(0), Some(LABEL), LABEL, false)); + assert!(in_stream(&legacy_shard(3), Some(LABEL), LABEL, false)); + // The shape SQLite and PostgreSQL wrote before v0.1.14 names the table, + // not its id; the row is the table's, so it is a legacy shard too. + assert!(in_stream( + "shardId-Orders-0000000000000002", + Some(LABEL), + LABEL, + false + )); + } + #[test] - fn a_shard_belongs_to_its_tables_current_stream() { - assert!(shard_in_stream(TABLE, &shard(0), Some(LABEL), LABEL)); - assert!(shard_in_stream(TABLE, &shard(3), Some(LABEL), LABEL)); + fn an_earlier_label_on_legacy_shards_is_not_the_current_stream() { + // The table took a new label while keeping its legacy shard rows: the + // old ARN no longer names a stream of the table. + assert!(!in_stream(&legacy_shard(0), Some(LATER), LABEL, false)); + // The new label is a labelled stream of its own (an enable gave it + // shards), so the legacy rows are the earlier, closed stream and the + // newer ARN does not name them. + assert!(!in_stream(&legacy_shard(0), Some(LATER), LATER, true)); } #[test] - fn an_earlier_label_on_the_same_shards_is_not_the_current_stream() { - // The table took a new label while keeping its shard rows: the old - // ARN no longer names a stream of the table. - assert!(!shard_in_stream(TABLE, &shard(0), Some(LATER), LABEL)); - assert!(shard_in_stream(TABLE, &shard(0), Some(LATER), LATER)); + fn a_labelled_shard_is_decided_without_reading_the_tables_shards() { + let reads = Cell::new(0); + let answer = futures::executor::block_on(shard_in_stream( + TABLE, + &labelled_shard(LABEL, 0), + Some(LABEL), + LABEL, + || { + reads.set(reads.get() + 1); + async { Ok::(true) } + }, + )); + assert!(matches!(answer, Ok(true))); + assert_eq!(reads.get(), 0); } #[test] fn a_table_without_a_stream_owns_no_stream_shard() { - assert!(!shard_in_stream(TABLE, &shard(0), None, LABEL)); + assert!(!in_stream(&legacy_shard(0), None, LABEL, false)); + assert!(!in_stream(&labelled_shard(LABEL, 0), None, LATER, false)); } } diff --git a/docs/design/07-component-streams.md b/docs/design/07-component-streams.md index dbff306b5..edba96f83 100755 --- a/docs/design/07-component-streams.md +++ b/docs/design/07-component-streams.md @@ -91,6 +91,18 @@ DynamoDB retains stream records for 24 hours. Options: **Recommended:** Application-level background task (same pattern as TTL cleanup), configurable retention period (default: 24 hours). +### 4.5 Enable and Disable + +A table has a sequence of streams. `UpdateTable StreamEnabled=false` closes the current one: the stored specification becomes `{StreamEnabled: false}` with the stream's view type (DescribeTable omits a disabled `StreamSpecification`), `LatestStreamArn` and `LatestStreamLabel` keep naming the closed stream, DescribeStream reports it `DISABLED` with its original view type, and every open shard gets an `EndingSequenceNumber` (its last record's sequence number, or its starting number when it holds none). `StreamEnabled=true` afterwards creates a new stream: a new label later than the previous one, four new shard ids, the requested view type. The closed stream stays describable and listed; ListStreams lists every stream of a table, newest first. Enabling a table whose stream is enabled, and disabling a table with no enabled stream, are refused with `ValidationException`. + +Each stream's identity is carried in its shard ids, `shardId-{table_id}-{label_millis:013}{view_type:1}{index:02}`, so no backend stores a stream table. Shard ids from before this scheme (`shardId-{table_id}-{index}`, and the `shardId-{table_name}-{index}` form SQLite and PostgreSQL wrote before v0.1.14) belong to the table's current stream. Writes record only to open shards. The rules shared by the four backends live in `extenddb_storage::stream_lineage`; each backend reads its table row and shard rows and calls them. + +On Cassandra the write paths compute a record's partition from the table id and partition key alone, inside the data write's LOGGED BATCH, so records stay in one partition per shard index and a stream's shard is a range of that partition: after its `StartingSequenceNumber`, up to its `EndingSequenceNumber` once closed. A new stream's shards start one above the larger of the enabling server's next clock value and the last `EndingSequenceNumber` among the table's shards. + +A stream change travels alone: an `UpdateTable` that carries `StreamSpecification` beside `BillingMode` or `ProvisionedThroughput` is refused with `You cannot modify stream status while updating table IOPS`, beside any `GlobalSecondaryIndexUpdates` entry with `You cannot create or delete index while changing stream status`, and nothing is applied. A table created with `StreamSpecification {StreamEnabled: false}` stores nothing of it. + +Writes racing a disable. The service's rule (measured 2026-10-24, 569 writes against one disable): every write accepted before the stream reports `DISABLED` lands in the closed stream before its shard's `EndingSequenceNumber`, and a write after that records nothing. ExtendDB closes the shards in the `UpdateTable` call. On SQLite the engine write lock serialises the close against every write. On PostgreSQL the write path reads the open shards without a lock. It locks the one shard its record goes to `FOR KEY SHARE`, re-checks that the shard is still open, and stores the record in `stream_pending`. The record gets its sequence number after the write commits. The close runs in one READ COMMITTED data transaction. It locks the open rows `FOR UPDATE`, numbers the pending records of each shard it locked, and then takes the ending numbers. The record of a write that locked its row before the close therefore gets a number at or below the ending number. A later write finds the row closed and records nothing. Before that transaction, the close numbers each shard's backlog in the numbering worker's batches. Writers do not wait for the row lock of those batches, so a write waits only while the close numbers the records committed in between. The close locks the rows in shard id order, and `TransactWriteItems` stores its records in the same order, so the two cannot deadlock. On Cassandra a new stream's shards start one above the larger of the enabling server's next clock value and the last `EndingSequenceNumber` among the table's shards, so on one server a write of the closed stream that commits after the close falls between the old shard's ending number and the new shard's starting number and is read by neither stream (the sequence is drawn in the write's LOGGED BATCH, which the close cannot wait for). A record of the old stream committed before the close by a server whose clock is ahead of the enabling server's is in the closed shard's ending number, so it is below the start and in one stream only; with several servers, a raced record from a server whose clock is ahead can land above the start, in the new stream. A server whose clock is behind the start can write a new-stream record below it, which no stream reads. Both cases are part of the deferred cross-instance consistency item below. On MongoDB the window is one write transaction wide: a write whose transaction read the shard as open and commits after the close lands past the ending number; closing the gap needs the write transaction to touch the shard document and the stream path to classify the resulting write conflict as retryable, which it does not today. On every backend a write that found no open shard because its key info pre-dated the disable is acknowledged and recorded in no stream; the service records every write it acknowledges while the disable is in progress. A table toggled many times accumulates four closed shard rows per cycle; nothing sweeps a live table's closed shards. On PostgreSQL, MongoDB and Cassandra an enable commits its catalog row before inserting its shards, so a disable that reads the row in between closes whatever shards exist at that instant; the enable re-reads the row after its inserts and closes its own shards when the row no longer says enabled with its label, so a stream never reports `DISABLED` with open shards. + ## 5. Integration Points ### 5.1 Write Path Integration @@ -158,7 +170,7 @@ GetRecords handler: | Cross-instance stream consistency | Deferred | Relevant for multi-instance deployments | | Stream record deduplication | Deferred | Relevant for at-least-once delivery guarantees | | Kinesis adapter compatibility | Deferred | DynamoDB Streams has a Kinesis-compatible adapter | -| Stream enable/disable lifecycle | Deferred | What happens to in-flight records when streams are disabled | +| Stream enable/disable lifecycle | Decided | See 4.5: disable closes the shards, enable creates a new stream | --- diff --git a/docs/design/13-storage-mongodb.md b/docs/design/13-storage-mongodb.md index cc4f74af3..3ebba2fb2 100644 --- a/docs/design/13-storage-mongodb.md +++ b/docs/design/13-storage-mongodb.md @@ -373,7 +373,9 @@ Simple collation is used whenever the tuple contains a string component. } ``` -Unique index on `shard_id`. Four shards per stream-enabled table. +Unique index on `shard_id`, and an index on `(table_id, shard_id)`, by which +every streamed write and every stream read finds a table's shards in shard id +order. Four shards per stream-enabled table. `shard_id` embeds `table_id` (a UUID) rather than `table_name`. Table names are only unique per-account; a name-derived scheme would let one diff --git a/docs/differences-from-dynamodb.md b/docs/differences-from-dynamodb.md index 063022578..60a9f381c 100755 --- a/docs/differences-from-dynamodb.md +++ b/docs/differences-from-dynamodb.md @@ -52,6 +52,7 @@ from a request sent to that service or backend. |------|----------|------| | Table creation delay | Returns `CREATING` immediately; transitions to `ACTIVE` typically within seconds. Same behavior for on-demand and provisioned | Configurable via `control_plane_delay_seconds` runtime setting (default: 5s) | | DeletionProtectionEnabled | Enforced | Enforced (accepted and stored, DeleteTable rejects when enabled) | +| Stream disable and enable | `UpdateTable StreamEnabled=false` answers `UPDATING`; the stream reports `DISABLED` a few seconds later with every shard closed, and every write acknowledged before that lands in the closed stream. `StreamEnabled=true` afterwards creates a new stream with a later label; both streams stay listed, and a disabled stream stays describable after its table is deleted | The shards close inside the `UpdateTable` call and the response says `ACTIVE`. A write that reaches storage after the close is acknowledged and recorded in no stream (on MongoDB it is recorded past the shard's `EndingSequenceNumber` instead); the window is one write wide. A disabled stream is removed with its table. A stream created by a release before this one loses its ARN once the table is disabled and re-enabled (its label lived only in the table row) | ## Time to Live (TTL) diff --git a/docs/manuals/07-upgrade-manual.md b/docs/manuals/07-upgrade-manual.md index 89eaa07d5..c77370543 100755 --- a/docs/manuals/07-upgrade-manual.md +++ b/docs/manuals/07-upgrade-manual.md @@ -319,6 +319,39 @@ Catalog upgrades change the schema; behavior changes alter how the running serve interprets existing data or configuration. Review these before upgrading a live deployment, even when no catalog migration is required. +### Streams: disabling closes the shards, enabling creates a new stream + +**What changed.** `UpdateTable` with `StreamEnabled: false` now closes the +stream's shards (each gets an `EndingSequenceNumber`), and `StreamEnabled: true` +afterwards creates a new stream with a new ARN and four new shard ids, as on +DynamoDB. Before this release a disable left the shards open and a re-enable +kept the same ARN. Each stream's shard ids carry its identity, so the change +needs no catalog migration of its own. On MongoDB its one new index comes with +migration 1 (below). + +**Servers of the earlier release.** Migrations 1 to 7 are breaking, so a +server of the earlier release must not run against the migrated catalog (see +[The Catalog Status](#the-catalog-status)). Such a server does not know closed +shards. It writes records past a shard's `EndingSequenceNumber`, and it +re-enables a disabled stream under its old ARN instead of creating a new one. +A stream that the earlier release disabled keeps its four shards open under +the new release. The first `UpdateTable` that sets a `StreamSpecification` on +the table closes them: an enable, or a disable, which the server refuses as +before. + +**MongoDB.** Migration 1 creates the `{table_id: 1, shard_id: 1}` index on the +data database's `stream_shards`: `extenddb init` runs it on a new deployment, +and `extenddb migrate --yes` runs it on a 0.1.x deployment. +A deployment that a build without the index has already migrated gets it from +`extenddb migrate --yes --reconverge`, which runs migration 1 again. + +**Rollback.** Going back to the earlier release means restoring the +pre-upgrade database as in [Rollback Procedure](#rollback-procedure); the +earlier release does not read the closed shards' ending numbers or the labelled +shard ids the new release writes. + +--- + ### v0.1.15: Point-in-time recovery is reported as unsupported (catalog migration required) **What changed.** `UpdateContinuousBackups` no longer accepts an enable request, and `DescribeContinuousBackups` no longer reports point-in-time recovery `ENABLED` with a 35-day window. Point-in-time recovery is reported as unsupported: describe answers `PointInTimeRecoveryStatus: DISABLED`, an enable request returns `ContinuousBackupsUnavailableException`, and `RestoreTableToPointInTime` checks the source table and then returns `PointInTimeRecoveryUnavailableException`. A table that does not exist is `TableNotFoundException` on all three operations, where `DescribeContinuousBackups` and `UpdateContinuousBackups` answered `ResourceNotFoundException`, and `UpdateContinuousBackups` without `PointInTimeRecoverySpecification` is a `ValidationException`, where it was taken as disabling. See [Differences from DynamoDB](../differences-from-dynamodb.md) for the full surface. The catalog migration removes the storage of a per-table enable flag that only the description read. diff --git a/docs/schema/mongodb.txt b/docs/schema/mongodb.txt index 14488e8fb..518bce418 100644 --- a/docs/schema/mongodb.txt +++ b/docs/schema/mongodb.txt @@ -70,3 +70,4 @@ index extenddb_data.stream_records shard_id_1_sequence_number_1 key {"shard_id": collection extenddb_data.stream_shards type collection options {} index extenddb_data.stream_shards _id_ key {"_id":1} options {} index extenddb_data.stream_shards shard_id_1 key {"shard_id":1} options {"unique":true} +index extenddb_data.stream_shards table_id_1_shard_id_1 key {"table_id":1,"shard_id":1} options {} diff --git a/tests/python/test_stream_disable.py b/tests/python/test_stream_disable.py new file mode 100644 index 000000000..1b7c175e2 --- /dev/null +++ b/tests/python/test_stream_disable.py @@ -0,0 +1,416 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""Disabling a table's stream closes it; enabling it again creates a new one. + +One test per row measured on Amazon DynamoDB on 2026-10-24 (us-east-1, client +retries off), on a table with a NEW_AND_OLD_IMAGES stream and three items. The +module fixture runs the whole sequence once, in the order it was measured, and +records every answer; each test asserts one row. + +Runs against ExtendDB (DYNAMODB_ENDPOINT) and against the service (streams +client on its default endpoint). On the service the disable completes +asynchronously, so the fixture waits for DISABLED. +""" + +from __future__ import annotations + +import os +import time + +import boto3 +import botocore.config +import pytest +from botocore.exceptions import ClientError + +from helpers import unique_name, wait_for_active, wait_for_deleted + +NEW_AND_OLD = {"StreamEnabled": True, "StreamViewType": "NEW_AND_OLD_IMAGES"} +ITEMS = ["k0", "k1", "k2"] + + +def _streams_client(endpoint_url): + kwargs: dict = { + "service_name": "dynamodbstreams", + "region_name": os.environ.get("AWS_DEFAULT_REGION", "us-east-1"), + "config": botocore.config.Config(retries={"max_attempts": 0}), + } + if endpoint_url: + kwargs["endpoint_url"] = endpoint_url + if endpoint_url.startswith("https://"): + ca_cert = os.environ.get("EXTENDDB_CA_CERT", "") + kwargs["verify"] = ca_cert if ca_cert else False + return boto3.client(**kwargs) + + +def _call(fn, **kwargs) -> dict: + """The response without its metadata, or the error's status, code and message.""" + try: + resp = fn(**kwargs) + resp.pop("ResponseMetadata", None) + return {"status": 200, "response": resp} + except ClientError as e: + return { + "status": e.response["ResponseMetadata"]["HTTPStatusCode"], + "code": e.response["Error"]["Code"], + "message": e.response["Error"]["Message"], + } + + +def _stream(streams, arn: str, **kwargs) -> dict: + return streams.describe_stream(StreamArn=arn, **kwargs)["StreamDescription"] + + +def _wait_disabled(streams, arn: str, timeout: float = 120.0) -> dict: + deadline = time.monotonic() + timeout + while True: + desc = _stream(streams, arn) + if desc["StreamStatus"] == "DISABLED" or time.monotonic() > deadline: + return desc + time.sleep(3) + + +def _iterator(streams, arn: str, shard: str, kind: str, seq: str | None = None) -> dict: + kwargs = {"StreamArn": arn, "ShardId": shard, "ShardIteratorType": kind} + if seq is not None: + kwargs["SequenceNumber"] = seq + return _call(streams.get_shard_iterator, **kwargs) + + +def _reread(streams, arn: str, shards: list[str]) -> dict: + """Every record of every shard, read from a fresh TRIM_HORIZON iterator.""" + out = {} + for shard in shards: + it = _iterator(streams, arn, shard, "TRIM_HORIZON")["response"]["ShardIterator"] + got = [] + for _ in range(50): + page = streams.get_records(ShardIterator=it, Limit=1000) + got += [r["dynamodb"]["SequenceNumber"] for r in page["Records"]] + it = page.get("NextShardIterator") + if not it or not page["Records"]: + break + out[shard] = got + return out + + +def _closed_shard_page_ends(page: dict) -> None: + """The page that reaches the end of a closed shard omits NextShardIterator. + + That is PR #372's rule (a closed and drained shard ends the iterator); + until it lands ExtendDB keeps answering an iterator, so on ExtendDB this + one assertion is an expected failure. Every other assertion of the row + stays unconditional. + """ + if "NextShardIterator" not in page["response"]: + return + if os.environ.get("DYNAMODB_ENDPOINT", "").strip(): + pytest.xfail("closed shard terminal GetRecords page: PR #372") + raise AssertionError(f"NextShardIterator present on a closed shard's final page: {page}") + + +def _iterate(streams, arn: str, shard: str, kind: str, seq: str | None = None): + """GetShardIterator, then one GetRecords with the iterator it issued.""" + it = _iterator(streams, arn, shard, kind, seq) + if it["status"] != 200: + return it, None + return it, _call(streams.get_records, ShardIterator=it["response"]["ShardIterator"]) + + +@pytest.fixture(scope="module") +def run(dynamodb_client, endpoint_url): + """Run the measured sequence once and record every answer by row name.""" + ddb = dynamodb_client + streams = _streams_client(endpoint_url) + name = unique_name("t482") + nostream = unique_name("x482-nostream") + key = { + "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}], + "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}], + "BillingMode": "PAY_PER_REQUEST", + } + ddb.create_table(TableName=name, StreamSpecification=NEW_AND_OLD, **key) + ddb.create_table(TableName=nostream, **key) + wait_for_active(ddb, name) + wait_for_active(ddb, nostream) + rows: dict = {"name": name, "nostream": nostream} + try: + table = ddb.describe_table(TableName=name)["Table"] + rows["describe_enabled"] = table + old_arn = table["LatestStreamArn"] + rows["old_arn"] = old_arn + for pk in ITEMS: + ddb.put_item(TableName=name, Item={"pk": {"S": pk}, "v": {"S": pk}}) + shards = [s["ShardId"] for s in _stream(streams, old_arn)["Shards"]] + rows["old_shards"] = shards + rows["records"] = {} + rows["pre_disable_iterators"] = {} + for shard in shards: + it = _iterator(streams, old_arn, shard, "TRIM_HORIZON")["response"]["ShardIterator"] + got = streams.get_records(ShardIterator=it, Limit=1000)["Records"] + rows["records"][shard] = [r["dynamodb"]["SequenceNumber"] for r in got] + rows["pre_disable_iterators"][shard] = _iterator( + streams, old_arn, shard, "TRIM_HORIZON" + )["response"]["ShardIterator"] + + for view in ["NEW_AND_OLD_IMAGES", "KEYS_ONLY"]: + rows[f"enable_while_enabled_{view}"] = _call( + ddb.update_table, + TableName=name, + StreamSpecification={"StreamEnabled": True, "StreamViewType": view}, + ) + rows["disable_never_streamed"] = _call( + ddb.update_table, TableName=nostream, StreamSpecification={"StreamEnabled": False} + ) + + rows["disable"] = _call( + ddb.update_table, TableName=name, StreamSpecification={"StreamEnabled": False} + ) + wait_for_active(ddb, name) + rows["describe_disabled"] = ddb.describe_table(TableName=name)["Table"] + rows["stream_disabled"] = _wait_disabled(streams, old_arn) + rows["disable_while_disabled"] = _call( + ddb.update_table, TableName=name, StreamSpecification={"StreamEnabled": False} + ) + + rows["latest"] = {} + rows["pre_disable_reads"] = {} + rows["after_last"] = {} + rows["at_ending"] = {} + for s in rows["stream_disabled"]["Shards"]: + shard = s["ShardId"] + rows["latest"][shard] = _iterate(streams, old_arn, shard, "LATEST") + rows["pre_disable_reads"][shard] = _call( + streams.get_records, ShardIterator=rows["pre_disable_iterators"][shard] + ) + if rows["records"][shard]: + last = rows["records"][shard][-1] + rows["after_last"][shard] = _iterate( + streams, old_arn, shard, "AFTER_SEQUENCE_NUMBER", last + ) + # Absent on a server that does not close the shard; the row + # for the ending number reports that on its own. + ending = s["SequenceNumberRange"].get("EndingSequenceNumber", last) + rows["at_ending"][shard] = _iterate( + streams, old_arn, shard, "AT_SEQUENCE_NUMBER", ending + ) + + page1 = _stream(streams, old_arn, Limit=2) + page2 = _stream( + streams, old_arn, Limit=2, ExclusiveStartShardId=page1.get("LastEvaluatedShardId", "") + ) + rows["paged"] = (page1, page2) + + # A write while the stream is disabled records nothing (0 of 1 on the + # service); every old shard is re-read from TRIM_HORIZON afterwards. + ddb.put_item(TableName=name, Item={"pk": {"S": "while-disabled"}}) + rows["old_records_after_disabled_write"] = _reread(streams, old_arn, shards) + + rows["reenable"] = _call(ddb.update_table, TableName=name, StreamSpecification=NEW_AND_OLD) + wait_for_active(ddb, name) + rows["describe_reenabled"] = ddb.describe_table(TableName=name)["Table"] + new_arn = rows["describe_reenabled"]["LatestStreamArn"] + rows["new_arn"] = new_arn + rows["stream_new"] = _stream(streams, new_arn) + ddb.put_item(TableName=name, Item={"pk": {"S": "after"}}) + rows["stream_old_after_write"] = _stream(streams, old_arn) + rows["old_records_after_reenable_write"] = _reread(streams, old_arn, shards) + + rows["list_all"] = streams.list_streams()["Streams"] + rows["list_table"] = streams.list_streams(TableName=name)["Streams"] + new_shard = rows["stream_new"]["Shards"][0]["ShardId"] + rows["cross_old_arn"] = _iterator(streams, old_arn, new_shard, "TRIM_HORIZON") + rows["cross_new_arn"] = _iterator(streams, new_arn, shards[0], "TRIM_HORIZON") + yield rows + finally: + for t in (name, nostream): + try: + ddb.delete_table(TableName=t) + wait_for_deleted(ddb, t) + except ClientError: + pass + + +def _refused(row: dict, sentence: str) -> None: + assert (row.get("status"), row.get("code"), row.get("message")) == ( + 400, + "ValidationException", + sentence, + ), row + + +def test_enable_while_enabled_same_view_type_is_refused(run): + _refused( + run["enable_while_enabled_NEW_AND_OLD_IMAGES"], + f"Table already has an enabled stream: TableName: {run['name']}", + ) + + +def test_enable_while_enabled_other_view_type_is_refused(run): + _refused( + run["enable_while_enabled_KEYS_ONLY"], + f"Table already has an enabled stream: TableName: {run['name']}", + ) + + +def test_disable_on_a_table_that_never_had_a_stream_is_refused(run): + _refused( + run["disable_never_streamed"], + f"Table has no stream to disable: TableName: {run['nostream']}", + ) + + +def test_disable_while_disabled_is_refused(run): + _refused( + run["disable_while_disabled"], + f"Table has no stream to disable: TableName: {run['name']}", + ) + + +def test_disable_response_omits_the_specification_and_keeps_the_latest_stream(run): + assert run["disable"]["status"] == 200, run["disable"] + desc = run["disable"]["response"]["TableDescription"] + # The service answers TableStatus UPDATING here; ExtendDB applies every + # UpdateTable change at once and answers ACTIVE, for stream changes as for + # billing-mode changes, so the status is not asserted. + assert "StreamSpecification" not in desc + enabled = run["describe_enabled"] + assert desc["LatestStreamArn"] == enabled["LatestStreamArn"] + assert desc["LatestStreamLabel"] == enabled["LatestStreamLabel"] + + +def test_reenable_response_names_a_new_stream(run): + assert run["reenable"]["status"] == 200, run["reenable"] + desc = run["reenable"]["response"]["TableDescription"] + assert desc["StreamSpecification"] == NEW_AND_OLD + assert desc["LatestStreamArn"] != run["old_arn"] + assert desc["LatestStreamLabel"] > run["describe_enabled"]["LatestStreamLabel"] + + +def test_describe_table_while_enabled(run): + t = run["describe_enabled"] + assert t["StreamSpecification"] == NEW_AND_OLD + assert t["LatestStreamArn"].endswith("/stream/" + t["LatestStreamLabel"]) + + +def test_describe_table_while_disabled(run): + t = run["describe_disabled"] + assert t["TableStatus"] == "ACTIVE" + assert "StreamSpecification" not in t + assert t["LatestStreamArn"] == run["old_arn"] + assert t["LatestStreamLabel"] == run["describe_enabled"]["LatestStreamLabel"] + + +def test_describe_table_after_reenable(run): + t = run["describe_reenabled"] + assert t["StreamSpecification"] == NEW_AND_OLD + assert t["LatestStreamArn"] == run["new_arn"] != run["old_arn"] + assert run["new_arn"].endswith("/stream/" + t["LatestStreamLabel"]) + + +def test_describe_stream_of_a_disabled_stream(run): + d = run["stream_disabled"] + assert d["StreamStatus"] == "DISABLED" + assert d["StreamViewType"] == "NEW_AND_OLD_IMAGES" + assert d["StreamArn"] == run["old_arn"] + assert d["TableName"] == run["name"] + assert d["KeySchema"] == [{"AttributeName": "pk", "KeyType": "HASH"}] + assert "LastEvaluatedShardId" not in d + assert [s["ShardId"] for s in d["Shards"]] == run["old_shards"] + assert all("ParentShardId" not in s for s in d["Shards"]) + + +def test_every_closed_shard_carries_its_ending_sequence_number(run): + for s in run["stream_disabled"]["Shards"]: + r = s["SequenceNumberRange"] + written = run["records"][s["ShardId"]] + # A shard with records ends at its last record; an empty one ends at + # its starting number. + expected = written[-1] if written else r["StartingSequenceNumber"] + assert r.get("EndingSequenceNumber") == expected, (s, written) + assert sum(len(v) for v in run["records"].values()) == len(ITEMS) + + +def test_describe_stream_pages_a_disabled_stream(run): + page1, page2 = run["paged"] + assert [s["ShardId"] for s in page1["Shards"]] == run["old_shards"][:2] + assert page1["LastEvaluatedShardId"] == run["old_shards"][1] + assert [s["ShardId"] for s in page2["Shards"]] == run["old_shards"][2:] + assert "LastEvaluatedShardId" not in page2 + + +def test_latest_on_a_closed_shard_reads_an_empty_final_page(run): + for shard, (it, page) in run["latest"].items(): + assert it["status"] == 200, (shard, it) + assert page["status"] == 200, (shard, page) + assert page["response"]["Records"] == [], (shard, page) + _closed_shard_page_ends(page) + + +def test_an_iterator_from_before_the_disable_reads_to_the_end(run): + for shard, page in run["pre_disable_reads"].items(): + assert page["status"] == 200, (shard, page) + got = [r["dynamodb"]["SequenceNumber"] for r in page["response"]["Records"]] + assert got == run["records"][shard], (shard, page) + _closed_shard_page_ends(page) + + +def test_after_the_last_record_of_a_closed_shard_is_an_empty_final_page(run): + assert run["after_last"], "no shard held a record" + for shard, (it, page) in run["after_last"].items(): + assert it["status"] == 200, (shard, it) + assert page["status"] == 200, (shard, page) + assert page["response"]["Records"] == [], (shard, page) + _closed_shard_page_ends(page) + + +def test_at_the_ending_number_reads_the_last_record_and_ends(run): + assert run["at_ending"], "no shard held a record" + for shard, (it, page) in run["at_ending"].items(): + assert it["status"] == 200, (shard, it) + assert page["status"] == 200, (shard, page) + got = [r["dynamodb"]["SequenceNumber"] for r in page["response"]["Records"]] + assert got == run["records"][shard][-1:], (shard, page) + _closed_shard_page_ends(page) + + +def test_a_write_while_the_stream_is_disabled_records_nothing(run): + assert run["old_records_after_disabled_write"] == run["records"] + + +def test_reenable_creates_a_new_stream_with_new_shards(run): + new = run["stream_new"] + assert new["StreamStatus"] == "ENABLED" + assert new["StreamArn"] == run["new_arn"] + assert new["StreamLabel"] > run["describe_enabled"]["LatestStreamLabel"] + ids = [s["ShardId"] for s in new["Shards"]] + assert ids and not set(ids) & set(run["old_shards"]) + assert all("EndingSequenceNumber" not in s["SequenceNumberRange"] for s in new["Shards"]) + + +def test_the_old_stream_is_unchanged_after_reenable_and_a_write(run): + before, after = run["stream_disabled"], run["stream_old_after_write"] + assert after["StreamStatus"] == "DISABLED" + assert after["Shards"] == before["Shards"] + # The records of every old shard, read afresh, are the pre-disable set. + assert run["old_records_after_reenable_write"] == run["records"] + + +def test_list_streams_lists_both_streams_newest_first(run): + expected = [ + {"StreamArn": arn, "TableName": run["name"], "StreamLabel": arn.rsplit("/", 1)[1]} + for arn in (run["new_arn"], run["old_arn"]) + ] + assert run["list_table"] == expected + # The account-wide call is paged at 100 streams; only the table's own ARNs + # are asserted, when the page reached them. + own = [s for s in run["list_all"] if s["TableName"] == run["name"]] + assert own in (expected, []), own + + +def test_a_shard_of_the_other_stream_does_not_exist(run): + for row in (run["cross_old_arn"], run["cross_new_arn"]): + assert row == { + "status": 400, + "code": "ResourceNotFoundException", + "message": "Requested resource not found: Shard does not exist", + }, row diff --git a/tests/python/test_stream_disable_fold.py b/tests/python/test_stream_disable_fold.py new file mode 100644 index 000000000..9620a0954 --- /dev/null +++ b/tests/python/test_stream_disable_fold.py @@ -0,0 +1,329 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""Stream lifecycle rows measured on Amazon DynamoDB on 2026-10-24, second round. + +A stream change beside a billing, throughput or index change; ListStreams +paging over a table with two streams; GetShardIterator on an unknown shard or +stream; a table created with StreamSpecification {StreamEnabled: false}. +Companion to test_stream_disable.py. One module fixture runs the measured +sequence once; each test asserts one row. +""" + +from __future__ import annotations + +import os +import time + +import boto3 +import botocore.config +import pytest +from botocore.exceptions import ClientError + +from helpers import unique_name, wait_for_active, wait_for_deleted + +KEYS_ONLY = {"StreamEnabled": True, "StreamViewType": "KEYS_ONLY"} +NEW_IMAGE = {"StreamEnabled": True, "StreamViewType": "NEW_IMAGE"} +DISABLED = {"StreamEnabled": False} +INVALID = "One or more parameter values were invalid: " + +ON_SERVICE = not os.environ.get("DYNAMODB_ENDPOINT", "").strip() + + +def _streams_client(endpoint_url): + kwargs: dict = { + "service_name": "dynamodbstreams", + "region_name": os.environ.get("AWS_DEFAULT_REGION", "us-east-1"), + # Client-side parameter validation off: the short ShardId row measures + # the server's own length check. + "config": botocore.config.Config( + retries={"max_attempts": 0}, parameter_validation=False + ), + } + if endpoint_url: + kwargs["endpoint_url"] = endpoint_url + if endpoint_url.startswith("https://"): + ca_cert = os.environ.get("EXTENDDB_CA_CERT", "") + kwargs["verify"] = ca_cert if ca_cert else False + return boto3.client(**kwargs) + + +def _call(fn, **kwargs) -> dict: + try: + resp = fn(**kwargs) + resp.pop("ResponseMetadata", None) + return {"status": 200, "response": resp} + except ClientError as e: + return { + "status": e.response["ResponseMetadata"]["HTTPStatusCode"], + "code": e.response["Error"]["Code"], + "message": e.response["Error"]["Message"], + } + + +def _list(streams, **kwargs) -> dict: + # The service rate-limits ListStreams within a burst. + if ON_SERVICE: + time.sleep(6) + return _call(streams.list_streams, **kwargs) + + +def _wait_disabled(streams, arn: str, timeout: float = 120.0) -> None: + deadline = time.monotonic() + timeout + while streams.describe_stream(StreamArn=arn)["StreamDescription"]["StreamStatus"] != "DISABLED": + if time.monotonic() > deadline: + return + time.sleep(3) + + +def _set_stream(ddb, name: str, spec: dict, **extra) -> dict: + row = _call(ddb.update_table, TableName=name, StreamSpecification=spec, **extra) + wait_for_active(ddb, name) + return row + + +@pytest.fixture(scope="module") +def run(dynamodb_client, endpoint_url): + ddb = dynamodb_client + streams = _streams_client(endpoint_url) + key = { + "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}], + "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}], + "BillingMode": "PAY_PER_REQUEST", + } + a, b, off = unique_name("f482-a"), unique_name("f482-b"), unique_name("f482-off") + ddb.create_table(TableName=a, StreamSpecification=KEYS_ONLY, **key) + ddb.create_table(TableName=b, StreamSpecification=KEYS_ONLY, **key) + ddb.create_table(TableName=off, StreamSpecification=DISABLED, **key) + for t in (a, b, off): + wait_for_active(ddb, t) + rows: dict = {"a": a, "b": b, "off": off} + try: + # F1: a stream change beside another change, while enabled. + rows["disable_plus_billing"] = _set_stream( + ddb, + a, + DISABLED, + BillingMode="PROVISIONED", + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + gsi = { + "IndexName": "gsi1", + "KeySchema": [{"AttributeName": "g", "KeyType": "HASH"}], + "Projection": {"ProjectionType": "ALL"}, + } + rows["disable_plus_gsi"] = _set_stream( + ddb, + a, + DISABLED, + AttributeDefinitions=[{"AttributeName": "g", "AttributeType": "S"}], + GlobalSecondaryIndexUpdates=[{"Create": gsi}], + ) + rows["disable_plus_gsi_undefined_attribute"] = _set_stream( + ddb, + a, + DISABLED, + GlobalSecondaryIndexUpdates=[{"Create": {**gsi, "IndexName": "gsi2"}}], + ) + rows["enable_plus_billing_while_enabled"] = _set_stream( + ddb, a, KEYS_ONLY, BillingMode="PAY_PER_REQUEST" + ) + rows["a_after_refusals"] = ddb.describe_table(TableName=a)["Table"] + + # F3: GetShardIterator on unknown shard or stream. + a_arn = rows["a_after_refusals"]["LatestStreamArn"] + a_shard = streams.describe_stream(StreamArn=a_arn)["StreamDescription"]["Shards"][0]["ShardId"] + rows["unknown_shard"] = _call( + streams.get_shard_iterator, + StreamArn=a_arn, + ShardId="shardId-00000001700000000000-00000000", + ShardIteratorType="TRIM_HORIZON", + ) + rows["short_shard"] = _call( + streams.get_shard_iterator, + StreamArn=a_arn, + ShardId="nope", + ShardIteratorType="TRIM_HORIZON", + ) + unknown_arn = a_arn[:-3] + ("000" if not a_arn.endswith("000") else "001") + rows["unknown_arn"] = unknown_arn + rows["unknown_stream"] = _call( + streams.get_shard_iterator, + StreamArn=unknown_arn, + ShardId=a_shard, + ShardIteratorType="TRIM_HORIZON", + ) + + # F2: two streams on A (and on B), then ListStreams paging. + _set_stream(ddb, a, DISABLED) + _wait_disabled(streams, a_arn) + _set_stream(ddb, a, NEW_IMAGE) + new_arn = ddb.describe_table(TableName=a)["Table"]["LatestStreamArn"] + rows["a_old"], rows["a_new"] = a_arn, new_arn + b_arn = ddb.describe_table(TableName=b)["Table"]["LatestStreamArn"] + rows["list_a"] = _list(streams, TableName=a) + rows["list_a_page1"] = _list(streams, TableName=a, Limit=1) + rows["list_a_page2"] = _list(streams, TableName=a, Limit=1, ExclusiveStartStreamArn=new_arn) + rows["list_a_page3"] = _list(streams, TableName=a, Limit=1, ExclusiveStartStreamArn=a_arn) + rows["list_a_from_older"] = _list(streams, TableName=a, ExclusiveStartStreamArn=a_arn) + rows["list_a_from_newest"] = _list(streams, TableName=a, ExclusiveStartStreamArn=new_arn) + rows["b_arn"] = b_arn + rows["list_a_from_b"] = _list(streams, TableName=a, ExclusiveStartStreamArn=b_arn) + + # F4: the table created with a disabled specification. + rows["off_describe"] = ddb.describe_table(TableName=off)["Table"] + rows["off_disable"] = _set_stream(ddb, off, DISABLED) + rows["off_enable"] = _set_stream(ddb, off, KEYS_ONLY) + rows["off_after_enable"] = ddb.describe_table(TableName=off)["Table"] + yield rows + finally: + for t in (a, b, off): + try: + ddb.delete_table(TableName=t) + wait_for_deleted(ddb, t) + except ClientError: + pass + + +def _refused(row: dict, code: str, sentence: str) -> None: + assert (row.get("status"), row.get("code"), row.get("message")) == (400, code, sentence), row + + +# F1 + + +def test_a_stream_change_beside_a_billing_change_is_refused(run): + _refused( + run["disable_plus_billing"], + "ValidationException", + INVALID + "You cannot modify stream status while updating table IOPS", + ) + + +def test_a_stream_change_beside_an_index_create_is_refused(run): + _refused( + run["disable_plus_gsi"], + "ValidationException", + INVALID + "You cannot create or delete index while changing stream status", + ) + + +def test_the_index_definition_check_precedes_the_companion_rule(run): + row = run["disable_plus_gsi_undefined_attribute"] + assert (row["status"], row["code"]) == (400, "ValidationException"), row + # The index definition check comes first. The service's sentence is + # "AttributeDefinitions is not specified for index: gsi2"; ExtendDB's own + # sentence for an undefined index key ("Some index key attributes are not + # defined in AttributeDefinitions. Keys: [g], ...") pre-dates this change, + # so only the ordering is asserted here. + assert row["message"].startswith(INVALID), row + assert "AttributeDefinitions" in row["message"] and "stream" not in row["message"], row + + +def test_the_already_enabled_check_precedes_the_companion_rule(run): + _refused( + run["enable_plus_billing_while_enabled"], + "ValidationException", + f"Table already has an enabled stream: TableName: {run['a']}", + ) + + +def test_a_refused_combined_update_changes_nothing(run): + t = run["a_after_refusals"] + assert t["StreamSpecification"] == KEYS_ONLY + assert t.get("BillingModeSummary", {}).get("BillingMode", "PAY_PER_REQUEST") == "PAY_PER_REQUEST" + assert "GlobalSecondaryIndexes" not in t + + +# F2 + + +def test_list_streams_of_a_table_with_two_streams(run): + row = run["list_a"] + assert row["status"] == 200, row + assert [s["StreamArn"] for s in row["response"]["Streams"]] == [run["a_new"], run["a_old"]] + assert "LastEvaluatedStreamArn" not in row["response"] + + +def test_list_streams_limit_one_carries_the_last_arn_on_every_cut_page(run): + p1, p2, p3 = run["list_a_page1"], run["list_a_page2"], run["list_a_page3"] + assert [s["StreamArn"] for s in p1["response"]["Streams"]] == [run["a_new"]], p1 + assert p1["response"]["LastEvaluatedStreamArn"] == run["a_new"] + assert [s["StreamArn"] for s in p2["response"]["Streams"]] == [run["a_old"]], p2 + assert p2["response"]["LastEvaluatedStreamArn"] == run["a_old"] + assert p3 == {"status": 200, "response": {"Streams": []}}, p3 + + +def test_list_streams_after_the_older_stream_is_empty(run): + assert run["list_a_from_older"] == {"status": 200, "response": {"Streams": []}} + + +def test_list_streams_after_the_newest_stream_is_the_older_one(run): + row = run["list_a_from_newest"] + assert [s["StreamArn"] for s in row["response"]["Streams"]] == [run["a_old"]], row + assert "LastEvaluatedStreamArn" not in row["response"] + + +def test_list_streams_start_arn_of_another_table_is_refused(run): + _refused( + run["list_a_from_b"], + "ValidationException", + INVALID + f"ExclusiveStartStreamArn provided is invalid: {run['b_arn']}", + ) + + +# F3 + + +def test_an_unknown_shard_id_on_an_existing_stream_does_not_exist(run): + _refused( + run["unknown_shard"], + "ResourceNotFoundException", + "Requested resource not found: Shard does not exist", + ) + + +def test_a_short_shard_id_fails_the_length_constraint(run): + _refused( + run["short_shard"], + "ValidationException", + "1 validation error detected: Value at 'shardId' failed to satisfy constraint: " + "Member must have length greater than or equal to 28", + ) + + +def test_an_unknown_stream_arn_does_not_exist(run): + _refused( + run["unknown_stream"], + "ResourceNotFoundException", + "Requested resource not found: Stream does not exist", + ) + + +# F4 + + +def test_a_table_created_with_a_disabled_specification_stores_none(run): + t = run["off_describe"] + assert "StreamSpecification" not in t + assert "LatestStreamArn" not in t + assert "LatestStreamLabel" not in t + + +def test_disabling_it_is_refused_like_a_table_without_the_member(run): + _refused( + run["off_disable"], + "ValidationException", + f"Table has no stream to disable: TableName: {run['off']}", + ) + + +def test_enabling_it_creates_the_first_stream(run): + row = run["off_enable"] + assert row["status"] == 200, row + desc = row["response"]["TableDescription"] + assert desc["StreamSpecification"] == KEYS_ONLY + assert desc["LatestStreamArn"].endswith("/stream/" + desc["LatestStreamLabel"]) + after = run["off_after_enable"] + assert after["StreamSpecification"] == KEYS_ONLY + assert after["LatestStreamArn"] == desc["LatestStreamArn"] diff --git a/tests/python/test_stream_disable_validation.py b/tests/python/test_stream_disable_validation.py new file mode 100644 index 000000000..2eb1b9acf --- /dev/null +++ b/tests/python/test_stream_disable_validation.py @@ -0,0 +1,272 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""Stream lifecycle rows measured on Amazon DynamoDB on 2026-10-24, third round. + +Request validation on CreateTable, UpdateTable and the streams API: a stream +enabled without a view type, a specification without StreamEnabled, a disable +beside a view type or beside DeletionProtection, the ShardId upper bound, the +SequenceNumber rules, DescribeStream and ListStreams member constraints, and +the DescribeStream sentence for an unknown stream of an existing table. +Companion to test_stream_disable.py and test_stream_disable_fold.py. +""" + +from __future__ import annotations + +import os + +import boto3 +import botocore.config +import pytest +from botocore.exceptions import ClientError + +from helpers import unique_name, wait_for_active, wait_for_deleted + +KEYS_ONLY = {"StreamEnabled": True, "StreamViewType": "KEYS_ONLY"} +INVALID = "One or more parameter values were invalid: " +VALIDATION = "1 validation error detected: Value " + + +def _client(service: str, endpoint_url): + kwargs: dict = { + "service_name": service, + "region_name": os.environ.get("AWS_DEFAULT_REGION", "us-east-1"), + # Client-side parameter validation off: every row here measures the + # server's own checks. + "config": botocore.config.Config( + retries={"max_attempts": 0}, parameter_validation=False + ), + } + if endpoint_url: + kwargs["endpoint_url"] = endpoint_url + if endpoint_url.startswith("https://"): + ca_cert = os.environ.get("EXTENDDB_CA_CERT", "") + kwargs["verify"] = ca_cert if ca_cert else False + return boto3.client(**kwargs) + + +def _call(fn, **kwargs) -> dict: + try: + resp = fn(**kwargs) + resp.pop("ResponseMetadata", None) + return {"status": 200, "response": resp} + except ClientError as e: + return { + "status": e.response["ResponseMetadata"]["HTTPStatusCode"], + "code": e.response["Error"]["Code"], + "message": e.response["Error"]["Message"], + } + + +def _refused(row: dict, code: str, sentence: str) -> None: + assert (row.get("status"), row.get("code"), row.get("message")) == (400, code, sentence), row + + +@pytest.fixture(scope="module") +def run(endpoint_url): + ddb = _client("dynamodb", endpoint_url) + streams = _client("dynamodbstreams", endpoint_url) + key = { + "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}], + "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}], + "BillingMode": "PAY_PER_REQUEST", + } + on, off = unique_name("v482-on"), unique_name("v482-off") + rows: dict = {"on": on, "off": off} + created = [] + try: + # CreateTable request validation. + never = unique_name("v482-never") + rows["create_enabled_no_view"] = _call( + ddb.create_table, TableName=never, StreamSpecification={"StreamEnabled": True}, **key + ) + rows["create_never_exists"] = _call(ddb.describe_table, TableName=never) + rows["create_no_enabled"] = _call( + ddb.create_table, TableName=never, StreamSpecification={"StreamViewType": "KEYS_ONLY"}, **key + ) + ddb.create_table(TableName=on, StreamSpecification=KEYS_ONLY, **key) + ddb.create_table(TableName=off, **key) + created += [on, off] + wait_for_active(ddb, on) + wait_for_active(ddb, off) + + # UpdateTable request validation, on both tables. + for label, name in (("on", on), ("off", off)): + rows[f"update_no_enabled_{label}"] = _call( + ddb.update_table, TableName=name, StreamSpecification={} + ) + rows[f"disable_with_view_{label}"] = _call( + ddb.update_table, + TableName=name, + StreamSpecification={"StreamEnabled": False, "StreamViewType": "KEYS_ONLY"}, + ) + rows["disable_plus_deletion_protection"] = _call( + ddb.update_table, + TableName=on, + StreamSpecification={"StreamEnabled": False}, + DeletionProtectionEnabled=True, + ) + rows["on_after"] = ddb.describe_table(TableName=on)["Table"] + + # Streams API member constraints. + arn = rows["on_after"]["LatestStreamArn"] + shard = streams.describe_stream(StreamArn=arn)["StreamDescription"]["Shards"][0]["ShardId"] + rows["shard_66"] = _call( + streams.get_shard_iterator, + StreamArn=arn, ShardId="s" * 66, ShardIteratorType="TRIM_HORIZON", + ) + rows["shard_65_unknown"] = _call( + streams.get_shard_iterator, + StreamArn=arn, ShardId="shardId-" + "0" * 57, ShardIteratorType="TRIM_HORIZON", + ) + rows["at_no_sequence"] = _call( + streams.get_shard_iterator, + StreamArn=arn, ShardId=shard, ShardIteratorType="AT_SEQUENCE_NUMBER", + ) + rows["at_sequence_0"] = _call( + streams.get_shard_iterator, + StreamArn=arn, ShardId=shard, ShardIteratorType="AT_SEQUENCE_NUMBER", SequenceNumber="0", + ) + unknown = arn[:-3] + ("000" if not arn.endswith("000") else "001") + rows["unknown_arn"] = unknown + rows["describe_unknown"] = _call(streams.describe_stream, StreamArn=unknown) + rows["describe_short_start"] = _call( + streams.describe_stream, StreamArn=arn, ExclusiveStartShardId="short" + ) + rows["list_limit_0"] = _call(streams.list_streams, TableName=on, Limit=0) + rows["list_short_start"] = _call( + streams.list_streams, TableName=on, ExclusiveStartStreamArn="arn:aws:dynamodb:x" + ) + rows["list_never_existed"] = _call(streams.list_streams, TableName=unique_name("v482-none")) + yield rows + finally: + for t in created: + try: + ddb.delete_table(TableName=t) + wait_for_deleted(ddb, t) + except ClientError: + pass + + +# CreateTable + + +def test_create_table_with_a_stream_enabled_and_no_view_type_is_refused(run): + _refused( + run["create_enabled_no_view"], + "ValidationException", + INVALID + "Table is being created with a stream enabled, UpdateViewType is required", + ) + assert run["create_never_exists"]["code"] == "ResourceNotFoundException", "no table created" + + +def test_create_table_without_stream_enabled_is_refused(run): + _refused( + run["create_no_enabled"], + "ValidationException", + VALIDATION + "null at 'streamSpecification.streamEnabled' failed to satisfy constraint: " + "Member must not be null", + ) + + +# UpdateTable + + +def test_update_table_without_stream_enabled_is_refused(run): + for label in ("on", "off"): + _refused( + run[f"update_no_enabled_{label}"], + "ValidationException", + VALIDATION + "null at 'streamSpecification.streamEnabled' failed to satisfy constraint: " + "Member must not be null", + ) + + +def test_a_disable_beside_a_view_type_is_refused_before_the_no_stream_rule(run): + for label in ("on", "off"): + _refused( + run[f"disable_with_view_{label}"], + "ValidationException", + INVALID + "If stream is being disabled, then UpdateViewType must not be specified", + ) + + +def test_a_disable_beside_deletion_protection_is_refused(run): + _refused( + run["disable_plus_deletion_protection"], + "ValidationException", + INVALID + "DeletionProtection modification must be the only operation in the request", + ) + t = run["on_after"] + assert t["StreamSpecification"] == KEYS_ONLY, "nothing applied" + assert t.get("DeletionProtectionEnabled", False) is False + + +# Streams API + + +def test_a_shard_id_over_65_characters_fails_the_length_constraint(run): + _refused( + run["shard_66"], + "ValidationException", + VALIDATION + "at 'shardId' failed to satisfy constraint: Member must have length less than or equal to 65", + ) + + +def test_a_65_character_unknown_shard_id_does_not_exist(run): + _refused( + run["shard_65_unknown"], + "ResourceNotFoundException", + "Requested resource not found: Shard does not exist", + ) + + +def test_at_sequence_number_without_a_sequence_number_is_refused(run): + _refused(run["at_no_sequence"], "ValidationException", INVALID + "SequenceNumber cannot be null") + + +def test_a_sequence_number_under_21_characters_fails_the_length_constraint(run): + _refused( + run["at_sequence_0"], + "ValidationException", + VALIDATION + "at 'sequenceNumber' failed to satisfy constraint: " + "Member must have length greater than or equal to 21", + ) + + +def test_describe_stream_of_an_unknown_stream_of_an_existing_table(run): + _refused( + run["describe_unknown"], + "ResourceNotFoundException", + f"Requested resource not found: Stream: {run['unknown_arn']} not found", + ) + + +def test_describe_stream_with_a_short_exclusive_start_shard_id_is_refused(run): + _refused( + run["describe_short_start"], + "ValidationException", + VALIDATION + "at 'exclusiveStartShardId' failed to satisfy constraint: " + "Member must have length greater than or equal to 28", + ) + + +def test_list_streams_limit_zero_is_refused(run): + _refused( + run["list_limit_0"], + "ValidationException", + VALIDATION + "at 'limit' failed to satisfy constraint: Member must have value greater than or equal to 1", + ) + + +def test_list_streams_with_a_short_start_arn_is_refused(run): + _refused( + run["list_short_start"], + "ValidationException", + VALIDATION + "at 'exclusiveStartStreamArn' failed to satisfy constraint: " + "Member must have length greater than or equal to 37", + ) + + +def test_list_streams_of_a_table_that_never_existed_is_empty(run): + assert run["list_never_existed"] == {"status": 200, "response": {"Streams": []}} diff --git a/tests/python/test_stream_shard_ownership.py b/tests/python/test_stream_shard_ownership.py index 6a9974b81..3ebfbde79 100644 --- a/tests/python/test_stream_shard_ownership.py +++ b/tests/python/test_stream_shard_ownership.py @@ -143,12 +143,15 @@ def test_allowed_stream_arn_with_another_streams_shard_reads_nothing( for name in (name_a, name_b): wait_for_deleted(dynamodb_client, name) - def test_iterator_reads_only_while_its_arn_is_the_tables_stream( + def test_iterator_reads_only_the_stream_its_arn_names( self, dynamodb_client, streams_client ): """An iterator issued before the stream was disabled and enabled again - keeps reading while the table keeps its stream ARN: across the four - shards it reads the same records a fresh iterator on each shard reads.""" + keeps reading the stream it was issued for. Re-enabling creates a + stream with a new ARN, as on the service, so the writes after it are + in the new stream's shards alone: across the four old shards the old + iterators read the same records a fresh iterator on each reads, and + none of the new keys.""" name, old_arn = _create_stream_table(dynamodb_client) try: old_shards = _shards(streams_client, old_arn) @@ -160,14 +163,14 @@ def test_iterator_reads_only_while_its_arn_is_the_tables_stream( dynamodb_client.update_table(TableName=name, StreamSpecification=STREAM_SPEC) wait_for_active(dynamodb_client, name) new_arn = dynamodb_client.describe_table(TableName=name)["Table"]["LatestStreamArn"] - # A re-enabled stream with a new ARN, as on the service, becomes reachable with #515. - assert new_arn == old_arn, f"re-enabling made a new stream ARN: {new_arn}" + assert new_arn != old_arn, f"re-enabling kept the stream ARN: {new_arn}" new_keys = {f"n{i}" for i in range(8)} for key in sorted(new_keys): dynamodb_client.put_item(TableName=name, Item={"pk": {"S": key}}) time.sleep(1) new_shards = _shards(streams_client, new_arn) + assert not set(new_shards) & set(old_shards), (old_shards, new_shards) new_records = [ r for s in new_shards @@ -183,7 +186,15 @@ def test_iterator_reads_only_while_its_arn_is_the_tables_stream( old_seqs = sorted( r["dynamodb"]["SequenceNumber"] for p in old_pages.values() for r in p["Records"] ) - assert old_seqs == sorted(r["dynamodb"]["SequenceNumber"] for r in new_records) + fresh_seqs = sorted( + r["dynamodb"]["SequenceNumber"] + for s in old_shards + for r in streams_client.get_records( + ShardIterator=_trim_horizon(streams_client, old_arn, s) + )["Records"] + ) + assert old_seqs == fresh_seqs + assert not set(old_seqs) & {r["dynamodb"]["SequenceNumber"] for r in new_records} finally: dynamodb_client.delete_table(TableName=name) wait_for_deleted(dynamodb_client, name)