diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 8da033213..ecd644b84 100755 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -442,10 +442,15 @@ impl OperationContext { /// Dispatch an operation by name. /// +/// `raw_body` is the request text `body` was parsed from. `Scan` reads the +/// member order of its `ExpressionAttributeValues`, `ExclusiveStartKey` and +/// `ScanFilter` from it. +/// /// Returns the JSON response body and sideband metrics for the metrics collector. pub async fn dispatch( operation: &str, body: serde_json::Value, + raw_body: &[u8], ctx: &OperationContext, server_addr: &str, ) -> Result { @@ -474,7 +479,7 @@ pub async fn dispatch( "DeleteItem" => handle_delete_item(body, ctx).await, "UpdateItem" => handle_update_item(body, ctx).await, "Query" => handle_query(body, ctx).await, - "Scan" => handle_scan(body, ctx).await, + "Scan" => handle_scan(body, raw_body, ctx).await, "SearchVectors" => handle_search_vectors(body, ctx).await, "BatchGetItem" => handle_batch_get_item(body, ctx).await, "BatchWriteItem" => handle_batch_write_item(body, ctx).await, diff --git a/crates/engine/src/query_key_values.rs b/crates/engine/src/query_key_values.rs index 6948b5db1..e3c324b2b 100644 --- a/crates/engine/src/query_key_values.rs +++ b/crates/engine/src/query_key_values.rs @@ -98,17 +98,23 @@ pub(crate) fn shape_fault_first( check: DynamoDbError, ) -> DynamoDbError { match serde_json::from_value::(body.clone()) { - Err(e) => match crate::deserialize_error(e) { - DynamoDbError::SerializationException(msg) if msg.contains(UNKNOWN_TYPE_DESCRIPTOR) => { - check - } - shape @ DynamoDbError::SerializationException(_) => shape, - _ => check, - }, + Err(e) => { + let fault = crate::deserialize_error(e); + if is_shape_fault(&fault) { fault } else { check } + } Ok(_) => check, } } +/// Whether `fault`, from [`crate::deserialize_error`], is a fault in the shape +/// of the body (a field of the wrong JSON type), which the service reports +/// ahead of every check, rather than an attribute value the deserializer +/// refuses with a `ValidationException` or an unknown type descriptor; see +/// [`shape_fault_first`]. +pub(crate) fn is_shape_fault(fault: &DynamoDbError) -> bool { + matches!(fault, DynamoDbError::SerializationException(msg) if !msg.contains(UNKNOWN_TYPE_DESCRIPTOR)) +} + /// The deserializer's text for an AttributeValue whose type descriptor is not /// one of the service's. const UNKNOWN_TYPE_DESCRIPTOR: &str = "unknown AttributeValue type descriptor"; @@ -205,7 +211,8 @@ pub(crate) fn non_empty(body: &Value, field: &str) -> bool { } /// The type descriptors an AttributeValue may carry. -const TYPE_DESCRIPTORS: &[&str] = &["S", "N", "B", "SS", "NS", "BS", "M", "L", "NULL", "BOOL"]; +pub(crate) const TYPE_DESCRIPTORS: &[&str] = + &["S", "N", "B", "SS", "NS", "BS", "M", "L", "NULL", "BOOL"]; /// The legacy condition maps of `Scan` whose `AttributeValueList` entries /// [`has_unreadable_attribute_value`] reads. @@ -315,7 +322,7 @@ pub fn validate_expression_attribute_value_numbers( /// Every number inside one attribute value (`N`, `NS` members, and the values /// nested in `L` and `M`), with Unicode decimal digits folded for the check. -fn validate_value_numbers(value: &AttributeValue) -> Result<(), DynamoDbError> { +pub(crate) fn validate_value_numbers(value: &AttributeValue) -> Result<(), DynamoDbError> { match value { AttributeValue::N(n) => validate_number_text(n), AttributeValue::NS(set) => set.iter().try_for_each(|n| validate_number_text(n)), @@ -327,7 +334,7 @@ fn validate_value_numbers(value: &AttributeValue) -> Result<(), DynamoDbError> { /// One number text against the number rules, with the service's `Query` /// message for an empty value and the original text echoed for a malformed one. -fn validate_number_text(text: &str) -> Result<(), DynamoDbError> { +pub(crate) fn validate_number_text(text: &str) -> Result<(), DynamoDbError> { let folded = crate::decimal_digits::fold_decimal_digits(text); match extenddb_core::validation::number::validate_and_normalize_number(&folded) { Ok(_) => Ok(()), diff --git a/crates/engine/src/scan.rs b/crates/engine/src/scan.rs index 9f4f8836d..77212cc24 100755 --- a/crates/engine/src/scan.rs +++ b/crates/engine/src/scan.rs @@ -24,6 +24,9 @@ use crate::index_helpers::{ }; use crate::legacy_filter::desugar_filter; use crate::read_helpers::{PostRead, read_page}; +use crate::scan_request::{ + CheckOrder, ExpressionValuesBody, ScanFilterBody, SelectBody, StartKeyBody, member_order, +}; use crate::serialize_output; use crate::start_key::validate_scan_exclusive_start_key; use crate::{DispatchMetrics, DispatchResult}; @@ -32,27 +35,71 @@ use crate::{DispatchMetrics, DispatchResult}; /// /// Reads all items from the table (or segment), applies `FilterExpression` /// post-read, applies `ProjectionExpression`, and enforces the 1 MB limit. +/// `raw_body` is the request text `body` was parsed from; the checks read the +/// request's order of the `ExclusiveStartKey` members, the +/// `ExpressionAttributeValues` entries and the `ScanFilter` conditions from it +/// (see `scan_request::member_order`). /// /// # Errors /// /// Returns `DynamoDbError` for validation failures, missing tables, or storage errors. #[allow(clippy::cast_possible_wrap)] // item counts won't exceed i64::MAX pub async fn handle_scan( - body: Value, + mut body: Value, + raw_body: &[u8], ctx: &OperationContext, ) -> Result { - // Request-time checks on the raw body, in the service's order: the Limit, - // Segment and TotalSegments ranges, the expression and non-expression - // parameter mix, the ReturnConsumedCapacity enum, every number in - // ExpressionAttributeValues, the Select enum. They run before typed - // deserialization so a field that does not deserialize does not stop the - // ones before it; a field of the wrong JSON type anywhere in the body is - // reported ahead of them; see `scan_request`. - crate::scan_request::validate_scan_request_body(&body)?; + // The checks run in one of the service's two orders, measured 2026-10-25 + // to 2026-11-05 (see `scan_request`): order U when the body holds a value + // that is not one AttributeValue anywhere, order R otherwise. Order R: + // - request-time, on the raw body: the AttributesToGet length and the + // Limit, Segment and TotalSegments ranges in one message, the + // ReturnConsumedCapacity enum, the expression and non-expression + // parameter mix, every number in the ExpressionAttributeValues entries + // that deserialize (the entries in the hash order of the key), the + // first empty or duplicate set in ExpressionAttributeValues (the + // entries in sorted name order), the nesting depth of the entries that + // deserialize (the numbers of the ScanFilter lists are #507's check); + // - a field of the wrong JSON type anywhere in the body; + // - a malformed number inside an ExpressionAttributeValues entry the + // deserializer refuses; + // - the request parameter checks: ExpressionAttributeNames and + // ExpressionAttributeValues with no expression, ConditionalOperator with + // no ScanFilter, a ScanFilter condition's value count; + // - the Select enum; + // - the expression and request checks: the filter syntax, the projection + // syntax, an undefined value, the operand types, the Select rules, the + // unused values, COUNT with AttributesToGet, the Segment pairing; + // - the ExclusiveStartKey's value faults (a number, an empty or duplicate + // set), in the fitted member order; + // - the table lookup, the index resolution and the index rules, the key + // schema. + // Order U differs in these places: the constraint clauses take the older + // form and the Select enum joins them at request time; the numbers of + // ExpressionAttributeValues and the nesting check leave request time; the + // ScanFilter list values (a value that is not one AttributeValue) are + // checked after the no-expression checks and ahead of + // ConditionalOperator; after the request parameter checks come the + // ExclusiveStartKey's faults, shape and value together, then the + // faulty ExpressionAttributeValues entries in the same member order (an + // unreadable entry, a malformed number, a set fault, each in the + // invalid-value form), then the nesting depth, then the expression and + // request checks and the table. + let order = CheckOrder::of(&body); + let values_order = member_order(raw_body, "ExpressionAttributeValues"); + crate::scan_request::validate_scan_request_body(&body, order, values_order.as_deref())?; + // A key, an ExpressionAttributeValues entry, a ScanFilter list value (order + // U) or a Select value (order R) that the service reports later is held + // back here, so that the typed deserialization does not report it ahead of + // the checks the service makes first. + let key_order = member_order(raw_body, "ExclusiveStartKey"); + let start_key = StartKeyBody::take(&mut body, key_order.as_deref()); + let expression_values = ExpressionValuesBody::take(&mut body, order, values_order.as_deref()); + let filter_order = member_order(raw_body, "ScanFilter"); + let scan_filter_values = ScanFilterBody::take(&mut body, order, filter_order.as_deref()); + let select = SelectBody::take(&mut body, order); let input: ScanInput = serde_json::from_value(body).map_err(crate::deserialize_error)?; - // Validate Filter/Projection expressions before the existence - // check; index resolution, Segment pairing, Select and key checks stay after. let has_cond_op = input.conditional_operator.is_some(); let maps = build_expression_maps( @@ -60,6 +107,13 @@ pub async fn handle_scan( input.expression_attribute_values.as_ref(), ); + // Order R: a malformed number inside an ExpressionAttributeValues entry + // the deserializer refuses, ahead of the no-expression check; the set + // faults are request-time checks (measured 2026-11-05). + if order == CheckOrder::Readable { + expression_values.validate()?; + } + let has_filter_expr = input .filter_expression .as_ref() @@ -76,29 +130,60 @@ pub async fn handle_scan( &[ExpressionKind::Filter], )?; + // Order U: a ScanFilter list value that is not one AttributeValue, after + // the no-expression checks and ahead of ConditionalOperator, the value + // count and the key's faults (measured 2026-11-05; see + // `scan_request::ScanFilterBody`). + scan_filter_values.validate()?; + // ConditionalOperator requires a ScanFilter with two or more conditions. extenddb_core::validation::validate_conditional_operator_usage( has_cond_op, input.scan_filter.as_ref().map_or(0, HashMap::len), )?; - // Parse FilterExpression or desugar legacy ScanFilter - let (filter, filter_maps) = if let Some(ref sf) = input.scan_filter { - if sf.is_empty() { - ( - parse_optional_filter(input.filter_expression.as_deref(), &ctx.limits)?, - None, - ) - } else { + // Desugar a legacy ScanFilter: its value count is a request parameter + // check, ahead of every fault in the key (measured 2026-11-05). The mix + // check refuses a ScanFilter beside a FilterExpression, so the two + // branches below never both apply. + let legacy_filter = match input.scan_filter { + Some(ref sf) if !sf.is_empty() => { let cond_op = input.conditional_operator.unwrap_or_default(); - let (expr, fmaps) = desugar_filter(sf, cond_op)?; - (Some(expr), Some(fmaps)) + Some(desugar_filter(sf, cond_op)?) } - } else { - ( + _ => None, + }; + + match order { + CheckOrder::Unreadable => { + // The key's faults, shape and value together, then the faulty + // ExpressionAttributeValues entries in the fitted order (an + // unreadable entry, a malformed number in the invalid-value form, + // a set fault), then the nesting depth (measured 2026-10-31 and + // 2026-11-05; see `scan_request::StartKeyBody` and + // `ExpressionValuesBody`). + start_key.validate(order)?; + expression_values.validate()?; + if let Some(ref values) = input.expression_attribute_values { + extenddb_core::validation::validate_attribute_values_nesting_depth( + values.values(), + )?; + } + } + CheckOrder::Readable => { + // The Select enum, after the no-expression check and the nesting + // check and ahead of the filter syntax (measured 2026-11-05). + select.validate()?; + } + } + + // Parse FilterExpression, or take the desugared ScanFilter. + let (filter, filter_maps) = match legacy_filter { + Some((expr, fmaps)) => (Some(expr), Some(fmaps)), + None => ( parse_optional_filter(input.filter_expression.as_deref(), &ctx.limits)?, None, - ) + ), }; // Parse ProjectionExpression or desugar legacy AttributesToGet @@ -148,14 +233,48 @@ pub async fn handle_scan( None => None, }; + // Build the combined expression maps for the operand checks and the + // post-read evaluation. + let combined_maps = { + let mut names = maps.names.clone(); + let mut values = maps.values.clone(); + + if let Some(ref fm) = filter_maps { + values.extend(fm.values.clone()); + } + if !extra_proj_names.is_empty() { + for (k, v) in &extra_proj_names { + let stripped = k.strip_prefix('#').unwrap_or(k); + names.insert(stripped.to_owned(), v.clone()); + } + } + ExpressionMaps::new(names, values) + }; + + // Validate the filter's placeholders and operand types, ahead of the + // Select rules, the unused values and the Segment pairing (measured + // 2026-11-05). + if let Some(ref f) = filter { + let mut placeholders = Vec::new(); + extenddb_core::expression::collect_value_placeholders(f, &mut placeholders); + for placeholder in placeholders { + combined_maps.resolve_value_for(&placeholder, "FilterExpression")?; + } + extenddb_core::expression::validate_begins_with_operands(f, &combined_maps).map_err( + |e| crate::expression_helpers::prefix_expression_error(e, ExpressionKind::Filter), + )?; + extenddb_core::expression::validate_ordering_operand_types(f, &combined_maps).map_err( + |e| crate::expression_helpers::prefix_expression_error(e, ExpressionKind::Filter), + )?; + } + // Validate Select vs ProjectionExpression and index requirements (shared - // with Query so both reject the same combinations identically). + // with Query so both reject the same combinations identically). COUNT with + // a ProjectionExpression is checked below, after the unused values + // (measured 2026-11-05). extenddb_core::validation::validate_select_projection( input.select, - input - .projection_expression - .as_deref() - .is_some_and(|s| !s.is_empty()), + has_proj_expr && !matches!(input.select, Some(Select::Count)), input .attributes_to_get .as_ref() @@ -189,6 +308,64 @@ pub async fn handle_scan( )?; } + // Select COUNT with a projection: a ProjectionExpression after the unused + // values, AttributesToGet ahead of the Segment pairing, both ahead of the + // key's value faults and the table lookup (measured 2026-11-05). The + // texts are the service's for Scan; the AttributesToGet one differs from + // the service's (#838). + if let Some(Select::Count) = input.select { + if has_proj_expr { + return Err(DynamoDbError::ValidationException( + "Cannot specify the ProjectionExpression when choosing to get only the Count" + .to_owned(), + )); + } + if effective_projection_str.is_some() { + return Err(DynamoDbError::ValidationException( + "Cannot specify the ProjectionExpression when Select is COUNT".to_owned(), + )); + } + } + + // Validate Segment/TotalSegments pairing. The range of each field is + // checked at request time in validate_scan_request_body; the service + // reports a missing partner with a message per direction, after the unused + // values and ahead of the key's value faults and of the table lookup + // (measured 2026-11-05). + match (input.segment, input.total_segments) { + (Some(_), None) => { + return Err(DynamoDbError::ValidationException( + "The TotalSegments parameter is required but was not present in the request when Segment parameter is present" + .to_owned(), + )); + } + (None, Some(_)) => { + return Err(DynamoDbError::ValidationException( + "The Segment parameter is required but was not present in the request when parameter TotalSegments is present" + .to_owned(), + )); + } + (Some(seg), Some(total)) => { + if seg >= total { + return Err(DynamoDbError::ValidationException(format!( + "The Segment parameter is zero-based and must be less than parameter TotalSegments: \ + Segment: {seg} is not less than TotalSegments: {total}" + ))); + } + } + (None, None) => {} + } + + // Limit >= 1 is checked at request time in validate_scan_request_body. + + // Order R: the ExclusiveStartKey's value faults, every number in it with + // the value check's text and the set faults, in the fitted member order, + // ahead of the table lookup (measured 2026-11-05; see + // `scan_request::StartKeyBody`). + if order == CheckOrder::Readable { + start_key.validate(order)?; + } + // P118: Fetch key_info first so we can use table_id for index lookup. let key_info = ctx .table_key_info(&input.table_name) @@ -261,35 +438,6 @@ pub async fn handle_scan( )?; } - // Validate Segment/TotalSegments pairing. The range of each field is - // checked at request time in validate_scan_request_body; the service - // reports a missing partner with a message per direction. - match (input.segment, input.total_segments) { - (Some(_), None) => { - return Err(DynamoDbError::ValidationException( - "The TotalSegments parameter is required but was not present in the request when Segment parameter is present" - .to_owned(), - )); - } - (None, Some(_)) => { - return Err(DynamoDbError::ValidationException( - "The Segment parameter is required but was not present in the request when parameter TotalSegments is present" - .to_owned(), - )); - } - (Some(seg), Some(total)) => { - if seg >= total { - return Err(DynamoDbError::ValidationException(format!( - "The Segment parameter is zero-based and must be less than parameter TotalSegments: \ - Segment: {seg} is not less than TotalSegments: {total}" - ))); - } - } - (None, None) => {} - } - - // Limit >= 1 is checked at request time in validate_scan_request_body. - // For index scans, build a key_info that reflects the index's key schema. let scan_key_info = if let Some(ref idx) = index_info { TableKeyInfo { @@ -317,37 +465,12 @@ pub async fn handle_scan( "ALL_PROJECTED_ATTRIBUTES can be used only when Querying using an IndexName".to_owned(), )); } - if let Some(Select::Count) = input.select - && effective_projection_str.is_some() - { - return Err(DynamoDbError::ValidationException( - "Cannot specify the ProjectionExpression when Select is COUNT".to_owned(), - )); - } - let index_proj = if matches!(input.select, Some(Select::AllProjectedAttributes)) { index_info.as_ref() } else { None }; - // Build the combined expression maps for post-read evaluation. - let combined_maps = { - let mut names = maps.names.clone(); - let mut values = maps.values.clone(); - - if let Some(ref fm) = filter_maps { - values.extend(fm.values.clone()); - } - if !extra_proj_names.is_empty() { - for (k, v) in &extra_proj_names { - let stripped = k.strip_prefix('#').unwrap_or(k); - names.insert(stripped.to_owned(), v.clone()); - } - } - ExpressionMaps::new(names, values) - }; - // Validate ExclusiveStartKey matches the key schema if let Some(ref start_key) = input.exclusive_start_key { validate_scan_exclusive_start_key(start_key, &key_info, index_info.as_ref())?; @@ -381,16 +504,6 @@ pub async fn handle_scan( } } - // Validate begins_with operand types upfront (before any rows are scanned). - if let Some(ref f) = filter { - extenddb_core::expression::validate_begins_with_operands(f, &combined_maps).map_err( - |e| crate::expression_helpers::prefix_expression_error(e, ExpressionKind::Filter), - )?; - extenddb_core::expression::validate_ordering_operand_types(f, &combined_maps).map_err( - |e| crate::expression_helpers::prefix_expression_error(e, ExpressionKind::Filter), - )?; - } - // Determine which key schema to use for LastEvaluatedKey extraction. // For index scans the LEK carries the index key and the base table key, // and the same shape addresses each chunk read from storage. diff --git a/crates/engine/src/scan_request.rs b/crates/engine/src/scan_request.rs index 2fb57e206..b938736eb 100644 --- a/crates/engine/src/scan_request.rs +++ b/crates/engine/src/scan_request.rs @@ -2,54 +2,204 @@ // SPDX-License-Identifier: Apache-2.0 //! The `Scan` checks that run on the request body alone, before the body is -//! deserialized and before the table is looked up, in the service's order. +//! deserialized and before the table is looked up, and the checks on the +//! `ExclusiveStartKey` and `ExpressionAttributeValues` values the handler makes +//! later, each in the service's order. //! -//! Measured 2026-10-25 and 2026-10-30 (us-east-1, each request twice): +//! The service follows one of two orders, measured 2026-10-25 to 2026-11-05 +//! (us-east-1, each request twice; see [`CheckOrder`]). A request that holds a +//! value that is not one AttributeValue anywhere (empty, with a type descriptor +//! the service does not have, or with two, at the top or inside an `L` or `M`, +//! in `ExpressionAttributeValues`, `ExclusiveStartKey` or a `ScanFilter` list; +//! see [`has_unreadable_attribute_value`]) follows order U; every other request +//! follows order R. No single order fits the measured rows: `{"pk": {}, "sk": +//! {"S": "x"}}` with `Select: "BAD"` answers the `Select` clause, a value nested +//! 32 levels with `Select: "BAD"` answers the nesting text, and the same nested +//! value beside that key answers the key's text. //! -//! 1. the range constraints on `Limit` (at least 1), `Segment` (0 to 999999) -//! and `TotalSegments` (1 to 1000000), each checked when present, whether or -//! not the other segment field is, every fault in one message: `1 -//! validation error detected: Value at 'Limit' failed to satisfy -//! constraint: Member must have value greater than or equal to 1`, or `2 -//! validation errors detected: ...; ...` in the order `Limit`, `Segment`, -//! `TotalSegments`. A float or a value outside int32 in any of the three, -//! or a value that is not one AttributeValue in `ExpressionAttributeValues`, -//! in `ExclusiveStartKey` or in a `ScanFilter` `AttributeValueList`, -//! switches the message to the older form, `Value '-2147483648' at 'limit' -//! ...`, with the value saturated or truncated, in the order -//! `totalSegments`, `segment`, `limit`; see `validate_ranges`; -//! 2. the expression and non-expression parameter mix; -//! 3. the `ReturnConsumedCapacity` enum; -//! 4. every number in `ExpressionAttributeValues`, `NS` members and values -//! nested in `L` and `M` included, whether a `FilterExpression` reads the -//! value, nothing reads it, or the request carries no expression at all; a -//! Unicode decimal digit counts as the digit it stands for; -//! 5. the `Select` enum. Both enums invalid answers the `ReturnConsumedCapacity` -//! error alone, so the two are not aggregated. +//! Order R (every value readable): //! -//! Each of these is reported ahead of the next one, ahead of the table lookup, -//! and ahead of the checks the handler makes after deserializing the body -//! (Segment without TotalSegments and the reverse, Segment not below -//! TotalSegments, the undefined placeholder, unused value, filter syntax, -//! IndexName, ConsistentRead, Select with ProjectionExpression and -//! ExclusiveStartKey checks). A field of the wrong JSON type anywhere in the -//! body is reported ahead of all of them, as a `SerializationException`. +//! 1. the constraints the service aggregates into one message, in the newer +//! form: the length of `AttributesToGet` and the ranges of `Limit`, +//! `Segment` and `TotalSegments` (see [`validate_constraints`]); then the +//! `ReturnConsumedCapacity` enum, ahead of the parameter mix (2026-11-05); +//! 2. the expression and non-expression parameter mix, in which a `ScanFilter` +//! object counts as present whether or not it is empty; +//! 3. every number in `ExpressionAttributeValues`, `NS` members and values +//! nested in `L` and `M` included, whether an expression reads the value or +//! not; a Unicode decimal digit counts as the digit it stands for. The +//! entries are visited in the hash order of [`fitted_member_order`] +//! (2026-11-05: `{":n": {"N": "fred"}, ":a": {"N": "bob"}}` answers `fred`, +//! which sorted order would not). An entry the deserializer refuses (a set +//! fault anywhere in it) is skipped here: its set fault is reported at step +//! 4, and a malformed number inside it after the typed deserialization +//! (2026-11-05: `{"NS": ["bob", "1", "1"]}` beside a value nested 32 levels +//! answers the nesting text; `{"L": [{"N": "bob"}, {"NS": []}]}` beside it +//! answered the number in one send and the nesting text in the other). The +//! numbers of the `ScanFilter` lists are #507's check (#564 item 3, measured +//! 2026-11-05: the number text ahead of a number in the key) and are not +//! checked on this branch; +//! 4. an empty or duplicate set in `ExpressionAttributeValues`, the entries in +//! sorted name order whatever order the backend's parsed body keeps +//! (2026-11-05: `{":e": {"NS": []}}` with no expression answers the set text +//! ahead of the no-expression check; beside a value nested 32 levels in +//! another entry the empty `NS` text comes ahead of the nesting text in both +//! sends and the `SS` duplicates text did in one of two; `{":n": {"NS": []}, +//! ":a": {"SS": []}}` answered the `SS` text in one send and the `NS` text +//! in the other, so the sorted order matches one of the two), see +//! [`expression_values_set_fault`] and [`ExpressionValuesBody`]; +//! 5. the nesting depth of `ExpressionAttributeValues`: a value nested 32 +//! levels deep answers `Nesting Levels have exceeded supported limits: +//! Attributes in the item have nested levels beyond supported limit`, 31 +//! levels pass, read or unused, with or without an expression (2026-10-30 +//! and 2026-10-31: ahead of the no-expression check, B4-5a; 2026-11-05: +//! ahead of the filter syntax, the `Select` enum and the key's number); +//! 6. the request parameter checks: `ExpressionAttributeNames` with no +//! expression, then `ExpressionAttributeValues` with no expression, then +//! `ConditionalOperator` with no `ScanFilter`, then a `ScanFilter` +//! condition's value count (2026-11-05); +//! 7. the `Select` enum (2026-11-05: after the no-expression check and the +//! nesting check, ahead of the filter syntax); +//! 8. the expression and request checks: the `FilterExpression` syntax, the +//! `ProjectionExpression` syntax, an undefined value, the operand types, the +//! three `Select` rules, the unused values, `COUNT` with `AttributesToGet`, +//! the `Segment` pairing (2026-11-05, where two were sent together: the +//! filter syntax first; an undefined value and an operand type ahead of the +//! unused values, the `Segment` pairing and `SPECIFIC_ATTRIBUTES`; the +//! unused values ahead of `COUNT` with `ProjectionExpression`; the unused +//! values and `COUNT` with `AttributesToGet` ahead of the `Segment` +//! pairing); +//! 9. the `ExclusiveStartKey` value faults: a number that does not parse, an +//! empty set, a set with duplicates, in the member order of +//! [`StartKeyBody`], with no prefix and the set members JSON-quoted; +//! 10. the table lookup, the index, `ConsistentRead` on a GSI, the key schema. +//! +//! Order U (the request holds an unreadable value): +//! +//! 1. the constraints in the older form (the value echoed, the field in lower +//! camel case, `totalSegments`, `segment`, `limit`), then the +//! `ReturnConsumedCapacity` enum, then the `Select` enum (2026-11-05: ahead +//! of the key's shape fault and of an unreadable expression value); +//! 2. the parameter mix, then `ExpressionAttributeNames` and +//! `ExpressionAttributeValues` with no expression, then the `ScanFilter` +//! list values (see [`ScanFilterBody`]: a value that is not one +//! AttributeValue answers `Supplied AttributeValue is empty, must contain +//! exactly one of the supported datatypes` with no prefix, for `{}` and for +//! an unknown descriptor alike, 2026-11-05), then `ConditionalOperator` +//! with no `ScanFilter` and a `ScanFilter` condition's value count +//! (2026-11-05: each ahead of the key's faults). The numbers of +//! `ExpressionAttributeValues` are not checked at request time in this +//! order; +//! 3. the `ExclusiveStartKey` faults, shape and value together, the first +//! faulty member in the member order of [`StartKeyBody`]: a shape fault +//! with the prefix `The provided starting key is invalid: `, a number with +//! the value check's text, a set fault with the texts of order U (the +//! members unquoted, the `NS` duplicate text bare, the empty `SS` and `BS` +//! texts with the key prefix); +//! 4. the `ExpressionAttributeValues` entries, in the same member order +//! ([`fitted_member_order`]), the first faulty one named, every fault in +//! the form `ExpressionAttributeValues contains invalid value: for +//! key :e`: an entry that is not one AttributeValue, read by an expression +//! or not, `Supplied AttributeValue is empty, must contain exactly one of +//! the supported datatypes` for an empty value and for an unknown +//! descriptor, the more-than-one text for two descriptors (2026-10-31 B4-7 +//! and 2026-11-05: ahead of the nesting check and after the key's faults); +//! a malformed number, `The parameter cannot be converted to a numeric +//! value: bob`, at the top of the entry or inside its `L` or `NS`, and the +//! 38-digit text (2026-11-05: after the key's faults); a set fault with the +//! texts of order U, `One or more parameter values were invalid: An number +//! set may not be empty` and `... Input collection [a, a] contains +//! duplicates.` (2026-11-05). Inside one entry the first fault in the +//! value's own element order is named, a number or a shape fault +//! (2026-11-05: `{"L": [{"N": "bob"}, {}]}` answers the number); +//! 5. the nesting depth, then the expression and request checks of order R +//! step 8, then the table. +//! +//! Pairs never sent together keep the code's order: in order R the nesting +//! check against `ExpressionAttributeNames` with no expression, +//! `ConditionalOperator` and the `ScanFilter` value count; a set fault in +//! `ExpressionAttributeValues` against a malformed number in another entry +//! (the number first, at request time; the service's two sends disagreed, +//! 2026-11-05); which of two set faults in `ExpressionAttributeValues` is +//! named (sorted name order; the service's two sends disagreed, 2026-11-05); +//! the `ProjectionExpression` syntax against an undefined value, an operand +//! type and the unused values. In order U the `Select` enum against the mix +//! (the enum first; the mix beside `Select: "BAD"` and an empty entry answers +//! the `Select` clause, 2026-11-05); an empty value ahead of a malformed +//! number inside one `ExpressionAttributeValues` entry (the empty value, the +//! element order; the other way round is measured); a set fault in a +//! `ScanFilter` list value in order U (at the list's step, the deserializer's +//! text in order R, 2026-11-05); a `BS` with duplicates in the key (no key +//! prefix, the members unquoted); nine or more names in one hash bucket (see +//! [`fitted_member_order`]). The service refuses a member or entry name written +//! twice as a `SerializationException` (`Duplicate key found in payload`, +//! 2026-11-05, #630); the parser here keeps the last value. +//! +//! A field of the wrong JSON type anywhere in the body is reported ahead of +//! all of them, as a `SerializationException` (`"Limit": "5"` with `Select: +//! "BAD"`, 2026-11-05; the service's texts differ from the deserializer's). The +//! service reports a key shape fault ahead of `"ConsistentRead": "yes"` +//! (2026-11-05); the code reads the whole body first and does not. In order U +//! the service reports an invalid `ScanFilter` `ComparisonOperator` as a +//! constraint clause (`Value 'BAD' at 'scanFilter.a.member.comparisonOperator' +//! failed to satisfy constraint: Member must satisfy enum value set: [...]`, +//! 2026-11-05) ahead of the list's values; the legacy condition checks are +//! #507's and the clause is not made here. +//! +//! The JSON parser accepts 128 levels of nesting, which is 62 `L` levels; a +//! body nested deeper is a `SerializationException` from the parser, ahead of +//! every check here. The service answers the nesting text from 32 to 62 levels +//! and, from 63 to 100 levels, `ExpressionAttributeValues contains invalid +//! value: Nesting Levels have exceeded supported limits: Attributes in the +//! item have nested levels beyond supported limit for key :d` (measured +//! 2026-10-30), which this module does not reach. use std::collections::HashMap; use extenddb_core::error::DynamoDbError; -use extenddb_core::types::{AttributeValue, ScanInput, saturated_int32}; +use extenddb_core::types::{AttributeValue, Item, ScanInput, saturated_int32}; +use serde::Deserialize; use serde_json::Value; use crate::query_key_values::{ - SCAN_CONDITIONS, has_unreadable_attribute_value, non_empty, present, shape_fault_first, - validate_expression_attribute_value_numbers, + SCAN_CONDITIONS, TYPE_DESCRIPTORS, has_unreadable_attribute_value, is_shape_fault, non_empty, + present, shape_fault_first, validate_number_text, validate_value_numbers, }; -/// The `Scan` checks that run on the raw request body, in the service's order: -/// the `Limit`, `Segment` and `TotalSegments` ranges, the expression and -/// non-expression parameter mix, the `ReturnConsumedCapacity` enum, every -/// number in `ExpressionAttributeValues`, then the `Select` enum. +/// Which of the service's two check orders a `Scan` request follows (see the +/// module doc): order U when the request holds a value that is not one +/// AttributeValue anywhere [`has_unreadable_attribute_value`] reads +/// (`ExpressionAttributeValues`, `ExclusiveStartKey`, the `ScanFilter` lists), +/// order R otherwise. The same test picks the older form of the constraint +/// clauses (563, D1). Computed once per request by [`CheckOrder::of`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum CheckOrder { + /// Every value is readable. + Readable, + /// The request holds an unreadable value. + Unreadable, +} + +impl CheckOrder { + /// The order `body` follows. + pub(crate) fn of(body: &Value) -> Self { + if has_unreadable_attribute_value(body, SCAN_CONDITIONS) { + Self::Unreadable + } else { + Self::Readable + } + } +} + +/// The `Scan` checks that run on the raw request body, in the service's order +/// for `order` (see the module doc): the `AttributesToGet` length and the +/// `Limit`, `Segment` and `TotalSegments` ranges in one message, the +/// `ReturnConsumedCapacity` enum, in order U the `Select` enum, the expression +/// and non-expression parameter mix, every number in +/// `ExpressionAttributeValues`, and in order R the nesting depth of +/// `ExpressionAttributeValues`. The `ExclusiveStartKey` values are checked +/// later, by [`StartKeyBody`], and in order R the `Select` enum by +/// [`SelectBody`]. /// /// The checks read the raw body, so a field that does not deserialize (an /// invalid `Select`, for one) does not stop the ones before it. When a check @@ -63,19 +213,45 @@ use crate::query_key_values::{ /// /// Returns `DynamoDbError::ValidationException` with the service's text, or /// `DynamoDbError::SerializationException` for a body of the wrong shape. -pub fn validate_scan_request_body(body: &Value) -> Result<(), DynamoDbError> { - scan_request_checks(body).map_err(|check| shape_fault_first::(body, check)) +pub(crate) fn validate_scan_request_body( + body: &Value, + order: CheckOrder, + values_order: Option<&[String]>, +) -> Result<(), DynamoDbError> { + scan_request_checks(body, order, values_order) + .map_err(|check| shape_fault_first::(body, check)) } /// The `Scan` checks in the service's order; see -/// [`validate_scan_request_body`]. -fn scan_request_checks(body: &Value) -> Result<(), DynamoDbError> { - validate_ranges(body)?; +/// [`validate_scan_request_body`]. `values_order` is the +/// `ExpressionAttributeValues` entry names as the request wrote them +/// ([`member_order`]), for the hash order of the number check. +fn scan_request_checks( + body: &Value, + order: CheckOrder, + values_order: Option<&[String]>, +) -> Result<(), DynamoDbError> { + validate_constraints(body, order)?; + + crate::validate_enum_fields( + body, + &[crate::EnumField { + json_name: "ReturnConsumedCapacity", + valid: &["INDEXES", "TOTAL", "NONE"], + clause: crate::EnumClause::Named("returnConsumedCapacity"), + }], + )?; + + if order == CheckOrder::Unreadable { + validate_select_enum(body)?; + } extenddb_core::validation::validate_no_expression_param_mixing( &[ ("AttributesToGet", non_empty(body, "AttributesToGet")), - ("ScanFilter", non_empty(body, "ScanFilter")), + // An empty ScanFilter object counts as present, as a typed field + // that deserialized to an empty map is. + ("ScanFilter", present(body, "ScanFilter")), ("ConditionalOperator", present(body, "ConditionalOperator")), ], &[ @@ -87,21 +263,85 @@ fn scan_request_checks(body: &Value) -> Result<(), DynamoDbError> { ], )?; - crate::validate_enum_fields( - body, - &[crate::EnumField { - json_name: "ReturnConsumedCapacity", - valid: &["INDEXES", "TOTAL", "NONE"], - clause: crate::EnumClause::Named("returnConsumedCapacity"), - }], - )?; + if order == CheckOrder::Readable { + // Every number in ExpressionAttributeValues, the entries in the hash + // order the key and the order-U step use (an entry the deserializer + // refuses, one with a set fault, is skipped here), then the set faults + // in sorted name order, then the nesting depth of the entries that + // deserialize. In order U the numbers and the set faults are reported + // at the map's own step (see the module doc). The numbers of the + // ScanFilter lists are #507's check (#564 item 3). + let values = readable_attribute_values(body, "ExpressionAttributeValues"); + for name in fitted_order_of(body, "ExpressionAttributeValues", values_order) { + if let Some(value) = values.get(&name) { + validate_value_numbers(value)?; + } + } + expression_values_set_fault(body)?; + extenddb_core::validation::validate_attribute_values_nesting_depth(values.values())?; + } + Ok(()) +} - if let Some(raw) = body.get("ExpressionAttributeValues") - && let Ok(values) = serde_json::from_value::>(raw.clone()) - { - validate_expression_attribute_value_numbers(&values)?; +/// The entries of the object `field` of the body that deserialize as one +/// attribute value each; the others are left to [`ExpressionValuesBody`]. +fn readable_attribute_values(body: &Value, field: &str) -> HashMap { + body.get(field) + .and_then(Value::as_object) + .map(|entries| { + entries + .iter() + .filter_map(|(name, value)| { + serde_json::from_value::(value.clone()) + .ok() + .map(|value| (name.clone(), value)) + }) + .collect() + }) + .unwrap_or_default() +} + +/// The first empty or duplicate set among the `ExpressionAttributeValues` +/// entries, in sorted name order whatever order the backend's parsed body +/// keeps, with the texts of order R (see [`ValueFault::text`]). Measured +/// 2026-11-05: `{":d": <32 levels>, ":e": {"NS": []}}` answers the empty-set +/// text ahead of the nesting text in both sends, and `{":d": <32 levels>, ":s": +/// {"SS": ["a", "a"]}}` the duplicates text in one of two; beside a malformed +/// number in another entry the service's two sends disagreed, and the number +/// comes first here. With two set faults, `{":n": {"NS": []}, ":a": {"SS": +/// []}}`, the service answered the `SS` text in one send and the `NS` text in +/// the other; the sorted order names `:a`. An entry whose first fault, depth +/// first, is a malformed number is left to [`ExpressionValuesBody`]. +/// +/// # Errors +/// +/// Returns `DynamoDbError::ValidationException` with the set text. +fn expression_values_set_fault(body: &Value) -> Result<(), DynamoDbError> { + let Some(entries) = body + .get("ExpressionAttributeValues") + .and_then(Value::as_object) + else { + return Ok(()); + }; + let mut names: Vec<&String> = entries.keys().collect(); + names.sort_unstable(); + for name in names { + match value_fault(&entries[name.as_str()]) { + None | Some(ValueFault::Number(_)) => {} + Some(fault) => return Err(fault.text(CheckOrder::Readable, Place::ExpressionValue)), + } } + Ok(()) +} +/// The `Select` enum clause: `1 validation error detected: Value 'BAD' at +/// 'select' failed to satisfy constraint: Member must satisfy enum value set: +/// [...]`. +/// +/// # Errors +/// +/// Returns `DynamoDbError::ValidationException` with the clause. +fn validate_select_enum(body: &Value) -> Result<(), DynamoDbError> { crate::validate_enum_fields( body, &[crate::EnumField { @@ -117,23 +357,783 @@ fn scan_request_checks(body: &Value) -> Result<(), DynamoDbError> { ) } -/// The range constraints on `Limit`, `Segment` and `TotalSegments`, every -/// fault in one message, in one of the service's two forms (measured -/// 2026-10-25, us-east-1). +/// The service's prefix on a shape fault in an `ExclusiveStartKey` value +/// (measured 2026-10-31 and 2026-11-05: `{"pk": {}, "sk": {"S": "x"}}` answers +/// `The provided starting key is invalid: Supplied AttributeValue is empty, +/// must contain exactly one of the supported datatypes`), and on an empty `SS` +/// in the key in order U (2026-11-05: `{"pk": {}, "sk": {"SS": []}}` answers +/// `The provided starting key is invalid: One or more parameter values were +/// invalid: An string set may not be empty`). A number, an empty `NS` or a +/// set with duplicates in the key carries no prefix in either order. +const START_KEY_PREFIX: &str = "The provided starting key is invalid: "; + +/// The service's text for a value that is empty or carries a type descriptor +/// it does not have (an unknown or lower-case one, measured 2026-11-05). +const EMPTY_VALUE: &str = + "Supplied AttributeValue is empty, must contain exactly one of the supported datatypes"; + +/// The service's text for a value with more than one type descriptor. +const MORE_THAN_ONE_DESCRIPTOR: &str = "Supplied AttributeValue has more than one datatypes set, must contain exactly one of the supported datatypes"; + +/// The prefix of the service's set texts. +const INVALID_PARAMETER_VALUES: &str = "One or more parameter values were invalid: "; + +/// The member names of the body's object `field` (`ExclusiveStartKey`, +/// `ExpressionAttributeValues` or `ScanFilter`) in the order the request wrote +/// them, read from the raw request text, since the parsed body sorts object +/// members on every build but the MongoDB one. `None` when the text does not +/// parse, has no such field, or the field is not an object; a name that +/// appears twice keeps its first position; a field that appears twice is read +/// at its first position. +pub(crate) fn member_order(raw_body: &[u8], field: &str) -> Option> { + /// The member names of one JSON object, in document order. + struct MemberNames(Vec); + + impl<'de> Deserialize<'de> for MemberNames { + fn deserialize>(deserializer: D) -> Result { + struct NamesVisitor; + + impl<'de> serde::de::Visitor<'de> for NamesVisitor { + type Value = MemberNames; + + fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("an object") + } + + fn visit_map>( + self, + mut map: A, + ) -> Result { + let mut names: Vec = Vec::new(); + while let Some(name) = map.next_key::()? { + map.next_value::()?; + if !names.contains(&name) { + names.push(name); + } + } + Ok(MemberNames(names)) + } + } + + deserializer.deserialize_map(NamesVisitor) + } + } + + /// The top-level object, with only the member names of `field` read. + struct FieldNames<'f>(&'f str); + + impl<'de> serde::de::Visitor<'de> for FieldNames<'_> { + type Value = Option>; + + fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("an object") + } + + fn visit_map>( + self, + mut map: A, + ) -> Result { + let mut found: Option> = None; + while let Some(name) = map.next_key::()? { + if name == self.0 && found.is_none() { + found = Some(map.next_value::()?.0); + } else { + map.next_value::()?; + } + } + Ok(found) + } + } + + let mut deserializer = serde_json::Deserializer::from_slice(raw_body); + serde::Deserializer::deserialize_map(&mut deserializer, FieldNames(field)) + .ok() + .flatten() +} + +/// The order in which the service visits the members of an `ExclusiveStartKey` +/// with faults in more than one member, and the entries of +/// `ExpressionAttributeValues` in order U, fitted to the measured rows: the +/// iteration order of a Java `HashMap` over the names, that is +/// `String.hashCode()` of the name, spread as `h ^ (h >>> 16)`, masked to the +/// table's buckets, the buckets in ascending order and the request's order +/// inside a bucket. The table has four buckets for up to three names and +/// doubles when it holds more than three quarters of its size +/// ([`hash_table_mask`]): eight buckets for four to six names. With four +/// buckets `sk` falls in bucket 0, `a` in 1, `b` in 2, `pk`, `gpk`, `id` and +/// `ts` in 3; with eight, `sk` in 0, `gpk` and `pk` in 3, `gsk` in 6. +/// +/// Measured (2026-10-31, 2026-11-04 and 2026-11-05, six tables, every row +/// with a shape fault in one of two faulty members, both request orders): on +/// the `pk`, `sk` table the service names `sk`; on HASH `a`, RANGE `b` it +/// names `a`; on HASH `pk`, RANGE `a` it names `a`; on `ts`, `id` and on `id`, +/// `ts` it names the member first in the request; on HASH `ä`, RANGE `é` it +/// names `ä`; on the GSI `{"gpk": {}, "pk": {"N": "bob"}, "sk": {"S": "x"}}` +/// it names `gpk`; on a GSI with a range key, where the key has four members, +/// it names `gpk` ahead of `gsk` and `pk` ahead of `gsk`, which four buckets +/// would reverse. In `ExpressionAttributeValues` (order U, 2026-11-05) it +/// names `:n` ahead of `:e`, `:b` ahead of `:p` and `:a`, and `:a` ahead of +/// `:e` when `:a` comes first in the request. The rule is fitted to those +/// rows. The service does not document it. With two value faults and no shape +/// fault the service's own two sends disagreed on 19 of 40 matrix cells, so no +/// order is pinned there. Measured 2026-11-05 as well: a key of five members +/// and one of seven on the same GSI (eight and sixteen buckets), four and +/// seven `ExpressionAttributeValues` entries, two `ScanFilter` conditions, a +/// key name outside the Basic Multilingual Plane (`𝄞`, hashed as its two +/// UTF-16 code units) and the request's order inside one bucket for the +/// entries (`:e` ahead of `:a` when `:e` comes first); each as the rule gives. +/// Java's `HashMap` also doubles a table under 64 buckets when one bucket +/// reaches nine names; the model sizes the table by the count of names alone +/// and departs from the JDK there (not measured). +fn fitted_member_order<'a>(names_in_request_order: impl Iterator) -> Vec { + let names: Vec<&str> = names_in_request_order.collect(); + let mask = hash_table_mask(names.len()); + let mut names: Vec<(u32, usize, &str)> = names + .into_iter() + .enumerate() + .map(|(position, name)| (java_hash_bucket(name, mask), position, name)) + .collect(); + names.sort_unstable(); + names + .into_iter() + .map(|(_, _, name)| name.to_owned()) + .collect() +} + +/// The bucket mask of a Java `HashMap` that starts with four buckets and +/// doubles whenever it holds more than three quarters of its size: 3 for up to +/// three names, 7 for four to six, 15 for seven to twelve, and so on. +fn hash_table_mask(names: usize) -> u32 { + let mut buckets: usize = 4; + while names > buckets * 3 / 4 { + buckets *= 2; + } + u32::try_from(buckets - 1).unwrap_or(u32::MAX) +} + +/// The bucket of `name` in a Java `HashMap` whose bucket mask is `mask`: +/// `String.hashCode()` over the UTF-16 code units, spread as `h ^ (h >>> 16)`, +/// masked. The arithmetic wraps as Java's `int` does. +fn java_hash_bucket(name: &str, mask: u32) -> u32 { + let hash = name.encode_utf16().fold(0u32, |h, unit| { + h.wrapping_mul(31).wrapping_add(u32::from(unit)) + }); + (hash ^ (hash >> 16)) & mask +} + +/// The names of the object `field` of the body in the fitted order +/// ([`fitted_member_order`]): the request's order when `request_order` names +/// exactly the object's members, else the parsed body's order. +fn fitted_order_of(body: &Value, field: &str, request_order: Option<&[String]>) -> Vec { + let Some(members) = body.get(field).and_then(Value::as_object) else { + return Vec::new(); + }; + let parsed: Vec<&str> = members.keys().map(String::as_str).collect(); + let in_request: Vec<&str> = match request_order { + Some(names) + if names.len() == parsed.len() + && names.iter().all(|name| members.contains_key(name)) => + { + names.iter().map(String::as_str).collect() + } + _ => parsed, + }; + fitted_member_order(in_request.into_iter()) +} + +/// The `ExclusiveStartKey` of a `Scan` body as the request wrote it, with the +/// checks the service makes on the key (measured 2026-10-31 to 2026-11-05, +/// us-east-1; see the module doc): +/// +/// - in order U, after the request parameter checks and ahead of the +/// expression values and checks: the first faulty member in the fitted +/// member order ([`fitted_member_order`]), a shape fault (a value that is +/// empty, carries an unknown or lower-case type descriptor or more than one +/// descriptor, at the top of the member or inside its `L` or `M`) with the +/// prefix `The provided starting key is invalid: `, a number that does not +/// parse with the value check's text, a set fault with the order-U texts; +/// - in order R, after the expression and request checks and ahead of the +/// table lookup: the first member in the same order with a number or set +/// fault, with the order-R texts. /// -/// When each field present is an integer inside the int32 range, the newer -/// form: `Value at '' failed to satisfy constraint: ...` with the field -/// capitalised and no value echoed, in the order `Limit`, `Segment`, -/// `TotalSegments`. When any of the three is a float or outside int32, or a -/// value in `ExpressionAttributeValues`, `ExclusiveStartKey` or a `ScanFilter` -/// `AttributeValueList` is not one AttributeValue (see -/// [`has_unreadable_attribute_value`]), the older form: `Value '' at +/// A key with an unreadable or set-faulty value does not deserialize as a map +/// of attribute values. [`StartKeyBody::take`] takes it out of the body, so +/// that the typed deserialization of the body does not report it ahead of the +/// checks the service makes first, and keeps the deserializer's fault for a +/// value the checks here do not classify. A key of the wrong JSON shape (not +/// an object, a value that is not an object, an `N` that is not a string) +/// stays in the body, as the service reports a shape fault ahead of every +/// check; see [`shape_fault_first`]. +pub(crate) struct StartKeyBody { + /// The key as the request wrote it; `None` when the body has none. + raw: Option, + /// The member names in the fitted visiting order. + members: Vec, + /// The deserializer's fault in a key taken out of the body, reported at the + /// key's stage when the checks here classify no member (an `SS` member + /// that is not a string, for one). + deferred: Option, +} + +impl StartKeyBody { + /// Reads the key from `body` and takes it out of the body when it does not + /// deserialize for a reason that is not a shape fault. `request_order` is + /// the key's member names as the request wrote them ([`member_order`]); + /// when it is `None` or does not name the key's members, the parsed body's + /// order stands in for it. + pub(crate) fn take(body: &mut Value, request_order: Option<&[String]>) -> Self { + let members = fitted_order_of(body, "ExclusiveStartKey", request_order); + let raw = body.get("ExclusiveStartKey").cloned(); + let deferred = match raw + .as_ref() + .map(|key| serde_json::from_value::(key.clone())) + { + Some(Err(e)) => match crate::deserialize_error(e) { + shape if is_shape_fault(&shape) => None, + DynamoDbError::ValidationException(msg) + if msg == EMPTY_VALUE || msg == MORE_THAN_ONE_DESCRIPTOR => + { + Some(DynamoDbError::ValidationException(format!( + "{START_KEY_PREFIX}{msg}" + ))) + } + other => Some(other), + }, + Some(Ok(_)) | None => None, + }; + if deferred.is_some() + && let Some(fields) = body.as_object_mut() + { + fields.remove("ExclusiveStartKey"); + } + Self { + raw, + members, + deferred, + } + } + + /// The key's members as `(name, value)` in the fitted visiting order. + fn members(&self) -> impl Iterator { + let key = self.raw.as_ref().and_then(Value::as_object); + self.members + .iter() + .filter_map(move |name| key?.get(name).map(|value| (name.as_str(), value))) + } + + /// The key's checks for `order`: the first faulty member in the fitted + /// order, a shape fault (order U only) ahead of a value fault inside one + /// member; then the deserializer's fault in a key [`StartKeyBody::take`] + /// took out of the body that the checks here did not classify. + /// + /// # Errors + /// + /// Returns `DynamoDbError::ValidationException` with the service's text. + pub(crate) fn validate(&self, order: CheckOrder) -> Result<(), DynamoDbError> { + for (_, value) in self.members() { + if order == CheckOrder::Unreadable + && let Some(text) = shape_fault(value) + { + return Err(DynamoDbError::ValidationException(format!( + "{START_KEY_PREFIX}{text}" + ))); + } + if let Some(fault) = value_fault(value) { + return Err(fault.text(order, Place::Key)); + } + } + match &self.deferred { + Some(fault) => Err(fault.clone()), + None => Ok(()), + } + } +} + +/// The service's text for the first shape fault in one raw key value, depth +/// first: an object with no type descriptor or with one the service does not +/// have is an empty value, one with more than one descriptor is the +/// more-than-one text, and the members of an `L` or `M` are read the same way. +/// A value that is not an object is a shape fault the deserializer reports. +fn shape_fault(value: &Value) -> Option<&'static str> { + let fields = value.as_object()?; + let mut fields = fields.iter(); + match (fields.next(), fields.next()) { + (None, _) => Some(EMPTY_VALUE), + (Some(_), Some(_)) => Some(MORE_THAN_ONE_DESCRIPTOR), + (Some((descriptor, inner)), None) => match (descriptor.as_str(), inner) { + ("L", Value::Array(members)) => members.iter().find_map(shape_fault), + ("M", Value::Object(members)) => members.values().find_map(shape_fault), + (descriptor, _) if TYPE_DESCRIPTORS.contains(&descriptor) => None, + _ => Some(EMPTY_VALUE), + }, + } +} + +/// Where a faulty value sits, for the texts that differ by place. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Place { + /// A member of `ExclusiveStartKey`. + Key, + /// An entry of `ExpressionAttributeValues`. + ExpressionValue, +} + +/// A value fault in one AttributeValue: a number that does not parse (with the +/// value check's error), an empty set, or a set with duplicates (with its +/// members as written). +enum ValueFault { + Number(DynamoDbError), + NumberSetEmpty, + StringSetEmpty, + BinarySetEmpty, + NumberSetDuplicates, + StringSetDuplicates(Vec), + BinarySetDuplicates(Vec), +} + +impl ValueFault { + /// The service's text for the fault in `order` at `place` (measured + /// 2026-11-05 unless noted): + /// + /// - a number: `The parameter cannot be converted to a numeric value: bob`; + /// - an empty `NS`: `One or more parameter values were invalid: An number + /// set may not be empty` (the service's two spaces), both orders; + /// - an empty `SS`: `... An string set may not be empty`, with the key + /// prefix for a key member in order U; + /// - an empty `BS`: `... Binary sets should not be empty`, with the key + /// prefix for a key member in order U (2026-11-05, `{"pk": {}, "sk": + /// {"BS": []}}`); an empty `NS` carries no prefix in either order; + /// - an `NS` with duplicates: `... Input collection contains duplicates.` + /// in order R, `Input collection contains duplicates` in order U; + /// - an `SS` with duplicates: `... Input collection ["a", "a"] contains + /// duplicates.` in order R (the members JSON-quoted, `["a\"b", "a\"b"]` + /// for a member with a quote), `... Input collection [a, a] contains + /// duplicates.` in order U; + /// - a `BS` with duplicates: `... Input collection ["YQ==","YQ=="]of type + /// BS contains duplicates.` in order R (no space after the comma or + /// before `of`); order U is not measured and gets the members unquoted. + fn text(&self, order: CheckOrder, place: Place) -> DynamoDbError { + let quoted = |members: &[String], separator: &str| -> String { + members + .iter() + .map(|member| match order { + CheckOrder::Readable => serde_json::to_string(member).unwrap_or_default(), + CheckOrder::Unreadable => member.clone(), + }) + .collect::>() + .join(separator) + }; + match self { + Self::Number(fault) => fault.clone(), + Self::NumberSetEmpty => invalid_parameter("An number set may not be empty"), + Self::StringSetEmpty => key_prefixed_in_order_u( + invalid_parameter("An string set may not be empty"), + order, + place, + ), + Self::BinarySetEmpty => key_prefixed_in_order_u( + invalid_parameter("Binary sets should not be empty"), + order, + place, + ), + Self::NumberSetDuplicates => match order { + CheckOrder::Readable => invalid_parameter("Input collection contains duplicates."), + CheckOrder::Unreadable => DynamoDbError::ValidationException( + "Input collection contains duplicates".to_owned(), + ), + }, + Self::StringSetDuplicates(members) => invalid_parameter(&format!( + "Input collection [{}] contains duplicates.", + quoted(members, ", ") + )), + Self::BinarySetDuplicates(members) => { + let separator = match order { + CheckOrder::Readable => ",", + CheckOrder::Unreadable => ", ", + }; + invalid_parameter(&format!( + "Input collection [{}]of type BS contains duplicates.", + quoted(members, separator) + )) + } + } + } +} + +/// `text` with the key prefix when it is a key member's fault in order U +/// (measured 2026-11-05 for an empty `SS` and an empty `BS` beside an +/// unreadable key member), else as it is. +fn key_prefixed_in_order_u(text: DynamoDbError, order: CheckOrder, place: Place) -> DynamoDbError { + match (order, place, text) { + (CheckOrder::Unreadable, Place::Key, DynamoDbError::ValidationException(msg)) => { + DynamoDbError::ValidationException(format!("{START_KEY_PREFIX}{msg}")) + } + (_, _, text) => text, + } +} + +/// The first value fault in one raw AttributeValue, depth first: an `N` text, +/// the members of an `NS` (every number first, then an empty set, then +/// duplicates by numeric value: `["1", "1.0"]` has a duplicate), an empty `SS` +/// or `BS` and their duplicates, and the values nested in `L` and `M`. A +/// member of a set that is not a string holds nothing to check here; the +/// deserializer reports it. +fn value_fault(value: &Value) -> Option { + let fields = value.as_object()?; + let mut fields = fields.iter(); + let (Some((descriptor, inner)), None) = (fields.next(), fields.next()) else { + return None; + }; + match (descriptor.as_str(), inner) { + ("N", Value::String(text)) => validate_number_text(text).err().map(ValueFault::Number), + ("NS", Value::Array(members)) => number_set_fault(members), + ("SS", Value::Array(members)) => string_set_fault(members), + ("BS", Value::Array(members)) => binary_set_fault(members), + ("L", Value::Array(members)) => members.iter().find_map(value_fault), + ("M", Value::Object(members)) => members.values().find_map(value_fault), + _ => None, + } +} + +/// A shape fault ([`shape_fault`]) or a value fault ([`value_fault`]) in one +/// raw AttributeValue. +enum Fault { + Shape(&'static str), + Value(ValueFault), +} + +/// The first fault of either kind in one raw AttributeValue, depth first in +/// the value's own element order: at each value, a shape fault of the value +/// itself (empty, an unknown descriptor, two descriptors), else the members of +/// an `L` or `M` in turn, else the value's number or set fault. Measured +/// 2026-11-05 (order U, `ExpressionAttributeValues`): `{"L": [{"N": "bob"}, +/// {}]}` answers the number, which [`shape_fault`] alone would pass over. The +/// empty value ahead of the number is not measured. +fn first_fault(value: &Value) -> Option { + let fields = value.as_object()?; + let mut fields = fields.iter(); + match (fields.next(), fields.next()) { + (None, _) => Some(Fault::Shape(EMPTY_VALUE)), + (Some(_), Some(_)) => Some(Fault::Shape(MORE_THAN_ONE_DESCRIPTOR)), + (Some((descriptor, inner)), None) => match (descriptor.as_str(), inner) { + ("L", Value::Array(members)) => members.iter().find_map(first_fault), + ("M", Value::Object(members)) => members.values().find_map(first_fault), + (descriptor, _) if TYPE_DESCRIPTORS.contains(&descriptor) => { + value_fault(value).map(Fault::Value) + } + _ => Some(Fault::Shape(EMPTY_VALUE)), + }, + } +} + +/// A `ValidationException` with the service's prefix for an invalid parameter +/// value. +fn invalid_parameter(text: &str) -> DynamoDbError { + DynamoDbError::ValidationException(format!("{INVALID_PARAMETER_VALUES}{text}")) +} + +/// The members of a set as written, skipping members that are not strings. +fn set_members(members: &[Value]) -> Vec { + members + .iter() + .filter_map(Value::as_str) + .map(str::to_owned) + .collect() +} + +/// An `NS`: every member's number text first (2026-11-05: `["bob", "1", "1"]` +/// and `["1", "1", "bob"]` answer `bob`), then an empty set, then two members +/// with one numeric value (`["1", "1"]` and `["1", "1.0"]`). +fn number_set_fault(members: &[Value]) -> Option { + let texts = set_members(members); + if let Some(fault) = texts + .iter() + .find_map(|text| validate_number_text(text).err()) + { + return Some(ValueFault::Number(fault)); + } + if members.is_empty() { + return Some(ValueFault::NumberSetEmpty); + } + let mut normalized: Vec = texts + .iter() + .map(|text| { + let folded = crate::decimal_digits::fold_decimal_digits(text); + extenddb_core::validation::number::validate_and_normalize_number(&folded) + .unwrap_or_else(|_| folded.into_owned()) + }) + .collect(); + normalized.sort_unstable(); + normalized.dedup(); + (normalized.len() != texts.len()).then_some(ValueFault::NumberSetDuplicates) +} + +/// An `SS`: an empty set, then duplicates. +fn string_set_fault(members: &[Value]) -> Option { + if members.is_empty() { + return Some(ValueFault::StringSetEmpty); + } + let texts = set_members(members); + if has_duplicates(&texts) { + return Some(ValueFault::StringSetDuplicates(texts)); + } + None +} + +/// A `BS`: an empty set, then duplicates. +fn binary_set_fault(members: &[Value]) -> Option { + if members.is_empty() { + return Some(ValueFault::BinarySetEmpty); + } + let texts = set_members(members); + if has_duplicates(&texts) { + return Some(ValueFault::BinarySetDuplicates(texts)); + } + None +} + +/// Whether two members of a set are equal. +fn has_duplicates(texts: &[String]) -> bool { + let mut distinct: Vec<&String> = texts.iter().collect(); + distinct.sort_unstable(); + distinct.dedup(); + distinct.len() != texts.len() +} + +/// The `ExpressionAttributeValues` of a `Scan` body whose entries the handler +/// reports at the map's own step rather than at request time. In order R the +/// numbers, the set faults and the nesting depth are request-time checks +/// ([`scan_request_checks`]); an empty or duplicate set is reported there ahead +/// of the no-expression check (2026-11-05, `{":e": {"NS": []}}` with no +/// expression answers `One or more parameter values were invalid: An number set +/// may not be empty`; `{":a": {"SS": ["a", "a"]}}` read by the filter answers +/// `... Input collection ["a", "a"] contains duplicates.`), and here only an +/// entry the request-time checks did not reach is held back, with the same +/// texts: a malformed number inside an entry the deserializer refuses for a set +/// fault (2026-11-05: `{"NS": ["bob", "1", "1"]}` beside a value nested 32 +/// levels answers the nesting text, so the number is reported after it). In +/// order U, after the key's faults and ahead of the nesting check, read by an +/// expression or not (2026-10-31 B4-7 and 2026-11-05), every fault in the form +/// `ExpressionAttributeValues contains invalid value: for key :e`: an +/// entry that is not one AttributeValue (empty, an unknown or lower-case type +/// descriptor, two descriptors, at the top or inside an `L` or `M`), `Supplied +/// AttributeValue is empty, must contain exactly one of the supported datatypes` +/// for `{}` and for `{"X": "1"}`, `... has more than one datatypes set, ...` +/// for two descriptors; a malformed number, `The parameter cannot be converted +/// to a numeric value: bob`, at the top or inside an `L` or `NS`, and the +/// 38-digit text; a set fault with the set texts of order U (`One or more +/// parameter values were invalid: An number set may not be empty`, `... Input +/// collection [a, a] contains duplicates.`). Inside one entry the first fault +/// in the value's element order is named ([`first_fault`]). The entries are +/// visited in the fitted order ([`fitted_member_order`]), and the first faulty +/// one is named (2026-11-05: `:n` ahead of `:e`, `:b` ahead of `:p` and `:a`, +/// `:a` ahead of `:e` when `:a` comes first in the request and `:e` ahead of +/// `:a` when `:e` does, `:s` ahead of `:e`; with four and seven entries the +/// eight and sixteen buckets). +/// +/// [`Self::take`] puts a readable stand-in in the body for each such entry, so +/// that the typed deserialization and the expression checks run on the other +/// values, and keeps the entry's fault; [`Self::validate`] reports it. The +/// request is refused before the scan reads the stand-in. An entry of the wrong +/// JSON shape stays in the body. +pub(crate) struct ExpressionValuesBody { + /// The faults of the entries taken out, in the fitted order of their names. + faults: Vec, +} + +impl ExpressionValuesBody { + /// Replaces each entry the checks here classify with a readable stand-in + /// and keeps its fault for `order`. `request_order` is the entry names as + /// the request wrote them ([`member_order`]). + pub(crate) fn take( + body: &mut Value, + order: CheckOrder, + request_order: Option<&[String]>, + ) -> Self { + let names = fitted_order_of(body, "ExpressionAttributeValues", request_order); + let Some(Value::Object(entries)) = body.get_mut("ExpressionAttributeValues") else { + return Self { faults: Vec::new() }; + }; + let mut faults = Vec::new(); + for name in names { + let Some(value) = entries.get_mut(&name) else { + continue; + }; + let fault = match (order, first_fault(value)) { + (_, None) => None, + (_, Some(Fault::Shape(text))) => Some(invalid_expression_value(text, &name)), + // In order R a number is reported here only when the request-time + // check skipped its entry (a set fault below it), with the plain + // text; the set faults were reported at request time. + (CheckOrder::Readable, Some(Fault::Value(ValueFault::Number(fault)))) => { + Some(fault) + } + (CheckOrder::Readable, Some(Fault::Value(fault))) => { + Some(fault.text(order, Place::ExpressionValue)) + } + (CheckOrder::Unreadable, Some(Fault::Value(fault))) => { + match fault.text(order, Place::ExpressionValue) { + DynamoDbError::ValidationException(msg) => { + Some(invalid_expression_value(&msg, &name)) + } + other => Some(other), + } + } + }; + if let Some(fault) = fault { + *value = serde_json::json!({"S": ""}); + faults.push(fault); + } + } + Self { faults } + } + + /// The first entry's fault, at the stage of `order`. + /// + /// # Errors + /// + /// Returns the fault of the first entry taken out, in the fitted order. + pub(crate) fn validate(&self) -> Result<(), DynamoDbError> { + match self.faults.first() { + Some(fault) => Err(fault.clone()), + None => Ok(()), + } + } +} + +/// The service's text for a faulty `ExpressionAttributeValues` entry at the +/// map's own step: `ExpressionAttributeValues contains invalid value: +/// for key `. +fn invalid_expression_value(text: &str, name: &str) -> DynamoDbError { + DynamoDbError::ValidationException(format!( + "ExpressionAttributeValues contains invalid value: {text} for key {name}" + )) +} + +/// The `ScanFilter` lists of a `Scan` body in order U, whose values the service +/// checks after the no-expression checks and ahead of `ConditionalOperator`, +/// the lists' value counts, the key's faults and the table (2026-11-05): a +/// value that is not one AttributeValue answers `Supplied AttributeValue is +/// empty, must contain exactly one of the supported datatypes` for `{}` and +/// for `{"X": "1"}`, with no prefix. A malformed number in a list is #507's +/// check (#564 item 3) and stays in the body. Two descriptors in a list value, +/// a set fault in a list value, and which value is named when several are +/// faulty (the conditions in the fitted order, the list in its order) are not +/// measured. [`Self::take`] puts a readable stand-in in the body for each such +/// value and keeps the fault; [`Self::validate`] reports it. In order R nothing +/// is taken. +pub(crate) struct ScanFilterBody { + faults: Vec, +} + +impl ScanFilterBody { + /// Replaces each list value that is not one AttributeValue with a readable + /// stand-in in order U and keeps its fault. `request_order` is the + /// condition names as the request wrote them ([`member_order`]). + pub(crate) fn take( + body: &mut Value, + order: CheckOrder, + request_order: Option<&[String]>, + ) -> Self { + if order == CheckOrder::Readable { + return Self { faults: Vec::new() }; + } + let names = fitted_order_of(body, "ScanFilter", request_order); + let Some(Value::Object(conditions)) = body.get_mut("ScanFilter") else { + return Self { faults: Vec::new() }; + }; + let mut faults = Vec::new(); + for name in names { + let Some(Value::Array(values)) = conditions + .get_mut(&name) + .and_then(|condition| condition.get_mut("AttributeValueList")) + else { + continue; + }; + for value in values.iter_mut() { + if let Some(text) = shape_fault(value) { + *value = serde_json::json!({"S": ""}); + faults.push(DynamoDbError::ValidationException(text.to_owned())); + } + } + } + Self { faults } + } + + /// The first list value's fault, at the step of order U. + /// + /// # Errors + /// + /// Returns the fault of the first value taken out. + pub(crate) fn validate(&self) -> Result<(), DynamoDbError> { + match self.faults.first() { + Some(fault) => Err(fault.clone()), + None => Ok(()), + } + } +} + +/// The `Select` field of a `Scan` body in order R, where the service checks +/// the enum after the no-expression check and the nesting check and ahead of +/// the filter syntax (2026-11-05): [`Self::take`] takes an invalid value out of +/// the body, so that the typed deserialization does not report it first, and +/// keeps the clause; [`Self::validate`] reports it. In order U the enum is a +/// request-time check ([`validate_scan_request_body`]) and nothing is taken. +pub(crate) struct SelectBody { + clause: Option, +} + +impl SelectBody { + /// Takes an invalid `Select` string out of `body` in order R. + pub(crate) fn take(body: &mut Value, order: CheckOrder) -> Self { + if order == CheckOrder::Unreadable { + return Self { clause: None }; + } + let clause = validate_select_enum(body).err(); + if clause.is_some() + && let Some(fields) = body.as_object_mut() + { + fields.remove("Select"); + } + Self { clause } + } + + /// The enum clause, at its stage. + /// + /// # Errors + /// + /// Returns `DynamoDbError::ValidationException` with the clause. + pub(crate) fn validate(&self) -> Result<(), DynamoDbError> { + match &self.clause { + Some(clause) => Err(clause.clone()), + None => Ok(()), + } + } +} + +/// The constraints the service aggregates into one message: the length of +/// `AttributesToGet` and the ranges of `Limit`, `Segment` and `TotalSegments`, +/// every fault in one message, in one of the service's two forms (measured +/// 2026-10-25 and 2026-10-30, us-east-1). +/// +/// When each number field present is an integer inside the int32 range, the +/// newer form: `Value at '' failed to satisfy constraint: ...` with the +/// field capitalised and no value echoed, in the order `AttributesToGet`, +/// `Limit`, `Segment`, `TotalSegments` (`Limit: 0` with `AttributesToGet: []` +/// answers the length clause, then the `Limit` clause, measured 2026-10-30 +/// 12:40 UTC; the length clause beside a `Segment` or `TotalSegments` fault +/// alone is not measured). When any of the three is a float or outside int32, +/// or the request follows order U (a value in `ExpressionAttributeValues`, +/// `ExclusiveStartKey` or a `ScanFilter` `AttributeValueList` is not one +/// AttributeValue, see [`CheckOrder`]), the older form: `Value '' at /// '' ...` with the value as [`saturated_int32`] reads it and the field /// in lower camel case, in the reverse of the newer form's order: /// `totalSegments`, `segment`, `limit` (measured 2026-10-30 12:40 UTC with a /// fault in each of the three, and on 2026-10-25 with `segment` then `limit` -/// and `totalSegments` then `limit`). -fn validate_ranges(body: &Value) -> Result<(), DynamoDbError> { +/// and `totalSegments` then `limit`). The older form of the length clause is +/// not measured: the code echoes the empty list, `Value '[]' at +/// 'attributesToGet' ...`, and lists it last. +fn validate_constraints(body: &Value, order: CheckOrder) -> Result<(), DynamoDbError> { let numbers: Vec<(&RangeField, &serde_json::Number)> = RANGE_FIELDS .iter() .filter_map(|range| { @@ -142,54 +1142,56 @@ fn validate_ranges(body: &Value) -> Result<(), DynamoDbError> { .map(|number| (range, number)) }) .collect(); - let older_form = has_unreadable_attribute_value(body, SCAN_CONDITIONS) + let older_form = order == CheckOrder::Unreadable || numbers.iter().any(|(_, number)| { number .as_i64() .is_none_or(|value| i32::try_from(value).is_err()) }); - let faults: Vec<(&RangeField, i32, String)> = numbers - .into_iter() - .filter_map(|(range, number)| { - let value = saturated_int32(number); - if value < range.min { - Some(( - range, - value, - format!("greater than or equal to {}", range.min), - )) - } else { - range - .max - .filter(|max| value > *max) - .map(|max| (range, value, format!("less than or equal to {max}"))) - } - }) - .collect(); + // Each fault as its newer-form and its older-form clause, in the newer + // form's order. + let mut faults: Vec<(String, String)> = Vec::new(); + if body + .get("AttributesToGet") + .and_then(Value::as_array) + .is_some_and(Vec::is_empty) + { + faults.push(( + "Value at 'AttributesToGet' failed to satisfy constraint: Member must have length greater than or equal to 1".to_owned(), + "Value '[]' at 'attributesToGet' failed to satisfy constraint: Member must have length greater than or equal to 1".to_owned(), + )); + } + for (range, number) in numbers { + let value = saturated_int32(number); + let constraint = if value < range.min { + format!("greater than or equal to {}", range.min) + } else if let Some(max) = range.max.filter(|max| value > *max) { + format!("less than or equal to {max}") + } else { + continue; + }; + faults.push(( + format!( + "Value at '{}' failed to satisfy constraint: Member must have value {constraint}", + range.field + ), + format!( + "Value '{value}' at '{}' failed to satisfy constraint: Member must have value {constraint}", + range.older_field + ), + )); + } if faults.is_empty() { return Ok(()); } - let clauses: Vec = if older_form { + let clauses: Vec<&str> = if older_form { faults .iter() .rev() - .map(|(range, value, constraint)| { - format!( - "Value '{value}' at '{}' failed to satisfy constraint: Member must have value {constraint}", - range.older_field - ) - }) + .map(|(_, older)| older.as_str()) .collect() } else { - faults - .iter() - .map(|(range, _, constraint)| { - format!( - "Value at '{}' failed to satisfy constraint: Member must have value {constraint}", - range.field - ) - }) - .collect() + faults.iter().map(|(newer, _)| newer.as_str()).collect() }; let count = clauses.len(); Err(DynamoDbError::ValidationException(format!( @@ -233,4 +1235,6 @@ const RANGE_FIELDS: [RangeField; 3] = [ #[cfg(test)] mod range_tests; #[cfg(test)] +mod request_tests; +#[cfg(test)] mod tests; diff --git a/crates/engine/src/scan_request/range_tests.rs b/crates/engine/src/scan_request/range_tests.rs index 0554cb1d5..3ef407f8d 100644 --- a/crates/engine/src/scan_request/range_tests.rs +++ b/crates/engine/src/scan_request/range_tests.rs @@ -11,7 +11,7 @@ use extenddb_core::error::DynamoDbError; use serde_json::Value; -use super::validate_scan_request_body; +use super::{CheckOrder, validate_scan_request_body}; const LIMIT: &str = "Value at 'Limit' failed to satisfy constraint: Member must have value greater than or equal to 1"; const SEGMENT_LOW: &str = "Value at 'Segment' failed to satisfy constraint: Member must have value greater than or equal to 0"; @@ -20,7 +20,7 @@ const TOTAL_LOW: &str = "Value at 'TotalSegments' failed to satisfy constraint: const TOTAL_HIGH: &str = "Value at 'TotalSegments' failed to satisfy constraint: Member must have value less than or equal to 1000000"; fn check(body: &Value) -> Option { - match validate_scan_request_body(body) { + match validate_scan_request_body(body, CheckOrder::of(body), None) { Ok(()) => None, Err(DynamoDbError::ValidationException(msg)) => Some(msg), Err(other) => panic!("unexpected error {other:?}"), diff --git a/crates/engine/src/scan_request/request_tests.rs b/crates/engine/src/scan_request/request_tests.rs new file mode 100644 index 000000000..b2692d9c6 --- /dev/null +++ b/crates/engine/src/scan_request/request_tests.rs @@ -0,0 +1,1253 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The `ScanFilter` in the parameter mix, the `AttributesToGet` length, the +//! nesting depth of `ExpressionAttributeValues`, the two check orders, and the +//! `ExclusiveStartKey` and `ExpressionAttributeValues` checks of each order. +//! +//! Every row is a request sent to the service (us-east-1, each request twice, +//! a Scan of a table the probe created) with the answer it gave: on 2026-10-25 +//! at 10:38 and 10:55 UTC, on 2026-10-30 at 12:40 UTC, on 2026-10-31 between +//! 23:54 and 23:56 UTC and on 2026-11-05 where a test says so. The tests of +//! `StartKeyBody::take` and `SelectBody::take` check which values they hold +//! back, with no service answer; the tests of `ExpressionValuesBody::take` and +//! `ScanFilterBody::take` also check the measured texts they keep; the tests of +//! the member order check the fitted rule's buckets and masks; the rows with +//! two value faults in two key members accept either member's fault. + +use extenddb_core::error::DynamoDbError; +use serde_json::{Value, json}; + +use super::{ + CheckOrder, ExpressionValuesBody, ScanFilterBody, SelectBody, StartKeyBody, + fitted_member_order, hash_table_mask, java_hash_bucket, member_order, + validate_scan_request_body, +}; + +const MIX: &str = "Can not use both expression and non-expression parameters in the same request: Non-expression parameters: {ScanFilter} Expression parameters: {FilterExpression}"; +const LENGTH: &str = "1 validation error detected: Value at 'AttributesToGet' failed to satisfy constraint: Member must have length greater than or equal to 1"; +const NUMERIC_BOB: &str = "The parameter cannot be converted to a numeric value: bob"; +const NESTING: &str = "Nesting Levels have exceeded supported limits: Attributes in the item have nested levels beyond supported limit"; +const SELECT_BAD: &str = "1 validation error detected: Value 'BAD' at 'select' failed to satisfy constraint: Member must satisfy enum value set: [SPECIFIC_ATTRIBUTES, COUNT, ALL_ATTRIBUTES, ALL_PROJECTED_ATTRIBUTES]"; +const KEY_PREFIX: &str = "The provided starting key is invalid: "; +const EMPTY_VALUE: &str = + "Supplied AttributeValue is empty, must contain exactly one of the supported datatypes"; +const MORE_THAN_ONE: &str = "Supplied AttributeValue has more than one datatypes set, must contain exactly one of the supported datatypes"; +const INVALID: &str = "One or more parameter values were invalid: "; + +/// The message of a `ValidationException`, `None` for `Ok`. +fn validation_message(result: Result<(), DynamoDbError>) -> Option { + match result { + Ok(()) => None, + Err(DynamoDbError::ValidationException(msg)) => Some(msg), + Err(other) => panic!("unexpected error {other:?}"), + } +} + +/// The request-time checks on `body`, in the order of the body's class. +fn check(body: &Value) -> Option { + validation_message(validate_scan_request_body(body, CheckOrder::of(body), None)) +} + +/// A Scan body with the fields of `extra` added to the table name. +fn scan(extra: Value) -> Value { + let mut body = json!({"TableName": "t"}); + for (field, value) in extra.as_object().expect("an object") { + body[field] = value.clone(); + } + body +} + +/// A filter that reads `:n`, which holds `number`. +fn filter(number: &str) -> Value { + json!({ + "FilterExpression": "a = :n", + "ExpressionAttributeValues": {":n": {"N": number}}, + }) +} + +fn with(mut body: Value, extra: Value) -> Value { + for (field, value) in extra.as_object().expect("an object") { + body[field] = value.clone(); + } + body +} + +/// `leaf` inside `levels` lists, each list one level. +fn nested(leaf: &Value, levels: usize) -> Value { + (0..levels).fold(leaf.clone(), |inner, _| json!({"L": [inner]})) +} + +/// A filter that reads `:p`, with `value` under `:d`, which nothing reads. +fn with_unused_value(value: Value) -> Value { + scan(json!({ + "FilterExpression": "a = :p", + "ExpressionAttributeValues": {":p": {"S": "x"}, ":d": value}, + })) +} + +/// A Scan body with `key` as its `ExclusiveStartKey`. +fn with_key(key: Value) -> Value { + scan(json!({"ExclusiveStartKey": key})) +} + +/// The key's check on `body` in the order of the body's class, as the handler +/// makes it: `StartKeyBody::take` on a copy of the body with the members in +/// `request_order` (the `json!` macro sorts them), then `validate`. +fn check_key_in(body: &Value, request_order: &[&str]) -> Option { + let mut body = body.clone(); + let order = CheckOrder::of(&body); + let names: Vec = request_order + .iter() + .map(|name| (*name).to_owned()) + .collect(); + let request_order = if names.is_empty() { + None + } else { + Some(names.as_slice()) + }; + validation_message(StartKeyBody::take(&mut body, request_order).validate(order)) +} + +/// [`check_key_in`] with the parsed body's member order. +fn check_key(body: &Value) -> Option { + check_key_in(body, &[]) +} + +#[test] +fn an_empty_scan_filter_counts_as_present_for_the_parameter_mix() { + let mix = with(scan(filter("1")), json!({"ScanFilter": {}})); + assert_eq!(check(&mix).as_deref(), Some(MIX)); + // Alone, an empty ScanFilter is accepted. + assert_eq!(check(&scan(json!({"ScanFilter": {}}))), None); + // Beside an unused malformed number, the number text. + let unused = scan(json!({ + "ScanFilter": {}, + "ExpressionAttributeValues": {":n": {"N": "bob"}}, + })); + assert_eq!(check(&unused).as_deref(), Some(NUMERIC_BOB)); +} + +#[test] +fn an_empty_attributes_to_get_is_a_length_fault_ahead_of_the_number_check() { + let empty = json!({"AttributesToGet": []}); + for body in [ + scan(empty.clone()), + with(scan(filter("1")), empty.clone()), + with(scan(filter("bob")), empty.clone()), + // Measured 2026-10-30 12:40 UTC: ahead of the parameter mix too. + with(scan(filter("1")), with(empty, json!({"ScanFilter": {}}))), + ] { + assert_eq!(check(&body).as_deref(), Some(LENGTH), "{body}"); + } + assert_eq!(check(&scan(json!({"AttributesToGet": ["a"]}))), None); +} + +/// Measured 2026-10-30 12:40 UTC: the length constraint is aggregated with the +/// range constraints, ahead of `Limit`. +#[test] +fn an_empty_attributes_to_get_joins_the_range_faults() { + let body = scan(json!({"AttributesToGet": [], "Limit": 0})); + assert_eq!( + check(&body).as_deref(), + Some( + "2 validation errors detected: Value at 'AttributesToGet' failed to satisfy constraint: Member must have length greater than or equal to 1; Value at 'Limit' failed to satisfy constraint: Member must have value greater than or equal to 1" + ) + ); +} + +/// The order flag reads `ExpressionAttributeValues`, `ExclusiveStartKey` and +/// the `ScanFilter` lists, at any depth of `L` and `M`; a set fault, a number +/// fault and a readable value of the wrong type for the key leave the request +/// in order R. +#[test] +fn the_order_follows_an_unreadable_value_anywhere() { + for body in [ + with_key(json!({"pk": {}, "sk": {"S": "x"}})), + with_key(json!({"pk": {"X": "1"}, "sk": {"S": "x"}})), + with_key(json!({"pk": {"s": "x"}, "sk": {"S": "x"}})), + with_key(json!({"pk": {"S": "x", "N": "1"}, "sk": {"S": "x"}})), + with_key(json!({"pk": {"L": [{"M": {"k": {}}}]}, "sk": {"S": "x"}})), + scan(json!({"ExpressionAttributeValues": {":e": {}}})), + scan( + json!({"FilterExpression": "a = :a", "ExpressionAttributeValues": {":a": {"X": "1"}}}), + ), + scan( + json!({"ScanFilter": {"a": {"ComparisonOperator": "EQ", "AttributeValueList": [{}]}}}), + ), + ] { + assert_eq!(CheckOrder::of(&body), CheckOrder::Unreadable, "{body}"); + } + for body in [ + scan(json!({})), + with_key(json!({"pk": {"N": "bob"}, "sk": {"NS": []}})), + with_key(json!({"pk": {"SS": ["a", "a"]}, "sk": {"S": "x"}})), + scan(json!({"ExpressionAttributeValues": {":e": {"NS": []}}})), + scan(json!({"ExpressionAttributeValues": {":n": {"N": "bob"}}})), + with_key(json!({"pk": {"N": "1"}, "sk": {"S": "x"}})), + ] { + assert_eq!(CheckOrder::of(&body), CheckOrder::Readable, "{body}"); + } +} + +#[test] +fn a_malformed_number_in_the_exclusive_start_key_is_reported_with_the_number_text() { + let key = |number: &str| json!({"pk": {"N": number}, "sk": {"S": "x"}}); + let bad = with(with_key(key("bob")), filter("1")); + // The request-time checks leave the number to the key's stage. + assert_eq!(check(&bad), None); + assert_eq!(check_key(&bad).as_deref(), Some(NUMERIC_BOB)); + assert_eq!(check_key(&with(with_key(key("1")), filter("1"))), None); + // Measured 2026-10-30 12:40 UTC: with no filter, the same text. + assert_eq!( + check_key(&with_key(key("bob"))).as_deref(), + Some(NUMERIC_BOB) + ); + // Measured 2026-10-31 between 23:54 and 23:56 UTC: a number as an NS + // member, as an L member, and in the L of an attribute that is not a key + // attribute; the number texts for a leading space, an empty text and 39 + // digits (on an N hash key). Measured 2026-11-05: `{"pk": {"N": "bob"}}` + // alone, with no `sk`, on the `pk`, `sk` table. + for key in [ + json!({"pk": {"NS": ["bob"]}, "sk": {"S": "x"}}), + json!({"pk": {"L": [{"N": "bob"}]}, "sk": {"S": "x"}}), + json!({"pk": {"S": "p0"}, "sk": {"S": "x"}, "z": {"L": [{"N": "bob"}]}}), + json!({"pk": {"N": "bob"}}), + ] { + assert_eq!( + check_key(&with_key(key.clone())).as_deref(), + Some(NUMERIC_BOB), + "{key}" + ); + } + for (number, text) in [ + ( + " 1", + "The parameter cannot be converted to a numeric value: 1", + ), + ("", "The parameter cannot be converted to a numeric value: "), + ( + "111111111111111111111111111111111111111", + "Attempting to store more than 38 significant digits in a Number", + ), + ] { + let key = with_key(json!({"pk": {"N": number}})); + assert_eq!(check_key(&key).as_deref(), Some(text), "{number:?}"); + } +} + +/// Measured 2026-10-31 between 23:54 and 23:56 UTC and 2026-11-05 with `{"pk": +/// {"N": "bob"}, "sk": {"S": "x"}}` as the key: each request-time check of +/// order R answers its own text ahead of the key's number, in the newer form; +/// beside a key value that is empty or carries an unknown type descriptor the +/// constraint clauses take the older form and the `Select` enum joins the +/// request-time checks (2026-11-05). The request parameter checks, the +/// expression checks and the Segment pairing, which the handler makes after +/// deserializing, come ahead of the key's number too; the wire tests send +/// those rows. +#[test] +fn every_request_time_check_comes_ahead_of_the_number_in_the_exclusive_start_key() { + let bob = json!({"pk": {"N": "bob"}, "sk": {"S": "x"}}); + for (extra, text) in [ + ( + json!({"Limit": 0}), + "1 validation error detected: Value at 'Limit' failed to satisfy constraint: Member must have value greater than or equal to 1", + ), + ( + json!({"ReturnConsumedCapacity": "BAD"}), + "1 validation error detected: Value 'BAD' at 'returnConsumedCapacity' failed to satisfy constraint: Member must satisfy enum value set: [INDEXES, TOTAL, NONE]", + ), + ( + json!({"Segment": -1, "TotalSegments": 2}), + "1 validation error detected: Value at 'Segment' failed to satisfy constraint: Member must have value greater than or equal to 0", + ), + (with(filter("1"), json!({"ScanFilter": {}})), MIX), + (json!({"AttributesToGet": []}), LENGTH), + ( + json!({ + "FilterExpression": "a = :p", + "ExpressionAttributeValues": {":p": {"S": "x"}, ":d": nested(&json!({"N": "1"}), 32)}, + }), + NESTING, + ), + // The values ahead of the key, with `fred` in the key. + ( + json!({ + "FilterExpression": "a = :n", + "ExpressionAttributeValues": {":n": {"N": "bob"}}, + "ExclusiveStartKey": {"pk": {"N": "fred"}, "sk": {"S": "x"}}, + }), + NUMERIC_BOB, + ), + ] { + let body = with(with_key(bob.clone()), extra); + assert_eq!(check(&body).as_deref(), Some(text), "{body}"); + } + // In order R the Select enum is not a request-time check: SelectBody + // holds it back for the handler (measured 2026-11-05: after the + // no-expression check and the nesting check). + let select = with(with_key(bob), json!({"Select": "BAD"})); + assert_eq!(check(&select), None); + assert_eq!(check_select(&select).as_deref(), Some(SELECT_BAD)); + // Beside a key value the shape stage refuses, the older form, and the + // Select enum at request time (2026-11-05). + for key in [ + json!({"pk": {}, "sk": {"S": "x"}}), + json!({"pk": {"X": "1"}, "sk": {"S": "x"}}), + ] { + assert_eq!( + check(&with(with_key(key.clone()), json!({"Limit": 0}))).as_deref(), + Some( + "1 validation error detected: Value '0' at 'limit' failed to satisfy constraint: Member must have value greater than or equal to 1" + ), + "{key}" + ); + assert_eq!( + check(&with(with_key(key.clone()), json!({"Select": "BAD"}))).as_deref(), + Some(SELECT_BAD), + "{key}" + ); + } + // An unused empty expression value beside Select BAD: the Select clause + // (2026-11-05, order U). + let unused_empty = with(with_unused_value(json!({})), json!({"Select": "BAD"})); + assert_eq!(check(&unused_empty).as_deref(), Some(SELECT_BAD)); +} + +/// `SelectBody::take` on a copy of `body`, then `validate`. +fn check_select(body: &Value) -> Option { + let mut body = body.clone(); + let order = CheckOrder::of(&body); + validation_message(SelectBody::take(&mut body, order).validate()) +} + +/// `SelectBody::take` holds an invalid `Select` back in order R, so that the +/// typed deserialization does not report it, and leaves it in order U, where +/// the request-time checks report it; no service answer is asserted here. +#[test] +fn an_invalid_select_is_held_back_in_order_r_only() { + let mut body = scan(json!({"Select": "BAD"})); + let select = SelectBody::take(&mut body, CheckOrder::Readable); + assert!(body.get("Select").is_none()); + assert_eq!( + validation_message(select.validate()).as_deref(), + Some(SELECT_BAD) + ); + assert!(serde_json::from_value::(body).is_ok()); + let mut body = scan(json!({"Select": "BAD", "ExclusiveStartKey": {"pk": {}}})); + let select = SelectBody::take(&mut body, CheckOrder::Unreadable); + assert!(body.get("Select").is_some()); + assert!(select.validate().is_ok()); + // A valid Select and a Select of the wrong JSON type stay as written. + for value in [json!("COUNT"), json!(5)] { + let mut body = scan(json!({"Select": value})); + let select = SelectBody::take(&mut body, CheckOrder::Readable); + assert_eq!(body["Select"], value); + assert!(select.validate().is_ok()); + } +} + +/// Measured 2026-11-05: a key value that is empty, carries an unknown or +/// lower-case type descriptor, or carries two, at its top or inside an `L` or +/// `M`, in a key member or in a member that is not a key attribute, answers the +/// service's text with the key prefix at the key's stage of order U. +#[test] +fn a_shape_fault_in_a_key_value_answers_the_prefixed_text() { + let empty = format!("{KEY_PREFIX}{EMPTY_VALUE}"); + for key in [ + json!({"pk": {}, "sk": {"S": "x"}}), + json!({"pk": {"X": "1"}, "sk": {"S": "x"}}), + json!({"pk": {"s": "x"}, "sk": {"S": "x"}}), + json!({"pk": {"L": [{}]}, "sk": {"S": "x"}}), + json!({"pk": {"M": {"k": {}}}, "sk": {"S": "x"}}), + json!({"pk": {"S": "p0"}, "sk": {"S": "x"}, "z": {}}), + json!({"pk": {}}), + ] { + let body = with_key(key.clone()); + assert_eq!(check(&body), None, "{key}"); + assert_eq!(check_key(&body).as_deref(), Some(empty.as_str()), "{key}"); + } + let two = with_key(json!({"pk": {"S": "x", "N": "1"}, "sk": {"S": "x"}})); + assert_eq!( + check_key(&two), + Some(format!("{KEY_PREFIX}{MORE_THAN_ONE}")) + ); + // Inside one member a shape fault comes ahead of a number (2026-11-05, + // `{"M": {"k": {"N": "bob"}, "j": {}}}` in a non-key member). + let inside = with_key(json!({ + "pk": {"S": "x"}, "sk": {"S": "x"}, "z": {"M": {"k": {"N": "bob"}, "j": {}}}, + })); + assert_eq!(check_key(&inside).as_deref(), Some(empty.as_str())); +} + +/// Measured 2026-11-05: in order R the set faults of a key answer the service's +/// texts with no prefix, the members JSON-quoted, after every number in the +/// key. +#[test] +fn a_set_fault_in_a_key_value_answers_the_order_r_text_without_a_prefix() { + for (key, text) in [ + (json!({"NS": []}), "An number set may not be empty"), + (json!({"SS": []}), "An string set may not be empty"), + (json!({"BS": []}), "Binary sets should not be empty"), + ( + json!({"NS": ["1", "1"]}), + "Input collection contains duplicates.", + ), + ( + json!({"NS": ["1", "1.0"]}), + "Input collection contains duplicates.", + ), + ( + json!({"SS": ["a", "a"]}), + r#"Input collection ["a", "a"] contains duplicates."#, + ), + ( + json!({"SS": ["a\"b", "a\"b"]}), + r#"Input collection ["a\"b", "a\"b"] contains duplicates."#, + ), + ( + json!({"BS": ["YQ==", "YQ=="]}), + r#"Input collection ["YQ==","YQ=="]of type BS contains duplicates."#, + ), + ] { + let body = with_key(json!({"pk": key, "sk": {"S": "x"}})); + assert_eq!(CheckOrder::of(&body), CheckOrder::Readable, "{key}"); + assert_eq!(check_key(&body), Some(format!("{INVALID}{text}")), "{key}"); + } + // A number in a set is reported with the number text, ahead of a + // duplicate in the same set (2026-11-05, `["bob", "1", "1"]` and `["1", + // "1", "bob"]`). + for members in [ + json!(["bob"]), + json!(["bob", "1", "1"]), + json!(["1", "1", "bob"]), + ] { + let bob = with_key(json!({"pk": {"S": "x"}, "sk": {"S": "x"}, "z": {"NS": members}})); + assert_eq!(check_key(&bob).as_deref(), Some(NUMERIC_BOB), "{members}"); + } + // A set with distinct numbers passes. + assert_eq!( + check_key(&with_key( + json!({"pk": {"NS": ["1", "2"]}, "sk": {"S": "x"}}) + )), + None + ); +} + +/// Measured 2026-11-05: in order U (here, the other member of the key is +/// unreadable and the set member comes first in the fitted order) the set texts +/// have no JSON quotes, the `NS` duplicate text is bare, and the empty `SS` +/// text carries the key prefix. The key's number keeps its text. +#[test] +fn a_set_fault_in_a_key_value_answers_the_order_u_text() { + for (sk, text) in [ + ( + json!({"NS": []}), + format!("{INVALID}An number set may not be empty"), + ), + ( + json!({"SS": []}), + format!("{KEY_PREFIX}{INVALID}An string set may not be empty"), + ), + ( + json!({"NS": ["1", "1"]}), + "Input collection contains duplicates".to_owned(), + ), + ( + json!({"SS": ["a", "a"]}), + format!("{INVALID}Input collection [a, a] contains duplicates."), + ), + (json!({"N": "bob"}), NUMERIC_BOB.to_owned()), + ] { + for pk in [json!({}), json!({"X": "1"}), json!({"S": "x", "N": "1"})] { + let body = with_key(json!({"pk": pk, "sk": sk})); + assert_eq!(CheckOrder::of(&body), CheckOrder::Unreadable); + assert_eq!(check_key(&body).as_deref(), Some(text.as_str()), "{body}"); + } + } + // Measured 2026-11-05 beside an unreadable key member: an empty BS with + // the key prefix, as the empty SS; a BS with duplicates with the members + // unquoted and no prefix. + let bs_empty = with_key(json!({"pk": {}, "sk": {"BS": []}})); + assert_eq!( + check_key(&bs_empty), + Some(format!( + "{KEY_PREFIX}{INVALID}Binary sets should not be empty" + )) + ); + let bs_dup = with_key(json!({"pk": {}, "sk": {"BS": ["YQ==", "YQ=="]}})); + assert_eq!( + check_key(&bs_dup), + Some(format!( + "{INVALID}Input collection [YQ==, YQ==]of type BS contains duplicates." + )) + ); +} + +/// The fitted member order: `String.hashCode()` of the name, spread and masked +/// to four buckets, buckets ascending, the request's order inside a bucket +/// (measured 2026-10-31, 2026-11-04 and 2026-11-05 on five tables). +#[test] +fn the_fitted_member_order_follows_the_hash_buckets_then_the_request() { + for (name, bucket) in [ + ("sk", 0), + ("a", 1), + ("b", 2), + ("pk", 3), + ("gpk", 3), + ("id", 3), + ("ts", 3), + ("\u{e4}", 0), + ("\u{e9}", 1), + (":n", 0), + (":b", 0), + (":p", 2), + (":a", 3), + (":e", 3), + ] { + assert_eq!(java_hash_bucket(name, 3), bucket, "{name}"); + } + // The table doubles when it holds more than three quarters of its size. + for (names, mask) in [ + (0, 3), + (1, 3), + (3, 3), + (4, 7), + (6, 7), + (7, 15), + (12, 15), + (13, 31), + ] { + assert_eq!(hash_table_mask(names), mask, "{names} names"); + } + for (name, bucket) in [("sk", 0), ("gpk", 3), ("pk", 3), ("gsk", 6)] { + assert_eq!(java_hash_bucket(name, 7), bucket, "{name}"); + } + let order = |names: &[&str]| fitted_member_order(names.iter().copied()); + // The pk, sk table: sk first in both request orders. + assert_eq!(order(&["pk", "sk"]), ["sk", "pk"]); + assert_eq!(order(&["sk", "pk"]), ["sk", "pk"]); + // HASH a, RANGE b: a first; HASH pk, RANGE a: a first. + assert_eq!(order(&["b", "a"]), ["a", "b"]); + assert_eq!(order(&["pk", "a"]), ["a", "pk"]); + assert_eq!(order(&["a", "pk"]), ["a", "pk"]); + // ts and id share a bucket: the request's order. + assert_eq!(order(&["ts", "id"]), ["ts", "id"]); + assert_eq!(order(&["id", "ts"]), ["id", "ts"]); + // The GSI key of three members: sk, then gpk and pk in the request's order. + assert_eq!(order(&["gpk", "pk", "sk"]), ["sk", "gpk", "pk"]); + assert_eq!(order(&["pk", "sk", "gpk"]), ["sk", "pk", "gpk"]); + // Four members (measured 2026-11-05 on a GSI with a range key): eight + // buckets, so gpk and pk (3) come ahead of gsk (6) in both request orders; + // with four buckets gsk (2) would come ahead of them. + assert_eq!( + order(&["gpk", "gsk", "pk", "sk"]), + ["sk", "gpk", "pk", "gsk"] + ); + assert_eq!( + order(&["sk", "pk", "gsk", "gpk"]), + ["sk", "pk", "gpk", "gsk"] + ); + // HASH \u{e4}, RANGE \u{e9}: \u{e4} first in both request orders. + assert_eq!(order(&["\u{e4}", "\u{e9}"]), ["\u{e4}", "\u{e9}"]); + assert_eq!(order(&["\u{e9}", "\u{e4}"]), ["\u{e4}", "\u{e9}"]); + // ExpressionAttributeValues entries (order U, 2026-11-05): :n ahead of :e, + // :b ahead of :p and :a, :a and :e in the request's order. + assert_eq!(order(&[":p", ":a", ":e"]), [":p", ":a", ":e"]); + assert_eq!(order(&[":p", ":n", ":e"]), [":n", ":p", ":e"]); + assert_eq!(order(&[":p", ":b", ":a"]), [":b", ":p", ":a"]); + // Seven names: sixteen buckets. + let seven = ["a", "b", "c", "d", "e", "f", "g"]; + assert_eq!(order(&seven).len(), 7); + assert_eq!(hash_table_mask(seven.len()), 15); +} + +/// `member_order` reads a top-level object's member names from the request +/// text in document order; a repeated name keeps its first position; a body +/// without the field, a field that is not an object or text that does not +/// parse gives `None`. +#[test] +fn the_request_order_of_the_members_is_read_from_the_raw_body() { + let raw = br#"{"TableName": "t", "ExclusiveStartKey": {"ts": {"N": "bob"}, "id": {}}, "ExpressionAttributeValues": {":p": {"S": "x"}, ":n": {"N": "bob"}}, "ScanFilter": {"b": {}, "a": {}}}"#; + let names = |names: &[&str]| Some(names.iter().map(|n| (*n).to_owned()).collect::>()); + assert_eq!(member_order(raw, "ExclusiveStartKey"), names(&["ts", "id"])); + assert_eq!( + member_order(raw, "ExpressionAttributeValues"), + names(&[":p", ":n"]) + ); + assert_eq!(member_order(raw, "ScanFilter"), names(&["b", "a"])); + let repeated = + br#"{"ExclusiveStartKey": {"b": {}, "a": {}, "b": {"S": "x"}}, "TableName": "t"}"#; + assert_eq!( + member_order(repeated, "ExclusiveStartKey"), + names(&["b", "a"]) + ); + assert_eq!(member_order(raw, "ProjectionExpression"), None); + assert_eq!( + member_order(br#"{"TableName": "t"}"#, "ExclusiveStartKey"), + None + ); + assert_eq!( + member_order( + br#"{"TableName": "t", "ExclusiveStartKey": "x"}"#, + "ExclusiveStartKey" + ), + None + ); + assert_eq!(member_order(b"not json", "ExclusiveStartKey"), None); + assert_eq!(member_order(b"", "ExclusiveStartKey"), None); +} + +/// Faults in two members of one key: the first faulty member in the fitted +/// order is reported. Measured rows: `{"pk": {}, "sk": {"N": "bob"}}` and the +/// reversed request order answer `bob` (2026-10-31); `{"pk": {"X": "1"}, "sk": +/// {"N": "bob"}}` answers `bob`; `{"pk": {}, "sk": {"NS": []}}` answers the +/// empty NS text and `{"pk": {"S": "x", "N": "1"}, "sk": {"SS": []}}` the +/// prefixed SS text (2026-11-05); on HASH `a`, RANGE `b` `{"a": {}, "b": {"N": +/// "bob"}}` answers the empty value and `{"a": {"N": "bob"}, "b": {}}` `bob`; +/// on HASH `pk`, RANGE `a` `{"pk": {}, "a": {"N": "bob"}}` answers `bob` in +/// both request orders; on HASH `id`, RANGE `ts` the member first in the +/// request; on the GSI `{"gpk": {}, "pk": {"N": "bob"}, "sk": {"S": "x"}}` +/// the empty value. With two value faults the service's two sends disagreed on +/// 8 of 22 rows: those rows accept either member's fault. +#[test] +fn two_faulty_members_answer_the_first_in_the_fitted_order() { + let empty = format!("{KEY_PREFIX}{EMPTY_VALUE}"); + let ns_empty = format!("{INVALID}An number set may not be empty"); + let ss_empty = format!("{KEY_PREFIX}{INVALID}An string set may not be empty"); + for (key, request_order, text) in [ + ( + json!({"pk": {}, "sk": {"N": "bob"}}), + ["pk", "sk"], + NUMERIC_BOB, + ), + ( + json!({"sk": {"N": "bob"}, "pk": {}}), + ["sk", "pk"], + NUMERIC_BOB, + ), + ( + json!({"pk": {"X": "1"}, "sk": {"N": "bob"}}), + ["pk", "sk"], + NUMERIC_BOB, + ), + ( + json!({"pk": {"N": "bob"}, "sk": {}}), + ["pk", "sk"], + empty.as_str(), + ), + ( + json!({"pk": {}, "sk": {"NS": []}}), + ["pk", "sk"], + ns_empty.as_str(), + ), + ( + json!({"pk": {"S": "x", "N": "1"}, "sk": {"SS": []}}), + ["pk", "sk"], + ss_empty.as_str(), + ), + ( + json!({"a": {}, "b": {"N": "bob"}}), + ["a", "b"], + empty.as_str(), + ), + (json!({"a": {"N": "bob"}, "b": {}}), ["a", "b"], NUMERIC_BOB), + ( + json!({"pk": {}, "a": {"N": "bob"}}), + ["pk", "a"], + NUMERIC_BOB, + ), + ( + json!({"a": {"N": "bob"}, "pk": {}}), + ["a", "pk"], + NUMERIC_BOB, + ), + ( + json!({"id": {}, "ts": {"N": "bob"}}), + ["id", "ts"], + empty.as_str(), + ), + ( + json!({"ts": {"N": "bob"}, "id": {}}), + ["ts", "id"], + NUMERIC_BOB, + ), + ] { + let body = with_key(key.clone()); + assert_eq!( + check_key_in(&body, &request_order).as_deref(), + Some(text), + "{key} in the order {request_order:?}" + ); + } + let gsi = with_key(json!({"gpk": {}, "pk": {"N": "bob"}, "sk": {"S": "x"}})); + assert_eq!( + check_key_in(&gsi, &["gpk", "pk", "sk"]).as_deref(), + Some(empty.as_str()) + ); + // Two value faults: either member's fault. + let fred = NUMERIC_BOB.replace("bob", "fred"); + for (key, either) in [ + ( + json!({"pk": {"N": "bob"}, "sk": {"N": "fred"}}), + [NUMERIC_BOB, fred.as_str()], + ), + ( + json!({"pk": {"NS": []}, "sk": {"N": "bob"}}), + [ns_empty.as_str(), NUMERIC_BOB], + ), + ] { + let answer = check_key_in(&with_key(key.clone()), &["pk", "sk"]); + assert!( + either.contains(&answer.as_deref().unwrap_or_default()), + "{key}: {answer:?}" + ); + } +} + +/// `StartKeyBody::take` holds back a key the deserializer refuses for its +/// value and leaves one of the wrong JSON shape in the body, where the typed +/// deserialization reports it ahead of every check; no service answer is +/// asserted here. +#[test] +fn a_key_of_the_wrong_shape_stays_in_the_body_and_an_unreadable_value_is_held_back() { + for key in [json!("x"), json!({"pk": "x"}), json!({"pk": {"N": 5}})] { + let mut body = with_key(key.clone()); + let order = CheckOrder::of(&body); + let start_key = StartKeyBody::take(&mut body, None); + assert!(body.get("ExclusiveStartKey").is_some(), "{key}"); + assert!(start_key.validate(order).is_ok(), "{key}"); + let shape = serde_json::from_value::(body) + .map_err(crate::deserialize_error); + assert!( + matches!(shape, Err(DynamoDbError::SerializationException(_))), + "{key}: {shape:?}" + ); + } + for key in [ + json!({"pk": {}}), + json!({"pk": {"X": "1"}}), + json!({"pk": {"S": "x", "N": "1"}}), + json!({"pk": {"NS": []}}), + json!({"pk": {"SS": ["a", "a"]}}), + ] { + let mut body = with_key(key.clone()); + let order = CheckOrder::of(&body); + let start_key = StartKeyBody::take(&mut body, None); + assert!(body.get("ExclusiveStartKey").is_none(), "{key}"); + assert!(start_key.validate(order).is_err(), "{key}"); + assert!( + serde_json::from_value::(body).is_ok(), + "{key}" + ); + } + // A readable key stays in the body and passes. + let mut body = with_key(json!({"pk": {"S": "p0"}, "sk": {"S": "x"}})); + let start_key = StartKeyBody::take(&mut body, None); + assert!(body.get("ExclusiveStartKey").is_some()); + assert!(start_key.validate(CheckOrder::Readable).is_ok()); + // No key at all. + let mut body = scan(json!({})); + let start_key = StartKeyBody::take(&mut body, None); + assert!(start_key.validate(CheckOrder::Readable).is_ok()); + // A request order that does not name the key's members is ignored. + let mut body = with_key(json!({"pk": {}, "sk": {"N": "bob"}})); + let names = ["a".to_owned(), "b".to_owned()]; + let start_key = StartKeyBody::take(&mut body, Some(&names)); + assert_eq!( + validation_message(start_key.validate(CheckOrder::Unreadable)).as_deref(), + Some(NUMERIC_BOB) + ); +} + +/// `ExpressionValuesBody::take` puts a readable stand-in in the body for an +/// entry the checks classify and keeps the fault with the service's text: in +/// order U the `ExpressionAttributeValues contains invalid value: ... for key +/// :a` text for an empty value, an unknown descriptor and two descriptors +/// (2026-10-31 B4-7, 2026-11-05); in order R the set texts with the members +/// JSON-quoted (2026-11-05). An entry of the wrong JSON shape stays in the +/// body. +#[test] +fn a_faulty_expression_value_is_held_back_and_a_stand_in_takes_its_place() { + let wrapper = + |text: &str| format!("ExpressionAttributeValues contains invalid value: {text} for key :a"); + for (value, text) in [ + (json!({}), wrapper(EMPTY_VALUE)), + (json!({"X": "1"}), wrapper(EMPTY_VALUE)), + (json!({"S": "x", "N": "1"}), wrapper(MORE_THAN_ONE)), + (json!({"L": [{}]}), wrapper(EMPTY_VALUE)), + ] { + let mut body = scan(json!({ + "FilterExpression": "a = :a", + "ExpressionAttributeValues": {":a": value}, + })); + let order = CheckOrder::of(&body); + assert_eq!(order, CheckOrder::Unreadable, "{value}"); + let values = ExpressionValuesBody::take(&mut body, order, None); + assert_eq!(body["ExpressionAttributeValues"], json!({":a": {"S": ""}})); + assert_eq!(validation_message(values.validate()), Some(text), "{value}"); + } + for (value, text) in [ + (json!({"NS": []}), "An number set may not be empty"), + ( + json!({"SS": ["a", "a"]}), + r#"Input collection ["a", "a"] contains duplicates."#, + ), + ] { + let mut body = scan(json!({"ExpressionAttributeValues": {":a": value}})); + let order = CheckOrder::of(&body); + assert_eq!(order, CheckOrder::Readable, "{value}"); + let values = ExpressionValuesBody::take(&mut body, order, None); + assert_eq!(body["ExpressionAttributeValues"], json!({":a": {"S": ""}})); + assert_eq!( + validation_message(values.validate()), + Some(format!("{INVALID}{text}")), + "{value}" + ); + } + // Two faulty entries: the first in the fitted order is reported (measured + // 2026-11-05: `:b` ahead of `:a`, which sorted order would reverse). + let mut body = + scan(json!({"ExpressionAttributeValues": {":p": {"S": "x"}, ":b": {}, ":a": {"X": "1"}}})); + let values = ExpressionValuesBody::take(&mut body, CheckOrder::Unreadable, None); + assert_eq!( + validation_message(values.validate()), + Some(wrapper(EMPTY_VALUE).replace(":a", ":b")) + ); + // A readable value and a value of the wrong JSON shape stay as written. + for value in [json!({"S": "x"}), json!({"N": 5})] { + let mut body = scan(json!({"ExpressionAttributeValues": {":a": value}})); + let values = ExpressionValuesBody::take(&mut body, CheckOrder::Readable, None); + assert_eq!(body["ExpressionAttributeValues"], json!({":a": value})); + assert!(values.validate().is_ok()); + } +} + +/// Measured 2026-11-05 (order U): a malformed number in `ExpressionAttributeValues` +/// is reported at the map's step in the invalid-value form, the entries visited +/// in the fitted order: `{":p": {"S": "x"}, ":a": {"N": "bob"}, ":e": {}}` +/// names `:a`, the same with `:n` names `:n`, and the request-time checks pass +/// over the number. In order R the number is a request-time check with the +/// plain text. +#[test] +fn a_malformed_expression_value_number_in_order_u_takes_the_invalid_value_form() { + let wrapper = |name: &str| { + format!("ExpressionAttributeValues contains invalid value: {NUMERIC_BOB} for key {name}") + }; + for (name, request_order) in [(":a", [":p", ":a", ":e"]), (":n", [":p", ":n", ":e"])] { + let mut body = scan(json!({ + "FilterExpression": "a = :p", + "ExpressionAttributeValues": {":p": {"S": "x"}, name: {"N": "bob"}, ":e": {}}, + })); + assert_eq!(CheckOrder::of(&body), CheckOrder::Unreadable); + assert_eq!(check(&body), None, "{name}"); + let names: Vec = request_order.iter().map(|n| (*n).to_owned()).collect(); + let values = ExpressionValuesBody::take(&mut body, CheckOrder::Unreadable, Some(&names)); + assert_eq!( + validation_message(values.validate()), + Some(wrapper(name)), + "{name}" + ); + assert_eq!(body["ExpressionAttributeValues"][name], json!({"S": ""})); + } + // With the unreadable value in the key, the number is still held back for + // the map's step, after the key's faults. + let mut body = scan(json!({ + "FilterExpression": "a = :n", + "ExpressionAttributeValues": {":n": {"N": "bob"}}, + "ExclusiveStartKey": {"pk": {}, "sk": {"S": "x"}}, + })); + assert_eq!(check(&body), None); + let values = ExpressionValuesBody::take(&mut body, CheckOrder::Unreadable, None); + assert_eq!(validation_message(values.validate()), Some(wrapper(":n"))); + // Order R: the request-time check, the plain text. + let body = scan(json!({"ExpressionAttributeValues": {":n": {"N": "bob"}}})); + assert_eq!(CheckOrder::of(&body), CheckOrder::Readable); + assert_eq!(check(&body).as_deref(), Some(NUMERIC_BOB)); +} + +/// Measured 2026-11-05 (order R): the request-time number check visits the +/// `ExpressionAttributeValues` entries in the hash order the key and the +/// order-U step use. `{":n": {"N": "fred"}, ":a": {"N": "bob"}}` answers +/// `fred` (`:n` in bucket 0, `:a` in 3), whichever order the request wrote and +/// whatever order the parsed body keeps; sorted name order would answer `bob`. +#[test] +fn the_order_r_number_check_visits_the_entries_in_the_hash_order() { + let numeric = + |text: &str| format!("The parameter cannot be converted to a numeric value: {text}"); + let body = scan(json!({ + "FilterExpression": "a = :a", + "ExpressionAttributeValues": {":n": {"N": "fred"}, ":a": {"N": "bob"}}, + })); + assert_eq!(CheckOrder::of(&body), CheckOrder::Readable); + assert_eq!(check(&body), Some(numeric("fred"))); + for request_order in [[":n", ":a"], [":a", ":n"]] { + let names: Vec = request_order.iter().map(|n| (*n).to_owned()).collect(); + assert_eq!( + validation_message(validate_scan_request_body( + &body, + CheckOrder::Readable, + Some(&names) + )), + Some(numeric("fred")), + "{request_order:?}" + ); + } + // Two names in one bucket: the request's order decides (`:a` and `:e` + // share bucket 3). + let body = scan(json!({ + "FilterExpression": "a = :p", + "ExpressionAttributeValues": {":p": {"S": "x"}, ":a": {"N": "bob"}, ":e": {"N": "fred"}}, + })); + for (request_order, first) in [([":p", ":a", ":e"], "bob"), ([":p", ":e", ":a"], "fred")] { + let names: Vec = request_order.iter().map(|n| (*n).to_owned()).collect(); + assert_eq!( + validation_message(validate_scan_request_body( + &body, + CheckOrder::Readable, + Some(&names) + )), + Some(numeric(first)), + "{request_order:?}" + ); + } +} + +/// Measured 2026-11-05 (order R): with two set faults, `{":n": {"NS": []}, +/// ":a": {"SS": []}}` answered the empty `SS` text in one send and the empty +/// `NS` text in the other. The set pass visits the entries in sorted name +/// order, whatever order the backend's parsed body keeps, so that the answer +/// does not depend on the backend: it names `:a`. A number in an entry is not +/// the set pass's to report (`eavR-NSbobdup-and-nest`: the number inside a +/// refused entry comes after the nesting check). +#[test] +fn the_order_r_set_pass_visits_the_entries_in_sorted_name_order() { + let body = scan(json!({"ExpressionAttributeValues": {":n": {"NS": []}, ":a": {"SS": []}}})); + assert_eq!(CheckOrder::of(&body), CheckOrder::Readable); + assert_eq!( + check(&body), + Some(format!("{INVALID}An string set may not be empty")) + ); + let body = scan(json!({"ExpressionAttributeValues": {":a": {"NS": []}, ":n": {"SS": []}}})); + assert_eq!( + check(&body), + Some(format!("{INVALID}An number set may not be empty")) + ); + // A malformed number inside an entry with a set fault below it: skipped by + // the request-time checks (the nesting check of another entry answers + // first), reported by `ExpressionValuesBody` with the plain text. + let one = json!({"N": "1"}); + let mut body = scan(json!({ + "FilterExpression": "a = :p", + "ExpressionAttributeValues": {":p": {"S": "x"}, ":d": nested(&one, 32), ":a": {"NS": ["bob", "1", "1"]}}, + })); + assert_eq!(check(&body).as_deref(), Some(NESTING)); + body["ExpressionAttributeValues"] + .as_object_mut() + .expect("an object") + .remove(":d"); + assert_eq!(check(&body), None); + let values = ExpressionValuesBody::take(&mut body, CheckOrder::Readable, None); + assert_eq!( + validation_message(values.validate()).as_deref(), + Some(NUMERIC_BOB) + ); +} + +/// Measured 2026-11-05 (order U): a set fault in an `ExpressionAttributeValues` +/// entry takes the invalid-value form at the map's step, with the set texts of +/// order U, in the entries' hash order: `{":p": {"S": "x"}, ":s": {"NS": []}, +/// ":e": {}}` names `:s` (bucket 1, ahead of `:e` in 3); an `SS` with +/// duplicates answers `... Input collection [a, a] contains duplicates. for key +/// :s`. Inside one entry the first fault in the value's element order is named: +/// `{"L": [{"N": "bob"}, {}]}` answers the number (measured), `{"L": [{}, +/// {"N": "bob"}]}` the empty value (not measured, the element order). +#[test] +fn an_order_u_set_fault_in_an_expression_value_takes_the_invalid_value_form() { + let wrapper = |text: &str, name: &str| { + format!("ExpressionAttributeValues contains invalid value: {text} for key {name}") + }; + for (set, text) in [ + ( + json!({"NS": []}), + format!("{INVALID}An number set may not be empty"), + ), + ( + json!({"SS": ["a", "a"]}), + format!("{INVALID}Input collection [a, a] contains duplicates."), + ), + ( + json!({"BS": []}), + format!("{INVALID}Binary sets should not be empty"), + ), + ] { + let mut body = scan(json!({ + "FilterExpression": "a = :p", + "ExpressionAttributeValues": {":p": {"S": "x"}, ":s": set, ":e": {}}, + })); + assert_eq!(CheckOrder::of(&body), CheckOrder::Unreadable); + assert_eq!(check(&body), None, "{set}"); + let values = ExpressionValuesBody::take(&mut body, CheckOrder::Unreadable, None); + assert_eq!( + validation_message(values.validate()), + Some(wrapper(&text, ":s")), + "{set}" + ); + assert_eq!(body["ExpressionAttributeValues"][":s"], json!({"S": ""})); + assert_eq!(body["ExpressionAttributeValues"][":e"], json!({"S": ""})); + } + // Inside one entry: the element order. + for (value, text) in [ + (json!({"L": [{"N": "bob"}, {}]}), NUMERIC_BOB), + (json!({"L": [{}, {"N": "bob"}]}), EMPTY_VALUE), + ( + json!({"L": [{"NS": []}, {}]}), + "One or more parameter values were invalid: An number set may not be empty", + ), + ( + json!({"L": [{"L": [{"N": "bob"}]}, {"S": "x", "N": "1"}]}), + NUMERIC_BOB, + ), + ] { + let mut body = scan(json!({ + "FilterExpression": "a = :p", + "ExpressionAttributeValues": {":p": {"S": "x"}, ":l": value}, + })); + assert_eq!(CheckOrder::of(&body), CheckOrder::Unreadable, "{value}"); + let values = ExpressionValuesBody::take(&mut body, CheckOrder::Unreadable, None); + assert_eq!( + validation_message(values.validate()), + Some(wrapper(text, ":l")), + "{value}" + ); + } +} + +/// Measured 2026-11-05: in order U `ScanFilterBody::take` holds a `ScanFilter` +/// list value back for the step after the no-expression checks: a value that is +/// not one AttributeValue answers `Supplied AttributeValue is empty, must +/// contain exactly one of the supported datatypes` with no prefix, for `{}` and +/// for `{"X": "1"}`. In order R nothing is taken. A malformed number in a list +/// is #507's check (#564 item 3): the request-time checks pass it and `take` +/// leaves it in the body in both orders. +#[test] +fn a_scan_filter_list_value_is_checked_at_its_step() { + let list = |values: Value| { + scan( + json!({"ScanFilter": {"a": {"ComparisonOperator": "EQ", "AttributeValueList": values}}}), + ) + }; + // The numbers of the lists are #507's: not a request-time check here. + let mut bob = list(json!([{"N": "bob"}, {"NS": ["bob"]}])); + assert_eq!(CheckOrder::of(&bob), CheckOrder::Readable); + assert_eq!(check(&bob), None); + let beside_key = with( + list(json!([{"N": "bob"}])), + json!({"ExclusiveStartKey": {"pk": {"N": "fred"}, "sk": {"S": "x"}}}), + ); + assert_eq!(check(&beside_key), None); + assert!( + ScanFilterBody::take(&mut bob, CheckOrder::Readable, None) + .validate() + .is_ok() + ); + assert_eq!( + bob["ScanFilter"]["a"]["AttributeValueList"], + json!([{"N": "bob"}, {"NS": ["bob"]}]) + ); + // Order U: the list value is held back. + for value in [json!({}), json!({"X": "1"}), json!({"L": [{}]})] { + let mut body = list(json!([value])); + assert_eq!(CheckOrder::of(&body), CheckOrder::Unreadable, "{value}"); + assert_eq!(check(&body), None, "{value}"); + let filter = ScanFilterBody::take(&mut body, CheckOrder::Unreadable, None); + assert_eq!( + body["ScanFilter"]["a"]["AttributeValueList"], + json!([{"S": ""}]), + "{value}" + ); + assert_eq!( + validation_message(filter.validate()).as_deref(), + Some(EMPTY_VALUE), + "{value}" + ); + assert!( + serde_json::from_value::(body).is_ok(), + "{value}" + ); + } + let mut two = list(json!([{"S": "x", "N": "1"}])); + let filter = ScanFilterBody::take(&mut two, CheckOrder::Unreadable, None); + assert_eq!( + validation_message(filter.validate()).as_deref(), + Some(MORE_THAN_ONE) + ); + // A number beside an unreadable key member stays in the body in order U + // too (#507's check). + let mut number = with( + list(json!([{"N": "bob"}])), + json!({"ExclusiveStartKey": {"pk": {}, "sk": {"S": "x"}}}), + ); + assert_eq!(check(&number), None); + let filter = ScanFilterBody::take(&mut number, CheckOrder::Unreadable, None); + assert!(filter.validate().is_ok()); + assert_eq!( + number["ScanFilter"]["a"]["AttributeValueList"], + json!([{"N": "bob"}]) + ); + // Order R: nothing is taken, a readable list stays as written. + let mut readable = list(json!([{"N": "1"}])); + let filter = ScanFilterBody::take(&mut readable, CheckOrder::Readable, None); + assert_eq!( + readable["ScanFilter"]["a"]["AttributeValueList"], + json!([{"N": "1"}]) + ); + assert!(filter.validate().is_ok()); + // No ScanFilter at all. + let mut none = scan(json!({})); + assert!( + ScanFilterBody::take(&mut none, CheckOrder::Unreadable, None) + .validate() + .is_ok() + ); +} + +#[test] +fn a_value_nested_32_levels_deep_is_refused_and_31_levels_pass() { + let one = json!({"N": "1"}); + for levels in [32, 33, 40, 60] { + let body = with_unused_value(nested(&one, levels)); + assert_eq!(check(&body).as_deref(), Some(NESTING), "{levels} levels"); + } + // 31 levels reach the checks the handler makes after deserializing. + assert_eq!(check(&with_unused_value(nested(&one, 31))), None); + // Measured 2026-10-30 12:40 UTC: ahead of the Select enum. + let select = with( + with_unused_value(nested(&one, 32)), + json!({"Select": "BAD"}), + ); + assert_eq!(check(&select).as_deref(), Some(NESTING)); + // Measured 2026-10-31 (B4-5a): with no expression at all, the nesting text + // ahead of the no-expression check. + let no_expression = scan(json!({"ExpressionAttributeValues": {":d": nested(&one, 32)}})); + assert_eq!(check(&no_expression).as_deref(), Some(NESTING)); + // Measured 2026-11-05 (eavR-NSe-and-nest, both sends): an empty set in + // another entry answers the set text ahead of the nesting text, with the + // text of order R; a duplicate set too (not measured beside a nested + // value). A number in a third entry comes ahead of both. + let beside_empty_set = scan( + json!({"FilterExpression": "a = :p", "ExpressionAttributeValues": + {":p": {"S": "x"}, ":d": nested(&one, 32), ":e": {"NS": []}}}), + ); + assert_eq!(CheckOrder::of(&beside_empty_set), CheckOrder::Readable); + assert_eq!( + check(&beside_empty_set).as_deref(), + Some("One or more parameter values were invalid: An number set may not be empty") + ); + let beside_duplicates = scan( + json!({"FilterExpression": "a = :p", "ExpressionAttributeValues": + {":p": {"S": "x"}, ":d": nested(&one, 32), ":a": {"SS": ["a", "a"]}}}), + ); + assert_eq!( + check(&beside_duplicates).as_deref(), + Some( + "One or more parameter values were invalid: Input collection [\"a\", \"a\"] contains duplicates." + ) + ); + let number_first = scan( + json!({"FilterExpression": "a = :p", "ExpressionAttributeValues": + {":p": {"S": "x"}, ":d": nested(&one, 32), ":e": {"NS": []}, ":n": {"N": "bob"}}}), + ); + assert_eq!(check(&number_first).as_deref(), Some(NUMERIC_BOB)); + // In order U the nesting check leaves request time: beside an unused + // empty value (B4-7) or a key shape fault (2026-11-05) the request-time + // checks pass and the handler reports those first. + let beside_empty = scan(json!({ + "FilterExpression": "a = :p", + "ExpressionAttributeValues": {":p": {"S": "x"}, ":d": nested(&one, 32), ":e": {}}, + })); + assert_eq!(check(&beside_empty), None); + let beside_key = with( + with_unused_value(nested(&one, 32)), + json!({"ExclusiveStartKey": {"pk": {}, "sk": {"S": "x"}}}), + ); + assert_eq!(check(&beside_key), None); +} + +#[test] +fn the_number_check_runs_before_the_nesting_check() { + let bob = json!({"N": "bob"}); + for levels in [31, 32, 33, 40, 60] { + let body = with_unused_value(nested(&bob, levels)); + assert_eq!( + check(&body).as_deref(), + Some(NUMERIC_BOB), + "{levels} levels" + ); + } + // An M at the top, 31 lists below it. + let body = with_unused_value(json!({"M": {"k": nested(&bob, 31)}})); + assert_eq!(check(&body).as_deref(), Some(NUMERIC_BOB)); +} + +/// Combinations the other tests do not send, each measured 2026-10-31 between +/// 23:54 and 23:56 UTC with the answer asserted here (the key row with `{"pk": +/// {"N": "fred"}}` and with `"sk": {"S": "x"}` added to it). +#[test] +fn combinations_measured_on_2026_10_31_follow_the_order_of_the_checks() { + // The mix names whichever expression parameter is present. + let projection = scan(json!({"ScanFilter": {}, "ProjectionExpression": "a"})); + assert_eq!( + check(&projection).as_deref(), + Some( + "Can not use both expression and non-expression parameters in the same request: Non-expression parameters: {ScanFilter} Expression parameters: {ProjectionExpression}" + ) + ); + // A null ScanFilter is absent. + let null_filter = with(scan(filter("1")), json!({"ScanFilter": null})); + assert_eq!(check(&null_filter), None); + // The length constraint applies beside a ProjectionExpression too. + let beside_projection = scan(json!({"AttributesToGet": [], "ProjectionExpression": "a"})); + assert_eq!(check(&beside_projection).as_deref(), Some(LENGTH)); + // The length clause keeps its place ahead of the range clauses beside a + // Segment fault, as measured beside a Limit fault. + let segment = scan(json!({"AttributesToGet": [], "Segment": -1, "TotalSegments": 2})); + assert_eq!( + check(&segment).as_deref(), + Some( + "2 validation errors detected: Value at 'AttributesToGet' failed to satisfy constraint: Member must have length greater than or equal to 1; Value at 'Segment' failed to satisfy constraint: Member must have value greater than or equal to 0" + ) + ); + // In the older form the clauses are reversed and the values echoed, so + // the empty list comes last. + let older = scan(json!({"AttributesToGet": [], "Limit": 0.0})); + assert_eq!( + check(&older).as_deref(), + Some( + "2 validation errors detected: Value '0' at 'limit' failed to satisfy constraint: Member must have value greater than or equal to 1; Value '[]' at 'attributesToGet' failed to satisfy constraint: Member must have length greater than or equal to 1" + ) + ); + // The values are checked ahead of the key. + let both = with( + scan(filter("bob")), + json!({"ExclusiveStartKey": {"pk": {"N": "fred"}}}), + ); + assert_eq!(check(&both).as_deref(), Some(NUMERIC_BOB)); + // A map nests as a list does. + let map = + |levels: usize| (0..levels).fold(json!({"N": "1"}), |inner, _| json!({"M": {"k": inner}})); + assert_eq!(check(&with_unused_value(map(32))).as_deref(), Some(NESTING)); + assert_eq!(check(&with_unused_value(map(31))), None); + // The ReturnConsumedCapacity enum comes ahead of the parameter mix + // (2026-11-05). + let rcc_mix = with( + with(scan(filter("1")), json!({"ScanFilter": {}})), + json!({"ReturnConsumedCapacity": "BAD"}), + ); + assert_eq!( + check(&rcc_mix).as_deref(), + Some( + "1 validation error detected: Value 'BAD' at 'returnConsumedCapacity' failed to satisfy constraint: Member must satisfy enum value set: [INDEXES, TOTAL, NONE]" + ) + ); +} diff --git a/crates/engine/src/scan_request/tests.rs b/crates/engine/src/scan_request/tests.rs index 91005eb22..d576b4e98 100644 --- a/crates/engine/src/scan_request/tests.rs +++ b/crates/engine/src/scan_request/tests.rs @@ -9,7 +9,7 @@ use extenddb_core::error::DynamoDbError; use serde_json::{Value, json}; -use super::validate_scan_request_body; +use super::{CheckOrder, validate_scan_request_body}; const NUMERIC_BOB: &str = "The parameter cannot be converted to a numeric value: bob"; const LIMIT_RANGE: &str = "Value at 'Limit' failed to satisfy constraint: Member must have value greater than or equal to 1"; @@ -25,7 +25,7 @@ fn message(result: Result<(), DynamoDbError>) -> Option { } fn check(body: &Value) -> Option { - message(validate_scan_request_body(body)) + message(validate_scan_request_body(body, CheckOrder::of(body), None)) } /// A Scan whose filter reads `:n = {"N": "bob"}`. @@ -204,12 +204,11 @@ fn parameter_mix_beats_the_enum_and_the_numbers() { Non-expression parameters: {ScanFilter} Expression parameters: {FilterExpression}" ) ); + // The ReturnConsumedCapacity enum comes ahead of the mix (measured + // 2026-11-05). assert_eq!( check(&with(mix, "ReturnConsumedCapacity", json!("NOPE"))).as_deref(), - Some( - "Can not use both expression and non-expression parameters in the same request: \ - Non-expression parameters: {ScanFilter} Expression parameters: {FilterExpression}" - ) + Some(RCC_ENUM) ); let atg = with( with(bad_number(), "AttributesToGet", json!(["a"])), @@ -272,9 +271,18 @@ fn enum_checks_sit_on_either_side_of_the_numbers() { json!("NOPE"), ); assert_eq!(check(&both).as_deref(), Some(RCC_ENUM)); - // With the numbers valid, an invalid Select is reported. + // With every value readable (order R) the Select enum is not a + // request-time check: the handler reports it after the no-expression check + // and the nesting check (measured 2026-11-05); beside an unreadable value + // (order U) it is one. + assert_eq!(check(&json!({"TableName": "t", "Select": "NOPE"})), None); assert_eq!( - check(&json!({"TableName": "t", "Select": "NOPE"})).as_deref(), + check(&json!({ + "TableName": "t", + "Select": "NOPE", + "ExclusiveStartKey": {"pk": {}, "sk": {"S": "x"}}, + })) + .as_deref(), Some(SELECT_ENUM) ); // A valid request passes. @@ -308,7 +316,7 @@ fn a_shape_fault_anywhere_in_the_body_is_reported_first() { for body in shape_faults { assert!( matches!( - validate_scan_request_body(&body), + validate_scan_request_body(&body, CheckOrder::of(&body), None), Err(DynamoDbError::SerializationException(_)) ), "{body}" diff --git a/crates/server/src/handler.rs b/crates/server/src/handler.rs index aeef4ccd1..055a44aed 100755 --- a/crates/server/src/handler.rs +++ b/crates/server/src/handler.rs @@ -226,7 +226,7 @@ pub(crate) async fn handle_request( // --- Dispatch segment --- let dispatch_start = std::time::Instant::now(); let dispatch_result = - extenddb_engine::dispatch(&operation, input, &ctx, &state.server_addr).await; + extenddb_engine::dispatch(&operation, input, &body, &ctx, &state.server_addr).await; #[allow(clippy::cast_precision_loss)] let dispatch_us = dispatch_start.elapsed().as_micros() as f64; // P120c: Record storage query metrics for the request's dispatch phase. diff --git a/crates/storage-sqlite/src/backup_catalog.rs b/crates/storage-sqlite/src/backup_catalog.rs index 459c76f86..727b483c2 100644 --- a/crates/storage-sqlite/src/backup_catalog.rs +++ b/crates/storage-sqlite/src/backup_catalog.rs @@ -1610,7 +1610,7 @@ mod write_lock_tests { operation: &str, body: serde_json::Value, ) -> serde_json::Value { - dispatch(operation, body, ctx, "127.0.0.1:0") + dispatch(operation, body, &[], ctx, "127.0.0.1:0") .await .unwrap_or_else(|e| panic!("{operation}: {e:?}")) .body diff --git a/crates/storage-sqlite/tests/backup_job_panic.rs b/crates/storage-sqlite/tests/backup_job_panic.rs index 14bd52ed8..b39d3be15 100644 --- a/crates/storage-sqlite/tests/backup_job_panic.rs +++ b/crates/storage-sqlite/tests/backup_job_panic.rs @@ -101,7 +101,7 @@ async fn call( operation: &str, body: serde_json::Value, ) -> serde_json::Value { - dispatch(operation, body, ctx, "127.0.0.1:0") + dispatch(operation, body, &[], ctx, "127.0.0.1:0") .await .unwrap_or_else(|e| panic!("{operation}: {e:?}")) .body @@ -316,6 +316,7 @@ async fn a_job_task_that_panics_leaves_a_row_the_next_cycle_resolves_without_a_r let described = dispatch( "DescribeBackup", json!({"BackupArn": arn_text}), + &[], &ctx, "127.0.0.1:0", ) diff --git a/crates/storage-sqlite/tests/restore_job_resume.rs b/crates/storage-sqlite/tests/restore_job_resume.rs index 2b1a5dd81..ebad4cbb5 100644 --- a/crates/storage-sqlite/tests/restore_job_resume.rs +++ b/crates/storage-sqlite/tests/restore_job_resume.rs @@ -62,7 +62,7 @@ async fn call( operation: &str, body: serde_json::Value, ) -> serde_json::Value { - dispatch(operation, body, ctx, "127.0.0.1:0") + dispatch(operation, body, &[], ctx, "127.0.0.1:0") .await .unwrap_or_else(|e| panic!("{operation}: {e:?}")) .body diff --git a/tests/python/test_scan_request_validation.py b/tests/python/test_scan_request_validation.py new file mode 100644 index 000000000..27b3bbb23 --- /dev/null +++ b/tests/python/test_scan_request_validation.py @@ -0,0 +1,1092 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""Scan request validation: an empty ``ScanFilter`` in the parameter mix, an +empty ``AttributesToGet``, a malformed number in ``ExclusiveStartKey`` and the +nesting depth of ``ExpressionAttributeValues``. + +Every request below is sent as raw signed JSON. Each is a row of the table +measured against DynamoDB (us-east-1) on 2026-10-25 at 10:38 and 10:55 UTC on +a Scan of an existing table of 29 items, or of the ones measured on 2026-10-30 +at 12:40 UTC and on 2026-10-31 between 23:54 and 23:56 UTC where a row says +so, each sent twice, and each expected answer is the service's. The table here +holds 12 items, so the count the accepted request returns is 12. + +Depth is the number of ``L`` wrappers around the leaf: 32 or more answers the +nesting text, 31 reaches the unused-value check. A malformed number at the +bottom is reported first, at the depths 31 to 60 that were measured. + +The service follows one of two check orders (measured 2026-10-31 and +2026-11-05, each request twice). A request that +holds a value that is not one AttributeValue (empty, an unknown or lower-case +type descriptor, two descriptors) in ``ExpressionAttributeValues``, +``ExclusiveStartKey`` or a ``ScanFilter`` list follows order U: the constraint +clauses in the older form with the ``Select`` enum among them, the request +parameter checks, the key's faults (shape and value together, the first faulty +member in the fitted member order, a shape fault with the prefix ``The provided +starting key is invalid: ``), the unreadable ``ExpressionAttributeValues`` +entries, the nesting depth, the expression checks, the table. Every other +request follows order R: the constraint clauses in the newer form, the parameter +mix, the nesting depth, a set fault in ``ExpressionAttributeValues``, the request +parameter checks, the ``Select`` enum, the expression checks, the key's number +and set faults, the table. The set texts quote their members in order R and not +in order U. With two value faults in two key members the service's own sends +disagreed, so those tests accept either fault. + +A malformed number in a ``ScanFilter`` list answers the number text ahead of +the key's faults (measured 2026-11-05); that check is #564 item 3, which PR +#507 makes, so its two rows are expected to fail here. In order U the +``ExpressionAttributeValues`` entries are visited in the same fitted order as +the key's members (the hash buckets, then the request's order), a malformed +number among them answers ``ExpressionAttributeValues contains invalid value: +... for key :a``, and a ``ScanFilter`` list value that is not one AttributeValue +answers the bare empty-value text after the no-expression checks. A key of four +members is visited with eight buckets (measured 2026-11-05 on a GSI with a +range key). +""" + +from __future__ import annotations + +import json + +import pytest +from botocore.awsrequest import AWSRequest + +from helpers import unique_name, wait_for_active, wait_for_deleted + +ITEMS = 12 + +MIX = ( + "Can not use both expression and non-expression parameters in the same " + "request: Non-expression parameters: {ScanFilter} " + "Expression parameters: {FilterExpression}" +) +LENGTH = ( + "1 validation error detected: Value at 'AttributesToGet' failed to " + "satisfy constraint: Member must have length greater than or equal to 1" +) +LENGTH_AND_LIMIT = ( + "2 validation errors detected: Value at 'AttributesToGet' failed to " + "satisfy constraint: Member must have length greater than or equal to 1; " + "Value at 'Limit' failed to satisfy constraint: Member must have value " + "greater than or equal to 1" +) +NUMERIC_BOB = "The parameter cannot be converted to a numeric value: bob" +RCC_ENUM = ( + "1 validation error detected: Value 'BAD' at 'returnConsumedCapacity' failed " + "to satisfy constraint: Member must satisfy enum value set: [INDEXES, TOTAL, NONE]" +) +NESTING = ( + "Nesting Levels have exceeded supported limits: " + "Attributes in the item have nested levels beyond supported limit" +) +UNUSED = "Value provided in ExpressionAttributeValues unused in expressions: keys: {:d}" +LIMIT = ( + "1 validation error detected: Value at 'Limit' failed to satisfy constraint: " + "Member must have value greater than or equal to 1" +) +SEGMENT = ( + "1 validation error detected: Value at 'Segment' failed to satisfy constraint: " + "Member must have value greater than or equal to 0" +) +SELECT_ENUM = ( + "1 validation error detected: Value 'BAD' at 'select' failed to satisfy constraint: " + "Member must satisfy enum value set: " + "[SPECIFIC_ATTRIBUTES, COUNT, ALL_ATTRIBUTES, ALL_PROJECTED_ATTRIBUTES]" +) +SELECT_SPECIFIC = ( + "Must specify the AttributesToGet or ProjectionExpression when choosing to get " + "SPECIFIC_ATTRIBUTES" +) +UNUSED_U = "Value provided in ExpressionAttributeValues unused in expressions: keys: {:u}" +KEY_PREFIX = "The provided starting key is invalid: " +EMPTY_KEY_VALUE = ( + KEY_PREFIX + "Supplied AttributeValue is empty, must contain exactly one of the " + "supported datatypes" +) +TWO_DESCRIPTORS = ( + KEY_PREFIX + "Supplied AttributeValue has more than one datatypes set, must contain " + "exactly one of the supported datatypes" +) +INVALID = "One or more parameter values were invalid: " +NS_EMPTY = INVALID + "An number set may not be empty" +SS_EMPTY = INVALID + "An string set may not be empty" +BS_EMPTY = INVALID + "Binary sets should not be empty" +NS_DUP = INVALID + "Input collection contains duplicates." +SS_DUP = INVALID + 'Input collection ["a", "a"] contains duplicates.' +NUMERIC_FRED = NUMERIC_BOB.replace("bob", "fred") +EAV_NO_EXPRESSION = "ExpressionAttributeValues can only be specified when using expressions" +MISSING_INDEX = "The table does not have the specified index: nope" + +VALID_FILTER = '"FilterExpression": "a = :n", "ExpressionAttributeValues": {":n": {"N": "1"}}' +BOB_FILTER = '"FilterExpression": "a = :n", "ExpressionAttributeValues": {":n": {"N": "bob"}}' +ONE = {"N": "1"} +BOB = {"N": "bob"} +BOB_KEY = '"ExclusiveStartKey": {"pk": {"N": "bob"}, "sk": {"S": "x"}}' + +# (id, the fields after TableName as JSON text, the expected message) +REFUSED = [ + ("ScanFilter-empty-with-FilterExpression", '"ScanFilter": {}, ' + VALID_FILTER, MIX), + ("ScanFilter-empty-with-unused-bob", + '"ScanFilter": {}, "ExpressionAttributeValues": {":n": {"N": "bob"}}', NUMERIC_BOB), + ("AttributesToGet-empty", '"AttributesToGet": []', LENGTH), + ("AttributesToGet-empty-with-FilterExpression", + '"AttributesToGet": [], ' + VALID_FILTER, LENGTH), + ("AttributesToGet-empty-with-FilterExpression-and-bob", + '"AttributesToGet": [], ' + BOB_FILTER, LENGTH), + ("ExclusiveStartKey-bob-on-an-S-key", + '"ExclusiveStartKey": {"pk": {"N": "bob"}, "sk": {"S": "x"}}, ' + VALID_FILTER, + NUMERIC_BOB), + # Measured 2026-10-30 12:40 UTC. + ("AttributesToGet-empty-with-Limit-0", '"AttributesToGet": [], "Limit": 0', + LENGTH_AND_LIMIT), + ("AttributesToGet-empty-with-ScanFilter-empty-and-FilterExpression", + '"AttributesToGet": [], "ScanFilter": {}, ' + VALID_FILTER, LENGTH), + ("ExclusiveStartKey-bob-with-no-filter", + '"ExclusiveStartKey": {"pk": {"N": "bob"}, "sk": {"S": "x"}}', NUMERIC_BOB), + ("ExclusiveStartKey-bob-in-the-index-key-of-a-GSI-scan", + '"IndexName": "gsi", "ExclusiveStartKey": {"gpk": {"N": "bob"}, "pk": {"S": "p"}, ' + '"sk": {"S": "x"}}', NUMERIC_BOB), + ("ExclusiveStartKey-bob-with-ReturnConsumedCapacity-BAD", + '"ExclusiveStartKey": {"pk": {"N": "bob"}, "sk": {"S": "x"}}, ' + '"ReturnConsumedCapacity": "BAD"', RCC_ENUM), +] + +# Measured 2026-10-31 between 23:54 and 23:56 UTC: each check sent beside the +# key comes ahead of the number in ExclusiveStartKey. +KEY_AFTER_THE_OTHER_CHECKS = [ + ("Limit-0", BOB_KEY + ', "Limit": 0', LIMIT), + ("Segment-minus-1", BOB_KEY + ', "Segment": -1, "TotalSegments": 2', SEGMENT), + ("Select-BAD", BOB_KEY + ', "Select": "BAD"', SELECT_ENUM), + ("ScanFilter-empty-with-FilterExpression", BOB_KEY + ', "ScanFilter": {}, ' + VALID_FILTER, + MIX), + ("AttributesToGet-empty", BOB_KEY + ', "AttributesToGet": []', LENGTH), + ("unused-value", + BOB_KEY + ', "FilterExpression": "a = :p", ' + '"ExpressionAttributeValues": {":p": {"S": "x"}, ":u": {"S": "y"}}', UNUSED_U), + ("bob-in-the-values-and-fred-in-the-key", + '"ExclusiveStartKey": {"pk": {"N": "fred"}, "sk": {"S": "x"}}, ' + BOB_FILTER, NUMERIC_BOB), +] + +# Measured 2026-10-31 and 2026-11-05: a number fault anywhere in the key, in +# an NS, in an L, in a non-key attribute and in an LSI key, answers the number +# text with no prefix. +KEY_NUMBER_FAULTS = [ + ("bob-in-an-NS", '"ExclusiveStartKey": {"pk": {"NS": ["bob"]}, "sk": {"S": "x"}}', + NUMERIC_BOB), + ("bob-in-an-L", '"ExclusiveStartKey": {"pk": {"L": [{"N": "bob"}]}, "sk": {"S": "x"}}', + NUMERIC_BOB), + ("bob-in-the-L-of-a-non-key-attribute", + '"ExclusiveStartKey": {"pk": {"S": "p0"}, "sk": {"S": "x"}, "z": {"L": [{"N": "bob"}]}}', + NUMERIC_BOB), + ("bob-in-the-index-key-of-an-LSI-scan", + '"IndexName": "lsi", "ExclusiveStartKey": {"pk": {"S": "p0"}, "sk": {"S": "x"}, ' + '"ln": {"N": "bob"}}', NUMERIC_BOB), +] + +# Measured 2026-11-05: each key alone. +# A shape fault answers the service's text with the key prefix; a set fault +# answers the service's text with no prefix. +KEYS_ALONE = [ + ("empty", '{"pk": {}, "sk": {"S": "x"}}', EMPTY_KEY_VALUE), + ("type-X", '{"pk": {"X": "1"}, "sk": {"S": "x"}}', EMPTY_KEY_VALUE), + ("lower-case-s", '{"pk": {"s": "x"}, "sk": {"S": "x"}}', EMPTY_KEY_VALUE), + ("empty-in-an-M", '{"pk": {"M": {"k": {}}}, "sk": {"S": "x"}}', EMPTY_KEY_VALUE), + ("empty-in-an-L", '{"pk": {"L": [{}]}, "sk": {"S": "x"}}', EMPTY_KEY_VALUE), + ("empty-non-key-attribute", '{"pk": {"S": "p0"}, "sk": {"S": "x"}, "z": {}}', EMPTY_KEY_VALUE), + ("two-descriptors", '{"pk": {"S": "x", "N": "1"}, "sk": {"S": "x"}}', TWO_DESCRIPTORS), + ("NS-empty", '{"pk": {"NS": []}, "sk": {"S": "x"}}', NS_EMPTY), + ("SS-empty", '{"pk": {"SS": []}, "sk": {"S": "x"}}', SS_EMPTY), + ("BS-empty", '{"pk": {"BS": []}, "sk": {"S": "x"}}', BS_EMPTY), + ("NS-duplicates", '{"pk": {"NS": ["1", "1"]}, "sk": {"S": "x"}}', NS_DUP), + ("NS-duplicates-by-value", '{"pk": {"NS": ["1", "1.0"]}, "sk": {"S": "x"}}', NS_DUP), + ("SS-duplicates", '{"pk": {"SS": ["a", "a"]}, "sk": {"S": "x"}}', SS_DUP), + ("NS-empty-non-key-attribute", '{"pk": {"S": "p0"}, "sk": {"S": "x"}, "z": {"NS": []}}', + NS_EMPTY), +] + +# Measured 2026-10-31 and 2026-11-05: faults in two members of one key, one of +# them a shape fault. The service names the first faulty member in the fitted +# member order (sk ahead of pk and of z in both +# request orders; the set texts of order U where the set member comes first). +TWO_MEMBER_FAULTS = [ + ("empty-pk-then-bob", '{"pk": {}, "sk": {"N": "bob"}}', NUMERIC_BOB), + ("bob-then-empty-pk", '{"sk": {"N": "bob"}, "pk": {}}', NUMERIC_BOB), + ("type-X-pk-then-bob", '{"pk": {"X": "1"}, "sk": {"N": "bob"}}', NUMERIC_BOB), + ("bob-then-empty-sk", '{"pk": {"N": "bob"}, "sk": {}}', EMPTY_KEY_VALUE), + ("two-descriptors-then-bob", '{"pk": {"S": "x", "N": "1"}, "sk": {"N": "bob"}}', NUMERIC_BOB), + ("empty-then-NS-empty", '{"pk": {}, "sk": {"NS": []}}', NS_EMPTY), + ("NS-empty-then-empty", '{"pk": {"NS": []}, "sk": {}}', EMPTY_KEY_VALUE), + ("empty-then-NS-bob", '{"pk": {}, "sk": {"NS": ["1", "bob"]}}', NUMERIC_BOB), + ("empty-in-an-L-then-bob", '{"pk": {"L": [{}]}, "sk": {"N": "bob"}}', NUMERIC_BOB), + ("type-X-then-bob-in-z", '{"pk": {"X": "1"}, "sk": {"S": "x"}, "z": {"N": "bob"}}', + NUMERIC_BOB), + ("two-descriptors-then-SS-empty", '{"pk": {"S": "x", "N": "1"}, "sk": {"SS": []}}', + KEY_PREFIX + SS_EMPTY), + ("empty-then-NS-duplicates", '{"pk": {}, "sk": {"NS": ["1", "1"]}}', + "Input collection contains duplicates"), + ("empty-then-SS-duplicates", '{"pk": {}, "sk": {"SS": ["a", "a"]}}', + INVALID + "Input collection [a, a] contains duplicates."), +] + +# Two value faults in two members: the service's two sends disagreed on 8 of +# 22 such rows (2026-11-05), so each row accepts either member's fault. +TWO_VALUE_FAULTS = [ + ("NS-empty-then-bob", '{"pk": {"NS": []}, "sk": {"N": "bob"}}', (NS_EMPTY, NUMERIC_BOB)), + ("SS-duplicates-then-bob", '{"pk": {"SS": ["a", "a"]}, "sk": {"N": "bob"}}', + (SS_DUP, NUMERIC_BOB)), +] + +# Measured 2026-11-05 on tables with other +# key names: (id, table fixture, key, expected). HASH a, RANGE b: a first; HASH +# pk, RANGE a: a first in both request orders; HASH id, RANGE ts: the member +# first in the request; the GSI key of three members: gpk ahead of pk. +MEMBER_ORDER_ROWS = [ + ("ab-empty-and-bob", "ab_table", '{"a": {}, "b": {"N": "bob"}}', EMPTY_KEY_VALUE), + ("ab-bob-and-empty", "ab_table", '{"a": {"N": "bob"}, "b": {}}', NUMERIC_BOB), + ("pa-empty-and-bob", "pa_table", '{"pk": {}, "a": {"N": "bob"}}', NUMERIC_BOB), + ("pa-bob-and-empty-reversed", "pa_table", '{"a": {"N": "bob"}, "pk": {}}', NUMERIC_BOB), + ("it-empty-and-bob", "it_table", '{"id": {}, "ts": {"N": "bob"}}', EMPTY_KEY_VALUE), + ("it-bob-and-empty-reversed", "it_table", '{"ts": {"N": "bob"}, "id": {}}', NUMERIC_BOB), +] + +# Measured 2026-11-05: beside each class of key +# fault, a check of each stage. The request parameter checks come ahead of +# every key fault; the shape faults come ahead of the expression checks and of +# the table lookup; the expression checks come ahead of the number and set +# faults, which come ahead of the table lookup. +KEY_CLASSES = [ + ("empty", '{"pk": {}, "sk": {"S": "x"}}', EMPTY_KEY_VALUE, True), + ("type-X", '{"pk": {"X": "1"}, "sk": {"S": "x"}}', EMPTY_KEY_VALUE, True), + ("two-descriptors", '{"pk": {"S": "x", "N": "1"}, "sk": {"S": "x"}}', TWO_DESCRIPTORS, True), + ("bob", '{"pk": {"N": "bob"}, "sk": {"S": "x"}}', NUMERIC_BOB, False), + ("NS-empty", '{"pk": {"NS": []}, "sk": {"S": "x"}}', NS_EMPTY, False), + ("SS-duplicates", '{"pk": {"SS": ["a", "a"]}, "sk": {"S": "x"}}', SS_DUP, False), +] +STAGES = [ + # A request parameter check (S1): ahead of every key fault. The head's + # text carries a suffix the service dropped (#722), so the test asserts the + # prefix of the text. + ("ExpressionAttributeValues-with-no-expression", + '"ExpressionAttributeValues": {":a": {"S": "x"}}', "parameter"), + # An expression check: ahead of a number or set fault (order R), after the + # key's faults (order U). + ("unused-value", + '"FilterExpression": "a = :p", "ExpressionAttributeValues": {":p": {"S": "x"}, ":u": {"S": "y"}}', + "expression"), + # The table's indexes (S5): after every key fault. + ("missing-index", '"IndexName": "nope"', "table"), +] + +# Measured 2026-10-31 between 23:54 and 23:56 UTC on a table with an N hash +# key: the number texts for a leading space, an empty text and 39 digits. +N_KEY_FAULTS = [ + ("leading-space", '"ExclusiveStartKey": {"pk": {"N": " 1"}}', + "The parameter cannot be converted to a numeric value: 1"), + ("empty-text", '"ExclusiveStartKey": {"pk": {"N": ""}}', + "The parameter cannot be converted to a numeric value: "), + ("39-digits", '"ExclusiveStartKey": {"pk": {"N": "111111111111111111111111111111111111111"}}', + "Attempting to store more than 38 significant digits in a Number"), +] + +def nest(leaf: dict, levels: int) -> dict: + """``leaf`` inside ``levels`` lists.""" + value = leaf + for _ in range(levels): + value = {"L": [value]} + return value + + +EAV_VALUE = "ExpressionAttributeValues contains invalid value: " +EAV_EMPTY_E = (EAV_VALUE + "Supplied AttributeValue is empty, must contain exactly one of the " + "supported datatypes for key :e") +EAV_EMPTY_A = EAV_EMPTY_E.replace(":e", ":a") +EAV_TWO_A = (EAV_VALUE + "Supplied AttributeValue has more than one datatypes set, must contain " + "exactly one of the supported datatypes for key :a") +EAN_NO_EXPRESSION = "ExpressionAttributeNames can only be specified when using expressions" +CONDOP = "ConditionalOperator can only be used when Filter or Expected has two or more elements" +UNDEFINED_ZZ = ("Invalid FilterExpression: An expression attribute value used in expression " + "is not defined; attribute value: :zz") +BEGINS_N = ("Invalid FilterExpression: Incorrect operand type for operator or function; " + "operator or function: begins_with, operand type: N") +SEGMENT_ALONE = ("The TotalSegments parameter is required but was not present in the request " + "when Segment parameter is present") +EMPTY_KEY = '"ExclusiveStartKey": {"pk": {}, "sk": {"S": "x"}}' +FILTER_P = '"FilterExpression": "a = :p"' +NEST32_S = json.dumps(nest({"S": "x"}, 32)) +NEST32_N = json.dumps(nest({"N": "1"}, 32)) + +# Measured 2026-10-31 (B4-7) and 2026-11-05: +# representative rows of each order. An expected text given as ("startswith", +# prefix) asserts the check's place where the text predates this branch (#722, +# #838, #839). "4th" names the rows that fail on 45705bf27. +TWO_ORDERS = [ + # Order U: an unreadable ExpressionAttributeValues entry, read or unused, + # after the Select enum, the request parameter checks and the key's faults, + # ahead of the nesting check, the unused values and the table. + ("B4-7", FILTER_P + ', "ExpressionAttributeValues": {":p": {"S": "x"}, ":d": ' + + NEST32_N + ', ":e": {}}', EAV_EMPTY_E), + ("B4-7-flat", FILTER_P + ', "ExpressionAttributeValues": {":p": {"S": "x"}, ":e": {}}', + EAV_EMPTY_E), + ("B4-7-X", FILTER_P + ', "ExpressionAttributeValues": {":p": {"S": "x"}, ":e": {"X": "1"}}', + EAV_EMPTY_E), + ("eav-empty-select-bad", FILTER_P + ', "ExpressionAttributeValues": {":p": {"S": "x"}, ' + '":e": {}}, "Select": "BAD"', SELECT_ENUM), + ("eav-empty-unused-and-key-empty", FILTER_P + ', "ExpressionAttributeValues": ' + '{":p": {"S": "x"}, ":e": {}}, ' + EMPTY_KEY, EMPTY_KEY_VALUE), + ("eav-empty-read-plus-unused", + '"FilterExpression": "a = :a", "ExpressionAttributeValues": {":a": {}, ":u": {"S": "x"}}', + EAV_EMPTY_A), + ("eav-read-X", '"FilterExpression": "a = :a", "ExpressionAttributeValues": {":a": {"X": "1"}}', + EAV_EMPTY_A), + ("eav-read-two", + '"FilterExpression": "a = :a", "ExpressionAttributeValues": {":a": {"S": "x", "N": "1"}}', + EAV_TWO_A), + ("eav-read-empty-begins", + '"FilterExpression": "begins_with(a, :a)", "ExpressionAttributeValues": {":a": {}}', + EAV_EMPTY_A), + ("B4-7-noexpr-empty", '"ExpressionAttributeValues": {":e": {}}', + ("startswith", EAV_NO_EXPRESSION)), + ("B4-7-noexpr-two", '"ExpressionAttributeValues": {":e": {"S": "x", "N": "1"}}', + ("startswith", EAV_NO_EXPRESSION)), + # Order U: the nesting check after the key's faults (the three b.*.nesting + # rows of 2026-11-05 and the nest-*-key-empty rows). + ("nesting-beside-empty-key", FILTER_P + ', "ExpressionAttributeValues": {":p": ' + + NEST32_S + '}, ' + EMPTY_KEY, EMPTY_KEY_VALUE), + ("nesting-beside-type-X-key", FILTER_P + ', "ExpressionAttributeValues": {":p": ' + + NEST32_S + '}, "ExclusiveStartKey": {"pk": {"X": "1"}, "sk": {"S": "x"}}', + EMPTY_KEY_VALUE), + ("nesting-beside-two-descriptors-key", FILTER_P + ', "ExpressionAttributeValues": {":p": ' + + NEST32_S + '}, "ExclusiveStartKey": {"pk": {"S": "x", "N": "1"}, "sk": {"S": "x"}}', + TWO_DESCRIPTORS), + ("nest-read-N-key-empty", FILTER_P + ', "ExpressionAttributeValues": {":p": ' + + NEST32_N + '}, ' + EMPTY_KEY, EMPTY_KEY_VALUE), + ("nest-unused-S-key-empty", FILTER_P + ', "ExpressionAttributeValues": {":p": {"S": "x"}, ' + '":d": ' + NEST32_S + '}, ' + EMPTY_KEY, EMPTY_KEY_VALUE), + ("nest-unused-N-key-empty", FILTER_P + ', "ExpressionAttributeValues": {":p": {"S": "x"}, ' + '":d": ' + NEST32_N + '}, ' + EMPTY_KEY, EMPTY_KEY_VALUE), + # Order U: the parameter mix and the other request parameter checks ahead of + # the key's shape fault (arity-key-empty-and-eav-noexpr). + ("mix-beside-empty-key", '"ScanFilter": {"a": {"ComparisonOperator": "EQ", ' + '"AttributeValueList": [{"N": "1"}]}}, "FilterExpression": "a = :p", ' + '"ExpressionAttributeValues": {":p": {"N": "1"}}, ' + EMPTY_KEY, MIX), + ("arity-key-empty-and-ean-noexpr", '"ScanFilter": {"a": {"ComparisonOperator": "EQ", ' + '"AttributeValueList": []}}, "ExpressionAttributeNames": {"#a": "a"}, ' + EMPTY_KEY, + EAN_NO_EXPRESSION), + # Order R: the nesting check ahead of the Select enum, the filter syntax, + # the no-expression check and the key's number. + ("nest-unused-S-select-bad", FILTER_P + ', "ExpressionAttributeValues": {":p": {"S": "x"}, ' + '":d": ' + NEST32_S + '}, "Select": "BAD"', NESTING), + ("nest-read-S-filter-syntax", '"FilterExpression": "a = = :p", "ExpressionAttributeValues": ' + '{":p": ' + NEST32_S + '}', NESTING), + ("nest-read-S-eav-noexpr-sibling", '"ProjectionExpression": "a", ' + '"ExpressionAttributeValues": {":p": ' + NEST32_S + '}', NESTING), + ("nest-read-S-key-bob", FILTER_P + ', "ExpressionAttributeValues": {":p": ' + NEST32_S + + '}, ' + BOB_KEY, NESTING), + ("B4-5a", '"ExpressionAttributeValues": {":d": ' + NEST32_N + '}', NESTING), + # Order R: a set fault in ExpressionAttributeValues ahead of the + # no-expression check; the set texts with the members JSON-quoted. + ("B4-7-noexpr-NSe", '"ExpressionAttributeValues": {":e": {"NS": []}}', NS_EMPTY), + ("eav-read-NSe", '"FilterExpression": "a = :a", "ExpressionAttributeValues": {":a": {"NS": []}}', + NS_EMPTY), + ("eav-read-SSdup", + '"FilterExpression": "a = :a", "ExpressionAttributeValues": {":a": {"SS": ["a", "a"]}}', + SS_DUP), + # Order R: the constraint enums and the request parameter checks among + # themselves, and the Select enum after the no-expression check. + ("mix-rcc-bad", '"ScanFilter": {"a": {"ComparisonOperator": "EQ", "AttributeValueList": ' + '[{"N": "1"}]}}, "FilterExpression": "a = :p", "ExpressionAttributeValues": ' + '{":p": {"N": "1"}}, "ReturnConsumedCapacity": "BAD"', RCC_ENUM), + ("rcc-bad-select-bad", '"ReturnConsumedCapacity": "BAD", "Select": "BAD"', RCC_ENUM), + ("mix-select-bad", '"ScanFilter": {"a": {"ComparisonOperator": "EQ", "AttributeValueList": ' + '[{"N": "1"}]}}, "FilterExpression": "a = :p", "ExpressionAttributeValues": ' + '{":p": {"N": "1"}}, "Select": "BAD"', MIX), + ("select-bad-eav-noexpr", '"ExpressionAttributeValues": {":p": {"S": "x"}}, "Select": "BAD"', + ("startswith", EAV_NO_EXPRESSION)), + ("eav-noexpr-ean-noexpr", '"ExpressionAttributeValues": {":p": {"S": "x"}}, ' + '"ExpressionAttributeNames": {"#a": "a"}', EAN_NO_EXPRESSION), + ("ean-noexpr-condop", '"ExpressionAttributeNames": {"#a": "a"}, "ConditionalOperator": "AND"', + EAN_NO_EXPRESSION), + ("condop-arity", '"ConditionalOperator": "AND", "ScanFilter": {"a": {"ComparisonOperator": ' + '"EQ", "AttributeValueList": []}}', CONDOP), + # Order R: the expression checks among themselves (section 6 step 5). + ("filter-syntax-segment-alone", '"FilterExpression": "a = = :p", ' + '"ExpressionAttributeValues": {":p": {"N": "1"}}, "Segment": 0', + ("startswith", "Invalid FilterExpression:")), + ("segment-alone-unused", FILTER_P + ', "ExpressionAttributeValues": {":p": {"S": "x"}, ' + '":u": {"S": "y"}}, "Segment": 0', UNUSED_U), + ("undefined-unused", '"FilterExpression": "a = :zz", "ExpressionAttributeValues": ' + '{":u": {"S": "y"}}', UNDEFINED_ZZ), + ("undefined-segment-alone", '"FilterExpression": "a = :zz", "Segment": 0', UNDEFINED_ZZ), + ("undefined-select-specific", '"FilterExpression": "a = :zz", "Select": "SPECIFIC_ATTRIBUTES"', + UNDEFINED_ZZ), + ("begins-n-segment-alone", '"FilterExpression": "begins_with(a, :p)", ' + '"ExpressionAttributeValues": {":p": {"N": "1"}}, "Segment": 0', BEGINS_N), + ("begins-n-unused", '"FilterExpression": "begins_with(a, :p)", ' + '"ExpressionAttributeValues": {":p": {"N": "1"}, ":u": {"S": "y"}}', BEGINS_N), + ("projection-syntax-filter-syntax", '"FilterExpression": "a = = :p", ' + '"ExpressionAttributeValues": {":p": {"N": "1"}}, "ProjectionExpression": "a,,b"', + ("startswith", "Invalid FilterExpression:")), + ("select-specific-filter-syntax", '"FilterExpression": "a = = :p", ' + '"ExpressionAttributeValues": {":p": {"N": "1"}}, "Select": "SPECIFIC_ATTRIBUTES"', + ("startswith", "Invalid FilterExpression:")), + ("allproj-noindex-filter-syntax", '"Select": "ALL_PROJECTED_ATTRIBUTES", ' + '"FilterExpression": "a = = :p", "ExpressionAttributeValues": {":p": {"N": "1"}}', + ("startswith", "Invalid FilterExpression:")), + ("select-count-pe-unused", '"Select": "COUNT", "ProjectionExpression": "a", ' + FILTER_P + + ', "ExpressionAttributeValues": {":p": {"S": "x"}, ":u": {"S": "y"}}', UNUSED_U), + ("segment-ge-total-unused", FILTER_P + ', "ExpressionAttributeValues": {":p": {"S": "x"}, ' + '":u": {"S": "y"}}, "Segment": 2, "TotalSegments": 2', UNUSED_U), + ("select-count-atg-segment-alone", '"Select": "COUNT", "AttributesToGet": ["a"], "Segment": 0', + ("startswith", "Cannot specify the ")), + ("select-count-atg-unused", '"Select": "COUNT", "AttributesToGet": ["a"], ' + '"ExpressionAttributeValues": {":u": {"S": "y"}}', ("startswith", EAV_NO_EXPRESSION)), + # Order R: the key's value faults after the expression checks, their texts. + ("key-bob-eav-bob", '"FilterExpression": "a = :n", "ExpressionAttributeValues": ' + '{":n": {"N": "fred"}}, ' + BOB_KEY, NUMERIC_FRED), + ("key-bob-and-NSe-one-member", + '"ExclusiveStartKey": {"pk": {"S": "x"}, "sk": {"S": "x"}, "z": {"NS": ["bob", "1", "1"]}}', + NUMERIC_BOB), + ("key-NS-dup-and-bob", + '"ExclusiveStartKey": {"pk": {"S": "x"}, "sk": {"S": "x"}, "z": {"NS": ["1", "1", "bob"]}}', + NUMERIC_BOB), + ("key-BS-dup", + '"ExclusiveStartKey": {"pk": {"S": "x"}, "sk": {"S": "x"}, "z": {"BS": ["YQ==", "YQ=="]}}', + INVALID + 'Input collection ["YQ==","YQ=="]of type BS contains duplicates.'), + ("key-SS-dup-quotes", + '"ExclusiveStartKey": {"pk": {"S": "x"}, "sk": {"S": "x"}, "z": {"SS": ["a\\"b", "a\\"b"]}}', + INVALID + 'Input collection ["a\\"b", "a\\"b"] contains duplicates.'), + ("key-bob-schema-mismatch", '"ExclusiveStartKey": {"pk": {"N": "bob"}}', NUMERIC_BOB), + # Order U: the key's set faults beside an unreadable expression value, the + # order-U texts (the NS duplicate text bare, the SS members unquoted). + ("key-bob-eav-empty-read-and-NSe", '"FilterExpression": "a = :a", ' + '"ExpressionAttributeValues": {":a": {}}, "ExclusiveStartKey": {"pk": {"NS": []}, ' + '"sk": {"S": "x"}}', NS_EMPTY), + ("key-NS-dup-beside-eav-empty", '"FilterExpression": "a = :a", ' + '"ExpressionAttributeValues": {":a": {}}, "ExclusiveStartKey": {"pk": {"NS": ["1", "1"]}, ' + '"sk": {"S": "x"}}', "Input collection contains duplicates"), + ("key-SS-dup-beside-eav-empty", '"FilterExpression": "a = :a", ' + '"ExpressionAttributeValues": {":a": {}}, "ExclusiveStartKey": {"pk": {"SS": ["a", "a"]}, ' + '"sk": {"S": "x"}}', INVALID + "Input collection [a, a] contains duplicates."), + # Order U: the key's shape fault ahead of the table's index rules. + ("key-empty-and-consistent-gsi", '"IndexName": "gsi", "ConsistentRead": true, ' + '"ExclusiveStartKey": {"gpk": {"S": "g"}, "pk": {"S": "p"}, "sk": {}}', EMPTY_KEY_VALUE), + ("key-shape-M-nested-N-bob", '"ExclusiveStartKey": {"pk": {"S": "x"}, "sk": {"S": "x"}, ' + '"z": {"M": {"k": {"N": "bob"}, "j": {}}}}', EMPTY_KEY_VALUE), +] + + +EMPTY_VALUE_BARE = ("Supplied AttributeValue is empty, must contain exactly one of the " + "supported datatypes") +MORE_THAN_ONE_BARE = ("Supplied AttributeValue has more than one datatypes set, must contain " + "exactly one of the supported datatypes") +EAV_EMPTY_B = EAV_EMPTY_E.replace(":e", ":b") +EAV_EMPTY_P = EAV_EMPTY_E.replace(":e", ":p") +EAV_NUMBER_A = EAV_VALUE + NUMERIC_BOB + " for key :a" +EAV_NUMBER_N = EAV_VALUE + NUMERIC_BOB + " for key :n" +SF_EMPTY = '"ScanFilter": {"a": {"ComparisonOperator": "EQ", "AttributeValueList": [{}]}}' +SF_BOB = '"ScanFilter": {"a": {"ComparisonOperator": "EQ", "AttributeValueList": [{"N": "bob"}]}}' +NONASCII_KEY_A_BOB = '"ExclusiveStartKey": {"\\u00e4": {"N": "bob"}, "\\u00e9": {}}' +NONASCII_KEY_E_EMPTY = '"ExclusiveStartKey": {"\\u00e9": {}, "\\u00e4": {"N": "bob"}}' + +# Measured 2026-11-05 at 05:31, 06:20 and 07:38 UTC: (id, table +# fixture or None for a missing table, the fields, the expected text). A text +# given as ("startswith", prefix) asserts the check's place where the text +# predates this branch (#838); ("either", a, b) accepts both answers where the +# service's two sends differed. +ROWS_2026_11_05 = [ + # 7.1: a malformed number in a ScanFilter list, ahead of the key's number. + # The check is #564 item 3, which PR #507 makes (SCAN_FILTER_NUMBER_ROWS). + ("sfR-N-bob", "table", SF_BOB, NUMERIC_BOB), + ("sfR-N-bob-and-key-bob", "table", + SF_BOB + ', "ExclusiveStartKey": {"pk": {"N": "fred"}, "sk": {"S": "x"}}', NUMERIC_BOB), + # 7.2: order U, the ExpressionAttributeValues entries in the fitted order, + # a number among them in the invalid-value form, after the key's faults. + ("eavU-num-first-sorted", "table", FILTER_P + ', "ExpressionAttributeValues": ' + '{":p": {"S": "x"}, ":a": {"N": "bob"}, ":e": {}}', EAV_NUMBER_A), + ("eavU-num-last-sorted", "table", FILTER_P + ', "ExpressionAttributeValues": ' + '{":p": {"S": "x"}, ":n": {"N": "bob"}, ":e": {}}', EAV_NUMBER_N), + ("eavU-num-and-key-empty", "table", '"FilterExpression": "a = :n", ' + '"ExpressionAttributeValues": {":n": {"N": "bob"}}, ' + EMPTY_KEY, EMPTY_KEY_VALUE), + ("eavU-num-and-key-empty-and-eav-empty", "table", '"FilterExpression": "a = :n", ' + '"ExpressionAttributeValues": {":n": {"N": "bob"}, ":e": {}}, ' + EMPTY_KEY, + EMPTY_KEY_VALUE), + ("eavU-two-faulty-entries", "table", FILTER_P + ', "ExpressionAttributeValues": ' + '{":p": {"S": "x"}, ":b": {}, ":a": {"X": "1"}}', EAV_EMPTY_B), + # 7.3: four members on a GSI with a range key (eight buckets: gpk and pk + # ahead of gsk), and names outside ASCII. + ("key4-gsk-bob-gpk-empty", "t4_table", '"IndexName": "gsi2", "ExclusiveStartKey": ' + '{"gpk": {}, "gsk": {"N": "bob"}, "pk": {"S": "p"}, "sk": {"S": "s"}}', EMPTY_KEY_VALUE), + ("key4-gsk-bob-gpk-empty-reversed", "t4_table", '"IndexName": "gsi2", "ExclusiveStartKey": ' + '{"sk": {"S": "s"}, "pk": {"S": "p"}, "gsk": {"N": "bob"}, "gpk": {}}', EMPTY_KEY_VALUE), + ("key4-pk-bob-gsk-empty", "t4_table", '"IndexName": "gsi2", "ExclusiveStartKey": ' + '{"gpk": {"S": "g"}, "gsk": {}, "pk": {"N": "bob"}, "sk": {"S": "s"}}', NUMERIC_BOB), + ("keyU-nonascii-a-bob-e-empty", "u_table", NONASCII_KEY_A_BOB, NUMERIC_BOB), + ("keyU-nonascii-e-empty-a-bob", "u_table", NONASCII_KEY_E_EMPTY, NUMERIC_BOB), + # 7.4: a ScanFilter list value that is not one AttributeValue, in order U: + # the bare empty-value text for {} and for an unknown descriptor, after the + # no-expression checks and ahead of ConditionalOperator, the value count, + # the key's faults and the table; the Select enum ahead of it. + ("sfU-empty-alone", "table", SF_EMPTY, EMPTY_VALUE_BARE), + ("sfU-X-alone", "table", + '"ScanFilter": {"a": {"ComparisonOperator": "EQ", "AttributeValueList": [{"X": "1"}]}}', + EMPTY_VALUE_BARE), + ("sfU-empty-and-ean-noexpr", "table", SF_EMPTY + ', "ExpressionAttributeNames": {"#a": "a"}', + EAN_NO_EXPRESSION), + ("sfU-empty-and-key-empty", "table", SF_EMPTY + ", " + EMPTY_KEY, EMPTY_VALUE_BARE), + ("sfU-empty-and-key-bob", "table", SF_EMPTY + ", " + BOB_KEY, EMPTY_VALUE_BARE), + ("sfU-empty-arity", "table", + '"ScanFilter": {"a": {"ComparisonOperator": "EQ", "AttributeValueList": [{}, {}]}}', + EMPTY_VALUE_BARE), + ("sfU-empty-condop", "table", '"ConditionalOperator": "AND", ' + SF_EMPTY, EMPTY_VALUE_BARE), + ("sfU-empty-select-bad", "table", SF_EMPTY + ', "Select": "BAD"', SELECT_ENUM), + ("sfU-empty-missing-table", None, SF_EMPTY, EMPTY_VALUE_BARE), + # 7.5 and the other rows the head answers as the service does. + ("keyU-SSe-beside-eav-empty", "table", '"FilterExpression": "a = :a", ' + '"ExpressionAttributeValues": {":a": {}}, "ExclusiveStartKey": {"pk": {"SS": []}, ' + '"sk": {"S": "x"}}', KEY_PREFIX + SS_EMPTY), + ("keyU-BSe-beside-pk-empty", "table", '"ExclusiveStartKey": {"pk": {}, "sk": {"BS": []}}', + KEY_PREFIX + BS_EMPTY), + ("keyU-BSdup-beside-pk-empty", "table", + '"ExclusiveStartKey": {"pk": {}, "sk": {"BS": ["YQ==", "YQ=="]}}', + INVALID + "Input collection [YQ==, YQ==]of type BS contains duplicates."), + ("eavR-NSe-and-num-first-sorted", "table", FILTER_P + ', "ExpressionAttributeValues": ' + '{":p": {"S": "x"}, ":a": {"N": "bob"}, ":e": {"NS": []}}', ("either", NS_EMPTY, NUMERIC_BOB)), + ("eavR-NSe-and-num-last-sorted", "table", FILTER_P + ', "ExpressionAttributeValues": ' + '{":p": {"S": "x"}, ":n": {"N": "bob"}, ":e": {"NS": []}}', ("either", NS_EMPTY, NUMERIC_BOB)), + ("eavR-NSe-and-ean-noexpr", "table", '"ExpressionAttributeNames": {"#a": "a"}, ' + '"ExpressionAttributeValues": {":e": {"NS": []}}', NS_EMPTY), + ("eavR-NSe-and-rcc-bad", "table", '"ReturnConsumedCapacity": "BAD", ' + '"ExpressionAttributeValues": {":e": {"NS": []}}', RCC_ENUM), + ("eavR-NSe-and-select-bad", "table", FILTER_P + ', "ExpressionAttributeValues": ' + '{":p": {"S": "x"}, ":e": {"NS": []}}, "Select": "BAD"', NS_EMPTY), + # Re-measured 2026-11-05 at 06:20 UTC with the nested value written out + # (32 L levels): the empty set ahead of the nesting check in order R, and + # the empty leaf of a nested entry named in order U. + ("eavR-NSe-and-nest", "table", FILTER_P + ', "ExpressionAttributeValues": ' + '{":p": {"S": "x"}, ":d": ' + NEST32_N + ', ":e": {"NS": []}}', NS_EMPTY), + ("eavU-nest32-empty-leaf", "table", FILTER_P + ', "ExpressionAttributeValues": ' + '{":p": {"S": "x"}, ":d": ' + json.dumps(nest({}, 32)) + '}', EAV_EMPTY_E.replace(":e", ":d")), + ("eavU-empty-and-filter-syntax", "table", '"FilterExpression": "a = = :p", ' + '"ExpressionAttributeValues": {":p": {}}', EAV_EMPTY_P), + ("eavU-empty-and-undefined", "table", '"FilterExpression": "a = :zz", ' + '"ExpressionAttributeValues": {":e": {}}', EAV_EMPTY_E), + ("eavU-empty-and-projection-syntax", "table", '"ProjectionExpression": "a,,b", ' + FILTER_P + + ', "ExpressionAttributeValues": {":p": {}}', EAV_EMPTY_P), + ("eavU-empty-and-select-bad-and-mix", "table", '"ScanFilter": {"a": {"ComparisonOperator": ' + '"EQ", "AttributeValueList": [{"N": "1"}]}}, ' + FILTER_P + ', "ExpressionAttributeValues": ' + '{":p": {"N": "1"}, ":e": {}}, "Select": "BAD"', SELECT_ENUM), + ("eavU-empty-and-rcc-bad", "table", '"ReturnConsumedCapacity": "BAD", ' + FILTER_P + + ', "ExpressionAttributeValues": {":p": {"S": "x"}, ":e": {}}', RCC_ENUM), + ("selR-condop-select-bad", "table", '"ConditionalOperator": "AND", "ScanFilter": {"a": ' + '{"ComparisonOperator": "EQ", "AttributeValueList": [{"N": "1"}]}}, "Select": "BAD"', CONDOP), + ("selR-arity-select-bad", "table", '"ScanFilter": {"a": {"ComparisonOperator": "EQ", ' + '"AttributeValueList": []}}, "Select": "BAD"', + ("either", "Invalid number of attribute values for comparison", + INVALID + "Invalid number of argument(s) for the EQ ComparisonOperator")), + ("flag-unknown-field-empty", "table", '"Limit": 0, "Foo": {}', LIMIT), + ("flag-queryfilter-on-scan", "table", '"Limit": 0, "QueryFilter": {"a": {"ComparisonOperator": ' + '"EQ", "AttributeValueList": [{}]}}', LIMIT), + # Measured 2026-11-05 at 07:38 UTC. Order U, an ExpressionAttributeValues + # set fault takes the invalid-value form at the map's step, in the entries' + # hash order (:s in bucket 1 ahead of :e in 3). + ("eavU-NSe-entry", "table", FILTER_P + ', "ExpressionAttributeValues": ' + '{":p": {"S": "x"}, ":s": {"NS": []}, ":e": {}}', EAV_VALUE + NS_EMPTY + " for key :s"), + ("eavU-SSdup-entry", "table", FILTER_P + ', "ExpressionAttributeValues": ' + '{":p": {"S": "x"}, ":s": {"SS": ["a", "a"]}, ":e": {}}', + EAV_VALUE + INVALID + "Input collection [a, a] contains duplicates. for key :s"), + # Inside one entry the first fault in the value's element order is named. + ("eavU-number-then-empty-in-one-L", "table", FILTER_P + ', "ExpressionAttributeValues": ' + '{":p": {"S": "x"}, ":l": {"L": [{"N": "bob"}, {}]}}', EAV_NUMBER_A.replace(":a", ":l")), + # Order U: the entries' hash order with the request's order inside a bucket + # (:e and :a share bucket 3), four and seven entries (eight and sixteen + # buckets), a number below the top of an entry, the 38-digit text. + ("eavU-reqorder-e-then-a", "table", FILTER_P + ', "ExpressionAttributeValues": ' + '{":p": {"S": "x"}, ":e": {}, ":a": {"N": "bob"}}', EAV_EMPTY_E), + ("eavU-four-entries", "table", FILTER_P + ', "ExpressionAttributeValues": ' + '{":p": {"S": "x"}, ":b": {"S": "y"}, ":a": {"N": "bob"}, ":e": {}}', EAV_EMPTY_E), + ("eavU-seven-entries", "table", FILTER_P + ', "ExpressionAttributeValues": ' + '{":p": {"S": "x"}, ":b": {}, ":e": {"S": "e"}, ":a": {"S": "a"}, ":n": {"N": "bob"}, ' + '":d": {"S": "d"}, ":z": {"S": "z"}}', EAV_NUMBER_N), + ("eavU-number-in-L", "table", FILTER_P + ', "ExpressionAttributeValues": ' + '{":p": {"S": "x"}, ":l": {"L": [{"N": "bob"}]}, ":e": {}}', EAV_NUMBER_A.replace(":a", ":l")), + ("eavU-number-in-NS", "table", FILTER_P + ', "ExpressionAttributeValues": ' + '{":p": {"S": "x"}, ":s": {"NS": ["bob"]}, ":e": {}}', EAV_NUMBER_A.replace(":a", ":s")), + ("eavU-39-digits", "table", FILTER_P + ', "ExpressionAttributeValues": {":p": {"S": "x"}, ' + '":a": {"N": "123456789012345678901234567890123456789"}, ":e": {}}', + EAV_VALUE + "Attempting to store more than 38 significant digits in a Number for key :a"), + # Order R, two faulty entries: the numbers in the hash order (:n in bucket 0 + # ahead of :a in 3); the set faults in sorted name order, where the + # service's two sends disagreed (the SS text, then the NS text) and the + # sorted order is pinned so that the answer does not depend on the backend. + ("eavR-two-numbers", "table", '"FilterExpression": "a = :a", "ExpressionAttributeValues": ' + '{":n": {"N": "fred"}, ":a": {"N": "bob"}}', NUMERIC_BOB.replace("bob", "fred")), + ("eavR-two-set-faults", "table", '"ExpressionAttributeValues": {":n": {"NS": []}, ":a": {"SS": []}}', + ("either", SS_EMPTY, NS_EMPTY)), + # Order R: a malformed number inside an entry the deserializer refuses comes + # after the nesting check of another entry; with the number ahead of the + # set fault in one L the service's two sends disagreed. The SS duplicates + # beside a nested value answered the duplicates text and the nesting text. + ("eavR-NSbobdup-and-nest", "table", FILTER_P + ', "ExpressionAttributeValues": ' + '{":p": {"S": "x"}, ":d": ' + NEST32_N + ', ":a": {"NS": ["bob", "1", "1"]}}', NESTING), + ("eavR-L-bob-then-NSe-and-nest", "table", FILTER_P + ', "ExpressionAttributeValues": ' + '{":p": {"S": "x"}, ":d": ' + NEST32_N + ', ":l": {"L": [{"N": "bob"}, {"NS": []}]}}', + ("either", NUMERIC_BOB, NESTING)), + ("eavR-SSdup-and-nest", "table", FILTER_P + ', "ExpressionAttributeValues": ' + '{":p": {"S": "x"}, ":d": ' + NEST32_N + ', ":s": {"SS": ["a", "a"]}}', + ("either", INVALID + 'Input collection ["a", "a"] contains duplicates.', NESTING)), + ("eavR-NSe-and-mix", "table", FILTER_P + ', "ScanFilter": {"a": {"ComparisonOperator": "EQ", ' + '"AttributeValueList": [{"N": "1"}]}}, "ExpressionAttributeValues": {":p": {"S": "x"}, ' + '":e": {"NS": []}}', MIX), + # Keys of five and seven members on the GSI (eight and sixteen buckets), and + # a key name outside the Basic Multilingual Plane. + ("key5-gpk-bob-gsk-empty", "t4_table", '"IndexName": "gsi2", "ExclusiveStartKey": ' + '{"gpk": {"N": "bob"}, "gsk": {}, "pk": {"S": "p"}, "sk": {"S": "s"}, "z": {"S": "z"}}', + NUMERIC_BOB), + ("key7-a-bob-sk-empty", "t4_table", '"IndexName": "gsi2", "ExclusiveStartKey": ' + '{"gpk": {"S": "g"}, "gsk": {"S": "h"}, "pk": {"S": "p"}, "sk": {}, "a": {"N": "bob"}, ' + '"b": {"S": "b"}, "c": {"S": "c"}}', NUMERIC_BOB), + ("keyU-supplementary-first", "v_table", + '"ExclusiveStartKey": {"\\ud834\\udd1e": {}, "a": {"N": "bob"}}', EMPTY_KEY_VALUE), + ("keyU-supplementary-second", "v_table", + '"ExclusiveStartKey": {"a": {"N": "bob"}, "\\ud834\\udd1e": {}}', NUMERIC_BOB), + # Order U, the ScanFilter lists: two descriptors, an empty value inside an + # L, which condition is named (the fitted order: pk ahead of sk, the + # request's order for ts and id in one bucket), which list value (the + # list's order), and ExpressionAttributeValues with no expression first. + ("sfU-two-descriptors", "table", '"ScanFilter": {"a": {"ComparisonOperator": "EQ", ' + '"AttributeValueList": [{"S": "x", "N": "1"}]}}', MORE_THAN_ONE_BARE), + ("sfU-empty-in-L", "table", '"ScanFilter": {"a": {"ComparisonOperator": "EQ", ' + '"AttributeValueList": [{"L": [{}]}]}}', EMPTY_VALUE_BARE), + ("sfU-two-conditions-pk-sk", "table", '"ScanFilter": {"pk": {"ComparisonOperator": "EQ", ' + '"AttributeValueList": [{}]}, "sk": {"ComparisonOperator": "EQ", "AttributeValueList": ' + '[{"S": "x", "N": "1"}]}}', MORE_THAN_ONE_BARE), + ("sfU-two-conditions-same-bucket", "table", '"ScanFilter": {"ts": {"ComparisonOperator": "EQ", ' + '"AttributeValueList": [{}]}, "id": {"ComparisonOperator": "EQ", "AttributeValueList": ' + '[{"S": "x", "N": "1"}]}}', EMPTY_VALUE_BARE), + ("sfU-list-two-values", "table", '"ScanFilter": {"a": {"ComparisonOperator": "BETWEEN", ' + '"AttributeValueList": [{"S": "x", "N": "1"}, {}]}}', MORE_THAN_ONE_BARE), + ("sfU-eav-noexpr-and-list-empty", "table", '"ExpressionAttributeValues": {":a": {"S": "x"}}, ' + '"ScanFilter": {"a": {"ComparisonOperator": "EQ", "AttributeValueList": [{}]}}', + ("startswith", EAV_NO_EXPRESSION)), + # Order R: a set fault in a ScanFilter list value, the deserializer's text. + ("sfR-NSe-in-list", "table", '"ScanFilter": {"a": {"ComparisonOperator": "CONTAINS", ' + '"AttributeValueList": [{"NS": []}]}}', NS_EMPTY), + # Order U: the empty BS of a key member keeps the key prefix beside an + # unreadable expression value, inside an L and in a member that is not a + # key attribute. + ("keyU-BSe-beside-eav-empty", "table", '"FilterExpression": "a = :a", ' + '"ExpressionAttributeValues": {":a": {}}, "ExclusiveStartKey": {"pk": {"BS": []}, ' + '"sk": {"S": "x"}}', KEY_PREFIX + INVALID + "Binary sets should not be empty"), + ("keyU-BSe-in-L-beside-pk-empty", "table", '"ExclusiveStartKey": {"pk": {}, "sk": {"L": ' + '[{"BS": []}]}}', KEY_PREFIX + INVALID + "Binary sets should not be empty"), + ("keyU-BSe-in-nonkey-member", "table", '"ExclusiveStartKey": {"pk": {}, "sk": {"S": "s"}, ' + '"b": {"BS": []}}', KEY_PREFIX + INVALID + "Binary sets should not be empty"), +] + + +def unused_value(value: dict) -> str: + """A filter that reads ``:p`` and ``value`` under ``:d``, which nothing reads.""" + return ('"FilterExpression": "a = :p", "ExpressionAttributeValues": ' + + json.dumps({":p": {"S": "x"}, ":d": value})) + + +def _raw(client, operation: str, body_text: str): + """POST ``body_text`` exactly as written, signed with the client's credentials.""" + req = AWSRequest(method="POST", url=client.meta.endpoint_url, data=body_text, + headers={"Content-Type": "application/x-amz-json-1.0", + "X-Amz-Target": f"DynamoDB_20120810.{operation}"}) + client._request_signer.sign(operation, req) + resp = client._endpoint.http_session.send(req.prepare()) + return resp.status_code, json.loads(resp.content.decode("utf-8", "replace") or "{}") + + +def _scan(client, table: str, fields: str): + return _raw(client, "Scan", '{"TableName": %s, %s}' % (json.dumps(table), fields)) + + +def _validation_message(status: int, payload: dict) -> str: + assert status == 400, payload + assert payload.get("__type", "").endswith("ValidationException"), payload + return payload.get("message", payload.get("Message")) + + +@pytest.fixture(scope="module") +def table(dynamodb_client): + """pk S, sk S, a GSI ``gsi`` on gpk S, an LSI ``lsi`` on ln N, with ITEMS items.""" + name = unique_name("scanreq") + dynamodb_client.create_table( + TableName=name, + AttributeDefinitions=[ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "sk", "AttributeType": "S"}, + {"AttributeName": "gpk", "AttributeType": "S"}, + {"AttributeName": "ln", "AttributeType": "N"}, + ], + KeySchema=[ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"}, + ], + GlobalSecondaryIndexes=[{ + "IndexName": "gsi", + "KeySchema": [{"AttributeName": "gpk", "KeyType": "HASH"}], + "Projection": {"ProjectionType": "ALL"}, + }], + LocalSecondaryIndexes=[{ + "IndexName": "lsi", + "KeySchema": [ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "ln", "KeyType": "RANGE"}, + ], + "Projection": {"ProjectionType": "ALL"}, + }], + BillingMode="PAY_PER_REQUEST", + ) + wait_for_active(dynamodb_client, name) + for i in range(ITEMS): + dynamodb_client.put_item(TableName=name, Item={ + "pk": {"S": f"p{i}"}, "sk": {"S": "s"}, "a": {"N": str(i)}}) + yield name + dynamodb_client.delete_table(TableName=name) + wait_for_deleted(dynamodb_client, name) + + +@pytest.fixture(scope="module") +def n_table(dynamodb_client): + """pk N, with items 1 to 6.""" + name = unique_name("scanreqn") + dynamodb_client.create_table( + TableName=name, + AttributeDefinitions=[{"AttributeName": "pk", "AttributeType": "N"}], + KeySchema=[{"AttributeName": "pk", "KeyType": "HASH"}], + BillingMode="PAY_PER_REQUEST", + ) + wait_for_active(dynamodb_client, name) + for i in range(1, 7): + dynamodb_client.put_item(TableName=name, Item={"pk": {"N": str(i)}}) + yield name + dynamodb_client.delete_table(TableName=name) + wait_for_deleted(dynamodb_client, name) + + +def _two_member_table(dynamodb_client, prefix: str, hash_key: str, range_key: str): + name = unique_name(prefix) + dynamodb_client.create_table( + TableName=name, + AttributeDefinitions=[{"AttributeName": hash_key, "AttributeType": "S"}, + {"AttributeName": range_key, "AttributeType": "S"}], + KeySchema=[{"AttributeName": hash_key, "KeyType": "HASH"}, + {"AttributeName": range_key, "KeyType": "RANGE"}], + BillingMode="PAY_PER_REQUEST", + ) + wait_for_active(dynamodb_client, name) + return name + + +@pytest.fixture(scope="module") +def ab_table(dynamodb_client): + """HASH a S, RANGE b S, no items.""" + name = _two_member_table(dynamodb_client, "scanreqab", "a", "b") + yield name + dynamodb_client.delete_table(TableName=name) + wait_for_deleted(dynamodb_client, name) + + +@pytest.fixture(scope="module") +def pa_table(dynamodb_client): + """HASH pk S, RANGE a S, no items.""" + name = _two_member_table(dynamodb_client, "scanreqpa", "pk", "a") + yield name + dynamodb_client.delete_table(TableName=name) + wait_for_deleted(dynamodb_client, name) + + +@pytest.fixture(scope="module") +def it_table(dynamodb_client): + """HASH id S, RANGE ts S, no items.""" + name = _two_member_table(dynamodb_client, "scanreqit", "id", "ts") + yield name + dynamodb_client.delete_table(TableName=name) + wait_for_deleted(dynamodb_client, name) + + +@pytest.fixture(scope="module") +def t4_table(dynamodb_client): + """pk S, sk S, with a GSI ``gsi2`` on gpk S HASH and gsk S RANGE: a four-member index key.""" + name = unique_name("scanreqt4") + dynamodb_client.create_table( + TableName=name, + AttributeDefinitions=[{"AttributeName": n, "AttributeType": "S"} + for n in ("pk", "sk", "gpk", "gsk")], + KeySchema=[{"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"}], + GlobalSecondaryIndexes=[{ + "IndexName": "gsi2", + "KeySchema": [{"AttributeName": "gpk", "KeyType": "HASH"}, + {"AttributeName": "gsk", "KeyType": "RANGE"}], + "Projection": {"ProjectionType": "ALL"}, + }], + BillingMode="PAY_PER_REQUEST", + ) + wait_for_active(dynamodb_client, name) + yield name + dynamodb_client.delete_table(TableName=name) + wait_for_deleted(dynamodb_client, name) + + +@pytest.fixture(scope="module") +def u_table(dynamodb_client): + """HASH \u00e4 S, RANGE \u00e9 S: member names outside ASCII.""" + name = _two_member_table(dynamodb_client, "scanrequ", "\u00e4", "\u00e9") + yield name + dynamodb_client.delete_table(TableName=name) + wait_for_deleted(dynamodb_client, name) + + +@pytest.fixture(scope="module") +def v_table(dynamodb_client): + """HASH \U0001d11e S, RANGE a S: a member name outside the Basic Multilingual Plane.""" + name = _two_member_table(dynamodb_client, "scanreqv", "\U0001d11e", "a") + yield name + dynamodb_client.delete_table(TableName=name) + wait_for_deleted(dynamodb_client, name) + + +class TestRequestChecks: + """The parameter mix, the length constraint and the numbers in the key.""" + + @pytest.mark.parametrize("fields, expected", [r[1:] for r in REFUSED], + ids=[r[0] for r in REFUSED]) + def test_rows(self, dynamodb_client, table, fields, expected): + assert _validation_message(*_scan(dynamodb_client, table, fields)) == expected + + def test_an_empty_scan_filter_alone_is_accepted(self, dynamodb_client, table): + status, payload = _scan(dynamodb_client, table, + '"ScanFilter": {}, "ConsistentRead": true') + assert status == 200, payload + assert (payload["Count"], payload["ScannedCount"]) == (ITEMS, ITEMS) + + +class TestExclusiveStartKey: + """The key's two stages (measured 2026-10-31 and 2026-11-05).""" + + @pytest.mark.parametrize("fields, expected", [r[1:] for r in KEY_AFTER_THE_OTHER_CHECKS], + ids=[r[0] for r in KEY_AFTER_THE_OTHER_CHECKS]) + def test_the_other_checks_come_first(self, dynamodb_client, table, fields, expected): + assert _validation_message(*_scan(dynamodb_client, table, fields)) == expected + + def test_32_levels_are_refused_ahead_of_the_key(self, dynamodb_client, table): + fields = BOB_KEY + ", " + unused_value(nest(ONE, 32)) + assert _validation_message(*_scan(dynamodb_client, table, fields)) == NESTING + + def test_the_select_rule_is_checked_ahead_of_the_key(self, dynamodb_client, table): + # The service answers '1 validation error detected: ' and then this + # text (#832). The Scan text here has no prefix, so only the check's + # place is asserted. + fields = BOB_KEY + ', "Select": "SPECIFIC_ATTRIBUTES"' + message = _validation_message(*_scan(dynamodb_client, table, fields)) + assert message.endswith(SELECT_SPECIFIC), message + + def test_the_filter_syntax_is_checked_ahead_of_the_key(self, dynamodb_client, table): + # The service answers 'Invalid FilterExpression: Syntax error; token: + # "=", near: "= = :p"'. The parser here words the syntax error + # differently, so only the check's place is asserted. + fields = (BOB_KEY + ', "FilterExpression": "a = = :p", ' + '"ExpressionAttributeValues": {":p": {"S": "x"}}') + message = _validation_message(*_scan(dynamodb_client, table, fields)) + assert message.startswith("Invalid FilterExpression:"), message + + @pytest.mark.parametrize("fields, expected", [r[1:] for r in KEY_NUMBER_FAULTS], + ids=[r[0] for r in KEY_NUMBER_FAULTS]) + def test_a_number_fault_anywhere_in_the_key(self, dynamodb_client, table, fields, expected): + assert _validation_message(*_scan(dynamodb_client, table, fields)) == expected + + @pytest.mark.parametrize("key, expected", [r[1:] for r in KEYS_ALONE], + ids=[r[0] for r in KEYS_ALONE]) + def test_each_key_alone(self, dynamodb_client, table, key, expected): + fields = '"ExclusiveStartKey": ' + key + assert _validation_message(*_scan(dynamodb_client, table, fields)) == expected + + @pytest.mark.parametrize("key, expected", [r[1:] for r in TWO_MEMBER_FAULTS], + ids=[r[0] for r in TWO_MEMBER_FAULTS]) + def test_two_faulty_members_answer_the_first_in_the_fitted_order( + self, dynamodb_client, table, key, expected): + fields = '"ExclusiveStartKey": ' + key + assert _validation_message(*_scan(dynamodb_client, table, fields)) == expected + + @pytest.mark.parametrize("key, either", [r[1:] for r in TWO_VALUE_FAULTS], + ids=[r[0] for r in TWO_VALUE_FAULTS]) + def test_two_value_faults_answer_one_of_the_two(self, dynamodb_client, table, key, either): + fields = '"ExclusiveStartKey": ' + key + message = _validation_message(*_scan(dynamodb_client, table, fields)) + assert message in either, message + + @pytest.mark.parametrize("fixture, key, expected", [r[1:] for r in MEMBER_ORDER_ROWS], + ids=[r[0] for r in MEMBER_ORDER_ROWS]) + def test_the_member_order_on_other_tables(self, dynamodb_client, request, fixture, key, + expected): + name = request.getfixturevalue(fixture) + fields = '"ExclusiveStartKey": ' + key + assert _validation_message(*_scan(dynamodb_client, name, fields)) == expected + + def test_the_member_order_on_a_gsi_key_of_three_members(self, dynamodb_client, table): + fields = ('"IndexName": "gsi", "ExclusiveStartKey": {"gpk": {}, "pk": {"N": "bob"}, ' + '"sk": {"S": "x"}}') + assert _validation_message(*_scan(dynamodb_client, table, fields)) == EMPTY_KEY_VALUE + + def test_two_number_faults_name_one_of_them(self, dynamodb_client, table): + # The service named bob twice on 2026-10-31 (23:54 to 23:56 UTC) and + # fred on 2026-11-04 (23:45 UTC) for this key. + fields = '"ExclusiveStartKey": {"pk": {"N": "bob"}, "sk": {"N": "fred"}}' + message = _validation_message(*_scan(dynamodb_client, table, fields)) + assert message in (NUMERIC_BOB, NUMERIC_FRED), message + + @pytest.mark.parametrize("key, key_text, shape", [r[1:] for r in KEY_CLASSES], + ids=[r[0] for r in KEY_CLASSES]) + @pytest.mark.parametrize("check, stage", [(r[1], r[2]) for r in STAGES], + ids=[r[0] for r in STAGES]) + def test_each_stage_beside_each_key_fault( + self, dynamodb_client, table, key, key_text, shape, check, stage): + fields = '"ExclusiveStartKey": ' + key + ", " + check + message = _validation_message(*_scan(dynamodb_client, table, fields)) + if stage == "parameter": + assert message.startswith(EAV_NO_EXPRESSION), message + elif stage == "expression" and not shape: + assert message == UNUSED_U + else: + # A shape fault comes ahead of the expression check; every key + # fault comes ahead of the index resolution. + assert message == key_text + + def test_the_segment_pairing_is_checked_ahead_of_a_number_in_the_key( + self, dynamodb_client, table): + # Measured 2026-11-05: Segment without TotalSegments answers its own + # text beside bob in the key, and the key's text beside an empty value. + fields = BOB_KEY + ', "Segment": 0' + message = _validation_message(*_scan(dynamodb_client, table, fields)) + assert message == ("The TotalSegments parameter is required but was not present in the " + "request when Segment parameter is present"), message + fields = '"ExclusiveStartKey": {"pk": {}, "sk": {"S": "x"}}, "Segment": 0' + assert _validation_message(*_scan(dynamodb_client, table, fields)) == EMPTY_KEY_VALUE + + def test_an_undefined_filter_value_is_checked_ahead_of_a_number_in_the_key( + self, dynamodb_client, table): + # Measured 2026-11-05. + fields = BOB_KEY + ', "FilterExpression": "a = :zz"' + message = _validation_message(*_scan(dynamodb_client, table, fields)) + assert message == ("Invalid FilterExpression: An expression attribute value used in " + "expression is not defined; attribute value: :zz"), message + + def test_a_missing_table_is_reported_after_a_number_in_the_key(self, dynamodb_client): + # Measured 2026-11-05: the key's number text ahead of + # ResourceNotFoundException. + status, payload = _scan(dynamodb_client, unique_name("nosuch"), BOB_KEY) + assert status == 400, payload + assert payload.get("__type", "").endswith("ValidationException"), payload + assert payload.get("message") == NUMERIC_BOB, payload + + def test_an_empty_expression_value_is_reported_after_the_key_and_ahead_of_the_table( + self, dynamodb_client, table): + # Measured 2026-11-05: {":a": {}} read by the filter answers its own + # text with no key and the key's text beside bob or an empty set. + eav = '"FilterExpression": "a = :a", "ExpressionAttributeValues": {":a": {}}' + text = ("ExpressionAttributeValues contains invalid value: Supplied AttributeValue is " + "empty, must contain exactly one of the supported datatypes for key :a") + assert _validation_message(*_scan(dynamodb_client, table, eav)) == text + assert _validation_message(*_scan(dynamodb_client, table, BOB_KEY + ", " + eav)) == NUMERIC_BOB + fields = '"ExclusiveStartKey": {"pk": {"NS": []}, "sk": {"S": "x"}}, ' + eav + assert _validation_message(*_scan(dynamodb_client, table, fields)) == NS_EMPTY + status, payload = _scan(dynamodb_client, unique_name("nosuch"), eav) + assert payload.get("message") == text, payload + + @pytest.mark.parametrize("fields, expected", [r[1:] for r in N_KEY_FAULTS], + ids=[r[0] for r in N_KEY_FAULTS]) + def test_the_number_texts_on_an_n_hash_key(self, dynamodb_client, n_table, fields, expected): + assert _validation_message(*_scan(dynamodb_client, n_table, fields)) == expected + + +class TestTwoOrders: + """Representative rows of each order (measured 2026-10-31 and 2026-11-05).""" + + @pytest.mark.parametrize("fields, expected", [r[1:] for r in TWO_ORDERS], + ids=[r[0] for r in TWO_ORDERS]) + def test_rows(self, dynamodb_client, table, fields, expected): + message = _validation_message(*_scan(dynamodb_client, table, fields)) + if isinstance(expected, tuple): + # The text predates this branch (#722, #838, #839): the check's + # place is asserted. + assert message.startswith(expected[1]), message + else: + assert message == expected + + def test_an_unreadable_expression_value_on_a_missing_table(self, dynamodb_client): + # Measured 2026-11-05: ahead of ResourceNotFoundException. + fields = '"FilterExpression": "a = :a", "ExpressionAttributeValues": {":a": {}}' + status, payload = _scan(dynamodb_client, unique_name("nosuch"), fields) + assert payload.get("message") == EAV_EMPTY_A, payload + + def test_bob_in_the_key_of_an_n_table(self, dynamodb_client, n_table): + # Measured 2026-11-05. + fields = '"ExclusiveStartKey": {"pk": {"N": "bob"}}' + assert _validation_message(*_scan(dynamodb_client, n_table, fields)) == NUMERIC_BOB + + +# The two ScanFilter number rows: the measured answers are PR #507's check +# (#564 item 3, measured 2026-11-05), so they are expected to fail here until +# #507 lands. +SCAN_FILTER_NUMBER_ROWS = {"sfR-N-bob", "sfR-N-bob-and-key-bob"} + + +def _rows_2026_11_05_params(): + return [ + pytest.param(*r[1:], marks=pytest.mark.xfail(reason="#507")) + if r[0] in SCAN_FILTER_NUMBER_ROWS else r[1:] + for r in ROWS_2026_11_05 + ] + + +class TestRowsOf20261105: + """The rows measured 2026-11-05 at 05:31, 06:20 and 07:38 UTC.""" + + @pytest.mark.parametrize("fixture, fields, expected", _rows_2026_11_05_params(), + ids=[r[0] for r in ROWS_2026_11_05]) + def test_rows(self, dynamodb_client, request, fixture, fields, expected): + name = request.getfixturevalue(fixture) if fixture else unique_name("nosuch") + message = _validation_message(*_scan(dynamodb_client, name, fields)) + if isinstance(expected, tuple) and expected[0] == "startswith": + # The text predates this branch (#838): the check's place is asserted. + assert message.startswith(expected[1]), message + elif isinstance(expected, tuple): + assert message in expected[1:], message + else: + assert message == expected + + +class TestNestingDepth: + """``ExpressionAttributeValues`` nested in lists.""" + + @pytest.mark.parametrize("levels", [32, 33, 40, 60]) + def test_a_valid_number_at_the_bottom_is_refused_from_32_levels( + self, dynamodb_client, table, levels): + fields = unused_value(nest(ONE, levels)) + assert _validation_message(*_scan(dynamodb_client, table, fields)) == NESTING + + def test_31_levels_reach_the_unused_value_check(self, dynamodb_client, table): + fields = unused_value(nest(ONE, 31)) + assert _validation_message(*_scan(dynamodb_client, table, fields)) == UNUSED + + def test_32_levels_are_refused_ahead_of_the_select_enum(self, dynamodb_client, table): + # Measured 2026-10-30 12:40 UTC. + fields = unused_value(nest(ONE, 32)) + ', "Select": "BAD"' + assert _validation_message(*_scan(dynamodb_client, table, fields)) == NESTING + + @pytest.mark.parametrize("levels", [31, 32, 33, 40, 60]) + def test_a_malformed_number_at_the_bottom_is_reported_first( + self, dynamodb_client, table, levels): + fields = unused_value(nest(BOB, levels)) + assert _validation_message(*_scan(dynamodb_client, table, fields)) == NUMERIC_BOB + + def test_a_map_at_32_levels_with_a_malformed_number_is_reported_first( + self, dynamodb_client, table): + fields = unused_value({"M": {"k": nest(BOB, 31)}}) + assert _validation_message(*_scan(dynamodb_client, table, fields)) == NUMERIC_BOB diff --git a/tests/test_scan_number_values.py b/tests/test_scan_number_values.py index 89ab066aa..a4fd3d3b7 100644 --- a/tests/test_scan_number_values.py +++ b/tests/test_scan_number_values.py @@ -258,14 +258,13 @@ def test_parameter_mix(self, dynamodb_client, table, extra, non_expression, expr f"Non-expression parameters: {{{non_expression}}} Expression parameters: {{{expression}}}", ) - def test_parameter_mix_beats_return_consumed_capacity(self, dynamodb_client_no_validation, table): + def test_return_consumed_capacity_beats_parameter_mix(self, dynamodb_client_no_validation, table): + # Measured 2026-11-05: the ReturnConsumedCapacity enum comes ahead of + # the parameter mix. assert _scan_error( dynamodb_client_no_validation, TableName=table, ScanFilter=SCAN_FILTER, ReturnConsumedCapacity="NOPE", **OK_FILTER, - ) == ("ValidationException", ( - "Can not use both expression and non-expression parameters in the same request: " - "Non-expression parameters: {ScanFilter} Expression parameters: {FilterExpression}" - )) + ) == ("ValidationException", RCC_ENUM) def test_limit_range_beats_parameter_mix(self, dynamodb_client_no_validation, table): assert _scan_error(