From 8e4aa2204626c4245e28320f81eefdfc9588cb2d Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Fri, 30 Oct 2026 10:37:43 +0000 Subject: [PATCH 1/9] fix(scan): refuse an empty ScanFilter mix and AttributesToGet, check key numbers and nesting Measured against the service on 2026-10-25 at 10:38 and 10:55 UTC, each request sent twice (#564, items 1, 2, 4 and 5): - An empty ScanFilter counts as present for the expression and non-expression parameter mix. ScanFilter {} with a FilterExpression answers the mix text; ScanFilter {} alone is accepted. - AttributesToGet [] answers "1 validation error detected: Value at 'AttributesToGet' failed to satisfy constraint: Member must have length greater than or equal to 1", alone, with a FilterExpression, and with a FilterExpression and a malformed number, so it comes ahead of the number check. - A malformed N in ExclusiveStartKey answers "The parameter cannot be converted to a numeric value: ". - An ExpressionAttributeValues value nested 32 or more L levels deep answers "Nesting Levels have exceeded supported limits: Attributes in the item have nested levels beyond supported limit". 31 levels pass. ExtendDB accepted the first two, answered the starting-key schema text for the third, and had no nesting check on Scan. The request-time checks in crates/engine/src/scan_request.rs now make the four checks in the order the module doc lists. M levels count the same as L levels; only L was measured with a valid number at the bottom. Unit tests are in crates/engine/src/scan_request/request_tests.rs. tests/python/test_scan_request_validation.py has 18 rows; it passes on SQLite and PostgreSQL servers built from this branch, and 9 of the 18 fail on a server built from the #563 branch, which this one is stacked on. One row of item 5 stays open. From 63 L levels the service answers "ExpressionAttributeValues contains invalid value: Nesting Levels have exceeded supported limits: Attributes in the item have nested levels beyond supported limit for key :d" (measured 2026-10-30 at 63, 70, 80, 99 and 100 levels; 32 to 62 levels answer the bare text), and ExtendDB answers SerializationException from the JSON parser, whose recursion limit is 128 levels, 62 of them L wrappers. That is a change to request parsing for every operation, not to the Scan checks, and is filed on its own. Item 3, the legacy AttributeValueList numbers, is #507's. Refs #564 --- crates/engine/src/scan_request.rs | 85 +++++++-- .../engine/src/scan_request/request_tests.rs | 167 ++++++++++++++++++ tests/python/test_scan_request_validation.py | 163 +++++++++++++++++ 3 files changed, 402 insertions(+), 13 deletions(-) create mode 100644 crates/engine/src/scan_request/request_tests.rs create mode 100644 tests/python/test_scan_request_validation.py diff --git a/crates/engine/src/scan_request.rs b/crates/engine/src/scan_request.rs index 2fb57e206..e4afcbbe2 100644 --- a/crates/engine/src/scan_request.rs +++ b/crates/engine/src/scan_request.rs @@ -18,15 +18,38 @@ //! switches the message to the older form, `Value '-2147483648' at 'limit' //! ...`, with the value saturated or truncated, in the order //! `totalSegments`, `segment`, `limit`; see `validate_ranges`; -//! 2. the expression and non-expression parameter mix; -//! 3. the `ReturnConsumedCapacity` enum; -//! 4. every number in `ExpressionAttributeValues`, `NS` members and values +//! 2. the length constraint on `AttributesToGet`: an empty list answers `1 +//! validation error detected: Value at 'AttributesToGet' failed to satisfy +//! constraint: Member must have length greater than or equal to 1`, alone, +//! with a `FilterExpression`, and with a `FilterExpression` and a number +//! fault, so it comes ahead of the number check; +//! 3. the expression and non-expression parameter mix, in which a `ScanFilter` +//! object counts as present whether or not it is empty (`ScanFilter: {}` +//! with a `FilterExpression` answers the mix, `ScanFilter: {}` alone is +//! accepted); +//! 4. the `ReturnConsumedCapacity` enum; +//! 5. every number in `ExpressionAttributeValues`, `NS` members and values //! nested in `L` and `M` included, whether a `FilterExpression` reads the //! value, nothing reads it, or the request carries no expression at all; a -//! Unicode decimal digit counts as the digit it stands for; -//! 5. the `Select` enum. Both enums invalid answers the `ReturnConsumedCapacity` +//! Unicode decimal digit counts as the digit it stands for; then every +//! number in the `ExclusiveStartKey` values, which answers the same text as +//! a malformed number in a value; +//! 6. the nesting depth of `ExpressionAttributeValues`: a value nested 32 +//! levels deep in `L` answers `Nesting Levels have exceeded supported +//! limits: Attributes in the item have nested levels beyond supported +//! limit`, 31 levels pass, and a malformed number anywhere is reported +//! first. The check counts `M` levels the same way as `L` levels; only `L` +//! was measured with a valid number at the bottom; +//! 7. the `Select` enum. Both enums invalid answers the `ReturnConsumedCapacity` //! error alone, so the two are not aggregated. //! +//! Not measured, and checked in the order listed above: the `AttributesToGet` +//! length against the range constraints and against the enums, the +//! `ExclusiveStartKey` number check against the `ExpressionAttributeValues` +//! one, and the nesting check against the `Select` enum. A body nested deeper +//! than the JSON parser accepts is a `SerializationException` from the parser, +//! ahead of every check here. +//! //! Each of these is reported ahead of the next one, ahead of the table lookup, //! and ahead of the checks the handler makes after deserializing the body //! (Segment without TotalSegments and the reverse, Segment not below @@ -47,9 +70,11 @@ use crate::query_key_values::{ }; /// The `Scan` checks that run on the raw request body, in the service's order: -/// the `Limit`, `Segment` and `TotalSegments` ranges, the expression and -/// non-expression parameter mix, the `ReturnConsumedCapacity` enum, every -/// number in `ExpressionAttributeValues`, then the `Select` enum. +/// the `Limit`, `Segment` and `TotalSegments` ranges, the `AttributesToGet` +/// length, the expression and non-expression parameter mix, the +/// `ReturnConsumedCapacity` enum, every number in `ExpressionAttributeValues` +/// and `ExclusiveStartKey`, the nesting depth of `ExpressionAttributeValues`, +/// then the `Select` enum. /// /// The checks read the raw body, so a field that does not deserialize (an /// invalid `Select`, for one) does not stop the ones before it. When a check @@ -71,11 +96,14 @@ pub fn validate_scan_request_body(body: &Value) -> Result<(), DynamoDbError> { /// [`validate_scan_request_body`]. fn scan_request_checks(body: &Value) -> Result<(), DynamoDbError> { validate_ranges(body)?; + validate_attributes_to_get_length(body)?; extenddb_core::validation::validate_no_expression_param_mixing( &[ ("AttributesToGet", non_empty(body, "AttributesToGet")), - ("ScanFilter", non_empty(body, "ScanFilter")), + // An empty ScanFilter object counts as present, as a typed field + // that deserialized to an empty map is. + ("ScanFilter", present(body, "ScanFilter")), ("ConditionalOperator", present(body, "ConditionalOperator")), ], &[ @@ -96,10 +124,16 @@ fn scan_request_checks(body: &Value) -> Result<(), DynamoDbError> { }], )?; - if let Some(raw) = body.get("ExpressionAttributeValues") - && let Ok(values) = serde_json::from_value::>(raw.clone()) - { - validate_expression_attribute_value_numbers(&values)?; + let values = attribute_value_map(body, "ExpressionAttributeValues"); + if let Some(values) = &values { + validate_expression_attribute_value_numbers(values)?; + } + // The key's values are numbers checked with the same rules and text. + if let Some(key) = attribute_value_map(body, "ExclusiveStartKey") { + validate_expression_attribute_value_numbers(&key)?; + } + if let Some(values) = &values { + extenddb_core::validation::validate_attribute_values_nesting_depth(values.values())?; } crate::validate_enum_fields( @@ -117,6 +151,29 @@ fn scan_request_checks(body: &Value) -> Result<(), DynamoDbError> { ) } +/// The length constraint on `AttributesToGet`: an empty list is refused in the +/// newer form of the constraint text, with no value echoed (measured +/// 2026-10-25 10:38 and 10:55 UTC: alone, with a `FilterExpression`, and with +/// a `FilterExpression` and a malformed number). +fn validate_attributes_to_get_length(body: &Value) -> Result<(), DynamoDbError> { + if body + .get("AttributesToGet") + .and_then(Value::as_array) + .is_some_and(Vec::is_empty) + { + return Err(DynamoDbError::ValidationException( + "1 validation error detected: Value at 'AttributesToGet' failed to satisfy constraint: Member must have length greater than or equal to 1".to_owned(), + )); + } + Ok(()) +} + +/// `field` of the body read as a map of attribute values, or `None` when the +/// field is absent or is not one; [`shape_fault_first`] reports the shape. +fn attribute_value_map(body: &Value, field: &str) -> Option> { + serde_json::from_value(body.get(field)?.clone()).ok() +} + /// The range constraints on `Limit`, `Segment` and `TotalSegments`, every /// fault in one message, in one of the service's two forms (measured /// 2026-10-25, us-east-1). @@ -233,4 +290,6 @@ const RANGE_FIELDS: [RangeField; 3] = [ #[cfg(test)] mod range_tests; #[cfg(test)] +mod request_tests; +#[cfg(test)] mod tests; diff --git a/crates/engine/src/scan_request/request_tests.rs b/crates/engine/src/scan_request/request_tests.rs new file mode 100644 index 000000000..20eb9b822 --- /dev/null +++ b/crates/engine/src/scan_request/request_tests.rs @@ -0,0 +1,167 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The `ScanFilter` in the parameter mix, the `AttributesToGet` length, the +//! numbers in `ExclusiveStartKey` and the nesting depth of +//! `ExpressionAttributeValues`. +//! +//! The rows in every test but the last are requests sent to the service on +//! 2026-10-25 (us-east-1, 10:38 and 10:55 UTC, each request twice, a Scan of +//! an existing table of 29 items) with the answers it gave. The last test +//! applies the same rules to requests that were not sent. + +use extenddb_core::error::DynamoDbError; +use serde_json::{Value, json}; + +use super::validate_scan_request_body; + +const MIX: &str = "Can not use both expression and non-expression parameters in the same request: Non-expression parameters: {ScanFilter} Expression parameters: {FilterExpression}"; +const LENGTH: &str = "1 validation error detected: Value at 'AttributesToGet' failed to satisfy constraint: Member must have length greater than or equal to 1"; +const NUMERIC_BOB: &str = "The parameter cannot be converted to a numeric value: bob"; +const NESTING: &str = "Nesting Levels have exceeded supported limits: Attributes in the item have nested levels beyond supported limit"; + +fn check(body: &Value) -> Option { + match validate_scan_request_body(body) { + Ok(()) => None, + Err(DynamoDbError::ValidationException(msg)) => Some(msg), + Err(other) => panic!("unexpected error {other:?}"), + } +} + +/// A Scan body with the fields of `extra` added to the table name. +fn scan(extra: Value) -> Value { + let mut body = json!({"TableName": "t"}); + for (field, value) in extra.as_object().expect("an object") { + body[field] = value.clone(); + } + body +} + +/// A filter that reads `:n`, which holds `number`. +fn filter(number: &str) -> Value { + json!({ + "FilterExpression": "a = :n", + "ExpressionAttributeValues": {":n": {"N": number}}, + }) +} + +fn with(mut body: Value, extra: Value) -> Value { + for (field, value) in extra.as_object().expect("an object") { + body[field] = value.clone(); + } + body +} + +/// `leaf` inside `levels` lists, each list one level. +fn nested(leaf: &Value, levels: usize) -> Value { + (0..levels).fold(leaf.clone(), |inner, _| json!({"L": [inner]})) +} + +/// A filter that reads `:p`, with `value` under `:d`, which nothing reads. +fn with_unused_value(value: Value) -> Value { + scan(json!({ + "FilterExpression": "a = :p", + "ExpressionAttributeValues": {":p": {"S": "x"}, ":d": value}, + })) +} + +#[test] +fn an_empty_scan_filter_counts_as_present_for_the_parameter_mix() { + let mix = with(scan(filter("1")), json!({"ScanFilter": {}})); + assert_eq!(check(&mix).as_deref(), Some(MIX)); + // Alone, an empty ScanFilter is accepted. + assert_eq!(check(&scan(json!({"ScanFilter": {}}))), None); + // Beside an unused malformed number, the number text. + let unused = scan(json!({ + "ScanFilter": {}, + "ExpressionAttributeValues": {":n": {"N": "bob"}}, + })); + assert_eq!(check(&unused).as_deref(), Some(NUMERIC_BOB)); +} + +#[test] +fn an_empty_attributes_to_get_is_a_length_fault_ahead_of_the_number_check() { + let empty = json!({"AttributesToGet": []}); + for body in [ + scan(empty.clone()), + with(scan(filter("1")), empty.clone()), + with(scan(filter("bob")), empty), + ] { + assert_eq!(check(&body).as_deref(), Some(LENGTH), "{body}"); + } + assert_eq!(check(&scan(json!({"AttributesToGet": ["a"]}))), None); +} + +#[test] +fn a_malformed_number_in_the_exclusive_start_key_is_reported_with_the_number_text() { + let key = |number: &str| json!({"ExclusiveStartKey": {"pk": {"N": number}, "sk": {"S": "x"}}}); + let bad = with(scan(filter("1")), key("bob")); + assert_eq!(check(&bad).as_deref(), Some(NUMERIC_BOB)); + assert_eq!(check(&with(scan(filter("1")), key("1"))), None); +} + +#[test] +fn a_value_nested_32_levels_deep_is_refused_and_31_levels_pass() { + let one = json!({"N": "1"}); + for levels in [32, 33, 40, 60] { + let body = with_unused_value(nested(&one, levels)); + assert_eq!(check(&body).as_deref(), Some(NESTING), "{levels} levels"); + } + // 31 levels reach the checks the handler makes after deserializing. + assert_eq!(check(&with_unused_value(nested(&one, 31))), None); +} + +#[test] +fn the_number_check_runs_before_the_nesting_check() { + let bob = json!({"N": "bob"}); + for levels in [31, 32, 33, 40, 60] { + let body = with_unused_value(nested(&bob, levels)); + assert_eq!( + check(&body).as_deref(), + Some(NUMERIC_BOB), + "{levels} levels" + ); + } + // An M at the top, 31 lists below it. + let body = with_unused_value(json!({"M": {"k": nested(&bob, 31)}})); + assert_eq!(check(&body).as_deref(), Some(NUMERIC_BOB)); +} + +/// Requests the service was not sent. The answers follow from the measured +/// rows and from the order the checks are made in. +#[test] +fn combinations_the_service_was_not_sent_follow_the_order_of_the_checks() { + // The mix names whichever expression parameter is present. + let projection = scan(json!({"ScanFilter": {}, "ProjectionExpression": "a"})); + assert_eq!( + check(&projection).as_deref(), + Some( + "Can not use both expression and non-expression parameters in the same request: Non-expression parameters: {ScanFilter} Expression parameters: {ProjectionExpression}" + ) + ); + // A null ScanFilter is absent. + let null_filter = with(scan(filter("1")), json!({"ScanFilter": null})); + assert_eq!(check(&null_filter), None); + // The length constraint applies beside a ProjectionExpression too. + let beside_projection = scan(json!({"AttributesToGet": [], "ProjectionExpression": "a"})); + assert_eq!(check(&beside_projection).as_deref(), Some(LENGTH)); + // A range fault is reported alone, ahead of the length constraint. + let range = scan(json!({"AttributesToGet": [], "Limit": 0})); + assert_eq!( + check(&range).as_deref(), + Some( + "1 validation error detected: Value at 'Limit' failed to satisfy constraint: Member must have value greater than or equal to 1" + ) + ); + // The values are checked ahead of the key. + let both = with( + scan(filter("bob")), + json!({"ExclusiveStartKey": {"pk": {"N": "fred"}}}), + ); + assert_eq!(check(&both).as_deref(), Some(NUMERIC_BOB)); + // A map nests as a list does. + let map = + |levels: usize| (0..levels).fold(json!({"N": "1"}), |inner, _| json!({"M": {"k": inner}})); + assert_eq!(check(&with_unused_value(map(32))).as_deref(), Some(NESTING)); + assert_eq!(check(&with_unused_value(map(31))), None); +} diff --git a/tests/python/test_scan_request_validation.py b/tests/python/test_scan_request_validation.py new file mode 100644 index 000000000..63cb555fd --- /dev/null +++ b/tests/python/test_scan_request_validation.py @@ -0,0 +1,163 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""Scan request validation: an empty ``ScanFilter`` in the parameter mix, an +empty ``AttributesToGet``, a malformed number in ``ExclusiveStartKey`` and the +nesting depth of ``ExpressionAttributeValues``. + +Every request below is sent as raw signed JSON. Each is a row of the table +measured against DynamoDB (us-east-1) on 2026-10-25 at 10:38 and 10:55 UTC, +each sent twice, on a Scan of an existing table of 29 items, and each expected +answer is the service's. The table here holds 12 items, so the count the +accepted request returns is 12. + +Depth is the number of ``L`` wrappers around the leaf: 32 or more answers the +nesting text, 31 reaches the unused-value check. A malformed number at the +bottom is reported first, at the depths 31 to 60 that were measured. +""" + +from __future__ import annotations + +import json + +import pytest +from botocore.awsrequest import AWSRequest + +from helpers import unique_name, wait_for_active, wait_for_deleted + +ITEMS = 12 + +MIX = ( + "Can not use both expression and non-expression parameters in the same " + "request: Non-expression parameters: {ScanFilter} " + "Expression parameters: {FilterExpression}" +) +LENGTH = ( + "1 validation error detected: Value at 'AttributesToGet' failed to " + "satisfy constraint: Member must have length greater than or equal to 1" +) +NUMERIC_BOB = "The parameter cannot be converted to a numeric value: bob" +NESTING = ( + "Nesting Levels have exceeded supported limits: " + "Attributes in the item have nested levels beyond supported limit" +) +UNUSED = "Value provided in ExpressionAttributeValues unused in expressions: keys: {:d}" + +VALID_FILTER = '"FilterExpression": "a = :n", "ExpressionAttributeValues": {":n": {"N": "1"}}' +BOB_FILTER = '"FilterExpression": "a = :n", "ExpressionAttributeValues": {":n": {"N": "bob"}}' +ONE = {"N": "1"} +BOB = {"N": "bob"} + +# (id, the fields after TableName as JSON text, the expected message) +REFUSED = [ + ("ScanFilter-empty-with-FilterExpression", '"ScanFilter": {}, ' + VALID_FILTER, MIX), + ("ScanFilter-empty-with-unused-bob", + '"ScanFilter": {}, "ExpressionAttributeValues": {":n": {"N": "bob"}}', NUMERIC_BOB), + ("AttributesToGet-empty", '"AttributesToGet": []', LENGTH), + ("AttributesToGet-empty-with-FilterExpression", + '"AttributesToGet": [], ' + VALID_FILTER, LENGTH), + ("AttributesToGet-empty-with-FilterExpression-and-bob", + '"AttributesToGet": [], ' + BOB_FILTER, LENGTH), + ("ExclusiveStartKey-bob-on-an-S-key", + '"ExclusiveStartKey": {"pk": {"N": "bob"}, "sk": {"S": "x"}}, ' + VALID_FILTER, + NUMERIC_BOB), +] + + +def nest(leaf: dict, levels: int) -> dict: + """``leaf`` inside ``levels`` lists.""" + value = leaf + for _ in range(levels): + value = {"L": [value]} + return value + + +def unused_value(value: dict) -> str: + """A filter that reads ``:p`` and ``value`` under ``:d``, which nothing reads.""" + return ('"FilterExpression": "a = :p", "ExpressionAttributeValues": ' + + json.dumps({":p": {"S": "x"}, ":d": value})) + + +def _raw(client, operation: str, body_text: str): + """POST ``body_text`` exactly as written, signed with the client's credentials.""" + req = AWSRequest(method="POST", url=client.meta.endpoint_url, data=body_text, + headers={"Content-Type": "application/x-amz-json-1.0", + "X-Amz-Target": f"DynamoDB_20120810.{operation}"}) + client._request_signer.sign(operation, req) + resp = client._endpoint.http_session.send(req.prepare()) + return resp.status_code, json.loads(resp.content.decode("utf-8", "replace") or "{}") + + +def _scan(client, table: str, fields: str): + return _raw(client, "Scan", '{"TableName": %s, %s}' % (json.dumps(table), fields)) + + +def _validation_message(status: int, payload: dict) -> str: + assert status == 400, payload + assert payload.get("__type", "").endswith("ValidationException"), payload + return payload.get("message", payload.get("Message")) + + +@pytest.fixture(scope="module") +def table(dynamodb_client): + """pk S, sk S, with ITEMS items.""" + name = unique_name("scanreq") + dynamodb_client.create_table( + TableName=name, + AttributeDefinitions=[ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "sk", "AttributeType": "S"}, + ], + KeySchema=[ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"}, + ], + BillingMode="PAY_PER_REQUEST", + ) + wait_for_active(dynamodb_client, name) + for i in range(ITEMS): + dynamodb_client.put_item(TableName=name, Item={ + "pk": {"S": f"p{i}"}, "sk": {"S": "s"}, "a": {"N": str(i)}}) + yield name + dynamodb_client.delete_table(TableName=name) + wait_for_deleted(dynamodb_client, name) + + +class TestRequestChecks: + """The parameter mix, the length constraint and the numbers in the key.""" + + @pytest.mark.parametrize("fields, expected", [r[1:] for r in REFUSED], + ids=[r[0] for r in REFUSED]) + def test_rows(self, dynamodb_client, table, fields, expected): + assert _validation_message(*_scan(dynamodb_client, table, fields)) == expected + + def test_an_empty_scan_filter_alone_is_accepted(self, dynamodb_client, table): + status, payload = _scan(dynamodb_client, table, + '"ScanFilter": {}, "ConsistentRead": true') + assert status == 200, payload + assert (payload["Count"], payload["ScannedCount"]) == (ITEMS, ITEMS) + + +class TestNestingDepth: + """``ExpressionAttributeValues`` nested in lists.""" + + @pytest.mark.parametrize("levels", [32, 33, 40, 60]) + def test_a_valid_number_at_the_bottom_is_refused_from_32_levels( + self, dynamodb_client, table, levels): + fields = unused_value(nest(ONE, levels)) + assert _validation_message(*_scan(dynamodb_client, table, fields)) == NESTING + + def test_31_levels_reach_the_unused_value_check(self, dynamodb_client, table): + fields = unused_value(nest(ONE, 31)) + assert _validation_message(*_scan(dynamodb_client, table, fields)) == UNUSED + + @pytest.mark.parametrize("levels", [31, 32, 33, 40, 60]) + def test_a_malformed_number_at_the_bottom_is_reported_first( + self, dynamodb_client, table, levels): + fields = unused_value(nest(BOB, levels)) + assert _validation_message(*_scan(dynamodb_client, table, fields)) == NUMERIC_BOB + + def test_a_map_at_32_levels_with_a_malformed_number_is_reported_first( + self, dynamodb_client, table): + fields = unused_value({"M": {"k": nest(BOB, 31)}}) + assert _validation_message(*_scan(dynamodb_client, table, fields)) == NUMERIC_BOB From e20ffeed02d21e093a3477792527ad9801c89481 Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Sat, 31 Oct 2026 15:00:54 +0000 Subject: [PATCH 2/9] fix(scan): aggregate the AttributesToGet length with the range faults Measured against the service on 2026-10-30 at 12:40 UTC, each request sent twice, as a raw signed Scan: - Limit 0 with AttributesToGet [] answers "2 validation errors detected: Value at 'AttributesToGet' failed to satisfy constraint: Member must have length greater than or equal to 1; Value at 'Limit' failed to satisfy constraint: Member must have value greater than or equal to 1". The length constraint is one of the constraints the service aggregates into one message, listed ahead of Limit. The check reported the Limit fault alone and never reached the length check. - AttributesToGet [] with ScanFilter {} and a FilterExpression answers the length clause alone, so the aggregated constraints come ahead of the parameter mix. - A malformed N in ExclusiveStartKey with no filter, and in the index key of a GSI scan, answers "The parameter cannot be converted to a numeric value: bob"; with ReturnConsumedCapacity BAD as well, the enum text alone. - An L nested 32 levels with Select BAD answers the nesting text, so the nesting check comes ahead of the Select enum. The length check joins the range check, and the module doc, the comment at the top of handle_scan and the tests cite the measured orders. Where the length clause sits beside a Segment or TotalSegments fault alone, and its older form, are not measured: the code lists it first in the newer form, as beside Limit, and last in the older form with the empty list echoed, as the older form reverses the newer one. Both are in the unit test of requests the service was not sent. tests/python/test_scan_request_validation.py has 24 rows: the six new ones send the measured requests above, and the table gains a GSI for the index-scan row. Refs #564 --- crates/engine/src/scan.rs | 16 +- crates/engine/src/scan_request.rs | 196 +++++++++--------- .../engine/src/scan_request/request_tests.rs | 60 +++++- tests/python/test_scan_request_validation.py | 45 +++- 4 files changed, 201 insertions(+), 116 deletions(-) diff --git a/crates/engine/src/scan.rs b/crates/engine/src/scan.rs index 9a8e15296..c2b51ee61 100755 --- a/crates/engine/src/scan.rs +++ b/crates/engine/src/scan.rs @@ -41,13 +41,15 @@ pub async fn handle_scan( body: Value, ctx: &OperationContext, ) -> Result { - // Request-time checks on the raw body, in the service's order: the Limit, - // Segment and TotalSegments ranges, the expression and non-expression - // parameter mix, the ReturnConsumedCapacity enum, every number in - // ExpressionAttributeValues, the Select enum. They run before typed - // deserialization so a field that does not deserialize does not stop the - // ones before it; a field of the wrong JSON type anywhere in the body is - // reported ahead of them; see `scan_request`. + // Request-time checks on the raw body, in the service's order: the + // AttributesToGet length and the Limit, Segment and TotalSegments ranges + // in one message, the expression and non-expression parameter mix, the + // ReturnConsumedCapacity enum, every number in ExpressionAttributeValues + // and ExclusiveStartKey, the nesting depth of ExpressionAttributeValues, + // the Select enum. They run before typed deserialization so a field that + // does not deserialize does not stop the ones before it; a field of the + // wrong JSON type anywhere in the body is reported ahead of them; see + // `scan_request`. crate::scan_request::validate_scan_request_body(&body)?; let input: ScanInput = serde_json::from_value(body).map_err(crate::deserialize_error)?; diff --git a/crates/engine/src/scan_request.rs b/crates/engine/src/scan_request.rs index e4afcbbe2..62362b6be 100644 --- a/crates/engine/src/scan_request.rs +++ b/crates/engine/src/scan_request.rs @@ -4,51 +4,67 @@ //! The `Scan` checks that run on the request body alone, before the body is //! deserialized and before the table is looked up, in the service's order. //! -//! Measured 2026-10-25 and 2026-10-30 (us-east-1, each request twice): +//! Measured 2026-10-25, 2026-10-30 and 2026-10-31 (us-east-1, each request +//! twice): //! -//! 1. the range constraints on `Limit` (at least 1), `Segment` (0 to 999999) -//! and `TotalSegments` (1 to 1000000), each checked when present, whether or -//! not the other segment field is, every fault in one message: `1 -//! validation error detected: Value at 'Limit' failed to satisfy -//! constraint: Member must have value greater than or equal to 1`, or `2 -//! validation errors detected: ...; ...` in the order `Limit`, `Segment`, -//! `TotalSegments`. A float or a value outside int32 in any of the three, -//! or a value that is not one AttributeValue in `ExpressionAttributeValues`, -//! in `ExclusiveStartKey` or in a `ScanFilter` `AttributeValueList`, +//! 1. the constraints the service aggregates into one message: the length of +//! `AttributesToGet` (at least 1) and the ranges of `Limit` (at least 1), +//! `Segment` (0 to 999999) and `TotalSegments` (1 to 1000000), each checked +//! when present, whether or not the other segment field is, every fault in +//! one message: `1 validation error detected: Value at 'Limit' failed to +//! satisfy constraint: Member must have value greater than or equal to 1`, +//! or `2 validation errors detected: ...; ...` in the order +//! `AttributesToGet`, `Limit`, `Segment`, `TotalSegments`. `Limit: 0` with +//! `AttributesToGet: []` answers the length clause and then the `Limit` +//! clause; the empty list alone, with a `FilterExpression`, and with a +//! `FilterExpression` and a number fault answers the length clause alone, +//! so the length comes ahead of the number check. A float or a value +//! outside int32 in any of the three number fields, or a value that is not +//! one AttributeValue in `ExpressionAttributeValues`, in +//! `ExclusiveStartKey` or in a `ScanFilter` `AttributeValueList`, //! switches the message to the older form, `Value '-2147483648' at 'limit' //! ...`, with the value saturated or truncated, in the order -//! `totalSegments`, `segment`, `limit`; see `validate_ranges`; -//! 2. the length constraint on `AttributesToGet`: an empty list answers `1 -//! validation error detected: Value at 'AttributesToGet' failed to satisfy -//! constraint: Member must have length greater than or equal to 1`, alone, -//! with a `FilterExpression`, and with a `FilterExpression` and a number -//! fault, so it comes ahead of the number check; -//! 3. the expression and non-expression parameter mix, in which a `ScanFilter` +//! `totalSegments`, `segment`, `limit`; see `validate_constraints`; +//! 2. the expression and non-expression parameter mix, in which a `ScanFilter` //! object counts as present whether or not it is empty (`ScanFilter: {}` //! with a `FilterExpression` answers the mix, `ScanFilter: {}` alone is -//! accepted); -//! 4. the `ReturnConsumedCapacity` enum; -//! 5. every number in `ExpressionAttributeValues`, `NS` members and values +//! accepted); `AttributesToGet: []` with `ScanFilter: {}` and a +//! `FilterExpression` answers the length clause, so the constraints come +//! ahead of the mix; +//! 3. the `ReturnConsumedCapacity` enum; +//! 4. every number in `ExpressionAttributeValues`, `NS` members and values //! nested in `L` and `M` included, whether a `FilterExpression` reads the //! value, nothing reads it, or the request carries no expression at all; a //! Unicode decimal digit counts as the digit it stands for; then every //! number in the `ExclusiveStartKey` values, which answers the same text as -//! a malformed number in a value; -//! 6. the nesting depth of `ExpressionAttributeValues`: a value nested 32 +//! a malformed number in a value, with no filter and on an index scan too, +//! and after the `ReturnConsumedCapacity` enum; +//! 5. the nesting depth of `ExpressionAttributeValues`: a value nested 32 //! levels deep in `L` answers `Nesting Levels have exceeded supported //! limits: Attributes in the item have nested levels beyond supported //! limit`, 31 levels pass, and a malformed number anywhere is reported //! first. The check counts `M` levels the same way as `L` levels; only `L` //! was measured with a valid number at the bottom; -//! 7. the `Select` enum. Both enums invalid answers the `ReturnConsumedCapacity` -//! error alone, so the two are not aggregated. +//! 6. the `Select` enum, after the nesting check (32 levels with `Select: +//! "BAD"` answers the nesting text). Both enums invalid answers the +//! `ReturnConsumedCapacity` error alone, so the two are not aggregated. //! //! Not measured, and checked in the order listed above: the `AttributesToGet` -//! length against the range constraints and against the enums, the -//! `ExclusiveStartKey` number check against the `ExpressionAttributeValues` -//! one, and the nesting check against the `Select` enum. A body nested deeper -//! than the JSON parser accepts is a `SerializationException` from the parser, -//! ahead of every check here. +//! length clause beside a `Segment` or `TotalSegments` fault without a `Limit` +//! fault (listed first, as beside `Limit`), the older form of the length +//! clause (the empty list echoed, `Value '[]' at 'attributesToGet' ...`, and +//! listed last, as the older form reverses the newer one), the length clause +//! against the enums, the `ExclusiveStartKey` number check against the +//! `ExpressionAttributeValues` one, `M` levels in the nesting count, and the +//! nesting of a value an expression reads or of a request with no expression. +//! +//! The JSON parser accepts 128 levels of nesting, which is 62 `L` levels; a +//! body nested deeper is a `SerializationException` from the parser, ahead of +//! every check here. The service answers the nesting text from 32 to 62 levels +//! and, from 63 to 100 levels, `ExpressionAttributeValues contains invalid +//! value: Nesting Levels have exceeded supported limits: Attributes in the +//! item have nested levels beyond supported limit for key :d` (measured +//! 2026-10-30), which this module does not reach. //! //! Each of these is reported ahead of the next one, ahead of the table lookup, //! and ahead of the checks the handler makes after deserializing the body @@ -70,8 +86,8 @@ use crate::query_key_values::{ }; /// The `Scan` checks that run on the raw request body, in the service's order: -/// the `Limit`, `Segment` and `TotalSegments` ranges, the `AttributesToGet` -/// length, the expression and non-expression parameter mix, the +/// the `AttributesToGet` length and the `Limit`, `Segment` and `TotalSegments` +/// ranges in one message, the expression and non-expression parameter mix, the /// `ReturnConsumedCapacity` enum, every number in `ExpressionAttributeValues` /// and `ExclusiveStartKey`, the nesting depth of `ExpressionAttributeValues`, /// then the `Select` enum. @@ -95,8 +111,7 @@ pub fn validate_scan_request_body(body: &Value) -> Result<(), DynamoDbError> { /// The `Scan` checks in the service's order; see /// [`validate_scan_request_body`]. fn scan_request_checks(body: &Value) -> Result<(), DynamoDbError> { - validate_ranges(body)?; - validate_attributes_to_get_length(body)?; + validate_constraints(body)?; extenddb_core::validation::validate_no_expression_param_mixing( &[ @@ -151,46 +166,35 @@ fn scan_request_checks(body: &Value) -> Result<(), DynamoDbError> { ) } -/// The length constraint on `AttributesToGet`: an empty list is refused in the -/// newer form of the constraint text, with no value echoed (measured -/// 2026-10-25 10:38 and 10:55 UTC: alone, with a `FilterExpression`, and with -/// a `FilterExpression` and a malformed number). -fn validate_attributes_to_get_length(body: &Value) -> Result<(), DynamoDbError> { - if body - .get("AttributesToGet") - .and_then(Value::as_array) - .is_some_and(Vec::is_empty) - { - return Err(DynamoDbError::ValidationException( - "1 validation error detected: Value at 'AttributesToGet' failed to satisfy constraint: Member must have length greater than or equal to 1".to_owned(), - )); - } - Ok(()) -} - /// `field` of the body read as a map of attribute values, or `None` when the /// field is absent or is not one; [`shape_fault_first`] reports the shape. fn attribute_value_map(body: &Value, field: &str) -> Option> { serde_json::from_value(body.get(field)?.clone()).ok() } -/// The range constraints on `Limit`, `Segment` and `TotalSegments`, every -/// fault in one message, in one of the service's two forms (measured -/// 2026-10-25, us-east-1). +/// The constraints the service aggregates into one message: the length of +/// `AttributesToGet` and the ranges of `Limit`, `Segment` and `TotalSegments`, +/// every fault in one message, in one of the service's two forms (measured +/// 2026-10-25 and 2026-10-30, us-east-1). /// -/// When each field present is an integer inside the int32 range, the newer -/// form: `Value at '' failed to satisfy constraint: ...` with the field -/// capitalised and no value echoed, in the order `Limit`, `Segment`, -/// `TotalSegments`. When any of the three is a float or outside int32, or a -/// value in `ExpressionAttributeValues`, `ExclusiveStartKey` or a `ScanFilter` -/// `AttributeValueList` is not one AttributeValue (see +/// When each number field present is an integer inside the int32 range, the +/// newer form: `Value at '' failed to satisfy constraint: ...` with the +/// field capitalised and no value echoed, in the order `AttributesToGet`, +/// `Limit`, `Segment`, `TotalSegments` (`Limit: 0` with `AttributesToGet: []` +/// answers the length clause, then the `Limit` clause, measured 2026-10-30 +/// 12:40 UTC; the length clause beside a `Segment` or `TotalSegments` fault +/// alone is not measured). When any of the three is a float or outside int32, +/// or a value in `ExpressionAttributeValues`, `ExclusiveStartKey` or a +/// `ScanFilter` `AttributeValueList` is not one AttributeValue (see /// [`has_unreadable_attribute_value`]), the older form: `Value '' at /// '' ...` with the value as [`saturated_int32`] reads it and the field /// in lower camel case, in the reverse of the newer form's order: /// `totalSegments`, `segment`, `limit` (measured 2026-10-30 12:40 UTC with a /// fault in each of the three, and on 2026-10-25 with `segment` then `limit` -/// and `totalSegments` then `limit`). -fn validate_ranges(body: &Value) -> Result<(), DynamoDbError> { +/// and `totalSegments` then `limit`). The older form of the length clause is +/// not measured: the code echoes the empty list, `Value '[]' at +/// 'attributesToGet' ...`, and lists it last. +fn validate_constraints(body: &Value) -> Result<(), DynamoDbError> { let numbers: Vec<(&RangeField, &serde_json::Number)> = RANGE_FIELDS .iter() .filter_map(|range| { @@ -205,48 +209,50 @@ fn validate_ranges(body: &Value) -> Result<(), DynamoDbError> { .as_i64() .is_none_or(|value| i32::try_from(value).is_err()) }); - let faults: Vec<(&RangeField, i32, String)> = numbers - .into_iter() - .filter_map(|(range, number)| { - let value = saturated_int32(number); - if value < range.min { - Some(( - range, - value, - format!("greater than or equal to {}", range.min), - )) - } else { - range - .max - .filter(|max| value > *max) - .map(|max| (range, value, format!("less than or equal to {max}"))) - } - }) - .collect(); + // Each fault as its newer-form and its older-form clause, in the newer + // form's order. + let mut faults: Vec<(String, String)> = Vec::new(); + if body + .get("AttributesToGet") + .and_then(Value::as_array) + .is_some_and(Vec::is_empty) + { + faults.push(( + "Value at 'AttributesToGet' failed to satisfy constraint: Member must have length greater than or equal to 1".to_owned(), + "Value '[]' at 'attributesToGet' failed to satisfy constraint: Member must have length greater than or equal to 1".to_owned(), + )); + } + for (range, number) in numbers { + let value = saturated_int32(number); + let constraint = if value < range.min { + format!("greater than or equal to {}", range.min) + } else if let Some(max) = range.max.filter(|max| value > *max) { + format!("less than or equal to {max}") + } else { + continue; + }; + faults.push(( + format!( + "Value at '{}' failed to satisfy constraint: Member must have value {constraint}", + range.field + ), + format!( + "Value '{value}' at '{}' failed to satisfy constraint: Member must have value {constraint}", + range.older_field + ), + )); + } if faults.is_empty() { return Ok(()); } - let clauses: Vec = if older_form { + let clauses: Vec<&str> = if older_form { faults .iter() .rev() - .map(|(range, value, constraint)| { - format!( - "Value '{value}' at '{}' failed to satisfy constraint: Member must have value {constraint}", - range.older_field - ) - }) + .map(|(_, older)| older.as_str()) .collect() } else { - faults - .iter() - .map(|(range, _, constraint)| { - format!( - "Value at '{}' failed to satisfy constraint: Member must have value {constraint}", - range.field - ) - }) - .collect() + faults.iter().map(|(newer, _)| newer.as_str()).collect() }; let count = clauses.len(); Err(DynamoDbError::ValidationException(format!( diff --git a/crates/engine/src/scan_request/request_tests.rs b/crates/engine/src/scan_request/request_tests.rs index 20eb9b822..3f3a568e2 100644 --- a/crates/engine/src/scan_request/request_tests.rs +++ b/crates/engine/src/scan_request/request_tests.rs @@ -5,10 +5,11 @@ //! numbers in `ExclusiveStartKey` and the nesting depth of //! `ExpressionAttributeValues`. //! -//! The rows in every test but the last are requests sent to the service on -//! 2026-10-25 (us-east-1, 10:38 and 10:55 UTC, each request twice, a Scan of -//! an existing table of 29 items) with the answers it gave. The last test -//! applies the same rules to requests that were not sent. +//! The rows in every test but the last are requests sent to the service +//! (us-east-1, each request twice, a Scan of an existing table) with the +//! answers it gave: on 2026-10-25 at 10:38 and 10:55 UTC, and on 2026-10-30 at +//! 12:40 UTC where a test says so. The last test applies the same rules to +//! requests that were not sent. use extenddb_core::error::DynamoDbError; use serde_json::{Value, json}; @@ -85,19 +86,44 @@ fn an_empty_attributes_to_get_is_a_length_fault_ahead_of_the_number_check() { for body in [ scan(empty.clone()), with(scan(filter("1")), empty.clone()), - with(scan(filter("bob")), empty), + with(scan(filter("bob")), empty.clone()), + // Measured 2026-10-30 12:40 UTC: ahead of the parameter mix too. + with(scan(filter("1")), with(empty, json!({"ScanFilter": {}}))), ] { assert_eq!(check(&body).as_deref(), Some(LENGTH), "{body}"); } assert_eq!(check(&scan(json!({"AttributesToGet": ["a"]}))), None); } +/// Measured 2026-10-30 12:40 UTC: the length constraint is aggregated with the +/// range constraints, ahead of `Limit`. +#[test] +fn an_empty_attributes_to_get_joins_the_range_faults() { + let body = scan(json!({"AttributesToGet": [], "Limit": 0})); + assert_eq!( + check(&body).as_deref(), + Some( + "2 validation errors detected: Value at 'AttributesToGet' failed to satisfy constraint: Member must have length greater than or equal to 1; Value at 'Limit' failed to satisfy constraint: Member must have value greater than or equal to 1" + ) + ); +} + #[test] fn a_malformed_number_in_the_exclusive_start_key_is_reported_with_the_number_text() { let key = |number: &str| json!({"ExclusiveStartKey": {"pk": {"N": number}, "sk": {"S": "x"}}}); let bad = with(scan(filter("1")), key("bob")); assert_eq!(check(&bad).as_deref(), Some(NUMERIC_BOB)); assert_eq!(check(&with(scan(filter("1")), key("1"))), None); + // Measured 2026-10-30 12:40 UTC: with no filter, the same text; the + // ReturnConsumedCapacity enum is checked first. + assert_eq!(check(&scan(key("bob"))).as_deref(), Some(NUMERIC_BOB)); + let enum_first = with(scan(key("bob")), json!({"ReturnConsumedCapacity": "BAD"})); + assert_eq!( + check(&enum_first).as_deref(), + Some( + "1 validation error detected: Value 'BAD' at 'returnConsumedCapacity' failed to satisfy constraint: Member must satisfy enum value set: [INDEXES, TOTAL, NONE]" + ) + ); } #[test] @@ -109,6 +135,12 @@ fn a_value_nested_32_levels_deep_is_refused_and_31_levels_pass() { } // 31 levels reach the checks the handler makes after deserializing. assert_eq!(check(&with_unused_value(nested(&one, 31))), None); + // Measured 2026-10-30 12:40 UTC: ahead of the Select enum. + let select = with( + with_unused_value(nested(&one, 32)), + json!({"Select": "BAD"}), + ); + assert_eq!(check(&select).as_deref(), Some(NESTING)); } #[test] @@ -145,12 +177,22 @@ fn combinations_the_service_was_not_sent_follow_the_order_of_the_checks() { // The length constraint applies beside a ProjectionExpression too. let beside_projection = scan(json!({"AttributesToGet": [], "ProjectionExpression": "a"})); assert_eq!(check(&beside_projection).as_deref(), Some(LENGTH)); - // A range fault is reported alone, ahead of the length constraint. - let range = scan(json!({"AttributesToGet": [], "Limit": 0})); + // The length clause keeps its place ahead of the range clauses beside a + // Segment fault, as measured beside a Limit fault. + let segment = scan(json!({"AttributesToGet": [], "Segment": -1, "TotalSegments": 2})); + assert_eq!( + check(&segment).as_deref(), + Some( + "2 validation errors detected: Value at 'AttributesToGet' failed to satisfy constraint: Member must have length greater than or equal to 1; Value at 'Segment' failed to satisfy constraint: Member must have value greater than or equal to 0" + ) + ); + // In the older form the clauses are reversed and the values echoed, so + // the empty list comes last. + let older = scan(json!({"AttributesToGet": [], "Limit": 0.0})); assert_eq!( - check(&range).as_deref(), + check(&older).as_deref(), Some( - "1 validation error detected: Value at 'Limit' failed to satisfy constraint: Member must have value greater than or equal to 1" + "2 validation errors detected: Value '0' at 'limit' failed to satisfy constraint: Member must have value greater than or equal to 1; Value '[]' at 'attributesToGet' failed to satisfy constraint: Member must have length greater than or equal to 1" ) ); // The values are checked ahead of the key. diff --git a/tests/python/test_scan_request_validation.py b/tests/python/test_scan_request_validation.py index 63cb555fd..4c57196de 100644 --- a/tests/python/test_scan_request_validation.py +++ b/tests/python/test_scan_request_validation.py @@ -6,10 +6,11 @@ nesting depth of ``ExpressionAttributeValues``. Every request below is sent as raw signed JSON. Each is a row of the table -measured against DynamoDB (us-east-1) on 2026-10-25 at 10:38 and 10:55 UTC, -each sent twice, on a Scan of an existing table of 29 items, and each expected -answer is the service's. The table here holds 12 items, so the count the -accepted request returns is 12. +measured against DynamoDB (us-east-1) on 2026-10-25 at 10:38 and 10:55 UTC on +a Scan of an existing table of 29 items, or of the one measured on 2026-10-30 +at 12:40 UTC where a row says so, each sent twice, and each expected answer is +the service's. The table here holds 12 items, so the count the accepted +request returns is 12. Depth is the number of ``L`` wrappers around the leaf: 32 or more answers the nesting text, 31 reaches the unused-value check. A malformed number at the @@ -36,7 +37,17 @@ "1 validation error detected: Value at 'AttributesToGet' failed to " "satisfy constraint: Member must have length greater than or equal to 1" ) +LENGTH_AND_LIMIT = ( + "2 validation errors detected: Value at 'AttributesToGet' failed to " + "satisfy constraint: Member must have length greater than or equal to 1; " + "Value at 'Limit' failed to satisfy constraint: Member must have value " + "greater than or equal to 1" +) NUMERIC_BOB = "The parameter cannot be converted to a numeric value: bob" +RCC_ENUM = ( + "1 validation error detected: Value 'BAD' at 'returnConsumedCapacity' failed " + "to satisfy constraint: Member must satisfy enum value set: [INDEXES, TOTAL, NONE]" +) NESTING = ( "Nesting Levels have exceeded supported limits: " "Attributes in the item have nested levels beyond supported limit" @@ -61,6 +72,19 @@ ("ExclusiveStartKey-bob-on-an-S-key", '"ExclusiveStartKey": {"pk": {"N": "bob"}, "sk": {"S": "x"}}, ' + VALID_FILTER, NUMERIC_BOB), + # Measured 2026-10-30 12:40 UTC. + ("AttributesToGet-empty-with-Limit-0", '"AttributesToGet": [], "Limit": 0', + LENGTH_AND_LIMIT), + ("AttributesToGet-empty-with-ScanFilter-empty-and-FilterExpression", + '"AttributesToGet": [], "ScanFilter": {}, ' + VALID_FILTER, LENGTH), + ("ExclusiveStartKey-bob-with-no-filter", + '"ExclusiveStartKey": {"pk": {"N": "bob"}, "sk": {"S": "x"}}', NUMERIC_BOB), + ("ExclusiveStartKey-bob-in-the-index-key-of-a-GSI-scan", + '"IndexName": "gsi", "ExclusiveStartKey": {"gpk": {"N": "bob"}, "pk": {"S": "p"}, ' + '"sk": {"S": "x"}}', NUMERIC_BOB), + ("ExclusiveStartKey-bob-with-ReturnConsumedCapacity-BAD", + '"ExclusiveStartKey": {"pk": {"N": "bob"}, "sk": {"S": "x"}}, ' + '"ReturnConsumedCapacity": "BAD"', RCC_ENUM), ] @@ -100,18 +124,24 @@ def _validation_message(status: int, payload: dict) -> str: @pytest.fixture(scope="module") def table(dynamodb_client): - """pk S, sk S, with ITEMS items.""" + """pk S, sk S, a GSI ``gsi`` on gpk S, with ITEMS items.""" name = unique_name("scanreq") dynamodb_client.create_table( TableName=name, AttributeDefinitions=[ {"AttributeName": "pk", "AttributeType": "S"}, {"AttributeName": "sk", "AttributeType": "S"}, + {"AttributeName": "gpk", "AttributeType": "S"}, ], KeySchema=[ {"AttributeName": "pk", "KeyType": "HASH"}, {"AttributeName": "sk", "KeyType": "RANGE"}, ], + GlobalSecondaryIndexes=[{ + "IndexName": "gsi", + "KeySchema": [{"AttributeName": "gpk", "KeyType": "HASH"}], + "Projection": {"ProjectionType": "ALL"}, + }], BillingMode="PAY_PER_REQUEST", ) wait_for_active(dynamodb_client, name) @@ -151,6 +181,11 @@ def test_31_levels_reach_the_unused_value_check(self, dynamodb_client, table): fields = unused_value(nest(ONE, 31)) assert _validation_message(*_scan(dynamodb_client, table, fields)) == UNUSED + def test_32_levels_are_refused_ahead_of_the_select_enum(self, dynamodb_client, table): + # Measured 2026-10-30 12:40 UTC. + fields = unused_value(nest(ONE, 32)) + ', "Select": "BAD"' + assert _validation_message(*_scan(dynamodb_client, table, fields)) == NESTING + @pytest.mark.parametrize("levels", [31, 32, 33, 40, 60]) def test_a_malformed_number_at_the_bottom_is_reported_first( self, dynamodb_client, table, levels): From 5741869d23d7e4d1a056318023984e3aeba528dd Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Wed, 4 Nov 2026 23:04:16 +0000 Subject: [PATCH 3/9] fix(scan): check the ExclusiveStartKey numbers at the key's own stage Measured against the service on 2026-10-31 between 23:54 and 23:56 UTC, each request sent twice, as a raw signed Scan with {"pk": {"N": "bob"}, "sk": {"S": "x"}} as the ExclusiveStartKey: - Every other check comes ahead of the key's number: the Limit, Segment and AttributesToGet constraint clauses, the ReturnConsumedCapacity and Select enum clauses, the Select rule for SPECIFIC_ATTRIBUTES, the parameter mix, the nesting text, the FilterExpression syntax error and the unused-value text each answer their own text, and a malformed number in ExpressionAttributeValues is reported ahead of one in the key. The check ran at request time, between the value numbers and the nesting check, so the Select enum, the nesting text, the SPECIFIC_ATTRIBUTES rule, the filter syntax and the unused value lost to the key's number. - Inside the key a number fault comes ahead of an empty value or an unknown type descriptor, in either member order, and a number nested in the L of an attribute that is not a key attribute counts: {"pk": {}, "sk": {"N": "bob"}} answers The parameter cannot be converted to a numeric value: bob. The typed deserialization of the body reported the empty value first, with the deserializer's text. - A key whose only fault is an empty value answers The provided starting key is invalid: Supplied AttributeValue is empty, must contain exactly one of the supported datatypes. The deserializer's text had no prefix. The handler now takes the key out of the body before the typed deserialization when its values do not deserialize for a reason other than a shape fault, and checks the key at its own stage, right before the key schema: every number in it, in the sorted order of its members, with the value check's rules and text, then the deserializer's fault with the service's prefix. A key of the wrong JSON shape stays in the body and is reported first, as before. Not measured: the key against the table lookup, the index resolution, the index rules and the Segment pairing, which the handler checks ahead of it; an unreadable-only key against the filter syntax and unused-value checks; the prefix on the deserializer's texts for two type descriptors and for an empty or duplicate set; and which of two number faults is named when the request lists the members the other way round (sorted order names bob either way). The unit test that applied the checks' order to rows the service had not been sent is renamed: all seven of its rows were sent on 2026-10-31 and answered as asserted. request_tests.rs has the measured rows, and tests/python/test_scan_request_validation.py sends them as raw signed requests: seven rows for the other checks ahead of the key, the nesting text, the SPECIFIC_ATTRIBUTES rule and the filter syntax ahead of it, nine rows for the faults inside the key, and three number texts on an N hash key. The parser here words the filter syntax error differently from the service, and the Scan text for the SPECIFIC_ATTRIBUTES rule has no "1 validation error detected" prefix here, so those two tests assert the check's place and not the text. --- crates/engine/src/query_key_values.rs | 22 +- crates/engine/src/scan.rs | 24 ++- crates/engine/src/scan_request.rs | 172 +++++++++++++-- .../engine/src/scan_request/request_tests.rs | 204 ++++++++++++++++-- tests/python/test_scan_request_validation.py | 158 +++++++++++++- 5 files changed, 519 insertions(+), 61 deletions(-) diff --git a/crates/engine/src/query_key_values.rs b/crates/engine/src/query_key_values.rs index 6948b5db1..ead1c93f9 100644 --- a/crates/engine/src/query_key_values.rs +++ b/crates/engine/src/query_key_values.rs @@ -98,17 +98,23 @@ pub(crate) fn shape_fault_first( check: DynamoDbError, ) -> DynamoDbError { match serde_json::from_value::(body.clone()) { - Err(e) => match crate::deserialize_error(e) { - DynamoDbError::SerializationException(msg) if msg.contains(UNKNOWN_TYPE_DESCRIPTOR) => { - check - } - shape @ DynamoDbError::SerializationException(_) => shape, - _ => check, - }, + Err(e) => { + let fault = crate::deserialize_error(e); + if is_shape_fault(&fault) { fault } else { check } + } Ok(_) => check, } } +/// Whether `fault`, from [`crate::deserialize_error`], is a fault in the shape +/// of the body (a field of the wrong JSON type), which the service reports +/// ahead of every check, rather than an attribute value the deserializer +/// refuses with a `ValidationException` or an unknown type descriptor; see +/// [`shape_fault_first`]. +pub(crate) fn is_shape_fault(fault: &DynamoDbError) -> bool { + matches!(fault, DynamoDbError::SerializationException(msg) if !msg.contains(UNKNOWN_TYPE_DESCRIPTOR)) +} + /// The deserializer's text for an AttributeValue whose type descriptor is not /// one of the service's. const UNKNOWN_TYPE_DESCRIPTOR: &str = "unknown AttributeValue type descriptor"; @@ -327,7 +333,7 @@ fn validate_value_numbers(value: &AttributeValue) -> Result<(), DynamoDbError> { /// One number text against the number rules, with the service's `Query` /// message for an empty value and the original text echoed for a malformed one. -fn validate_number_text(text: &str) -> Result<(), DynamoDbError> { +pub(crate) fn validate_number_text(text: &str) -> Result<(), DynamoDbError> { let folded = crate::decimal_digits::fold_decimal_digits(text); match extenddb_core::validation::number::validate_and_normalize_number(&folded) { Ok(_) => Ok(()), diff --git a/crates/engine/src/scan.rs b/crates/engine/src/scan.rs index c2b51ee61..44f84d586 100755 --- a/crates/engine/src/scan.rs +++ b/crates/engine/src/scan.rs @@ -38,19 +38,23 @@ use crate::{DispatchMetrics, DispatchResult}; /// Returns `DynamoDbError` for validation failures, missing tables, or storage errors. #[allow(clippy::cast_possible_wrap)] // item counts won't exceed i64::MAX pub async fn handle_scan( - body: Value, + mut body: Value, ctx: &OperationContext, ) -> Result { // Request-time checks on the raw body, in the service's order: the // AttributesToGet length and the Limit, Segment and TotalSegments ranges // in one message, the expression and non-expression parameter mix, the - // ReturnConsumedCapacity enum, every number in ExpressionAttributeValues - // and ExclusiveStartKey, the nesting depth of ExpressionAttributeValues, - // the Select enum. They run before typed deserialization so a field that - // does not deserialize does not stop the ones before it; a field of the - // wrong JSON type anywhere in the body is reported ahead of them; see - // `scan_request`. + // ReturnConsumedCapacity enum, every number in ExpressionAttributeValues, + // the nesting depth of ExpressionAttributeValues, the Select enum. They + // run before typed deserialization so a field that does not deserialize + // does not stop the ones before it; a field of the wrong JSON type + // anywhere in the body is reported ahead of them; see `scan_request`. crate::scan_request::validate_scan_request_body(&body)?; + // The ExclusiveStartKey is checked at its own stage, below, after the + // filter syntax, Select and unused-value checks; a key whose value does + // not deserialize is held back here so that typed deserialization does + // not report it first. + let start_key = crate::scan_request::StartKeyBody::take(&mut body); let input: ScanInput = serde_json::from_value(body).map_err(crate::deserialize_error)?; // Validate Filter/Projection expressions before the existence @@ -350,6 +354,12 @@ pub async fn handle_scan( ExpressionMaps::new(names, values) }; + // The ExclusiveStartKey's own checks: every number in it with the value + // check's text, then a value that is empty or has an unknown type + // descriptor, ahead of the key schema (measured 2026-10-31; see + // `scan_request::StartKeyBody`). + start_key.validate()?; + // Validate ExclusiveStartKey matches the key schema if let Some(ref start_key) = input.exclusive_start_key { validate_scan_exclusive_start_key(start_key, &key_info, index_info.as_ref())?; diff --git a/crates/engine/src/scan_request.rs b/crates/engine/src/scan_request.rs index 62362b6be..2427276e9 100644 --- a/crates/engine/src/scan_request.rs +++ b/crates/engine/src/scan_request.rs @@ -35,10 +35,7 @@ //! 4. every number in `ExpressionAttributeValues`, `NS` members and values //! nested in `L` and `M` included, whether a `FilterExpression` reads the //! value, nothing reads it, or the request carries no expression at all; a -//! Unicode decimal digit counts as the digit it stands for; then every -//! number in the `ExclusiveStartKey` values, which answers the same text as -//! a malformed number in a value, with no filter and on an index scan too, -//! and after the `ReturnConsumedCapacity` enum; +//! Unicode decimal digit counts as the digit it stands for; //! 5. the nesting depth of `ExpressionAttributeValues`: a value nested 32 //! levels deep in `L` answers `Nesting Levels have exceeded supported //! limits: Attributes in the item have nested levels beyond supported @@ -49,14 +46,38 @@ //! "BAD"` answers the nesting text). Both enums invalid answers the //! `ReturnConsumedCapacity` error alone, so the two are not aggregated. //! +//! The numbers in `ExclusiveStartKey` are checked by the handler at the key's +//! own stage, after the checks the service makes ahead of them (measured +//! 2026-10-31 between 23:54 and 23:56 UTC with `{"pk": {"N": "bob"}, "sk": +//! {"S": "x"}}` as the key): each of the six checks above, the `Select` rule +//! for `SPECIFIC_ATTRIBUTES`, the `FilterExpression` syntax and the +//! unused-value check answers its own text ahead of the number text, and a +//! malformed number in `ExpressionAttributeValues` is reported ahead of one in +//! the key. The number text is the value check's, with no filter and on an +//! index scan too (measured 2026-10-30). Inside the key a number fault +//! anywhere, an `NS` member or a value nested in the `L` of a non-key +//! attribute included, comes ahead of a value that is empty or has an unknown +//! type descriptor, in either member order; with two number faults the first +//! member is named (`{"pk": {"N": "bob"}, "sk": {"N": "fred"}}` names `bob`). +//! See [`StartKeyBody`]. +//! //! Not measured, and checked in the order listed above: the `AttributesToGet` //! length clause beside a `Segment` or `TotalSegments` fault without a `Limit` //! fault (listed first, as beside `Limit`), the older form of the length //! clause (the empty list echoed, `Value '[]' at 'attributesToGet' ...`, and //! listed last, as the older form reverses the newer one), the length clause -//! against the enums, the `ExclusiveStartKey` number check against the -//! `ExpressionAttributeValues` one, `M` levels in the nesting count, and the -//! nesting of a value an expression reads or of a request with no expression. +//! against the enums, `M` levels in the nesting count, and the nesting of a +//! value an expression reads or of a request with no expression. Not measured +//! at the key's stage: the key's checks against the table lookup, the index +//! resolution, the `ConsistentRead` and `Select` index rules and the `Segment` +//! pairing, which the handler makes ahead of them; a key whose only fault is +//! an empty or unknown-descriptor value against the filter syntax and +//! unused-value checks (the handler reports it after them, with the service's +//! prefix `The provided starting key is invalid: ` on the deserializer's +//! text, measured for the empty value alone); and which of two number faults +//! is named when +//! the request lists the members the other way round (the check reads members +//! in sorted order, which names `bob` either way). //! //! The JSON parser accepts 128 levels of nesting, which is 62 `L` levels; a //! body nested deeper is a `SerializationException` from the parser, ahead of @@ -77,20 +98,20 @@ use std::collections::HashMap; use extenddb_core::error::DynamoDbError; -use extenddb_core::types::{AttributeValue, ScanInput, saturated_int32}; +use extenddb_core::types::{AttributeValue, Item, ScanInput, saturated_int32}; use serde_json::Value; use crate::query_key_values::{ - SCAN_CONDITIONS, has_unreadable_attribute_value, non_empty, present, shape_fault_first, - validate_expression_attribute_value_numbers, + SCAN_CONDITIONS, has_unreadable_attribute_value, is_shape_fault, non_empty, present, + shape_fault_first, validate_expression_attribute_value_numbers, validate_number_text, }; /// The `Scan` checks that run on the raw request body, in the service's order: /// the `AttributesToGet` length and the `Limit`, `Segment` and `TotalSegments` /// ranges in one message, the expression and non-expression parameter mix, the -/// `ReturnConsumedCapacity` enum, every number in `ExpressionAttributeValues` -/// and `ExclusiveStartKey`, the nesting depth of `ExpressionAttributeValues`, -/// then the `Select` enum. +/// `ReturnConsumedCapacity` enum, every number in `ExpressionAttributeValues`, +/// the nesting depth of `ExpressionAttributeValues`, then the `Select` enum. +/// The numbers in `ExclusiveStartKey` are checked later, by [`StartKeyBody`]. /// /// The checks read the raw body, so a field that does not deserialize (an /// invalid `Select`, for one) does not stop the ones before it. When a check @@ -139,15 +160,8 @@ fn scan_request_checks(body: &Value) -> Result<(), DynamoDbError> { }], )?; - let values = attribute_value_map(body, "ExpressionAttributeValues"); - if let Some(values) = &values { - validate_expression_attribute_value_numbers(values)?; - } - // The key's values are numbers checked with the same rules and text. - if let Some(key) = attribute_value_map(body, "ExclusiveStartKey") { - validate_expression_attribute_value_numbers(&key)?; - } - if let Some(values) = &values { + if let Some(values) = attribute_value_map(body, "ExpressionAttributeValues") { + validate_expression_attribute_value_numbers(&values)?; extenddb_core::validation::validate_attribute_values_nesting_depth(values.values())?; } @@ -172,6 +186,120 @@ fn attribute_value_map(body: &Value, field: &str) -> Option, + /// The deserializer's fault in a key taken out of the body, to report + /// after the key's numbers. A `ValidationException` carries the service's + /// prefix for the key (measured for an empty value; the texts for more + /// than one descriptor and for an empty or duplicate set follow it, + /// unmeasured). The `SerializationException` for an unknown type + /// descriptor keeps the deserializer's text (unmeasured on its own; beside + /// a `Limit` fault or a number fault the service reports those). + deferred: Option, +} + +impl StartKeyBody { + /// Reads the key from `body` and takes it out of the body when it does not + /// deserialize for a reason that is not a shape fault. + pub(crate) fn take(body: &mut Value) -> Self { + let raw = body.get("ExclusiveStartKey").cloned(); + let deferred = match raw + .as_ref() + .map(|key| serde_json::from_value::(key.clone())) + { + Some(Err(e)) => match crate::deserialize_error(e) { + shape if is_shape_fault(&shape) => None, + DynamoDbError::ValidationException(msg) => Some( + DynamoDbError::ValidationException(format!("{START_KEY_PREFIX}{msg}")), + ), + other => Some(other), + }, + Some(Ok(_)) | None => None, + }; + if deferred.is_some() + && let Some(fields) = body.as_object_mut() + { + fields.remove("ExclusiveStartKey"); + } + Self { raw, deferred } + } + + /// The key's checks at its own stage: every number in the key with the + /// value check's rules and text (an `N`, the members of an `NS`, and the + /// values nested in `L` and `M`, in the sorted order of the key's members, + /// so that two faults name the same one on every backend), then the + /// deserializer's fault in a key [`StartKeyBody::take`] took out of the + /// body. + /// + /// # Errors + /// + /// Returns `DynamoDbError::ValidationException` with the service's text, or + /// the deserializer's `SerializationException` for an unknown type + /// descriptor. + pub(crate) fn validate(&self) -> Result<(), DynamoDbError> { + if let Some(members) = self.raw.as_ref().and_then(Value::as_object) { + let mut names: Vec<&String> = members.keys().collect(); + names.sort(); + for name in names { + validate_raw_value_numbers(&members[name])?; + } + } + match &self.deferred { + Some(fault) => Err(fault.clone()), + None => Ok(()), + } + } +} + +/// Every number in one raw attribute value: an `N` text, the members of an +/// `NS`, and the values nested in `L` and `M`. A value that is not one +/// attribute value, or whose inner JSON is not of the descriptor's shape, holds +/// no number to check here; the deserializer reports it. +fn validate_raw_value_numbers(value: &Value) -> Result<(), DynamoDbError> { + let Some(fields) = value.as_object() else { + return Ok(()); + }; + let mut fields = fields.iter(); + let (Some((descriptor, inner)), None) = (fields.next(), fields.next()) else { + return Ok(()); + }; + match (descriptor.as_str(), inner) { + ("N", Value::String(text)) => validate_number_text(text), + ("NS", Value::Array(texts)) => texts + .iter() + .filter_map(Value::as_str) + .try_for_each(validate_number_text), + ("L", Value::Array(members)) => members.iter().try_for_each(validate_raw_value_numbers), + ("M", Value::Object(members)) => members.values().try_for_each(validate_raw_value_numbers), + _ => Ok(()), + } +} + /// The constraints the service aggregates into one message: the length of /// `AttributesToGet` and the ranges of `Limit`, `Segment` and `TotalSegments`, /// every fault in one message, in one of the service's two forms (measured diff --git a/crates/engine/src/scan_request/request_tests.rs b/crates/engine/src/scan_request/request_tests.rs index 3f3a568e2..d1ea374b5 100644 --- a/crates/engine/src/scan_request/request_tests.rs +++ b/crates/engine/src/scan_request/request_tests.rs @@ -5,21 +5,21 @@ //! numbers in `ExclusiveStartKey` and the nesting depth of //! `ExpressionAttributeValues`. //! -//! The rows in every test but the last are requests sent to the service -//! (us-east-1, each request twice, a Scan of an existing table) with the -//! answers it gave: on 2026-10-25 at 10:38 and 10:55 UTC, and on 2026-10-30 at -//! 12:40 UTC where a test says so. The last test applies the same rules to -//! requests that were not sent. +//! Every row is a request sent to the service (us-east-1, each request twice, +//! a Scan of an existing table) with the answer it gave: on 2026-10-25 at +//! 10:38 and 10:55 UTC, on 2026-10-30 at 12:40 UTC, and on 2026-10-31 between +//! 23:54 and 23:56 UTC where a test says so. use extenddb_core::error::DynamoDbError; use serde_json::{Value, json}; -use super::validate_scan_request_body; +use super::{StartKeyBody, validate_scan_request_body}; const MIX: &str = "Can not use both expression and non-expression parameters in the same request: Non-expression parameters: {ScanFilter} Expression parameters: {FilterExpression}"; const LENGTH: &str = "1 validation error detected: Value at 'AttributesToGet' failed to satisfy constraint: Member must have length greater than or equal to 1"; const NUMERIC_BOB: &str = "The parameter cannot be converted to a numeric value: bob"; const NESTING: &str = "Nesting Levels have exceeded supported limits: Attributes in the item have nested levels beyond supported limit"; +const EMPTY_KEY_VALUE: &str = "The provided starting key is invalid: Supplied AttributeValue is empty, must contain exactly one of the supported datatypes"; fn check(body: &Value) -> Option { match validate_scan_request_body(body) { @@ -29,6 +29,19 @@ fn check(body: &Value) -> Option { } } +/// The key's own checks on `body`, as the handler makes them after the +/// request-time checks: `StartKeyBody::take` on a copy of the body, then +/// `validate`. Panics on a `SerializationException`, which no row here +/// expects. +fn check_key(body: &Value) -> Option { + let mut body = body.clone(); + match StartKeyBody::take(&mut body).validate() { + Ok(()) => None, + Err(DynamoDbError::ValidationException(msg)) => Some(msg), + Err(other) => panic!("unexpected error {other:?}"), + } +} + /// A Scan body with the fields of `extra` added to the table name. fn scan(extra: Value) -> Value { let mut body = json!({"TableName": "t"}); @@ -108,24 +121,176 @@ fn an_empty_attributes_to_get_joins_the_range_faults() { ); } +/// A Scan body with `key` as its `ExclusiveStartKey`. +fn with_key(key: Value) -> Value { + scan(json!({"ExclusiveStartKey": key})) +} + #[test] fn a_malformed_number_in_the_exclusive_start_key_is_reported_with_the_number_text() { - let key = |number: &str| json!({"ExclusiveStartKey": {"pk": {"N": number}, "sk": {"S": "x"}}}); - let bad = with(scan(filter("1")), key("bob")); - assert_eq!(check(&bad).as_deref(), Some(NUMERIC_BOB)); - assert_eq!(check(&with(scan(filter("1")), key("1"))), None); - // Measured 2026-10-30 12:40 UTC: with no filter, the same text; the - // ReturnConsumedCapacity enum is checked first. - assert_eq!(check(&scan(key("bob"))).as_deref(), Some(NUMERIC_BOB)); - let enum_first = with(scan(key("bob")), json!({"ReturnConsumedCapacity": "BAD"})); + let key = |number: &str| json!({"pk": {"N": number}, "sk": {"S": "x"}}); + let bad = with(with_key(key("bob")), filter("1")); + // The request-time checks leave the key to its own stage. + assert_eq!(check(&bad), None); + assert_eq!(check_key(&bad).as_deref(), Some(NUMERIC_BOB)); + assert_eq!(check_key(&with(with_key(key("1")), filter("1"))), None); + // Measured 2026-10-30 12:40 UTC: with no filter, the same text. assert_eq!( - check(&enum_first).as_deref(), + check_key(&with_key(key("bob"))).as_deref(), + Some(NUMERIC_BOB) + ); + // Measured 2026-10-31 between 23:54 and 23:56 UTC: a number as an NS + // member, as an L member, and in the L of an attribute that is not a key + // attribute; the number texts for a leading space, an empty text and 39 + // digits (on an N hash key). + for key in [ + json!({"pk": {"NS": ["bob"]}, "sk": {"S": "x"}}), + json!({"pk": {"L": [{"N": "bob"}]}, "sk": {"S": "x"}}), + json!({"pk": {"S": "p0"}, "sk": {"S": "x"}, "z": {"L": [{"N": "bob"}]}}), + ] { + assert_eq!( + check_key(&with_key(key.clone())).as_deref(), + Some(NUMERIC_BOB), + "{key}" + ); + } + for (number, text) in [ + ( + " 1", + "The parameter cannot be converted to a numeric value: 1", + ), + ("", "The parameter cannot be converted to a numeric value: "), + ( + "111111111111111111111111111111111111111", + "Attempting to store more than 38 significant digits in a Number", + ), + ] { + let key = with_key(json!({"pk": {"N": number}})); + assert_eq!(check_key(&key).as_deref(), Some(text), "{number:?}"); + } +} + +/// Measured 2026-10-31 between 23:54 and 23:56 UTC with `{"pk": {"N": "bob"}, +/// "sk": {"S": "x"}}` as the key: each request-time check answers its own text +/// ahead of the key's number (the `Select` rule for `SPECIFIC_ATTRIBUTES`, the +/// filter syntax and the unused-value check, which the handler makes after +/// deserializing, do too; the wire tests send those rows). Measured +/// 2026-10-30 12:40 UTC: the `ReturnConsumedCapacity` enum ahead of the key. +#[test] +fn every_request_time_check_comes_ahead_of_the_number_in_the_exclusive_start_key() { + let bob = json!({"pk": {"N": "bob"}, "sk": {"S": "x"}}); + for (extra, text) in [ + ( + json!({"Limit": 0}), + "1 validation error detected: Value at 'Limit' failed to satisfy constraint: Member must have value greater than or equal to 1", + ), + ( + json!({"ReturnConsumedCapacity": "BAD"}), + "1 validation error detected: Value 'BAD' at 'returnConsumedCapacity' failed to satisfy constraint: Member must satisfy enum value set: [INDEXES, TOTAL, NONE]", + ), + ( + json!({"Segment": -1, "TotalSegments": 2}), + "1 validation error detected: Value at 'Segment' failed to satisfy constraint: Member must have value greater than or equal to 0", + ), + ( + json!({"Select": "BAD"}), + "1 validation error detected: Value 'BAD' at 'select' failed to satisfy constraint: Member must satisfy enum value set: [SPECIFIC_ATTRIBUTES, COUNT, ALL_ATTRIBUTES, ALL_PROJECTED_ATTRIBUTES]", + ), + (with(filter("1"), json!({"ScanFilter": {}})), MIX), + (json!({"AttributesToGet": []}), LENGTH), + ( + json!({ + "FilterExpression": "a = :p", + "ExpressionAttributeValues": {":p": {"S": "x"}, ":d": nested(&json!({"N": "1"}), 32)}, + }), + NESTING, + ), + // The values ahead of the key, with `fred` in the key. + ( + json!({ + "FilterExpression": "a = :n", + "ExpressionAttributeValues": {":n": {"N": "bob"}}, + "ExclusiveStartKey": {"pk": {"N": "fred"}, "sk": {"S": "x"}}, + }), + NUMERIC_BOB, + ), + ] { + let body = with(with_key(bob.clone()), extra); + assert_eq!(check(&body).as_deref(), Some(text), "{body}"); + } +} + +/// Measured 2026-10-31 between 23:54 and 23:56 UTC: inside the key a number +/// fault comes ahead of a value that is empty or has an unknown type +/// descriptor, in either member order; with two number faults the first +/// member is named; a key whose only fault is an empty value answers the +/// service's key text. +#[test] +fn a_number_fault_in_the_key_comes_ahead_of_an_empty_or_unknown_descriptor_value() { + for key in [ + json!({"pk": {}, "sk": {"N": "bob"}}), + json!({"sk": {"N": "bob"}, "pk": {}}), + json!({"pk": {"X": "1"}, "sk": {"N": "bob"}}), + json!({"pk": {"N": "bob"}, "sk": {"N": "fred"}}), + ] { + let body = with_key(key.clone()); + // The request-time checks leave the key to its own stage. + assert_eq!(check(&body), None, "{key}"); + assert_eq!(check_key(&body).as_deref(), Some(NUMERIC_BOB), "{key}"); + } + let empty = with_key(json!({"pk": {}, "sk": {"S": "x"}})); + assert_eq!(check_key(&empty).as_deref(), Some(EMPTY_KEY_VALUE)); + // Measured 2026-10-31 between 23:54 and 23:56 UTC too: a Limit fault + // beside the empty value answers the Limit clause, in the older form. + assert_eq!( + check(&with(empty, json!({"Limit": 0}))).as_deref(), Some( - "1 validation error detected: Value 'BAD' at 'returnConsumedCapacity' failed to satisfy constraint: Member must satisfy enum value set: [INDEXES, TOTAL, NONE]" + "1 validation error detected: Value '0' at 'limit' failed to satisfy constraint: Member must have value greater than or equal to 1" ) ); } +/// `StartKeyBody::take` holds back a key the deserializer refuses for its +/// value and leaves one of the wrong JSON shape in the body, where the typed +/// deserialization reports it ahead of every check. +#[test] +fn a_key_of_the_wrong_shape_stays_in_the_body_and_an_unreadable_value_is_held_back() { + for key in [json!("x"), json!({"pk": "x"}), json!({"pk": {"N": 5}})] { + let mut body = with_key(key.clone()); + let start_key = StartKeyBody::take(&mut body); + assert!(body.get("ExclusiveStartKey").is_some(), "{key}"); + assert!(start_key.validate().is_ok(), "{key}"); + let shape = serde_json::from_value::(body) + .map_err(crate::deserialize_error); + assert!( + matches!(shape, Err(DynamoDbError::SerializationException(_))), + "{key}: {shape:?}" + ); + } + for key in [ + json!({"pk": {}}), + json!({"pk": {"X": "1"}}), + json!({"pk": {"S": "x", "N": "1"}}), + ] { + let mut body = with_key(key.clone()); + let start_key = StartKeyBody::take(&mut body); + assert!(body.get("ExclusiveStartKey").is_none(), "{key}"); + assert!(start_key.validate().is_err(), "{key}"); + assert!( + serde_json::from_value::(body).is_ok(), + "{key}" + ); + } + // A readable key stays in the body and passes. + let mut body = with_key(json!({"pk": {"S": "p0"}, "sk": {"S": "x"}})); + let start_key = StartKeyBody::take(&mut body); + assert!(body.get("ExclusiveStartKey").is_some()); + assert!(start_key.validate().is_ok()); + // No key at all. + let mut body = scan(json!({})); + assert!(StartKeyBody::take(&mut body).validate().is_ok()); +} + #[test] fn a_value_nested_32_levels_deep_is_refused_and_31_levels_pass() { let one = json!({"N": "1"}); @@ -159,10 +324,11 @@ fn the_number_check_runs_before_the_nesting_check() { assert_eq!(check(&body).as_deref(), Some(NUMERIC_BOB)); } -/// Requests the service was not sent. The answers follow from the measured -/// rows and from the order the checks are made in. +/// Combinations the other tests do not send, each measured 2026-10-31 between +/// 23:54 and 23:56 UTC with the answer asserted here (the key row with `{"pk": +/// {"N": "fred"}}` and with `"sk": {"S": "x"}` added to it). #[test] -fn combinations_the_service_was_not_sent_follow_the_order_of_the_checks() { +fn combinations_measured_on_2026_10_31_follow_the_order_of_the_checks() { // The mix names whichever expression parameter is present. let projection = scan(json!({"ScanFilter": {}, "ProjectionExpression": "a"})); assert_eq!( diff --git a/tests/python/test_scan_request_validation.py b/tests/python/test_scan_request_validation.py index 4c57196de..4c53ff255 100644 --- a/tests/python/test_scan_request_validation.py +++ b/tests/python/test_scan_request_validation.py @@ -7,14 +7,19 @@ Every request below is sent as raw signed JSON. Each is a row of the table measured against DynamoDB (us-east-1) on 2026-10-25 at 10:38 and 10:55 UTC on -a Scan of an existing table of 29 items, or of the one measured on 2026-10-30 -at 12:40 UTC where a row says so, each sent twice, and each expected answer is -the service's. The table here holds 12 items, so the count the accepted -request returns is 12. +a Scan of an existing table of 29 items, or of the ones measured on 2026-10-30 +at 12:40 UTC and on 2026-10-31 between 23:54 and 23:56 UTC where a row says +so, each sent twice, and each expected answer is the service's. The table here +holds 12 items, so the count the accepted request returns is 12. Depth is the number of ``L`` wrappers around the leaf: 32 or more answers the nesting text, 31 reaches the unused-value check. A malformed number at the bottom is reported first, at the depths 31 to 60 that were measured. + +The numbers in ``ExclusiveStartKey`` are checked after every other check in +this file and after the filter syntax and unused-value checks, and inside the +key a number fault comes ahead of an empty value or an unknown type +descriptor (2026-10-31). """ from __future__ import annotations @@ -53,11 +58,34 @@ "Attributes in the item have nested levels beyond supported limit" ) UNUSED = "Value provided in ExpressionAttributeValues unused in expressions: keys: {:d}" +LIMIT = ( + "1 validation error detected: Value at 'Limit' failed to satisfy constraint: " + "Member must have value greater than or equal to 1" +) +SEGMENT = ( + "1 validation error detected: Value at 'Segment' failed to satisfy constraint: " + "Member must have value greater than or equal to 0" +) +SELECT_ENUM = ( + "1 validation error detected: Value 'BAD' at 'select' failed to satisfy constraint: " + "Member must satisfy enum value set: " + "[SPECIFIC_ATTRIBUTES, COUNT, ALL_ATTRIBUTES, ALL_PROJECTED_ATTRIBUTES]" +) +SELECT_SPECIFIC = ( + "Must specify the AttributesToGet or ProjectionExpression when choosing to get " + "SPECIFIC_ATTRIBUTES" +) +UNUSED_U = "Value provided in ExpressionAttributeValues unused in expressions: keys: {:u}" +EMPTY_KEY_VALUE = ( + "The provided starting key is invalid: Supplied AttributeValue is empty, " + "must contain exactly one of the supported datatypes" +) VALID_FILTER = '"FilterExpression": "a = :n", "ExpressionAttributeValues": {":n": {"N": "1"}}' BOB_FILTER = '"FilterExpression": "a = :n", "ExpressionAttributeValues": {":n": {"N": "bob"}}' ONE = {"N": "1"} BOB = {"N": "bob"} +BOB_KEY = '"ExclusiveStartKey": {"pk": {"N": "bob"}, "sk": {"S": "x"}}' # (id, the fields after TableName as JSON text, the expected message) REFUSED = [ @@ -87,6 +115,57 @@ '"ReturnConsumedCapacity": "BAD"', RCC_ENUM), ] +# Measured 2026-10-31 between 23:54 and 23:56 UTC: every other check comes +# ahead of the number in ExclusiveStartKey. +KEY_AFTER_THE_OTHER_CHECKS = [ + ("Limit-0", BOB_KEY + ', "Limit": 0', LIMIT), + ("Segment-minus-1", BOB_KEY + ', "Segment": -1, "TotalSegments": 2', SEGMENT), + ("Select-BAD", BOB_KEY + ', "Select": "BAD"', SELECT_ENUM), + ("ScanFilter-empty-with-FilterExpression", BOB_KEY + ', "ScanFilter": {}, ' + VALID_FILTER, + MIX), + ("AttributesToGet-empty", BOB_KEY + ', "AttributesToGet": []', LENGTH), + ("unused-value", + BOB_KEY + ', "FilterExpression": "a = :p", ' + '"ExpressionAttributeValues": {":p": {"S": "x"}, ":u": {"S": "y"}}', UNUSED_U), + ("bob-in-the-values-and-fred-in-the-key", + '"ExclusiveStartKey": {"pk": {"N": "fred"}, "sk": {"S": "x"}}, ' + BOB_FILTER, NUMERIC_BOB), +] + +# Measured 2026-10-31 between 23:54 and 23:56 UTC: inside the key a number +# fault anywhere comes ahead of an empty value or an unknown type descriptor, +# in either member order; two number faults name the first member; a key +# whose only fault is an empty value answers the service's key text. +KEY_FAULTS = [ + ("empty-pk-then-bob", '"ExclusiveStartKey": {"pk": {}, "sk": {"N": "bob"}}', NUMERIC_BOB), + ("bob-then-empty-pk", '"ExclusiveStartKey": {"sk": {"N": "bob"}, "pk": {}}', NUMERIC_BOB), + ("type-X-pk-then-bob", '"ExclusiveStartKey": {"pk": {"X": "1"}, "sk": {"N": "bob"}}', + NUMERIC_BOB), + ("bob-then-fred", '"ExclusiveStartKey": {"pk": {"N": "bob"}, "sk": {"N": "fred"}}', + NUMERIC_BOB), + ("bob-in-an-NS", '"ExclusiveStartKey": {"pk": {"NS": ["bob"]}, "sk": {"S": "x"}}', + NUMERIC_BOB), + ("bob-in-an-L", '"ExclusiveStartKey": {"pk": {"L": [{"N": "bob"}]}, "sk": {"S": "x"}}', + NUMERIC_BOB), + ("bob-in-the-L-of-a-non-key-attribute", + '"ExclusiveStartKey": {"pk": {"S": "p0"}, "sk": {"S": "x"}, "z": {"L": [{"N": "bob"}]}}', + NUMERIC_BOB), + ("bob-in-the-index-key-of-an-LSI-scan", + '"IndexName": "lsi", "ExclusiveStartKey": {"pk": {"S": "p0"}, "sk": {"S": "x"}, ' + '"ln": {"N": "bob"}}', NUMERIC_BOB), + ("empty-pk-alone", '"ExclusiveStartKey": {"pk": {}, "sk": {"S": "x"}}', EMPTY_KEY_VALUE), +] + +# Measured 2026-10-31 between 23:54 and 23:56 UTC on a table with an N hash +# key: the number texts for a leading space, an empty text and 39 digits. +N_KEY_FAULTS = [ + ("leading-space", '"ExclusiveStartKey": {"pk": {"N": " 1"}}', + "The parameter cannot be converted to a numeric value: 1"), + ("empty-text", '"ExclusiveStartKey": {"pk": {"N": ""}}', + "The parameter cannot be converted to a numeric value: "), + ("39-digits", '"ExclusiveStartKey": {"pk": {"N": "111111111111111111111111111111111111111"}}', + "Attempting to store more than 38 significant digits in a Number"), +] + def nest(leaf: dict, levels: int) -> dict: """``leaf`` inside ``levels`` lists.""" @@ -124,7 +203,7 @@ def _validation_message(status: int, payload: dict) -> str: @pytest.fixture(scope="module") def table(dynamodb_client): - """pk S, sk S, a GSI ``gsi`` on gpk S, with ITEMS items.""" + """pk S, sk S, a GSI ``gsi`` on gpk S, an LSI ``lsi`` on ln N, with ITEMS items.""" name = unique_name("scanreq") dynamodb_client.create_table( TableName=name, @@ -132,6 +211,7 @@ def table(dynamodb_client): {"AttributeName": "pk", "AttributeType": "S"}, {"AttributeName": "sk", "AttributeType": "S"}, {"AttributeName": "gpk", "AttributeType": "S"}, + {"AttributeName": "ln", "AttributeType": "N"}, ], KeySchema=[ {"AttributeName": "pk", "KeyType": "HASH"}, @@ -142,6 +222,14 @@ def table(dynamodb_client): "KeySchema": [{"AttributeName": "gpk", "KeyType": "HASH"}], "Projection": {"ProjectionType": "ALL"}, }], + LocalSecondaryIndexes=[{ + "IndexName": "lsi", + "KeySchema": [ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "ln", "KeyType": "RANGE"}, + ], + "Projection": {"ProjectionType": "ALL"}, + }], BillingMode="PAY_PER_REQUEST", ) wait_for_active(dynamodb_client, name) @@ -153,6 +241,24 @@ def table(dynamodb_client): wait_for_deleted(dynamodb_client, name) +@pytest.fixture(scope="module") +def n_table(dynamodb_client): + """pk N, with items 1 to 6.""" + name = unique_name("scanreqn") + dynamodb_client.create_table( + TableName=name, + AttributeDefinitions=[{"AttributeName": "pk", "AttributeType": "N"}], + KeySchema=[{"AttributeName": "pk", "KeyType": "HASH"}], + BillingMode="PAY_PER_REQUEST", + ) + wait_for_active(dynamodb_client, name) + for i in range(1, 7): + dynamodb_client.put_item(TableName=name, Item={"pk": {"N": str(i)}}) + yield name + dynamodb_client.delete_table(TableName=name) + wait_for_deleted(dynamodb_client, name) + + class TestRequestChecks: """The parameter mix, the length constraint and the numbers in the key.""" @@ -168,6 +274,48 @@ def test_an_empty_scan_filter_alone_is_accepted(self, dynamodb_client, table): assert (payload["Count"], payload["ScannedCount"]) == (ITEMS, ITEMS) +class TestExclusiveStartKey: + """The numbers in the key, after every other check and ahead of the key's + other faults (measured 2026-10-31 between 23:54 and 23:56 UTC).""" + + @pytest.mark.parametrize("fields, expected", [r[1:] for r in KEY_AFTER_THE_OTHER_CHECKS], + ids=[r[0] for r in KEY_AFTER_THE_OTHER_CHECKS]) + def test_the_other_checks_come_first(self, dynamodb_client, table, fields, expected): + assert _validation_message(*_scan(dynamodb_client, table, fields)) == expected + + def test_32_levels_are_refused_ahead_of_the_key(self, dynamodb_client, table): + fields = BOB_KEY + ", " + unused_value(nest(ONE, 32)) + assert _validation_message(*_scan(dynamodb_client, table, fields)) == NESTING + + def test_the_select_rule_is_checked_ahead_of_the_key(self, dynamodb_client, table): + # The service answers '1 validation error detected: ' and then this + # text. The Scan text here has no prefix, so only the check's place + # is asserted. + fields = BOB_KEY + ', "Select": "SPECIFIC_ATTRIBUTES"' + message = _validation_message(*_scan(dynamodb_client, table, fields)) + assert message.endswith(SELECT_SPECIFIC), message + + def test_the_filter_syntax_is_checked_ahead_of_the_key(self, dynamodb_client, table): + # The service answers 'Invalid FilterExpression: Syntax error; token: + # "=", near: "= = :p"'. The parser here words the syntax error + # differently, so only the check's place is asserted. + fields = (BOB_KEY + ', "FilterExpression": "a = = :p", ' + '"ExpressionAttributeValues": {":p": {"S": "x"}}') + message = _validation_message(*_scan(dynamodb_client, table, fields)) + assert message.startswith("Invalid FilterExpression:"), message + + @pytest.mark.parametrize("fields, expected", [r[1:] for r in KEY_FAULTS], + ids=[r[0] for r in KEY_FAULTS]) + def test_a_number_fault_comes_ahead_of_the_other_faults_in_the_key( + self, dynamodb_client, table, fields, expected): + assert _validation_message(*_scan(dynamodb_client, table, fields)) == expected + + @pytest.mark.parametrize("fields, expected", [r[1:] for r in N_KEY_FAULTS], + ids=[r[0] for r in N_KEY_FAULTS]) + def test_the_number_texts_on_an_n_hash_key(self, dynamodb_client, n_table, fields, expected): + assert _validation_message(*_scan(dynamodb_client, n_table, fields)) == expected + + class TestNestingDepth: """``ExpressionAttributeValues`` nested in lists.""" From 45705bf2799355aee2aea1d567a58b20c1596482 Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Thu, 5 Nov 2026 01:47:51 +0000 Subject: [PATCH 4/9] fix(scan): check the ExclusiveStartKey at the service's two stages Measured against the service on 2026-11-05, each request sent twice, as a raw signed Scan: a check of each stage beside each class of key fault, each key alone, and pairs of faulty members on three tables. - A key value that is empty, carries an unknown or lower-case type descriptor or carries two descriptors, at its top or inside an L or M, in a key member or in another member, is reported after the request parameter checks (ExpressionAttributeValues or ExpressionAttributeNames with no expression, ConditionalOperator with no ScanFilter, a ScanFilter condition's value count, the parameter mix) and ahead of the expression checks, with the prefix The provided starting key is invalid: on the empty-value or more-than-one-datatypes text. An unknown descriptor answered the deserializer's SerializationException; the other shape faults answered after the expression checks. - A number that does not parse and the set faults are reported after the expression and request checks (the filter syntax, an undefined value, the operand types, the projection syntax, the Select rules, the unused values, the Segment pairing) and ahead of the table lookup, with no prefix: The parameter cannot be converted to a numeric value: bob; One or more parameter values were invalid: An number set may not be empty, An string set may not be empty, Binary sets should not be empty; Input collection contains duplicates. for an NS, also for ["1", "1.0"]; Input collection ["a", "a"] contains duplicates. for an SS. The set texts carried the key prefix, the NS duplicate text listed the members, and the key's number came after the table lookup and the index checks. - A value of ExpressionAttributeValues the deserializer refuses (an empty value read by the filter) is reported after the key's value faults and ahead of the table lookup. It was reported ahead of every handler check. - With faults in two members the service names one of them by the attribute names and, on some tables, the request's order: {"pk": {}, "sk": {"N": "bob"}} answered the number on 2026-10-31 and {"a": {}, "b": {"N": "bob"}} the empty value. The code reports a shape fault in any member ahead of a value fault in any member, then the members in sorted order, and no test pins which member. The handler holds back a key or an ExpressionAttributeValues value the deserializer refuses, checks the key's shape faults between the request parameter checks and the filter parse, and checks its numbers and sets, then the held-back value, right before the table lookup. The Segment pairing, the filter's placeholder and operand checks and the Select COUNT rule for AttributesToGet move ahead of the key's value stage from after the table lookup, as the measured rows beside a number in the key need. The nesting check keeps its request-time place: the service answered the key's text for a nested value the filter reads beside a key shape fault, and the nesting text ahead of the Select enum for an unused value, which one order cannot both give. Not measured: a BS with duplicates, a number and a duplicate in one NS, a set member that is not a string, an ExpressionAttributeValues value refused for a reason other than an empty value, and the nesting check beside a key shape fault for an unused value. request_tests.rs has the measured rows, and tests/python/test_scan_request_validation.py sends them as raw signed requests: each stage beside each class of key fault, each key alone, the pairs of faulty members (either member's fault accepted), the Segment pairing, an undefined filter value, a missing table and an empty expression value beside the key. The SPECIFIC_ATTRIBUTES rule and the filter syntax tests assert only the check's place, as the texts differ from the service's on main. --- crates/engine/src/query_key_values.rs | 3 +- crates/engine/src/scan.rs | 223 +++++----- crates/engine/src/scan_request.rs | 396 ++++++++++++++---- .../engine/src/scan_request/request_tests.rs | 296 ++++++++++--- tests/python/test_scan_request_validation.py | 210 ++++++++-- 5 files changed, 874 insertions(+), 254 deletions(-) diff --git a/crates/engine/src/query_key_values.rs b/crates/engine/src/query_key_values.rs index ead1c93f9..6a10523ed 100644 --- a/crates/engine/src/query_key_values.rs +++ b/crates/engine/src/query_key_values.rs @@ -211,7 +211,8 @@ pub(crate) fn non_empty(body: &Value, field: &str) -> bool { } /// The type descriptors an AttributeValue may carry. -const TYPE_DESCRIPTORS: &[&str] = &["S", "N", "B", "SS", "NS", "BS", "M", "L", "NULL", "BOOL"]; +pub(crate) const TYPE_DESCRIPTORS: &[&str] = + &["S", "N", "B", "SS", "NS", "BS", "M", "L", "NULL", "BOOL"]; /// The legacy condition maps of `Scan` whose `AttributeValueList` entries /// [`has_unreadable_attribute_value`] reads. diff --git a/crates/engine/src/scan.rs b/crates/engine/src/scan.rs index 44f84d586..bad2efe54 100755 --- a/crates/engine/src/scan.rs +++ b/crates/engine/src/scan.rs @@ -41,20 +41,33 @@ pub async fn handle_scan( mut body: Value, ctx: &OperationContext, ) -> Result { - // Request-time checks on the raw body, in the service's order: the - // AttributesToGet length and the Limit, Segment and TotalSegments ranges - // in one message, the expression and non-expression parameter mix, the - // ReturnConsumedCapacity enum, every number in ExpressionAttributeValues, - // the nesting depth of ExpressionAttributeValues, the Select enum. They - // run before typed deserialization so a field that does not deserialize - // does not stop the ones before it; a field of the wrong JSON type - // anywhere in the body is reported ahead of them; see `scan_request`. + // The checks run in the service's order, measured 2026-10-25 to 2026-11-05 + // (see `scan_request`): + // - request-time, on the raw body: the AttributesToGet length and the + // Limit, Segment and TotalSegments ranges in one message, the + // expression and non-expression parameter mix, the ReturnConsumedCapacity + // enum, every number in ExpressionAttributeValues, the nesting depth of + // ExpressionAttributeValues, the Select enum; + // - a field of the wrong JSON type anywhere in the body; + // - the request parameter checks: ExpressionAttributeValues and + // ExpressionAttributeNames with no expression, ConditionalOperator with + // no ScanFilter, a ScanFilter condition's value count; + // - the ExclusiveStartKey's shape faults (an empty value, an unknown type + // descriptor, two descriptors); + // - the expression and request checks: the filter syntax, an undefined + // value, the operand types, the projection syntax, the Select rules, + // the unused values, the Segment pairing; + // - the ExclusiveStartKey's value faults (a number, an empty or duplicate + // set), then a value of ExpressionAttributeValues that does not + // deserialize; + // - the table lookup, the index resolution and the index rules, the key + // schema. crate::scan_request::validate_scan_request_body(&body)?; - // The ExclusiveStartKey is checked at its own stage, below, after the - // filter syntax, Select and unused-value checks; a key whose value does - // not deserialize is held back here so that typed deserialization does - // not report it first. + // A key or an ExpressionAttributeValues value that does not deserialize + // is held back here, so that the typed deserialization does not report it + // ahead of the checks the service makes first. let start_key = crate::scan_request::StartKeyBody::take(&mut body); + let expression_values = crate::scan_request::ExpressionValuesBody::take(&mut body); let input: ScanInput = serde_json::from_value(body).map_err(crate::deserialize_error)?; // Validate Filter/Projection expressions before the existence @@ -88,23 +101,31 @@ pub async fn handle_scan( input.scan_filter.as_ref().map_or(0, HashMap::len), )?; - // Parse FilterExpression or desugar legacy ScanFilter - let (filter, filter_maps) = if let Some(ref sf) = input.scan_filter { - if sf.is_empty() { - ( - parse_optional_filter(input.filter_expression.as_deref(), &ctx.limits)?, - None, - ) - } else { + // Desugar a legacy ScanFilter: its value count is a request parameter + // check, ahead of every fault in the key (measured 2026-11-05). The mix + // check above refuses a ScanFilter beside a FilterExpression, so the two + // branches below never both apply. + let legacy_filter = match input.scan_filter { + Some(ref sf) if !sf.is_empty() => { let cond_op = input.conditional_operator.unwrap_or_default(); - let (expr, fmaps) = desugar_filter(sf, cond_op)?; - (Some(expr), Some(fmaps)) + Some(desugar_filter(sf, cond_op)?) } - } else { - ( + _ => None, + }; + + // The ExclusiveStartKey's shape stage: an empty value, an unknown type + // descriptor or two descriptors, after the request parameter checks and + // ahead of the expression checks (measured 2026-11-05; see + // `scan_request::StartKeyBody`). + start_key.validate_shape()?; + + // Parse FilterExpression, or take the desugared ScanFilter. + let (filter, filter_maps) = match legacy_filter { + Some((expr, fmaps)) => (Some(expr), Some(fmaps)), + None => ( parse_optional_filter(input.filter_expression.as_deref(), &ctx.limits)?, None, - ) + ), }; // Parse ProjectionExpression or desugar legacy AttributesToGet @@ -195,6 +216,88 @@ pub async fn handle_scan( )?; } + // Validate Segment/TotalSegments pairing. The range of each field is + // checked at request time in validate_scan_request_body; the service + // reports a missing partner with a message per direction, ahead of the + // key's value faults and of the table lookup (measured 2026-11-05). + match (input.segment, input.total_segments) { + (Some(_), None) => { + return Err(DynamoDbError::ValidationException( + "The TotalSegments parameter is required but was not present in the request when Segment parameter is present" + .to_owned(), + )); + } + (None, Some(_)) => { + return Err(DynamoDbError::ValidationException( + "The Segment parameter is required but was not present in the request when parameter TotalSegments is present" + .to_owned(), + )); + } + (Some(seg), Some(total)) => { + if seg >= total { + return Err(DynamoDbError::ValidationException(format!( + "The Segment parameter is zero-based and must be less than parameter TotalSegments: \ + Segment: {seg} is not less than TotalSegments: {total}" + ))); + } + } + (None, None) => {} + } + + // Limit >= 1 is checked at request time in validate_scan_request_body. + + // Build the combined expression maps for the operand checks and the + // post-read evaluation. + let combined_maps = { + let mut names = maps.names.clone(); + let mut values = maps.values.clone(); + + if let Some(ref fm) = filter_maps { + values.extend(fm.values.clone()); + } + if !extra_proj_names.is_empty() { + for (k, v) in &extra_proj_names { + let stripped = k.strip_prefix('#').unwrap_or(k); + names.insert(stripped.to_owned(), v.clone()); + } + } + ExpressionMaps::new(names, values) + }; + + // Validate the filter's placeholders and operand types, ahead of the key's + // value faults and of the table lookup (measured 2026-11-05). + if let Some(ref f) = filter { + let mut placeholders = Vec::new(); + extenddb_core::expression::collect_value_placeholders(f, &mut placeholders); + for placeholder in placeholders { + combined_maps.resolve_value_for(&placeholder, "FilterExpression")?; + } + extenddb_core::expression::validate_begins_with_operands(f, &combined_maps).map_err( + |e| crate::expression_helpers::prefix_expression_error(e, ExpressionKind::Filter), + )?; + extenddb_core::expression::validate_ordering_operand_types(f, &combined_maps).map_err( + |e| crate::expression_helpers::prefix_expression_error(e, ExpressionKind::Filter), + )?; + } + + // Select COUNT with a projection, from AttributesToGet too: ahead of the + // key's value faults and of the table lookup (measured 2026-11-05). + if let Some(Select::Count) = input.select + && effective_projection_str.is_some() + { + return Err(DynamoDbError::ValidationException( + "Cannot specify the ProjectionExpression when Select is COUNT".to_owned(), + )); + } + + // The ExclusiveStartKey's value stage: every number in it with the value + // check's text and the set faults, then a value of + // ExpressionAttributeValues that does not deserialize, both ahead of the + // table lookup (measured 2026-11-05; see `scan_request::StartKeyBody` and + // `scan_request::ExpressionValuesBody`). + start_key.validate_values()?; + expression_values.validate()?; + // P118: Fetch key_info first so we can use table_id for index lookup. let key_info = ctx .table_key_info(&input.table_name) @@ -267,35 +370,6 @@ pub async fn handle_scan( )?; } - // Validate Segment/TotalSegments pairing. The range of each field is - // checked at request time in validate_scan_request_body; the service - // reports a missing partner with a message per direction. - match (input.segment, input.total_segments) { - (Some(_), None) => { - return Err(DynamoDbError::ValidationException( - "The TotalSegments parameter is required but was not present in the request when Segment parameter is present" - .to_owned(), - )); - } - (None, Some(_)) => { - return Err(DynamoDbError::ValidationException( - "The Segment parameter is required but was not present in the request when parameter TotalSegments is present" - .to_owned(), - )); - } - (Some(seg), Some(total)) => { - if seg >= total { - return Err(DynamoDbError::ValidationException(format!( - "The Segment parameter is zero-based and must be less than parameter TotalSegments: \ - Segment: {seg} is not less than TotalSegments: {total}" - ))); - } - } - (None, None) => {} - } - - // Limit >= 1 is checked at request time in validate_scan_request_body. - // For index scans, build a key_info that reflects the index's key schema. let scan_key_info = if let Some(ref idx) = index_info { TableKeyInfo { @@ -323,43 +397,12 @@ pub async fn handle_scan( "ALL_PROJECTED_ATTRIBUTES can be used only when Querying using an IndexName".to_owned(), )); } - if let Some(Select::Count) = input.select - && effective_projection_str.is_some() - { - return Err(DynamoDbError::ValidationException( - "Cannot specify the ProjectionExpression when Select is COUNT".to_owned(), - )); - } - let index_proj = if matches!(input.select, Some(Select::AllProjectedAttributes)) { index_info.as_ref() } else { None }; - // Build the combined expression maps for post-read evaluation. - let combined_maps = { - let mut names = maps.names.clone(); - let mut values = maps.values.clone(); - - if let Some(ref fm) = filter_maps { - values.extend(fm.values.clone()); - } - if !extra_proj_names.is_empty() { - for (k, v) in &extra_proj_names { - let stripped = k.strip_prefix('#').unwrap_or(k); - names.insert(stripped.to_owned(), v.clone()); - } - } - ExpressionMaps::new(names, values) - }; - - // The ExclusiveStartKey's own checks: every number in it with the value - // check's text, then a value that is empty or has an unknown type - // descriptor, ahead of the key schema (measured 2026-10-31; see - // `scan_request::StartKeyBody`). - start_key.validate()?; - // Validate ExclusiveStartKey matches the key schema if let Some(ref start_key) = input.exclusive_start_key { validate_scan_exclusive_start_key(start_key, &key_info, index_info.as_ref())?; @@ -393,16 +436,6 @@ pub async fn handle_scan( } } - // Validate begins_with operand types upfront (before any rows are scanned). - if let Some(ref f) = filter { - extenddb_core::expression::validate_begins_with_operands(f, &combined_maps).map_err( - |e| crate::expression_helpers::prefix_expression_error(e, ExpressionKind::Filter), - )?; - extenddb_core::expression::validate_ordering_operand_types(f, &combined_maps).map_err( - |e| crate::expression_helpers::prefix_expression_error(e, ExpressionKind::Filter), - )?; - } - // Determine which key schema to use for LastEvaluatedKey extraction. // For index scans the LEK carries the index key and the base table key, // and the same shape addresses each chunk read from storage. diff --git a/crates/engine/src/scan_request.rs b/crates/engine/src/scan_request.rs index 2427276e9..177203a9b 100644 --- a/crates/engine/src/scan_request.rs +++ b/crates/engine/src/scan_request.rs @@ -46,20 +46,47 @@ //! "BAD"` answers the nesting text). Both enums invalid answers the //! `ReturnConsumedCapacity` error alone, so the two are not aggregated. //! -//! The numbers in `ExclusiveStartKey` are checked by the handler at the key's -//! own stage, after the checks the service makes ahead of them (measured -//! 2026-10-31 between 23:54 and 23:56 UTC with `{"pk": {"N": "bob"}, "sk": -//! {"S": "x"}}` as the key): each of the six checks above, the `Select` rule -//! for `SPECIFIC_ATTRIBUTES`, the `FilterExpression` syntax and the -//! unused-value check answers its own text ahead of the number text, and a -//! malformed number in `ExpressionAttributeValues` is reported ahead of one in -//! the key. The number text is the value check's, with no filter and on an -//! index scan too (measured 2026-10-30). Inside the key a number fault -//! anywhere, an `NS` member or a value nested in the `L` of a non-key -//! attribute included, comes ahead of a value that is empty or has an unknown -//! type descriptor, in either member order; with two number faults the first -//! member is named (`{"pk": {"N": "bob"}, "sk": {"N": "fred"}}` names `bob`). -//! See [`StartKeyBody`]. +//! The faults of `ExclusiveStartKey` come at two of the handler's stages, +//! measured 2026-11-05 (us-east-1, each request twice; a request parameter +//! check, an expression check and the table's indexes beside each class of +//! fault, and each key alone): +//! +//! - the shape faults, after the request parameter checks +//! (`ExpressionAttributeValues` or `ExpressionAttributeNames` with no +//! expression, `ConditionalOperator` with no `ScanFilter`, a `ScanFilter` +//! condition's value count, the parameter mix) and ahead of the expression +//! checks: a value that is empty, carries a type descriptor the service does +//! not have (`X`, a lower-case `s`) or more than one, at the top of a member +//! or inside its `L` or `M`, in a key member or in a member that is not a key +//! attribute. Text: `The provided starting key is invalid: Supplied +//! AttributeValue is empty, must contain exactly one of the supported +//! datatypes`, or the same prefix and `Supplied AttributeValue has more than +//! one datatypes set, must contain exactly one of the supported datatypes`. +//! Beside such a value the constraint clauses take the older form (D1); +//! - the value faults, after the expression and request checks (the filter +//! syntax, an undefined value, the operand types, the projection syntax, the +//! `Select` rules, the unused values, the `Segment` pairing) and ahead of the +//! table lookup, with no prefix: a number that does not parse (`The parameter +//! cannot be converted to a numeric value: bob`, in an `N`, an `NS` member, +//! an `L` or `M` member, a non-key member or an index key), an empty set +//! (`One or more parameter values were invalid: An number set may not be +//! empty`, `... An string set may not be empty`, `... Binary sets should +//! not be empty`) and a set with duplicates (`... Input collection contains +//! duplicates.` for an `NS`, `["1", "1"]` and `["1", "1.0"]` alike; `... +//! Input collection ["a", "a"] contains duplicates.` for an `SS`). A value +//! of `ExpressionAttributeValues` the deserializer refuses (an empty value) +//! is reported after them and ahead of the table lookup. +//! +//! See [`StartKeyBody`] and [`ExpressionValuesBody`]. With faults in two +//! members the service names one of them by the attribute names and, on some +//! tables, the request's order: `{"pk": {}, "sk": {"N": "bob"}}` answered the +//! number on 2026-10-31 and `{"a": {}, "b": {"N": "bob"}}` the empty value on +//! 2026-11-05. The code reports a shape fault in any member ahead of a value +//! fault in any member, then the members in sorted order; no test pins which +//! member. Inside one value the members of an `M` are read in the JSON map's +//! order and those of an `NS` in the request's order; the service named `bob` +//! and then `fred` for `{"M": {"b": {"N": "fred"}, "a": {"N": "bob"}}}` and +//! `fred` twice for `{"NS": ["fred", "bob"]}` in a non-key member. //! //! Not measured, and checked in the order listed above: the `AttributesToGet` //! length clause beside a `Segment` or `TotalSegments` fault without a `Limit` @@ -68,16 +95,15 @@ //! listed last, as the older form reverses the newer one), the length clause //! against the enums, `M` levels in the nesting count, and the nesting of a //! value an expression reads or of a request with no expression. Not measured -//! at the key's stage: the key's checks against the table lookup, the index -//! resolution, the `ConsistentRead` and `Select` index rules and the `Segment` -//! pairing, which the handler makes ahead of them; a key whose only fault is -//! an empty or unknown-descriptor value against the filter syntax and -//! unused-value checks (the handler reports it after them, with the service's -//! prefix `The provided starting key is invalid: ` on the deserializer's -//! text, measured for the empty value alone); and which of two number faults -//! is named when -//! the request lists the members the other way round (the check reads members -//! in sorted order, which names `bob` either way). +//! at the key's stages: a `BS` with duplicates (the deserializer's text); a +//! number fault and a duplicate in one `NS` (the number first); an `SS` or `BS` +//! member that is not a string (the deserializer's text at the value stage); a +//! value of `ExpressionAttributeValues` the deserializer refuses for a reason +//! other than an empty value (two descriptors, an unknown descriptor, a set), +//! reported with the empty value's; and the nesting check beside a key shape +//! fault, where the service answered the key's text for a value the filter +//! reads (2026-11-05) and the nesting text ahead of the `Select` enum for an +//! unused value (2026-10-30): the check keeps its request-time place. //! //! The JSON parser accepts 128 levels of nesting, which is 62 `L` levels; a //! body nested deeper is a `SerializationException` from the parser, ahead of @@ -102,8 +128,8 @@ use extenddb_core::types::{AttributeValue, Item, ScanInput, saturated_int32}; use serde_json::Value; use crate::query_key_values::{ - SCAN_CONDITIONS, has_unreadable_attribute_value, is_shape_fault, non_empty, present, - shape_fault_first, validate_expression_attribute_value_numbers, validate_number_text, + SCAN_CONDITIONS, TYPE_DESCRIPTORS, has_unreadable_attribute_value, is_shape_fault, non_empty, + present, shape_fault_first, validate_expression_attribute_value_numbers, validate_number_text, }; /// The `Scan` checks that run on the raw request body, in the service's order: @@ -186,40 +212,57 @@ fn attribute_value_map(body: &Value, field: &str) -> Option, - /// The deserializer's fault in a key taken out of the body, to report - /// after the key's numbers. A `ValidationException` carries the service's - /// prefix for the key (measured for an empty value; the texts for more - /// than one descriptor and for an empty or duplicate set follow it, - /// unmeasured). The `SerializationException` for an unknown type - /// descriptor keeps the deserializer's text (unmeasured on its own; beside - /// a `Limit` fault or a number fault the service reports those). + /// The deserializer's fault in a key taken out of the body, reported at the + /// value stage when neither stage classifies the value (an `SS` member that + /// is not a string, for one). deferred: Option, } @@ -234,9 +277,13 @@ impl StartKeyBody { { Some(Err(e)) => match crate::deserialize_error(e) { shape if is_shape_fault(&shape) => None, - DynamoDbError::ValidationException(msg) => Some( - DynamoDbError::ValidationException(format!("{START_KEY_PREFIX}{msg}")), - ), + DynamoDbError::ValidationException(msg) + if msg == EMPTY_VALUE || msg == MORE_THAN_ONE_DESCRIPTOR => + { + Some(DynamoDbError::ValidationException(format!( + "{START_KEY_PREFIX}{msg}" + ))) + } other => Some(other), }, Some(Ok(_)) | None => None, @@ -249,26 +296,56 @@ impl StartKeyBody { Self { raw, deferred } } - /// The key's checks at its own stage: every number in the key with the - /// value check's rules and text (an `N`, the members of an `NS`, and the - /// values nested in `L` and `M`, in the sorted order of the key's members, - /// so that two faults name the same one on every backend), then the - /// deserializer's fault in a key [`StartKeyBody::take`] took out of the - /// body. + /// The key's members in sorted order, so that two faults name the same + /// member on every backend (the MongoDB build keeps the request's order in + /// its JSON maps, the others sort them). + fn members(&self) -> Vec<&Value> { + let Some(members) = self.raw.as_ref().and_then(Value::as_object) else { + return Vec::new(); + }; + let mut names: Vec<&String> = members.keys().collect(); + names.sort(); + names.into_iter().map(|name| &members[name]).collect() + } + + /// The key's shape stage: the first member, in sorted order, whose value is + /// empty, carries a type descriptor the service does not have or more than + /// one, at its top or inside an `L` or `M` member, answers the service's + /// text with the key prefix. /// /// # Errors /// - /// Returns `DynamoDbError::ValidationException` with the service's text, or - /// the deserializer's `SerializationException` for an unknown type - /// descriptor. - pub(crate) fn validate(&self) -> Result<(), DynamoDbError> { - if let Some(members) = self.raw.as_ref().and_then(Value::as_object) { - let mut names: Vec<&String> = members.keys().collect(); - names.sort(); - for name in names { - validate_raw_value_numbers(&members[name])?; + /// Returns `DynamoDbError::ValidationException` with the prefixed text. + pub(crate) fn validate_shape(&self) -> Result<(), DynamoDbError> { + for member in self.members() { + if let Some(text) = shape_fault(member) { + return Err(DynamoDbError::ValidationException(format!( + "{START_KEY_PREFIX}{text}" + ))); } } + Ok(()) + } + + /// The key's value stage: every number in the key with the value check's + /// rules and text (an `N`, the members of an `NS`, and the values nested in + /// `L` and `M`), and the set faults, member by member in sorted order; then + /// the deserializer's fault in a key [`StartKeyBody::take`] took out of the + /// body that neither stage classified. + /// + /// Inside one value the members of an `M` are read in the JSON map's order + /// (the request's order on the MongoDB build) and those of an `NS` in the + /// request's order: the service named `bob` and then `fred` on two sends + /// of `{"M": {"b": {"N": "fred"}, "a": {"N": "bob"}}}` and `fred` twice for + /// `{"NS": ["fred", "bob"]}` (2026-11-05), so no order is pinned. + /// + /// # Errors + /// + /// Returns `DynamoDbError::ValidationException` with the service's text. + pub(crate) fn validate_values(&self) -> Result<(), DynamoDbError> { + for member in self.members() { + validate_key_value(member)?; + } match &self.deferred { Some(fault) => Err(fault.clone()), None => Ok(()), @@ -276,11 +353,32 @@ impl StartKeyBody { } } -/// Every number in one raw attribute value: an `N` text, the members of an -/// `NS`, and the values nested in `L` and `M`. A value that is not one -/// attribute value, or whose inner JSON is not of the descriptor's shape, holds -/// no number to check here; the deserializer reports it. -fn validate_raw_value_numbers(value: &Value) -> Result<(), DynamoDbError> { +/// The service's text for the first shape fault in one raw key value, depth +/// first: an object with no type descriptor or with one the service does not +/// have is an empty value, one with more than one descriptor is the +/// more-than-one text, and the members of an `L` or `M` are read the same way. +/// A value that is not an object is a shape fault the deserializer reports. +fn shape_fault(value: &Value) -> Option<&'static str> { + let fields = value.as_object()?; + let mut fields = fields.iter(); + match (fields.next(), fields.next()) { + (None, _) => Some(EMPTY_VALUE), + (Some(_), Some(_)) => Some(MORE_THAN_ONE_DESCRIPTOR), + (Some((descriptor, inner)), None) => match (descriptor.as_str(), inner) { + ("L", Value::Array(members)) => members.iter().find_map(shape_fault), + ("M", Value::Object(members)) => members.values().find_map(shape_fault), + (descriptor, _) if TYPE_DESCRIPTORS.contains(&descriptor) => None, + _ => Some(EMPTY_VALUE), + }, + } +} + +/// The value faults of one raw key value: an `N` text, the members of an `NS` +/// (every number first, then an empty set, then duplicates by numeric value: +/// `["1", "1.0"]` has a duplicate), an empty `SS` or `BS` and their duplicates, +/// and the values nested in `L` and `M`. A member of a set that is not a string +/// holds nothing to check here; the deserializer reports it. +fn validate_key_value(value: &Value) -> Result<(), DynamoDbError> { let Some(fields) = value.as_object() else { return Ok(()); }; @@ -290,16 +388,152 @@ fn validate_raw_value_numbers(value: &Value) -> Result<(), DynamoDbError> { }; match (descriptor.as_str(), inner) { ("N", Value::String(text)) => validate_number_text(text), - ("NS", Value::Array(texts)) => texts - .iter() - .filter_map(Value::as_str) - .try_for_each(validate_number_text), - ("L", Value::Array(members)) => members.iter().try_for_each(validate_raw_value_numbers), - ("M", Value::Object(members)) => members.values().try_for_each(validate_raw_value_numbers), + ("NS", Value::Array(members)) => validate_number_set(members), + ("SS", Value::Array(members)) => validate_string_set(members), + ("BS", Value::Array(members)) => validate_binary_set(members), + ("L", Value::Array(members)) => members.iter().try_for_each(validate_key_value), + ("M", Value::Object(members)) => members.values().try_for_each(validate_key_value), _ => Ok(()), } } +/// A `ValidationException` with the service's prefix for an invalid parameter +/// value. +fn invalid_parameter(text: &str) -> DynamoDbError { + DynamoDbError::ValidationException(format!("{INVALID_PARAMETER_VALUES}{text}")) +} + +/// An `NS` of the key: every member's number text, then `One or more parameter +/// values were invalid: An number set may not be empty` (the service's two +/// spaces), then `... Input collection contains duplicates.` for two members +/// with one numeric value (measured 2026-11-05 for `["1", "1"]` and `["1", +/// "1.0"]`). +fn validate_number_set(members: &[Value]) -> Result<(), DynamoDbError> { + let texts: Vec<&str> = members.iter().filter_map(Value::as_str).collect(); + texts + .iter() + .try_for_each(|text| validate_number_text(text))?; + if members.is_empty() { + return Err(invalid_parameter("An number set may not be empty")); + } + let mut normalized: Vec = texts + .iter() + .map(|text| { + let folded = crate::decimal_digits::fold_decimal_digits(text); + extenddb_core::validation::number::validate_and_normalize_number(&folded) + .unwrap_or_else(|_| folded.into_owned()) + }) + .collect(); + normalized.sort_unstable(); + normalized.dedup(); + if normalized.len() != texts.len() { + return Err(invalid_parameter("Input collection contains duplicates.")); + } + Ok(()) +} + +/// An `SS` of the key: `One or more parameter values were invalid: An string +/// set may not be empty`, then `... Input collection ["a", "a"] contains +/// duplicates.` with every member quoted (measured 2026-11-05). +fn validate_string_set(members: &[Value]) -> Result<(), DynamoDbError> { + if members.is_empty() { + return Err(invalid_parameter("An string set may not be empty")); + } + let texts: Vec<&str> = members.iter().filter_map(Value::as_str).collect(); + let mut distinct = texts.clone(); + distinct.sort_unstable(); + distinct.dedup(); + if distinct.len() != texts.len() { + let listed: Vec = texts + .iter() + .map(|text| serde_json::to_string(text).unwrap_or_default()) + .collect(); + return Err(invalid_parameter(&format!( + "Input collection [{}] contains duplicates.", + listed.join(", ") + ))); + } + Ok(()) +} + +/// A `BS` of the key: `One or more parameter values were invalid: Binary sets +/// should not be empty` (measured 2026-11-05); duplicates answer the +/// deserializer's text for a binary set, which is not measured for the key. +fn validate_binary_set(members: &[Value]) -> Result<(), DynamoDbError> { + if members.is_empty() { + return Err(invalid_parameter("Binary sets should not be empty")); + } + let texts: Vec<&str> = members.iter().filter_map(Value::as_str).collect(); + let mut distinct = texts.clone(); + distinct.sort_unstable(); + distinct.dedup(); + if distinct.len() != texts.len() { + return Err(invalid_parameter(&format!( + "Input collection [{}]of type BS contains duplicates.", + texts.join(", ") + ))); + } + Ok(()) +} + +/// The `ExpressionAttributeValues` of a `Scan` body whose values do not all +/// deserialize: an empty value, an unknown type descriptor, two descriptors, an +/// empty or duplicate set. The service reports such a value after the key's +/// value stage and ahead of the table lookup (measured 2026-11-05: `{":a": {}}` +/// read by the filter answers `ExpressionAttributeValues contains invalid +/// value: Supplied AttributeValue is empty, must contain exactly one of the +/// supported datatypes for key :a` with no key and with a valid key, and the +/// key's number or set text beside a key with such a fault). [`Self::take`] +/// puts a readable stand-in in the body for each such value, so that the typed +/// deserialization and the expression checks run on the other values, and +/// keeps the deserializer's fault for the map as the request wrote it; +/// [`Self::validate`] reports it. The request is refused before the scan reads +/// the stand-in. A value of the wrong JSON shape stays in the body. +pub(crate) struct ExpressionValuesBody { + deferred: Option, +} + +impl ExpressionValuesBody { + /// Replaces each value the deserializer refuses for a reason that is not a + /// shape fault with a readable stand-in and keeps the deserializer's fault. + pub(crate) fn take(body: &mut Value) -> Self { + let Some(Value::Object(entries)) = body.get_mut("ExpressionAttributeValues") else { + return Self { deferred: None }; + }; + let raw = Value::Object(entries.clone()); + let mut substituted = false; + for value in entries.values_mut() { + if let Err(e) = serde_json::from_value::(value.clone()) + && !is_shape_fault(&crate::deserialize_error(e)) + { + *value = serde_json::json!({"S": ""}); + substituted = true; + } + } + if !substituted { + return Self { deferred: None }; + } + // The deserializer's own wording and key order for the map as written. + let probe = serde_json::json!({"TableName": "t", "ExpressionAttributeValues": raw}); + let deferred = serde_json::from_value::(probe) + .err() + .map(crate::deserialize_error); + Self { deferred } + } + + /// The deferred fault, at the stage after the key's value checks. + /// + /// # Errors + /// + /// Returns the deserializer's fault for the first value it refused. + pub(crate) fn validate(&self) -> Result<(), DynamoDbError> { + match &self.deferred { + Some(fault) => Err(fault.clone()), + None => Ok(()), + } + } +} + /// The constraints the service aggregates into one message: the length of /// `AttributesToGet` and the ranges of `Limit`, `Segment` and `TotalSegments`, /// every fault in one message, in one of the service's two forms (measured diff --git a/crates/engine/src/scan_request/request_tests.rs b/crates/engine/src/scan_request/request_tests.rs index d1ea374b5..30e801146 100644 --- a/crates/engine/src/scan_request/request_tests.rs +++ b/crates/engine/src/scan_request/request_tests.rs @@ -6,9 +6,12 @@ //! `ExpressionAttributeValues`. //! //! Every row is a request sent to the service (us-east-1, each request twice, -//! a Scan of an existing table) with the answer it gave: on 2026-10-25 at -//! 10:38 and 10:55 UTC, on 2026-10-30 at 12:40 UTC, and on 2026-10-31 between -//! 23:54 and 23:56 UTC where a test says so. +//! a Scan of a table the probe created) with the answer it gave: on 2026-10-25 +//! at 10:38 and 10:55 UTC, on 2026-10-30 at 12:40 UTC, on 2026-10-31 between +//! 23:54 and 23:56 UTC and on 2026-11-05 where a test says so. The tests of +//! `StartKeyBody::take` and `ExpressionValuesBody::take` check which values +//! they hold back, with no service answer, and the test with faults in two key +//! members accepts either member's fault. use extenddb_core::error::DynamoDbError; use serde_json::{Value, json}; @@ -19,7 +22,6 @@ const MIX: &str = "Can not use both expression and non-expression parameters in const LENGTH: &str = "1 validation error detected: Value at 'AttributesToGet' failed to satisfy constraint: Member must have length greater than or equal to 1"; const NUMERIC_BOB: &str = "The parameter cannot be converted to a numeric value: bob"; const NESTING: &str = "Nesting Levels have exceeded supported limits: Attributes in the item have nested levels beyond supported limit"; -const EMPTY_KEY_VALUE: &str = "The provided starting key is invalid: Supplied AttributeValue is empty, must contain exactly one of the supported datatypes"; fn check(body: &Value) -> Option { match validate_scan_request_body(body) { @@ -29,19 +31,6 @@ fn check(body: &Value) -> Option { } } -/// The key's own checks on `body`, as the handler makes them after the -/// request-time checks: `StartKeyBody::take` on a copy of the body, then -/// `validate`. Panics on a `SerializationException`, which no row here -/// expects. -fn check_key(body: &Value) -> Option { - let mut body = body.clone(); - match StartKeyBody::take(&mut body).validate() { - Ok(()) => None, - Err(DynamoDbError::ValidationException(msg)) => Some(msg), - Err(other) => panic!("unexpected error {other:?}"), - } -} - /// A Scan body with the fields of `extra` added to the table name. fn scan(extra: Value) -> Value { let mut body = json!({"TableName": "t"}); @@ -126,13 +115,60 @@ fn with_key(key: Value) -> Value { scan(json!({"ExclusiveStartKey": key})) } +/// The key's shape stage on `body`, as the handler makes it after the request +/// parameter checks: `StartKeyBody::take` on a copy of the body, then +/// `validate_shape`. +fn check_key_shape(body: &Value) -> Option { + let mut body = body.clone(); + match StartKeyBody::take(&mut body).validate_shape() { + Ok(()) => None, + Err(DynamoDbError::ValidationException(msg)) => Some(msg), + Err(other) => panic!("unexpected error {other:?}"), + } +} + +/// The key's value stage on `body`, as the handler makes it after the +/// expression checks: `StartKeyBody::take` on a copy of the body, then +/// `validate_values`, for a key whose shape stage passes. +fn check_key_values(body: &Value) -> Option { + let mut body = body.clone(); + let key = StartKeyBody::take(&mut body); + assert!(key.validate_shape().is_ok(), "{body}"); + match key.validate_values() { + Ok(()) => None, + Err(DynamoDbError::ValidationException(msg)) => Some(msg), + Err(other) => panic!("unexpected error {other:?}"), + } +} + +/// The message of a `ValidationException`, `None` for `Ok`. +fn validation_message(result: Result<(), DynamoDbError>) -> Option { + match result { + Ok(()) => None, + Err(DynamoDbError::ValidationException(msg)) => Some(msg), + Err(other) => panic!("unexpected error {other:?}"), + } +} + +/// Both stages in the handler's order: the first fault of either. +fn check_key(body: &Value) -> Option { + check_key_shape(body).or_else(|| check_key_values(body)) +} + +const KEY_PREFIX: &str = "The provided starting key is invalid: "; +const EMPTY_VALUE: &str = + "Supplied AttributeValue is empty, must contain exactly one of the supported datatypes"; +const INVALID: &str = "One or more parameter values were invalid: "; + #[test] fn a_malformed_number_in_the_exclusive_start_key_is_reported_with_the_number_text() { let key = |number: &str| json!({"pk": {"N": number}, "sk": {"S": "x"}}); let bad = with(with_key(key("bob")), filter("1")); - // The request-time checks leave the key to its own stage. + // The request-time checks and the shape stage leave the number to the + // value stage. assert_eq!(check(&bad), None); - assert_eq!(check_key(&bad).as_deref(), Some(NUMERIC_BOB)); + assert_eq!(check_key_shape(&bad), None); + assert_eq!(check_key_values(&bad).as_deref(), Some(NUMERIC_BOB)); assert_eq!(check_key(&with(with_key(key("1")), filter("1"))), None); // Measured 2026-10-30 12:40 UTC: with no filter, the same text. assert_eq!( @@ -170,12 +206,14 @@ fn a_malformed_number_in_the_exclusive_start_key_is_reported_with_the_number_tex } } -/// Measured 2026-10-31 between 23:54 and 23:56 UTC with `{"pk": {"N": "bob"}, -/// "sk": {"S": "x"}}` as the key: each request-time check answers its own text -/// ahead of the key's number (the `Select` rule for `SPECIFIC_ATTRIBUTES`, the -/// filter syntax and the unused-value check, which the handler makes after -/// deserializing, do too; the wire tests send those rows). Measured -/// 2026-10-30 12:40 UTC: the `ReturnConsumedCapacity` enum ahead of the key. +/// Measured 2026-10-31 between 23:54 and 23:56 UTC and 2026-11-05 with `{"pk": +/// {"N": "bob"}, "sk": {"S": "x"}}` as the key: each request-time check answers +/// its own text ahead of the key's number, in the newer form; beside a key +/// value that is empty or carries an unknown type descriptor the constraint +/// clauses take the older form (2026-11-05). The request parameter checks, the +/// expression checks and the Segment pairing, which the handler makes after +/// deserializing, come ahead of the key's number too; the wire tests send +/// those rows. #[test] fn every_request_time_check_comes_ahead_of_the_number_in_the_exclusive_start_key() { let bob = json!({"pk": {"N": "bob"}, "sk": {"S": "x"}}); @@ -218,48 +256,160 @@ fn every_request_time_check_comes_ahead_of_the_number_in_the_exclusive_start_key let body = with(with_key(bob.clone()), extra); assert_eq!(check(&body).as_deref(), Some(text), "{body}"); } + // Beside a key value the shape stage refuses, the older form (2026-11-05). + for key in [ + json!({"pk": {}, "sk": {"S": "x"}}), + json!({"pk": {"X": "1"}, "sk": {"S": "x"}}), + ] { + assert_eq!( + check(&with(with_key(key.clone()), json!({"Limit": 0}))).as_deref(), + Some( + "1 validation error detected: Value '0' at 'limit' failed to satisfy constraint: Member must have value greater than or equal to 1" + ), + "{key}" + ); + } } -/// Measured 2026-10-31 between 23:54 and 23:56 UTC: inside the key a number -/// fault comes ahead of a value that is empty or has an unknown type -/// descriptor, in either member order; with two number faults the first -/// member is named; a key whose only fault is an empty value answers the -/// service's key text. +/// Measured 2026-11-05: a key value that is empty, carries an unknown or +/// lower-case type descriptor, or carries two, at its top or inside an `L` or +/// `M`, in a key member or in a member that is not a key attribute, answers the +/// service's text with the key prefix at the shape stage. #[test] -fn a_number_fault_in_the_key_comes_ahead_of_an_empty_or_unknown_descriptor_value() { +fn a_shape_fault_in_a_key_value_answers_the_prefixed_text_at_the_shape_stage() { + let empty = format!("{KEY_PREFIX}{EMPTY_VALUE}"); for key in [ - json!({"pk": {}, "sk": {"N": "bob"}}), - json!({"sk": {"N": "bob"}, "pk": {}}), - json!({"pk": {"X": "1"}, "sk": {"N": "bob"}}), - json!({"pk": {"N": "bob"}, "sk": {"N": "fred"}}), + json!({"pk": {}, "sk": {"S": "x"}}), + json!({"pk": {"X": "1"}, "sk": {"S": "x"}}), + json!({"pk": {"s": "x"}, "sk": {"S": "x"}}), + json!({"pk": {"L": [{}]}, "sk": {"S": "x"}}), + json!({"pk": {"M": {"k": {}}}, "sk": {"S": "x"}}), + json!({"pk": {"S": "p0"}, "sk": {"S": "x"}, "z": {}}), + json!({"pk": {}}), ] { let body = with_key(key.clone()); - // The request-time checks leave the key to its own stage. assert_eq!(check(&body), None, "{key}"); - assert_eq!(check_key(&body).as_deref(), Some(NUMERIC_BOB), "{key}"); + assert_eq!( + check_key_shape(&body).as_deref(), + Some(empty.as_str()), + "{key}" + ); } - let empty = with_key(json!({"pk": {}, "sk": {"S": "x"}})); - assert_eq!(check_key(&empty).as_deref(), Some(EMPTY_KEY_VALUE)); - // Measured 2026-10-31 between 23:54 and 23:56 UTC too: a Limit fault - // beside the empty value answers the Limit clause, in the older form. + let two = with_key(json!({"pk": {"S": "x", "N": "1"}, "sk": {"S": "x"}})); assert_eq!( - check(&with(empty, json!({"Limit": 0}))).as_deref(), + check_key_shape(&two).as_deref(), Some( - "1 validation error detected: Value '0' at 'limit' failed to satisfy constraint: Member must have value greater than or equal to 1" + "The provided starting key is invalid: Supplied AttributeValue has more than one datatypes set, must contain exactly one of the supported datatypes" ) ); + // A readable key passes the shape stage. + assert_eq!( + check_key_shape(&with_key(json!({"pk": {"N": "bob"}, "sk": {"NS": []}}))), + None + ); +} + +/// Measured 2026-11-05: the set faults of a key answer the service's texts with +/// no prefix at the value stage, after every number in the key. +#[test] +fn a_set_fault_in_a_key_value_answers_the_service_text_without_a_prefix() { + for (key, text) in [ + (json!({"NS": []}), "An number set may not be empty"), + (json!({"SS": []}), "An string set may not be empty"), + (json!({"BS": []}), "Binary sets should not be empty"), + ( + json!({"NS": ["1", "1"]}), + "Input collection contains duplicates.", + ), + ( + json!({"NS": ["1", "1.0"]}), + "Input collection contains duplicates.", + ), + ( + json!({"SS": ["a", "a"]}), + r#"Input collection ["a", "a"] contains duplicates."#, + ), + ] { + let body = with_key(json!({"pk": key, "sk": {"S": "x"}})); + assert_eq!( + check_key_values(&body), + Some(format!("{INVALID}{text}")), + "{key}" + ); + } + // A number in a set is reported with the number text. + let bob = with_key(json!({"pk": {"NS": ["bob"]}, "sk": {"S": "x"}})); + assert_eq!(check_key_values(&bob).as_deref(), Some(NUMERIC_BOB)); + // A set with distinct numbers passes. + assert_eq!( + check_key_values(&with_key( + json!({"pk": {"NS": ["1", "2"]}, "sk": {"S": "x"}}) + )), + None + ); +} + +/// Faults in two members of one key. The service's choice follows the attribute +/// names and, on some tables, the request's member order (measured 2026-11-05 +/// on three tables): `{"pk": {}, "sk": {"N": "bob"}}` answered `bob` on +/// 2026-10-31 and `{"a": {}, "b": {"N": "bob"}}` the empty value. The code +/// reports a shape fault in any member ahead of a value fault in any member, +/// then the members in sorted order; this test accepts either member's fault. +#[test] +fn two_faulty_members_answer_one_of_the_two_faults() { + let empty = format!("{KEY_PREFIX}{EMPTY_VALUE}"); + let ns_empty = format!("{INVALID}An number set may not be empty"); + let fred = NUMERIC_BOB.replace("bob", "fred"); + for (key, either) in [ + ( + json!({"pk": {}, "sk": {"N": "bob"}}), + [empty.as_str(), NUMERIC_BOB], + ), + ( + json!({"sk": {"N": "bob"}, "pk": {}}), + [empty.as_str(), NUMERIC_BOB], + ), + ( + json!({"pk": {"X": "1"}, "sk": {"N": "bob"}}), + [empty.as_str(), NUMERIC_BOB], + ), + ( + json!({"pk": {"N": "bob"}, "sk": {}}), + [empty.as_str(), NUMERIC_BOB], + ), + ( + json!({"pk": {"N": "bob"}, "sk": {"N": "fred"}}), + [NUMERIC_BOB, fred.as_str()], + ), + ( + json!({"pk": {"NS": []}, "sk": {"N": "bob"}}), + [ns_empty.as_str(), NUMERIC_BOB], + ), + ( + json!({"pk": {}, "sk": {"NS": []}}), + [empty.as_str(), ns_empty.as_str()], + ), + ] { + let answer = check_key(&with_key(key.clone())); + assert!( + either.contains(&answer.as_deref().unwrap_or_default()), + "{key}: {answer:?}" + ); + } } /// `StartKeyBody::take` holds back a key the deserializer refuses for its /// value and leaves one of the wrong JSON shape in the body, where the typed -/// deserialization reports it ahead of every check. +/// deserialization reports it ahead of every check; no service answer is +/// asserted here. #[test] fn a_key_of_the_wrong_shape_stays_in_the_body_and_an_unreadable_value_is_held_back() { for key in [json!("x"), json!({"pk": "x"}), json!({"pk": {"N": 5}})] { let mut body = with_key(key.clone()); let start_key = StartKeyBody::take(&mut body); assert!(body.get("ExclusiveStartKey").is_some(), "{key}"); - assert!(start_key.validate().is_ok(), "{key}"); + assert!(start_key.validate_shape().is_ok(), "{key}"); + assert!(start_key.validate_values().is_ok(), "{key}"); let shape = serde_json::from_value::(body) .map_err(crate::deserialize_error); assert!( @@ -271,24 +421,72 @@ fn a_key_of_the_wrong_shape_stays_in_the_body_and_an_unreadable_value_is_held_ba json!({"pk": {}}), json!({"pk": {"X": "1"}}), json!({"pk": {"S": "x", "N": "1"}}), + json!({"pk": {"NS": []}}), + json!({"pk": {"SS": ["a", "a"]}}), ] { let mut body = with_key(key.clone()); let start_key = StartKeyBody::take(&mut body); assert!(body.get("ExclusiveStartKey").is_none(), "{key}"); - assert!(start_key.validate().is_err(), "{key}"); + assert!( + start_key + .validate_shape() + .and(start_key.validate_values()) + .is_err(), + "{key}" + ); assert!( serde_json::from_value::(body).is_ok(), "{key}" ); } - // A readable key stays in the body and passes. + // A readable key stays in the body and passes both stages. let mut body = with_key(json!({"pk": {"S": "p0"}, "sk": {"S": "x"}})); let start_key = StartKeyBody::take(&mut body); assert!(body.get("ExclusiveStartKey").is_some()); - assert!(start_key.validate().is_ok()); + assert!(start_key.validate_shape().is_ok()); + assert!(start_key.validate_values().is_ok()); // No key at all. let mut body = scan(json!({})); - assert!(StartKeyBody::take(&mut body).validate().is_ok()); + let start_key = StartKeyBody::take(&mut body); + assert!( + start_key + .validate_shape() + .and(start_key.validate_values()) + .is_ok() + ); +} + +/// `ExpressionValuesBody::take` puts a readable stand-in in the body for a +/// value the deserializer refuses for its content and keeps the deserializer's +/// fault for the map as written (measured 2026-11-05: the fault comes after the +/// key's value stage); a value of the wrong JSON shape stays in the body. +#[test] +fn an_unreadable_expression_value_is_held_back_and_a_stand_in_takes_its_place() { + let mut body = scan(json!({ + "FilterExpression": "a = :a", + "ExpressionAttributeValues": {":a": {}}, + })); + let values = super::ExpressionValuesBody::take(&mut body); + assert_eq!(body["ExpressionAttributeValues"], json!({":a": {"S": ""}})); + assert_eq!( + validation_message(values.validate()).as_deref(), + Some( + "ExpressionAttributeValues contains invalid value: Supplied AttributeValue is empty, must contain exactly one of the supported datatypes for key :a" + ) + ); + let mut body = scan(json!({"ExpressionAttributeValues": {":a": {"X": "1"}}})); + let values = super::ExpressionValuesBody::take(&mut body); + assert_eq!(body["ExpressionAttributeValues"], json!({":a": {"S": ""}})); + assert!( + matches!(values.validate(), Err(DynamoDbError::SerializationException(msg)) if msg.contains("unknown AttributeValue type descriptor: X")) + ); + // A readable value and a value of the wrong JSON shape stay as written. + for value in [json!({"S": "x"}), json!({"N": 5})] { + let mut body = scan(json!({"ExpressionAttributeValues": {":a": value}})); + let values = super::ExpressionValuesBody::take(&mut body); + assert_eq!(body["ExpressionAttributeValues"], json!({":a": value})); + assert!(values.validate().is_ok()); + } } #[test] diff --git a/tests/python/test_scan_request_validation.py b/tests/python/test_scan_request_validation.py index 4c53ff255..b47d63e4d 100644 --- a/tests/python/test_scan_request_validation.py +++ b/tests/python/test_scan_request_validation.py @@ -16,10 +16,14 @@ nesting text, 31 reaches the unused-value check. A malformed number at the bottom is reported first, at the depths 31 to 60 that were measured. -The numbers in ``ExclusiveStartKey`` are checked after every other check in -this file and after the filter syntax and unused-value checks, and inside the -key a number fault comes ahead of an empty value or an unknown type -descriptor (2026-10-31). +The faults of ``ExclusiveStartKey`` come at two places (measured on +2026-11-05, each request twice): a value that is empty, carries an unknown type +descriptor or two descriptors is reported after the constraint clauses and the +request parameter checks, with the prefix ``The provided starting key is +invalid: ``, and ahead of the expression checks; a number or set fault is +reported after the expression and request checks, with no prefix, and ahead of +the table lookup. With faults in two members the service's choice follows the +attribute names and the request's order, so those tests accept either fault. """ from __future__ import annotations @@ -76,10 +80,24 @@ "SPECIFIC_ATTRIBUTES" ) UNUSED_U = "Value provided in ExpressionAttributeValues unused in expressions: keys: {:u}" +KEY_PREFIX = "The provided starting key is invalid: " EMPTY_KEY_VALUE = ( - "The provided starting key is invalid: Supplied AttributeValue is empty, " - "must contain exactly one of the supported datatypes" + KEY_PREFIX + "Supplied AttributeValue is empty, must contain exactly one of the " + "supported datatypes" ) +TWO_DESCRIPTORS = ( + KEY_PREFIX + "Supplied AttributeValue has more than one datatypes set, must contain " + "exactly one of the supported datatypes" +) +INVALID = "One or more parameter values were invalid: " +NS_EMPTY = INVALID + "An number set may not be empty" +SS_EMPTY = INVALID + "An string set may not be empty" +BS_EMPTY = INVALID + "Binary sets should not be empty" +NS_DUP = INVALID + "Input collection contains duplicates." +SS_DUP = INVALID + 'Input collection ["a", "a"] contains duplicates.' +NUMERIC_FRED = NUMERIC_BOB.replace("bob", "fred") +EAV_NO_EXPRESSION = "ExpressionAttributeValues can only be specified when using expressions" +MISSING_INDEX = "The table does not have the specified index: nope" VALID_FILTER = '"FilterExpression": "a = :n", "ExpressionAttributeValues": {":n": {"N": "1"}}' BOB_FILTER = '"FilterExpression": "a = :n", "ExpressionAttributeValues": {":n": {"N": "bob"}}' @@ -115,8 +133,8 @@ '"ReturnConsumedCapacity": "BAD"', RCC_ENUM), ] -# Measured 2026-10-31 between 23:54 and 23:56 UTC: every other check comes -# ahead of the number in ExclusiveStartKey. +# Measured 2026-10-31 between 23:54 and 23:56 UTC: each check sent beside the +# key comes ahead of the number in ExclusiveStartKey. KEY_AFTER_THE_OTHER_CHECKS = [ ("Limit-0", BOB_KEY + ', "Limit": 0', LIMIT), ("Segment-minus-1", BOB_KEY + ', "Segment": -1, "TotalSegments": 2', SEGMENT), @@ -131,17 +149,10 @@ '"ExclusiveStartKey": {"pk": {"N": "fred"}, "sk": {"S": "x"}}, ' + BOB_FILTER, NUMERIC_BOB), ] -# Measured 2026-10-31 between 23:54 and 23:56 UTC: inside the key a number -# fault anywhere comes ahead of an empty value or an unknown type descriptor, -# in either member order; two number faults name the first member; a key -# whose only fault is an empty value answers the service's key text. -KEY_FAULTS = [ - ("empty-pk-then-bob", '"ExclusiveStartKey": {"pk": {}, "sk": {"N": "bob"}}', NUMERIC_BOB), - ("bob-then-empty-pk", '"ExclusiveStartKey": {"sk": {"N": "bob"}, "pk": {}}', NUMERIC_BOB), - ("type-X-pk-then-bob", '"ExclusiveStartKey": {"pk": {"X": "1"}, "sk": {"N": "bob"}}', - NUMERIC_BOB), - ("bob-then-fred", '"ExclusiveStartKey": {"pk": {"N": "bob"}, "sk": {"N": "fred"}}', - NUMERIC_BOB), +# Measured 2026-10-31 and 2026-11-05: a number fault anywhere in the key, in +# an NS, in an L, in a non-key attribute and in an LSI key, answers the number +# text with no prefix. +KEY_NUMBER_FAULTS = [ ("bob-in-an-NS", '"ExclusiveStartKey": {"pk": {"NS": ["bob"]}, "sk": {"S": "x"}}', NUMERIC_BOB), ("bob-in-an-L", '"ExclusiveStartKey": {"pk": {"L": [{"N": "bob"}]}, "sk": {"S": "x"}}', @@ -152,7 +163,73 @@ ("bob-in-the-index-key-of-an-LSI-scan", '"IndexName": "lsi", "ExclusiveStartKey": {"pk": {"S": "p0"}, "sk": {"S": "x"}, ' '"ln": {"N": "bob"}}', NUMERIC_BOB), - ("empty-pk-alone", '"ExclusiveStartKey": {"pk": {}, "sk": {"S": "x"}}', EMPTY_KEY_VALUE), +] + +# Measured 2026-11-05: each key alone (the keys of MEASURED-scan-r3 section 4). +# A shape fault answers the service's text with the key prefix; a set fault +# answers the service's text with no prefix. +KEYS_ALONE = [ + ("empty", '{"pk": {}, "sk": {"S": "x"}}', EMPTY_KEY_VALUE), + ("type-X", '{"pk": {"X": "1"}, "sk": {"S": "x"}}', EMPTY_KEY_VALUE), + ("lower-case-s", '{"pk": {"s": "x"}, "sk": {"S": "x"}}', EMPTY_KEY_VALUE), + ("empty-in-an-M", '{"pk": {"M": {"k": {}}}, "sk": {"S": "x"}}', EMPTY_KEY_VALUE), + ("empty-in-an-L", '{"pk": {"L": [{}]}, "sk": {"S": "x"}}', EMPTY_KEY_VALUE), + ("empty-non-key-attribute", '{"pk": {"S": "p0"}, "sk": {"S": "x"}, "z": {}}', EMPTY_KEY_VALUE), + ("two-descriptors", '{"pk": {"S": "x", "N": "1"}, "sk": {"S": "x"}}', TWO_DESCRIPTORS), + ("NS-empty", '{"pk": {"NS": []}, "sk": {"S": "x"}}', NS_EMPTY), + ("SS-empty", '{"pk": {"SS": []}, "sk": {"S": "x"}}', SS_EMPTY), + ("BS-empty", '{"pk": {"BS": []}, "sk": {"S": "x"}}', BS_EMPTY), + ("NS-duplicates", '{"pk": {"NS": ["1", "1"]}, "sk": {"S": "x"}}', NS_DUP), + ("NS-duplicates-by-value", '{"pk": {"NS": ["1", "1.0"]}, "sk": {"S": "x"}}', NS_DUP), + ("SS-duplicates", '{"pk": {"SS": ["a", "a"]}, "sk": {"S": "x"}}', SS_DUP), + ("NS-empty-non-key-attribute", '{"pk": {"S": "p0"}, "sk": {"S": "x"}, "z": {"NS": []}}', + NS_EMPTY), +] + +# Measured 2026-11-05: faults in two members of one key. The service's choice +# follows the attribute names and, on some tables, the request's member order: +# {"pk": {}, "sk": {"N": "bob"}} answered bob on 2026-10-31, and {"a": {}, "b": +# {"N": "bob"}} the empty value. Each row accepts either member's fault. +TWO_MEMBER_FAULTS = [ + ("empty-pk-then-bob", '{"pk": {}, "sk": {"N": "bob"}}', (EMPTY_KEY_VALUE, NUMERIC_BOB)), + ("bob-then-empty-pk", '{"sk": {"N": "bob"}, "pk": {}}', (EMPTY_KEY_VALUE, NUMERIC_BOB)), + ("type-X-pk-then-bob", '{"pk": {"X": "1"}, "sk": {"N": "bob"}}', + (EMPTY_KEY_VALUE, NUMERIC_BOB)), + ("bob-then-empty-sk", '{"pk": {"N": "bob"}, "sk": {}}', (EMPTY_KEY_VALUE, NUMERIC_BOB)), + ("two-descriptors-then-bob", '{"pk": {"S": "x", "N": "1"}, "sk": {"N": "bob"}}', + (TWO_DESCRIPTORS, NUMERIC_BOB)), + ("NS-empty-then-bob", '{"pk": {"NS": []}, "sk": {"N": "bob"}}', (NS_EMPTY, NUMERIC_BOB)), + ("empty-then-NS-empty", '{"pk": {}, "sk": {"NS": []}}', (EMPTY_KEY_VALUE, NS_EMPTY)), + ("NS-empty-then-empty", '{"pk": {"NS": []}, "sk": {}}', (EMPTY_KEY_VALUE, NS_EMPTY)), + ("empty-then-NS-bob", '{"pk": {}, "sk": {"NS": ["1", "bob"]}}', (EMPTY_KEY_VALUE, NUMERIC_BOB)), +] + +# Measured 2026-11-05 (MEASURED-scan-r3 section 2): beside each class of key +# fault, a check of each stage. The request parameter checks come ahead of +# every key fault; the shape faults come ahead of the expression checks and of +# the table lookup; the expression checks come ahead of the number and set +# faults, which come ahead of the table lookup. +KEY_CLASSES = [ + ("empty", '{"pk": {}, "sk": {"S": "x"}}', EMPTY_KEY_VALUE, True), + ("type-X", '{"pk": {"X": "1"}, "sk": {"S": "x"}}', EMPTY_KEY_VALUE, True), + ("two-descriptors", '{"pk": {"S": "x", "N": "1"}, "sk": {"S": "x"}}', TWO_DESCRIPTORS, True), + ("bob", '{"pk": {"N": "bob"}, "sk": {"S": "x"}}', NUMERIC_BOB, False), + ("NS-empty", '{"pk": {"NS": []}, "sk": {"S": "x"}}', NS_EMPTY, False), + ("SS-duplicates", '{"pk": {"SS": ["a", "a"]}, "sk": {"S": "x"}}', SS_DUP, False), +] +STAGES = [ + # A request parameter check (S1): ahead of every key fault. The head's + # text carries a suffix the service dropped (#722), so the test asserts the + # prefix of the text. + ("ExpressionAttributeValues-with-no-expression", + '"ExpressionAttributeValues": {":a": {"S": "x"}}', "parameter"), + # An expression check (S3): ahead of a number or set fault, after a shape + # fault. + ("unused-value", + '"FilterExpression": "a = :p", "ExpressionAttributeValues": {":p": {"S": "x"}, ":u": {"S": "y"}}', + "expression"), + # The table's indexes (S5): after every key fault. + ("missing-index", '"IndexName": "nope"', "table"), ] # Measured 2026-10-31 between 23:54 and 23:56 UTC on a table with an N hash @@ -275,8 +352,7 @@ def test_an_empty_scan_filter_alone_is_accepted(self, dynamodb_client, table): class TestExclusiveStartKey: - """The numbers in the key, after every other check and ahead of the key's - other faults (measured 2026-10-31 between 23:54 and 23:56 UTC).""" + """The key's two stages (measured 2026-10-31 and 2026-11-05).""" @pytest.mark.parametrize("fields, expected", [r[1:] for r in KEY_AFTER_THE_OTHER_CHECKS], ids=[r[0] for r in KEY_AFTER_THE_OTHER_CHECKS]) @@ -289,8 +365,8 @@ def test_32_levels_are_refused_ahead_of_the_key(self, dynamodb_client, table): def test_the_select_rule_is_checked_ahead_of_the_key(self, dynamodb_client, table): # The service answers '1 validation error detected: ' and then this - # text. The Scan text here has no prefix, so only the check's place - # is asserted. + # text (#832). The Scan text here has no prefix, so only the check's + # place is asserted. fields = BOB_KEY + ', "Select": "SPECIFIC_ATTRIBUTES"' message = _validation_message(*_scan(dynamodb_client, table, fields)) assert message.endswith(SELECT_SPECIFIC), message @@ -304,12 +380,90 @@ def test_the_filter_syntax_is_checked_ahead_of_the_key(self, dynamodb_client, ta message = _validation_message(*_scan(dynamodb_client, table, fields)) assert message.startswith("Invalid FilterExpression:"), message - @pytest.mark.parametrize("fields, expected", [r[1:] for r in KEY_FAULTS], - ids=[r[0] for r in KEY_FAULTS]) - def test_a_number_fault_comes_ahead_of_the_other_faults_in_the_key( - self, dynamodb_client, table, fields, expected): + @pytest.mark.parametrize("fields, expected", [r[1:] for r in KEY_NUMBER_FAULTS], + ids=[r[0] for r in KEY_NUMBER_FAULTS]) + def test_a_number_fault_anywhere_in_the_key(self, dynamodb_client, table, fields, expected): + assert _validation_message(*_scan(dynamodb_client, table, fields)) == expected + + @pytest.mark.parametrize("key, expected", [r[1:] for r in KEYS_ALONE], + ids=[r[0] for r in KEYS_ALONE]) + def test_each_key_alone(self, dynamodb_client, table, key, expected): + fields = '"ExclusiveStartKey": ' + key assert _validation_message(*_scan(dynamodb_client, table, fields)) == expected + @pytest.mark.parametrize("key, either", [r[1:] for r in TWO_MEMBER_FAULTS], + ids=[r[0] for r in TWO_MEMBER_FAULTS]) + def test_two_faulty_members_answer_one_of_the_two_faults( + self, dynamodb_client, table, key, either): + fields = '"ExclusiveStartKey": ' + key + message = _validation_message(*_scan(dynamodb_client, table, fields)) + assert message in either, message + + def test_two_number_faults_name_one_of_them(self, dynamodb_client, table): + # The service named bob twice on 2026-10-31 (23:54 to 23:56 UTC) and + # fred on 2026-11-04 (23:45 UTC) for this key. + fields = '"ExclusiveStartKey": {"pk": {"N": "bob"}, "sk": {"N": "fred"}}' + message = _validation_message(*_scan(dynamodb_client, table, fields)) + assert message in (NUMERIC_BOB, NUMERIC_FRED), message + + @pytest.mark.parametrize("key, key_text, shape", [r[1:] for r in KEY_CLASSES], + ids=[r[0] for r in KEY_CLASSES]) + @pytest.mark.parametrize("check, stage", [(r[1], r[2]) for r in STAGES], + ids=[r[0] for r in STAGES]) + def test_each_stage_beside_each_key_fault( + self, dynamodb_client, table, key, key_text, shape, check, stage): + fields = '"ExclusiveStartKey": ' + key + ", " + check + message = _validation_message(*_scan(dynamodb_client, table, fields)) + if stage == "parameter": + assert message.startswith(EAV_NO_EXPRESSION), message + elif stage == "expression" and not shape: + assert message == UNUSED_U + else: + # A shape fault comes ahead of the expression check; every key + # fault comes ahead of the index resolution. + assert message == key_text + + def test_the_segment_pairing_is_checked_ahead_of_a_number_in_the_key( + self, dynamodb_client, table): + # Measured 2026-11-05: Segment without TotalSegments answers its own + # text beside bob in the key, and the key's text beside an empty value. + fields = BOB_KEY + ', "Segment": 0' + message = _validation_message(*_scan(dynamodb_client, table, fields)) + assert message == ("The TotalSegments parameter is required but was not present in the " + "request when Segment parameter is present"), message + fields = '"ExclusiveStartKey": {"pk": {}, "sk": {"S": "x"}}, "Segment": 0' + assert _validation_message(*_scan(dynamodb_client, table, fields)) == EMPTY_KEY_VALUE + + def test_an_undefined_filter_value_is_checked_ahead_of_a_number_in_the_key( + self, dynamodb_client, table): + # Measured 2026-11-05. + fields = BOB_KEY + ', "FilterExpression": "a = :zz"' + message = _validation_message(*_scan(dynamodb_client, table, fields)) + assert message == ("Invalid FilterExpression: An expression attribute value used in " + "expression is not defined; attribute value: :zz"), message + + def test_a_missing_table_is_reported_after_a_number_in_the_key(self, dynamodb_client): + # Measured 2026-11-05: the key's number text ahead of + # ResourceNotFoundException. + status, payload = _scan(dynamodb_client, unique_name("nosuch"), BOB_KEY) + assert status == 400, payload + assert payload.get("__type", "").endswith("ValidationException"), payload + assert payload.get("message") == NUMERIC_BOB, payload + + def test_an_empty_expression_value_is_reported_after_the_key_and_ahead_of_the_table( + self, dynamodb_client, table): + # Measured 2026-11-05: {":a": {}} read by the filter answers its own + # text with no key and the key's text beside bob or an empty set. + eav = '"FilterExpression": "a = :a", "ExpressionAttributeValues": {":a": {}}' + text = ("ExpressionAttributeValues contains invalid value: Supplied AttributeValue is " + "empty, must contain exactly one of the supported datatypes for key :a") + assert _validation_message(*_scan(dynamodb_client, table, eav)) == text + assert _validation_message(*_scan(dynamodb_client, table, BOB_KEY + ", " + eav)) == NUMERIC_BOB + fields = '"ExclusiveStartKey": {"pk": {"NS": []}, "sk": {"S": "x"}}, ' + eav + assert _validation_message(*_scan(dynamodb_client, table, fields)) == NS_EMPTY + status, payload = _scan(dynamodb_client, unique_name("nosuch"), eav) + assert payload.get("message") == text, payload + @pytest.mark.parametrize("fields, expected", [r[1:] for r in N_KEY_FAULTS], ids=[r[0] for r in N_KEY_FAULTS]) def test_the_number_texts_on_an_n_hash_key(self, dynamodb_client, n_table, fields, expected): From dba7827a23dc03c107d01b1276f0e7695819c966 Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Thu, 5 Nov 2026 04:04:44 +0000 Subject: [PATCH 5/9] fix(scan): follow the service's two check orders The service checks a Scan request in one of two orders. A value that is not one AttributeValue is empty, has an unknown or lower-case type descriptor, or has two descriptors. The fault can sit at the top or inside an L or M. A request with such a value in ExpressionAttributeValues, ExclusiveStartKey or a ScanFilter list follows order U. Every other request follows order R. The same test picks the older form of the constraint clauses. Measured against DynamoDB (us-east-1) on 2026-10-31 and 2026-11-05, each request twice. Order R: - The constraint clauses in the newer form, then the ReturnConsumedCapacity enum, which comes ahead of the parameter mix. Then the mix, the numbers in ExpressionAttributeValues and their nesting depth. The nesting check comes ahead of the no-expression check (B4-7, 2026-10-31). - An empty or duplicate set in ExpressionAttributeValues, ahead of the no-expression check. - The request parameter checks: ExpressionAttributeNames with no expression, ExpressionAttributeValues with no expression, ConditionalOperator with no ScanFilter, a ScanFilter condition's value count. Then the Select enum. - The expression checks: the filter syntax, the projection syntax, an undefined value, the operand types and the Select rules. Then the unused values, COUNT with a ProjectionExpression, COUNT with AttributesToGet and the Segment pairing. - The key's number and set faults, then the table and its indexes. Order U: - The constraint clauses in the older form, then the ReturnConsumedCapacity and Select enums. Then the mix and the other request parameter checks. - The key's faults, shape and value together: the first faulty member in the member order below. A shape fault carries the prefix The provided starting key is invalid: on the empty-value or more-than-one text. A number carries the value check's text. - The ExpressionAttributeValues entries the deserializer refuses, read or not, with the text ExpressionAttributeValues contains invalid value: ... for key :e. An unknown descriptor answers the empty-value text there. - The nesting depth, then the expression checks and the table. The member order inside ExclusiveStartKey is fitted to 112 measured rows on five tables. It is the order of a Java HashMap of capacity 4 over the member names. String.hashCode() of the name is spread as h ^ (h >>> 16) and masked to four buckets. The buckets come in ascending order, and the request's order holds inside a bucket. sk falls in bucket 0, a in 1, b in 2, pk, gpk, id and ts in 3. The parsed body sorts object members on every build but the MongoDB one. dispatch now passes the request text on, and the handler reads the request's member order from it. Two value faults in two members vary between sends on the service. The tests for those accept either answer. The set texts follow the order. Order R quotes the members: Input collection ["a", "a"] contains duplicates. and Input collection ["YQ==","YQ=="]of type BS contains duplicates. Order U does not: Input collection [a, a] contains duplicates. For an NS in order U the text is Input collection contains duplicates, with no prefix and no full stop. An empty SS in the key carries the key prefix in order U. Before this change the key's shape faults came at one stage and its value faults at another in every request. The unreadable ExpressionAttributeValues entries came after the key's value faults whether or not an expression read them. The Select enum was a request-time check in every request. Row B4-7 of 2026-10-31 (an unused empty value beside a nested value) answered the unused-value text instead of the empty value's. Not measured, and kept in the code's order. In order R, the nesting check against ExpressionAttributeNames with no expression, ConditionalOperator and the ScanFilter value count. A set fault in ExpressionAttributeValues against the nesting check. In order U, the Select enum against the mix. A malformed ExpressionAttributeValues number against the key's faults. An empty BS or a BS with duplicates in the key. An empty SS in the key beside an unreadable expression value. Which entry is named when several ExpressionAttributeValues entries are faulty: the first in sorted order. Keys of four members follow the same member order. The texts of #722 (the no-expression suffix), #832, #838 and #839 do not change here. Tests: request_tests.rs covers the order flag, the texts of each order, the member order and its buckets, and what the handler holds back. tests/python/test_scan_request_validation.py sends 161 rows as raw signed JSON, among them B4-7, the R5 rows of 2026-11-05 and the member-order rows on four tables. 41 of them fail on the previous commit. --- crates/engine/src/lib.rs | 6 +- crates/engine/src/scan.rs | 212 +++-- crates/engine/src/scan_request.rs | 866 ++++++++++++------ crates/engine/src/scan_request/range_tests.rs | 4 +- .../engine/src/scan_request/request_tests.rs | 615 +++++++++---- crates/engine/src/scan_request/tests.rs | 26 +- crates/server/src/handler.rs | 2 +- tests/python/test_scan_request_validation.py | 355 ++++++- tests/test_scan_number_values.py | 9 +- 9 files changed, 1507 insertions(+), 588 deletions(-) diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index a8ab1d5fc..04fae6189 100755 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -432,10 +432,14 @@ impl OperationContext { /// Dispatch an operation by name. /// +/// `raw_body` is the request text `body` was parsed from; `Scan` reads the +/// member order of its `ExclusiveStartKey` from it. +/// /// Returns the JSON response body and sideband metrics for the metrics collector. pub async fn dispatch( operation: &str, body: serde_json::Value, + raw_body: &[u8], ctx: &OperationContext, server_addr: &str, ) -> Result { @@ -464,7 +468,7 @@ pub async fn dispatch( "DeleteItem" => handle_delete_item(body, ctx).await, "UpdateItem" => handle_update_item(body, ctx).await, "Query" => handle_query(body, ctx).await, - "Scan" => handle_scan(body, ctx).await, + "Scan" => handle_scan(body, raw_body, ctx).await, "SearchVectors" => handle_search_vectors(body, ctx).await, "BatchGetItem" => handle_batch_get_item(body, ctx).await, "BatchWriteItem" => handle_batch_write_item(body, ctx).await, diff --git a/crates/engine/src/scan.rs b/crates/engine/src/scan.rs index bad2efe54..9642476c1 100755 --- a/crates/engine/src/scan.rs +++ b/crates/engine/src/scan.rs @@ -25,6 +25,7 @@ use crate::index_helpers::{ }; use crate::legacy_filter::desugar_filter; use crate::read_helpers::{PostRead, read_page}; +use crate::scan_request::{CheckOrder, ExpressionValuesBody, SelectBody, StartKeyBody}; use crate::serialize_output; use crate::{DispatchMetrics, DispatchResult}; @@ -32,6 +33,8 @@ use crate::{DispatchMetrics, DispatchResult}; /// /// Reads all items from the table (or segment), applies `FilterExpression` /// post-read, applies `ProjectionExpression`, and enforces the 1 MB limit. +/// `raw_body` is the request text `body` was parsed from; the checks on the +/// `ExclusiveStartKey` read its member order from it (see `scan_request`). /// /// # Errors /// @@ -39,39 +42,47 @@ use crate::{DispatchMetrics, DispatchResult}; #[allow(clippy::cast_possible_wrap)] // item counts won't exceed i64::MAX pub async fn handle_scan( mut body: Value, + raw_body: &[u8], ctx: &OperationContext, ) -> Result { - // The checks run in the service's order, measured 2026-10-25 to 2026-11-05 - // (see `scan_request`): + // The checks run in one of the service's two orders, measured 2026-10-25 + // to 2026-11-05 (see `scan_request`): order U when the body holds a value + // that is not one AttributeValue anywhere, order R otherwise. Order R: // - request-time, on the raw body: the AttributesToGet length and the // Limit, Segment and TotalSegments ranges in one message, the - // expression and non-expression parameter mix, the ReturnConsumedCapacity - // enum, every number in ExpressionAttributeValues, the nesting depth of - // ExpressionAttributeValues, the Select enum; + // ReturnConsumedCapacity enum, the expression and non-expression + // parameter mix, every number in ExpressionAttributeValues, the nesting + // depth of ExpressionAttributeValues; // - a field of the wrong JSON type anywhere in the body; - // - the request parameter checks: ExpressionAttributeValues and - // ExpressionAttributeNames with no expression, ConditionalOperator with + // - an empty or duplicate set in ExpressionAttributeValues; + // - the request parameter checks: ExpressionAttributeNames and + // ExpressionAttributeValues with no expression, ConditionalOperator with // no ScanFilter, a ScanFilter condition's value count; - // - the ExclusiveStartKey's shape faults (an empty value, an unknown type - // descriptor, two descriptors); - // - the expression and request checks: the filter syntax, an undefined - // value, the operand types, the projection syntax, the Select rules, - // the unused values, the Segment pairing; + // - the Select enum; + // - the expression and request checks: the filter syntax, the projection + // syntax, an undefined value, the operand types, the Select rules, the + // unused values, COUNT with AttributesToGet, the Segment pairing; // - the ExclusiveStartKey's value faults (a number, an empty or duplicate - // set), then a value of ExpressionAttributeValues that does not - // deserialize; + // set), in the fitted member order; // - the table lookup, the index resolution and the index rules, the key // schema. - crate::scan_request::validate_scan_request_body(&body)?; - // A key or an ExpressionAttributeValues value that does not deserialize - // is held back here, so that the typed deserialization does not report it - // ahead of the checks the service makes first. - let start_key = crate::scan_request::StartKeyBody::take(&mut body); - let expression_values = crate::scan_request::ExpressionValuesBody::take(&mut body); + // Order U differs in these places: the constraint clauses take the older + // form and the Select enum joins them at request time; the nesting check + // leaves request time; after the request parameter checks come the + // ExclusiveStartKey's faults, shape and value together, then the + // ExpressionAttributeValues entries the deserializer refuses, then the + // nesting depth, then the expression and request checks and the table. + let order = CheckOrder::of(&body); + crate::scan_request::validate_scan_request_body(&body, order)?; + // A key, an ExpressionAttributeValues entry or a Select value that does + // not deserialize is held back here, so that the typed deserialization + // does not report it ahead of the checks the service makes first. + let key_order = crate::scan_request::start_key_member_order(raw_body); + let start_key = StartKeyBody::take(&mut body, key_order.as_deref()); + let expression_values = ExpressionValuesBody::take(&mut body, order); + let select = SelectBody::take(&mut body, order); let input: ScanInput = serde_json::from_value(body).map_err(crate::deserialize_error)?; - // Validate Filter/Projection expressions before the existence - // check; index resolution, Segment pairing, Select and key checks stay after. let has_cond_op = input.conditional_operator.is_some(); let maps = build_expression_maps( @@ -79,6 +90,12 @@ pub async fn handle_scan( input.expression_attribute_values.as_ref(), ); + // Order R: an empty or duplicate set in ExpressionAttributeValues, ahead + // of the no-expression check (measured 2026-11-05). + if order == CheckOrder::Readable { + expression_values.validate()?; + } + let has_filter_expr = input .filter_expression .as_ref() @@ -103,7 +120,7 @@ pub async fn handle_scan( // Desugar a legacy ScanFilter: its value count is a request parameter // check, ahead of every fault in the key (measured 2026-11-05). The mix - // check above refuses a ScanFilter beside a FilterExpression, so the two + // check refuses a ScanFilter beside a FilterExpression, so the two // branches below never both apply. let legacy_filter = match input.scan_filter { Some(ref sf) if !sf.is_empty() => { @@ -113,11 +130,26 @@ pub async fn handle_scan( _ => None, }; - // The ExclusiveStartKey's shape stage: an empty value, an unknown type - // descriptor or two descriptors, after the request parameter checks and - // ahead of the expression checks (measured 2026-11-05; see - // `scan_request::StartKeyBody`). - start_key.validate_shape()?; + match order { + CheckOrder::Unreadable => { + // The key's faults, shape and value together, then the + // ExpressionAttributeValues entries the deserializer refuses, then + // the nesting depth (measured 2026-10-31 and 2026-11-05; see + // `scan_request::StartKeyBody` and `ExpressionValuesBody`). + start_key.validate(order)?; + expression_values.validate()?; + if let Some(ref values) = input.expression_attribute_values { + extenddb_core::validation::validate_attribute_values_nesting_depth( + values.values(), + )?; + } + } + CheckOrder::Readable => { + // The Select enum, after the no-expression check and the nesting + // check and ahead of the filter syntax (measured 2026-11-05). + select.validate()?; + } + } // Parse FilterExpression, or take the desugared ScanFilter. let (filter, filter_maps) = match legacy_filter { @@ -175,14 +207,48 @@ pub async fn handle_scan( None => None, }; + // Build the combined expression maps for the operand checks and the + // post-read evaluation. + let combined_maps = { + let mut names = maps.names.clone(); + let mut values = maps.values.clone(); + + if let Some(ref fm) = filter_maps { + values.extend(fm.values.clone()); + } + if !extra_proj_names.is_empty() { + for (k, v) in &extra_proj_names { + let stripped = k.strip_prefix('#').unwrap_or(k); + names.insert(stripped.to_owned(), v.clone()); + } + } + ExpressionMaps::new(names, values) + }; + + // Validate the filter's placeholders and operand types, ahead of the + // Select rules, the unused values and the Segment pairing (measured + // 2026-11-05). + if let Some(ref f) = filter { + let mut placeholders = Vec::new(); + extenddb_core::expression::collect_value_placeholders(f, &mut placeholders); + for placeholder in placeholders { + combined_maps.resolve_value_for(&placeholder, "FilterExpression")?; + } + extenddb_core::expression::validate_begins_with_operands(f, &combined_maps).map_err( + |e| crate::expression_helpers::prefix_expression_error(e, ExpressionKind::Filter), + )?; + extenddb_core::expression::validate_ordering_operand_types(f, &combined_maps).map_err( + |e| crate::expression_helpers::prefix_expression_error(e, ExpressionKind::Filter), + )?; + } + // Validate Select vs ProjectionExpression and index requirements (shared - // with Query so both reject the same combinations identically). + // with Query so both reject the same combinations identically). COUNT with + // a ProjectionExpression is checked below, after the unused values + // (measured 2026-11-05). extenddb_core::validation::validate_select_projection( input.select, - input - .projection_expression - .as_deref() - .is_some_and(|s| !s.is_empty()), + has_proj_expr && !matches!(input.select, Some(Select::Count)), input .attributes_to_get .as_ref() @@ -216,10 +282,30 @@ pub async fn handle_scan( )?; } + // Select COUNT with a projection: a ProjectionExpression after the unused + // values, AttributesToGet ahead of the Segment pairing, both ahead of the + // key's value faults and the table lookup (measured 2026-11-05). The + // texts are the service's for Scan; the AttributesToGet one differs from + // the service's (#838). + if let Some(Select::Count) = input.select { + if has_proj_expr { + return Err(DynamoDbError::ValidationException( + "Cannot specify the ProjectionExpression when choosing to get only the Count" + .to_owned(), + )); + } + if effective_projection_str.is_some() { + return Err(DynamoDbError::ValidationException( + "Cannot specify the ProjectionExpression when Select is COUNT".to_owned(), + )); + } + } + // Validate Segment/TotalSegments pairing. The range of each field is // checked at request time in validate_scan_request_body; the service - // reports a missing partner with a message per direction, ahead of the - // key's value faults and of the table lookup (measured 2026-11-05). + // reports a missing partner with a message per direction, after the unused + // values and ahead of the key's value faults and of the table lookup + // (measured 2026-11-05). match (input.segment, input.total_segments) { (Some(_), None) => { return Err(DynamoDbError::ValidationException( @@ -246,58 +332,14 @@ pub async fn handle_scan( // Limit >= 1 is checked at request time in validate_scan_request_body. - // Build the combined expression maps for the operand checks and the - // post-read evaluation. - let combined_maps = { - let mut names = maps.names.clone(); - let mut values = maps.values.clone(); - - if let Some(ref fm) = filter_maps { - values.extend(fm.values.clone()); - } - if !extra_proj_names.is_empty() { - for (k, v) in &extra_proj_names { - let stripped = k.strip_prefix('#').unwrap_or(k); - names.insert(stripped.to_owned(), v.clone()); - } - } - ExpressionMaps::new(names, values) - }; - - // Validate the filter's placeholders and operand types, ahead of the key's - // value faults and of the table lookup (measured 2026-11-05). - if let Some(ref f) = filter { - let mut placeholders = Vec::new(); - extenddb_core::expression::collect_value_placeholders(f, &mut placeholders); - for placeholder in placeholders { - combined_maps.resolve_value_for(&placeholder, "FilterExpression")?; - } - extenddb_core::expression::validate_begins_with_operands(f, &combined_maps).map_err( - |e| crate::expression_helpers::prefix_expression_error(e, ExpressionKind::Filter), - )?; - extenddb_core::expression::validate_ordering_operand_types(f, &combined_maps).map_err( - |e| crate::expression_helpers::prefix_expression_error(e, ExpressionKind::Filter), - )?; - } - - // Select COUNT with a projection, from AttributesToGet too: ahead of the - // key's value faults and of the table lookup (measured 2026-11-05). - if let Some(Select::Count) = input.select - && effective_projection_str.is_some() - { - return Err(DynamoDbError::ValidationException( - "Cannot specify the ProjectionExpression when Select is COUNT".to_owned(), - )); + // Order R: the ExclusiveStartKey's value faults, every number in it with + // the value check's text and the set faults, in the fitted member order, + // ahead of the table lookup (measured 2026-11-05; see + // `scan_request::StartKeyBody`). + if order == CheckOrder::Readable { + start_key.validate(order)?; } - // The ExclusiveStartKey's value stage: every number in it with the value - // check's text and the set faults, then a value of - // ExpressionAttributeValues that does not deserialize, both ahead of the - // table lookup (measured 2026-11-05; see `scan_request::StartKeyBody` and - // `scan_request::ExpressionValuesBody`). - start_key.validate_values()?; - expression_values.validate()?; - // P118: Fetch key_info first so we can use table_id for index lookup. let key_info = ctx .table_key_info(&input.table_name) diff --git a/crates/engine/src/scan_request.rs b/crates/engine/src/scan_request.rs index 177203a9b..daf7e1fac 100644 --- a/crates/engine/src/scan_request.rs +++ b/crates/engine/src/scan_request.rs @@ -2,108 +2,104 @@ // SPDX-License-Identifier: Apache-2.0 //! The `Scan` checks that run on the request body alone, before the body is -//! deserialized and before the table is looked up, in the service's order. +//! deserialized and before the table is looked up, and the checks on the +//! `ExclusiveStartKey` and `ExpressionAttributeValues` values the handler makes +//! later, each in the service's order. //! -//! Measured 2026-10-25, 2026-10-30 and 2026-10-31 (us-east-1, each request -//! twice): +//! The service follows one of two orders, measured 2026-10-25 to 2026-11-05 +//! (us-east-1, each request twice; see [`CheckOrder`]). A request that holds a +//! value that is not one AttributeValue anywhere (empty, with a type descriptor +//! the service does not have, or with two, at the top or inside an `L` or `M`, +//! in `ExpressionAttributeValues`, `ExclusiveStartKey` or a `ScanFilter` list; +//! see [`has_unreadable_attribute_value`]) follows order U; every other request +//! follows order R. No single order fits the measured rows: `{"pk": {}, "sk": +//! {"S": "x"}}` with `Select: "BAD"` answers the `Select` clause, a value nested +//! 32 levels with `Select: "BAD"` answers the nesting text, and the same nested +//! value beside that key answers the key's text. //! -//! 1. the constraints the service aggregates into one message: the length of -//! `AttributesToGet` (at least 1) and the ranges of `Limit` (at least 1), -//! `Segment` (0 to 999999) and `TotalSegments` (1 to 1000000), each checked -//! when present, whether or not the other segment field is, every fault in -//! one message: `1 validation error detected: Value at 'Limit' failed to -//! satisfy constraint: Member must have value greater than or equal to 1`, -//! or `2 validation errors detected: ...; ...` in the order -//! `AttributesToGet`, `Limit`, `Segment`, `TotalSegments`. `Limit: 0` with -//! `AttributesToGet: []` answers the length clause and then the `Limit` -//! clause; the empty list alone, with a `FilterExpression`, and with a -//! `FilterExpression` and a number fault answers the length clause alone, -//! so the length comes ahead of the number check. A float or a value -//! outside int32 in any of the three number fields, or a value that is not -//! one AttributeValue in `ExpressionAttributeValues`, in -//! `ExclusiveStartKey` or in a `ScanFilter` `AttributeValueList`, -//! switches the message to the older form, `Value '-2147483648' at 'limit' -//! ...`, with the value saturated or truncated, in the order -//! `totalSegments`, `segment`, `limit`; see `validate_constraints`; +//! Order R (every value readable): +//! +//! 1. the constraints the service aggregates into one message, in the newer +//! form: the length of `AttributesToGet` and the ranges of `Limit`, +//! `Segment` and `TotalSegments` (see [`validate_constraints`]); then the +//! `ReturnConsumedCapacity` enum, ahead of the parameter mix (2026-11-05); //! 2. the expression and non-expression parameter mix, in which a `ScanFilter` -//! object counts as present whether or not it is empty (`ScanFilter: {}` -//! with a `FilterExpression` answers the mix, `ScanFilter: {}` alone is -//! accepted); `AttributesToGet: []` with `ScanFilter: {}` and a -//! `FilterExpression` answers the length clause, so the constraints come -//! ahead of the mix; -//! 3. the `ReturnConsumedCapacity` enum; -//! 4. every number in `ExpressionAttributeValues`, `NS` members and values -//! nested in `L` and `M` included, whether a `FilterExpression` reads the -//! value, nothing reads it, or the request carries no expression at all; a -//! Unicode decimal digit counts as the digit it stands for; -//! 5. the nesting depth of `ExpressionAttributeValues`: a value nested 32 -//! levels deep in `L` answers `Nesting Levels have exceeded supported -//! limits: Attributes in the item have nested levels beyond supported -//! limit`, 31 levels pass, and a malformed number anywhere is reported -//! first. The check counts `M` levels the same way as `L` levels; only `L` -//! was measured with a valid number at the bottom; -//! 6. the `Select` enum, after the nesting check (32 levels with `Select: -//! "BAD"` answers the nesting text). Both enums invalid answers the -//! `ReturnConsumedCapacity` error alone, so the two are not aggregated. +//! object counts as present whether or not it is empty; +//! 3. every number in `ExpressionAttributeValues`, `NS` members and values +//! nested in `L` and `M` included, whether an expression reads the value or +//! not; a Unicode decimal digit counts as the digit it stands for; +//! 4. the nesting depth of `ExpressionAttributeValues`: a value nested 32 +//! levels deep answers `Nesting Levels have exceeded supported limits: +//! Attributes in the item have nested levels beyond supported limit`, 31 +//! levels pass, read or unused, with or without an expression (2026-10-30 +//! and 2026-10-31: ahead of the no-expression check, B4-5a; 2026-11-05: +//! ahead of the filter syntax, the `Select` enum and the key's number); +//! 5. an empty or duplicate set in `ExpressionAttributeValues` (2026-11-05: +//! `{":e": {"NS": []}}` with no expression answers the set text ahead of +//! the no-expression check), see [`ExpressionValuesBody`]; +//! 6. the request parameter checks: `ExpressionAttributeNames` with no +//! expression, then `ExpressionAttributeValues` with no expression, then +//! `ConditionalOperator` with no `ScanFilter`, then a `ScanFilter` +//! condition's value count (2026-11-05); +//! 7. the `Select` enum (2026-11-05: after the no-expression check and the +//! nesting check, ahead of the filter syntax); +//! 8. the expression and request checks: the `FilterExpression` syntax, the +//! `ProjectionExpression` syntax, an undefined value, the operand types, the +//! three `Select` rules, the unused values, `COUNT` with `AttributesToGet`, +//! the `Segment` pairing (2026-11-05, where two were sent together: the +//! filter syntax first; an undefined value and an operand type ahead of the +//! unused values, the `Segment` pairing and `SPECIFIC_ATTRIBUTES`; `COUNT` +//! with `ProjectionExpression` ahead of the unused values; the unused +//! values and `COUNT` with `AttributesToGet` ahead of the `Segment` +//! pairing); +//! 9. the `ExclusiveStartKey` value faults: a number that does not parse, an +//! empty set, a set with duplicates, in the member order of +//! [`StartKeyBody`], with no prefix and the set members JSON-quoted; +//! 10. the table lookup, the index, `ConsistentRead` on a GSI, the key schema. //! -//! The faults of `ExclusiveStartKey` come at two of the handler's stages, -//! measured 2026-11-05 (us-east-1, each request twice; a request parameter -//! check, an expression check and the table's indexes beside each class of -//! fault, and each key alone): +//! Order U (the request holds an unreadable value): //! -//! - the shape faults, after the request parameter checks -//! (`ExpressionAttributeValues` or `ExpressionAttributeNames` with no -//! expression, `ConditionalOperator` with no `ScanFilter`, a `ScanFilter` -//! condition's value count, the parameter mix) and ahead of the expression -//! checks: a value that is empty, carries a type descriptor the service does -//! not have (`X`, a lower-case `s`) or more than one, at the top of a member -//! or inside its `L` or `M`, in a key member or in a member that is not a key -//! attribute. Text: `The provided starting key is invalid: Supplied -//! AttributeValue is empty, must contain exactly one of the supported -//! datatypes`, or the same prefix and `Supplied AttributeValue has more than -//! one datatypes set, must contain exactly one of the supported datatypes`. -//! Beside such a value the constraint clauses take the older form (D1); -//! - the value faults, after the expression and request checks (the filter -//! syntax, an undefined value, the operand types, the projection syntax, the -//! `Select` rules, the unused values, the `Segment` pairing) and ahead of the -//! table lookup, with no prefix: a number that does not parse (`The parameter -//! cannot be converted to a numeric value: bob`, in an `N`, an `NS` member, -//! an `L` or `M` member, a non-key member or an index key), an empty set -//! (`One or more parameter values were invalid: An number set may not be -//! empty`, `... An string set may not be empty`, `... Binary sets should -//! not be empty`) and a set with duplicates (`... Input collection contains -//! duplicates.` for an `NS`, `["1", "1"]` and `["1", "1.0"]` alike; `... -//! Input collection ["a", "a"] contains duplicates.` for an `SS`). A value -//! of `ExpressionAttributeValues` the deserializer refuses (an empty value) -//! is reported after them and ahead of the table lookup. +//! 1. the constraints in the older form (the value echoed, the field in lower +//! camel case, `totalSegments`, `segment`, `limit`), then the +//! `ReturnConsumedCapacity` enum, then the `Select` enum (2026-11-05: ahead +//! of the key's shape fault and of an unreadable expression value); +//! 2. the parameter mix, then the request parameter checks of order R step 6, +//! with the mix first (2026-11-05: each ahead of the key's faults); +//! 3. the `ExclusiveStartKey` faults, shape and value together, the first +//! faulty member in the member order of [`StartKeyBody`]: a shape fault +//! with the prefix `The provided starting key is invalid: `, a number with +//! the value check's text, a set fault with the texts of order U (the +//! members unquoted, the `NS` duplicate text bare, the empty `SS` text with +//! the key prefix); +//! 4. the `ExpressionAttributeValues` entries the deserializer refuses, read by +//! an expression or not: `ExpressionAttributeValues contains invalid value: +//! Supplied AttributeValue is empty, must contain exactly one of the +//! supported datatypes for key :e` for an empty value and for an unknown +//! descriptor, the more-than-one text for two descriptors (2026-10-31 B4-7 +//! and 2026-11-05: ahead of the nesting check and after the key's faults); +//! 5. the nesting depth, then the expression and request checks of order R +//! step 8, then the table. //! -//! See [`StartKeyBody`] and [`ExpressionValuesBody`]. With faults in two -//! members the service names one of them by the attribute names and, on some -//! tables, the request's order: `{"pk": {}, "sk": {"N": "bob"}}` answered the -//! number on 2026-10-31 and `{"a": {}, "b": {"N": "bob"}}` the empty value on -//! 2026-11-05. The code reports a shape fault in any member ahead of a value -//! fault in any member, then the members in sorted order; no test pins which -//! member. Inside one value the members of an `M` are read in the JSON map's -//! order and those of an `NS` in the request's order; the service named `bob` -//! and then `fred` for `{"M": {"b": {"N": "fred"}, "a": {"N": "bob"}}}` and -//! `fred` twice for `{"NS": ["fred", "bob"]}` in a non-key member. +//! Pairs never sent together keep the code's order: in order R the nesting +//! check against `ExpressionAttributeNames` with no expression, +//! `ConditionalOperator` and the `ScanFilter` value count, a set fault in +//! `ExpressionAttributeValues` against the nesting check and against the +//! `ExpressionAttributeNames` check, the `ProjectionExpression` syntax against +//! an undefined value, an operand type and the unused values; in order U the +//! `Select` enum against the mix, a malformed `ExpressionAttributeValues` +//! number against the key's faults, a set fault in `ExpressionAttributeValues` +//! against the key's faults (reported with the unreadable entries, step 4), an +//! empty `BS` or a `BS` with duplicates in the key (no key prefix, the members +//! unquoted), an empty `SS` in the key beside an unreadable expression value +//! (the key prefix, as beside an unreadable key member). Which entry is named +//! when several `ExpressionAttributeValues` entries are faulty is not measured: +//! the first in sorted order. //! -//! Not measured, and checked in the order listed above: the `AttributesToGet` -//! length clause beside a `Segment` or `TotalSegments` fault without a `Limit` -//! fault (listed first, as beside `Limit`), the older form of the length -//! clause (the empty list echoed, `Value '[]' at 'attributesToGet' ...`, and -//! listed last, as the older form reverses the newer one), the length clause -//! against the enums, `M` levels in the nesting count, and the nesting of a -//! value an expression reads or of a request with no expression. Not measured -//! at the key's stages: a `BS` with duplicates (the deserializer's text); a -//! number fault and a duplicate in one `NS` (the number first); an `SS` or `BS` -//! member that is not a string (the deserializer's text at the value stage); a -//! value of `ExpressionAttributeValues` the deserializer refuses for a reason -//! other than an empty value (two descriptors, an unknown descriptor, a set), -//! reported with the empty value's; and the nesting check beside a key shape -//! fault, where the service answered the key's text for a value the filter -//! reads (2026-11-05) and the nesting text ahead of the `Select` enum for an -//! unused value (2026-10-30): the check keeps its request-time place. +//! A field of the wrong JSON type anywhere in the body is reported ahead of +//! all of them, as a `SerializationException` (`"Limit": "5"` with `Select: +//! "BAD"`, 2026-11-05; the service's texts differ from the deserializer's). The +//! service reports a key shape fault ahead of `"ConsistentRead": "yes"` +//! (2026-11-05); the code reads the whole body first and does not. //! //! The JSON parser accepts 128 levels of nesting, which is 62 `L` levels; a //! body nested deeper is a `SerializationException` from the parser, ahead of @@ -112,19 +108,12 @@ //! value: Nesting Levels have exceeded supported limits: Attributes in the //! item have nested levels beyond supported limit for key :d` (measured //! 2026-10-30), which this module does not reach. -//! -//! Each of these is reported ahead of the next one, ahead of the table lookup, -//! and ahead of the checks the handler makes after deserializing the body -//! (Segment without TotalSegments and the reverse, Segment not below -//! TotalSegments, the undefined placeholder, unused value, filter syntax, -//! IndexName, ConsistentRead, Select with ProjectionExpression and -//! ExclusiveStartKey checks). A field of the wrong JSON type anywhere in the -//! body is reported ahead of all of them, as a `SerializationException`. use std::collections::HashMap; use extenddb_core::error::DynamoDbError; use extenddb_core::types::{AttributeValue, Item, ScanInput, saturated_int32}; +use serde::Deserialize; use serde_json::Value; use crate::query_key_values::{ @@ -132,12 +121,40 @@ use crate::query_key_values::{ present, shape_fault_first, validate_expression_attribute_value_numbers, validate_number_text, }; -/// The `Scan` checks that run on the raw request body, in the service's order: -/// the `AttributesToGet` length and the `Limit`, `Segment` and `TotalSegments` -/// ranges in one message, the expression and non-expression parameter mix, the -/// `ReturnConsumedCapacity` enum, every number in `ExpressionAttributeValues`, -/// the nesting depth of `ExpressionAttributeValues`, then the `Select` enum. -/// The numbers in `ExclusiveStartKey` are checked later, by [`StartKeyBody`]. +/// Which of the service's two check orders a `Scan` request follows (see the +/// module doc): order U when the request holds a value that is not one +/// AttributeValue anywhere [`has_unreadable_attribute_value`] reads +/// (`ExpressionAttributeValues`, `ExclusiveStartKey`, the `ScanFilter` lists), +/// order R otherwise. The same test picks the older form of the constraint +/// clauses (563, D1). Computed once per request by [`CheckOrder::of`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum CheckOrder { + /// Every value is readable. + Readable, + /// The request holds an unreadable value. + Unreadable, +} + +impl CheckOrder { + /// The order `body` follows. + pub(crate) fn of(body: &Value) -> Self { + if has_unreadable_attribute_value(body, SCAN_CONDITIONS) { + Self::Unreadable + } else { + Self::Readable + } + } +} + +/// The `Scan` checks that run on the raw request body, in the service's order +/// for `order` (see the module doc): the `AttributesToGet` length and the +/// `Limit`, `Segment` and `TotalSegments` ranges in one message, the +/// `ReturnConsumedCapacity` enum, in order U the `Select` enum, the expression +/// and non-expression parameter mix, every number in +/// `ExpressionAttributeValues`, and in order R the nesting depth of +/// `ExpressionAttributeValues`. The `ExclusiveStartKey` values are checked +/// later, by [`StartKeyBody`], and in order R the `Select` enum by +/// [`SelectBody`]. /// /// The checks read the raw body, so a field that does not deserialize (an /// invalid `Select`, for one) does not stop the ones before it. When a check @@ -151,14 +168,30 @@ use crate::query_key_values::{ /// /// Returns `DynamoDbError::ValidationException` with the service's text, or /// `DynamoDbError::SerializationException` for a body of the wrong shape. -pub fn validate_scan_request_body(body: &Value) -> Result<(), DynamoDbError> { - scan_request_checks(body).map_err(|check| shape_fault_first::(body, check)) +pub(crate) fn validate_scan_request_body( + body: &Value, + order: CheckOrder, +) -> Result<(), DynamoDbError> { + scan_request_checks(body, order).map_err(|check| shape_fault_first::(body, check)) } /// The `Scan` checks in the service's order; see /// [`validate_scan_request_body`]. -fn scan_request_checks(body: &Value) -> Result<(), DynamoDbError> { - validate_constraints(body)?; +fn scan_request_checks(body: &Value, order: CheckOrder) -> Result<(), DynamoDbError> { + validate_constraints(body, order)?; + + crate::validate_enum_fields( + body, + &[crate::EnumField { + json_name: "ReturnConsumedCapacity", + valid: &["INDEXES", "TOTAL", "NONE"], + clause: crate::EnumClause::Named("returnConsumedCapacity"), + }], + )?; + + if order == CheckOrder::Unreadable { + validate_select_enum(body)?; + } extenddb_core::validation::validate_no_expression_param_mixing( &[ @@ -177,20 +210,23 @@ fn scan_request_checks(body: &Value) -> Result<(), DynamoDbError> { ], )?; - crate::validate_enum_fields( - body, - &[crate::EnumField { - json_name: "ReturnConsumedCapacity", - valid: &["INDEXES", "TOTAL", "NONE"], - clause: crate::EnumClause::Named("returnConsumedCapacity"), - }], - )?; - if let Some(values) = attribute_value_map(body, "ExpressionAttributeValues") { validate_expression_attribute_value_numbers(&values)?; - extenddb_core::validation::validate_attribute_values_nesting_depth(values.values())?; + if order == CheckOrder::Readable { + extenddb_core::validation::validate_attribute_values_nesting_depth(values.values())?; + } } + Ok(()) +} +/// The `Select` enum clause: `1 validation error detected: Value 'BAD' at +/// 'select' failed to satisfy constraint: Member must satisfy enum value set: +/// [...]`. +/// +/// # Errors +/// +/// Returns `DynamoDbError::ValidationException` with the clause. +fn validate_select_enum(body: &Value) -> Result<(), DynamoDbError> { crate::validate_enum_fields( body, &[crate::EnumField { @@ -215,8 +251,11 @@ fn attribute_value_map(body: &Value, field: &str) -> Option Option> { + /// The member names of one JSON object, in document order. + struct MemberNames(Vec); + + impl<'de> Deserialize<'de> for MemberNames { + fn deserialize>(deserializer: D) -> Result { + struct NamesVisitor; + + impl<'de> serde::de::Visitor<'de> for NamesVisitor { + type Value = MemberNames; + + fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("an object") + } + + fn visit_map>( + self, + mut map: A, + ) -> Result { + let mut names: Vec = Vec::new(); + while let Some(name) = map.next_key::()? { + map.next_value::()?; + if !names.contains(&name) { + names.push(name); + } + } + Ok(MemberNames(names)) + } + } + + deserializer.deserialize_map(NamesVisitor) + } + } + + /// The body with only the key's member names read. + #[derive(Deserialize)] + struct KeyNames { + #[serde(rename = "ExclusiveStartKey", default)] + key: Option, + } + + serde_json::from_slice::(raw_body) + .ok()? + .key + .map(|names| names.0) +} + +/// The order in which the service visits the members of an `ExclusiveStartKey` +/// with faults in more than one member, fitted to the measured rows: the order +/// of a Java `HashMap` of capacity 4 over the member names, that is +/// `String.hashCode()` of the name, spread as `h ^ (h >>> 16)`, masked to four +/// buckets, the buckets in ascending order and the request's order inside a +/// bucket. `sk` falls in bucket 0, `a` in 1, `b` in 2, `pk`, `gpk`, `id` and +/// `ts` in 3. /// -/// - [`StartKeyBody::validate_shape`], after the request parameter checks and -/// ahead of the expression checks: a value that is empty, carries an unknown -/// or lower-case type descriptor or more than one descriptor, at the top of a -/// member or inside its `L` or `M`, in any member of the key, with the prefix -/// `The provided starting key is invalid: `; -/// - [`StartKeyBody::validate_values`], after the expression and request checks -/// and ahead of the table lookup: every number in the key with the value -/// check's rules and text, and the set faults (an empty `NS`, `SS` or `BS`, -/// a set with duplicates), with no prefix. +/// Measured (2026-10-31, 2026-11-04 and 2026-11-05, five tables, every row +/// with a shape fault in one of two faulty members, both request orders): on +/// the `pk`, `sk` table the service names `sk`; on HASH `a`, RANGE `b` it +/// names `a`; on HASH `pk`, RANGE `a` it names `a`; on `ts`, `id` and on `id`, +/// `ts` it names the member first in the request; on the GSI `{"gpk": {}, +/// "pk": {"N": "bob"}, "sk": {"S": "x"}}` it names `gpk`. The rule is fitted +/// to those rows, not documented by the service. With two value faults and no +/// shape fault the service's own two sends disagreed on 8 of 22 rows, so no +/// order is pinned there. Keys of four members (an index scan with a range key +/// on the index and on the table) are not measured; the same rule applies. +fn fitted_member_order<'a>(names_in_request_order: impl Iterator) -> Vec { + let mut names: Vec<(u32, usize, &str)> = names_in_request_order + .enumerate() + .map(|(position, name)| (java_hash_bucket(name), position, name)) + .collect(); + names.sort_unstable(); + names + .into_iter() + .map(|(_, _, name)| name.to_owned()) + .collect() +} + +/// The bucket of `name` in a Java `HashMap` of capacity 4: +/// `String.hashCode()` over the UTF-16 code units, spread as `h ^ (h >>> 16)`, +/// masked with 3. The arithmetic wraps as Java's `int` does. +fn java_hash_bucket(name: &str) -> u32 { + let hash = name.encode_utf16().fold(0u32, |h, unit| { + h.wrapping_mul(31).wrapping_add(u32::from(unit)) + }); + (hash ^ (hash >> 16)) & 3 +} + +/// The `ExclusiveStartKey` of a `Scan` body as the request wrote it, with the +/// checks the service makes on the key (measured 2026-10-31 to 2026-11-05, +/// us-east-1; see the module doc): /// -/// A key with such a value does not deserialize as a map of attribute values. -/// [`StartKeyBody::take`] takes it out of the body, so that the typed -/// deserialization of the body does not report it ahead of the checks the -/// service makes first, and keeps the deserializer's fault for a value the two -/// stages do not classify. A key of the wrong JSON shape (not an object, a -/// value that is not an object, an `N` that is not a string) stays in the body, -/// as the service reports a shape fault ahead of every check; see -/// [`shape_fault_first`]. +/// - in order U, after the request parameter checks and ahead of the +/// expression values and checks: the first faulty member in the fitted +/// member order ([`fitted_member_order`]), a shape fault (a value that is +/// empty, carries an unknown or lower-case type descriptor or more than one +/// descriptor, at the top of the member or inside its `L` or `M`) with the +/// prefix `The provided starting key is invalid: `, a number that does not +/// parse with the value check's text, a set fault with the order-U texts; +/// - in order R, after the expression and request checks and ahead of the +/// table lookup: the first member in the same order with a number or set +/// fault, with the order-R texts. /// -/// With faults in two members the service's choice follows the attribute names -/// and, on some tables, the request's member order (measured 2026-11-05 on -/// three tables); the stages here read the members in sorted order, and a -/// shape fault in any member comes ahead of a value fault in any member. +/// A key with an unreadable or set-faulty value does not deserialize as a map +/// of attribute values. [`StartKeyBody::take`] takes it out of the body, so +/// that the typed deserialization of the body does not report it ahead of the +/// checks the service makes first, and keeps the deserializer's fault for a +/// value the checks here do not classify. A key of the wrong JSON shape (not +/// an object, a value that is not an object, an `N` that is not a string) +/// stays in the body, as the service reports a shape fault ahead of every +/// check; see [`shape_fault_first`]. pub(crate) struct StartKeyBody { /// The key as the request wrote it; `None` when the body has none. raw: Option, + /// The member names in the fitted visiting order. + members: Vec, /// The deserializer's fault in a key taken out of the body, reported at the - /// value stage when neither stage classifies the value (an `SS` member that - /// is not a string, for one). + /// key's stage when the checks here classify no member (an `SS` member + /// that is not a string, for one). deferred: Option, } impl StartKeyBody { /// Reads the key from `body` and takes it out of the body when it does not - /// deserialize for a reason that is not a shape fault. - pub(crate) fn take(body: &mut Value) -> Self { + /// deserialize for a reason that is not a shape fault. `request_order` is + /// the key's member names as the request wrote them + /// ([`start_key_member_order`]); when it is `None` or does not name the + /// key's members, the parsed body's order stands in for it. + pub(crate) fn take(body: &mut Value, request_order: Option<&[String]>) -> Self { let raw = body.get("ExclusiveStartKey").cloned(); let deferred = match raw .as_ref() @@ -293,58 +425,57 @@ impl StartKeyBody { { fields.remove("ExclusiveStartKey"); } - Self { raw, deferred } + let members = match raw.as_ref().and_then(Value::as_object) { + Some(key) => { + let parsed: Vec<&str> = key.keys().map(String::as_str).collect(); + let in_request: Vec<&str> = match request_order { + Some(names) + if names.len() == parsed.len() + && names.iter().all(|name| key.contains_key(name)) => + { + names.iter().map(String::as_str).collect() + } + _ => parsed, + }; + fitted_member_order(in_request.into_iter()) + } + None => Vec::new(), + }; + Self { + raw, + members, + deferred, + } } - /// The key's members in sorted order, so that two faults name the same - /// member on every backend (the MongoDB build keeps the request's order in - /// its JSON maps, the others sort them). - fn members(&self) -> Vec<&Value> { - let Some(members) = self.raw.as_ref().and_then(Value::as_object) else { - return Vec::new(); - }; - let mut names: Vec<&String> = members.keys().collect(); - names.sort(); - names.into_iter().map(|name| &members[name]).collect() + /// The key's members as `(name, value)` in the fitted visiting order. + fn members(&self) -> impl Iterator { + let key = self.raw.as_ref().and_then(Value::as_object); + self.members + .iter() + .filter_map(move |name| key?.get(name).map(|value| (name.as_str(), value))) } - /// The key's shape stage: the first member, in sorted order, whose value is - /// empty, carries a type descriptor the service does not have or more than - /// one, at its top or inside an `L` or `M` member, answers the service's - /// text with the key prefix. + /// The key's checks for `order`: the first faulty member in the fitted + /// order, a shape fault (order U only) ahead of a value fault inside one + /// member; then the deserializer's fault in a key [`StartKeyBody::take`] + /// took out of the body that the checks here did not classify. /// /// # Errors /// - /// Returns `DynamoDbError::ValidationException` with the prefixed text. - pub(crate) fn validate_shape(&self) -> Result<(), DynamoDbError> { - for member in self.members() { - if let Some(text) = shape_fault(member) { + /// Returns `DynamoDbError::ValidationException` with the service's text. + pub(crate) fn validate(&self, order: CheckOrder) -> Result<(), DynamoDbError> { + for (_, value) in self.members() { + if order == CheckOrder::Unreadable + && let Some(text) = shape_fault(value) + { return Err(DynamoDbError::ValidationException(format!( "{START_KEY_PREFIX}{text}" ))); } - } - Ok(()) - } - - /// The key's value stage: every number in the key with the value check's - /// rules and text (an `N`, the members of an `NS`, and the values nested in - /// `L` and `M`), and the set faults, member by member in sorted order; then - /// the deserializer's fault in a key [`StartKeyBody::take`] took out of the - /// body that neither stage classified. - /// - /// Inside one value the members of an `M` are read in the JSON map's order - /// (the request's order on the MongoDB build) and those of an `NS` in the - /// request's order: the service named `bob` and then `fred` on two sends - /// of `{"M": {"b": {"N": "fred"}, "a": {"N": "bob"}}}` and `fred` twice for - /// `{"NS": ["fred", "bob"]}` (2026-11-05), so no order is pinned. - /// - /// # Errors - /// - /// Returns `DynamoDbError::ValidationException` with the service's text. - pub(crate) fn validate_values(&self) -> Result<(), DynamoDbError> { - for member in self.members() { - validate_key_value(member)?; + if let Some(fault) = value_fault(value) { + return Err(fault.text(order, Place::Key)); + } } match &self.deferred { Some(fault) => Err(fault.clone()), @@ -373,27 +504,118 @@ fn shape_fault(value: &Value) -> Option<&'static str> { } } -/// The value faults of one raw key value: an `N` text, the members of an `NS` -/// (every number first, then an empty set, then duplicates by numeric value: -/// `["1", "1.0"]` has a duplicate), an empty `SS` or `BS` and their duplicates, -/// and the values nested in `L` and `M`. A member of a set that is not a string -/// holds nothing to check here; the deserializer reports it. -fn validate_key_value(value: &Value) -> Result<(), DynamoDbError> { - let Some(fields) = value.as_object() else { - return Ok(()); - }; +/// Where a faulty value sits, for the texts that differ by place. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Place { + /// A member of `ExclusiveStartKey`. + Key, + /// An entry of `ExpressionAttributeValues`. + ExpressionValue, +} + +/// A value fault in one AttributeValue: a number that does not parse (with the +/// value check's error), an empty set, or a set with duplicates (with its +/// members as written). +enum ValueFault { + Number(DynamoDbError), + NumberSetEmpty, + StringSetEmpty, + BinarySetEmpty, + NumberSetDuplicates, + StringSetDuplicates(Vec), + BinarySetDuplicates(Vec), +} + +impl ValueFault { + /// The service's text for the fault in `order` at `place` (measured + /// 2026-11-05 unless noted): + /// + /// - a number: `The parameter cannot be converted to a numeric value: bob`; + /// - an empty `NS`: `One or more parameter values were invalid: An number + /// set may not be empty` (the service's two spaces), both orders; + /// - an empty `SS`: `... An string set may not be empty`, with the key + /// prefix for a key member in order U; + /// - an empty `BS`: `... Binary sets should not be empty` (order R; order + /// U not measured); + /// - an `NS` with duplicates: `... Input collection contains duplicates.` + /// in order R, `Input collection contains duplicates` in order U; + /// - an `SS` with duplicates: `... Input collection ["a", "a"] contains + /// duplicates.` in order R (the members JSON-quoted, `["a\"b", "a\"b"]` + /// for a member with a quote), `... Input collection [a, a] contains + /// duplicates.` in order U; + /// - a `BS` with duplicates: `... Input collection ["YQ==","YQ=="]of type + /// BS contains duplicates.` in order R (no space after the comma or + /// before `of`); order U is not measured and gets the members unquoted. + fn text(&self, order: CheckOrder, place: Place) -> DynamoDbError { + let quoted = |members: &[String], separator: &str| -> String { + members + .iter() + .map(|member| match order { + CheckOrder::Readable => serde_json::to_string(member).unwrap_or_default(), + CheckOrder::Unreadable => member.clone(), + }) + .collect::>() + .join(separator) + }; + match self { + Self::Number(fault) => fault.clone(), + Self::NumberSetEmpty => invalid_parameter("An number set may not be empty"), + Self::StringSetEmpty => { + let text = invalid_parameter("An string set may not be empty"); + match (order, place, text) { + ( + CheckOrder::Unreadable, + Place::Key, + DynamoDbError::ValidationException(msg), + ) => DynamoDbError::ValidationException(format!("{START_KEY_PREFIX}{msg}")), + (_, _, text) => text, + } + } + Self::BinarySetEmpty => invalid_parameter("Binary sets should not be empty"), + Self::NumberSetDuplicates => match order { + CheckOrder::Readable => invalid_parameter("Input collection contains duplicates."), + CheckOrder::Unreadable => DynamoDbError::ValidationException( + "Input collection contains duplicates".to_owned(), + ), + }, + Self::StringSetDuplicates(members) => invalid_parameter(&format!( + "Input collection [{}] contains duplicates.", + quoted(members, ", ") + )), + Self::BinarySetDuplicates(members) => { + let separator = match order { + CheckOrder::Readable => ",", + CheckOrder::Unreadable => ", ", + }; + invalid_parameter(&format!( + "Input collection [{}]of type BS contains duplicates.", + quoted(members, separator) + )) + } + } + } +} + +/// The first value fault in one raw AttributeValue, depth first: an `N` text, +/// the members of an `NS` (every number first, then an empty set, then +/// duplicates by numeric value: `["1", "1.0"]` has a duplicate), an empty `SS` +/// or `BS` and their duplicates, and the values nested in `L` and `M`. A +/// member of a set that is not a string holds nothing to check here; the +/// deserializer reports it. +fn value_fault(value: &Value) -> Option { + let fields = value.as_object()?; let mut fields = fields.iter(); let (Some((descriptor, inner)), None) = (fields.next(), fields.next()) else { - return Ok(()); + return None; }; match (descriptor.as_str(), inner) { - ("N", Value::String(text)) => validate_number_text(text), - ("NS", Value::Array(members)) => validate_number_set(members), - ("SS", Value::Array(members)) => validate_string_set(members), - ("BS", Value::Array(members)) => validate_binary_set(members), - ("L", Value::Array(members)) => members.iter().try_for_each(validate_key_value), - ("M", Value::Object(members)) => members.values().try_for_each(validate_key_value), - _ => Ok(()), + ("N", Value::String(text)) => validate_number_text(text).err().map(ValueFault::Number), + ("NS", Value::Array(members)) => number_set_fault(members), + ("SS", Value::Array(members)) => string_set_fault(members), + ("BS", Value::Array(members)) => binary_set_fault(members), + ("L", Value::Array(members)) => members.iter().find_map(value_fault), + ("M", Value::Object(members)) => members.values().find_map(value_fault), + _ => None, } } @@ -403,18 +625,28 @@ fn invalid_parameter(text: &str) -> DynamoDbError { DynamoDbError::ValidationException(format!("{INVALID_PARAMETER_VALUES}{text}")) } -/// An `NS` of the key: every member's number text, then `One or more parameter -/// values were invalid: An number set may not be empty` (the service's two -/// spaces), then `... Input collection contains duplicates.` for two members -/// with one numeric value (measured 2026-11-05 for `["1", "1"]` and `["1", -/// "1.0"]`). -fn validate_number_set(members: &[Value]) -> Result<(), DynamoDbError> { - let texts: Vec<&str> = members.iter().filter_map(Value::as_str).collect(); - texts +/// The members of a set as written, skipping members that are not strings. +fn set_members(members: &[Value]) -> Vec { + members + .iter() + .filter_map(Value::as_str) + .map(str::to_owned) + .collect() +} + +/// An `NS`: every member's number text first (2026-11-05: `["bob", "1", "1"]` +/// and `["1", "1", "bob"]` answer `bob`), then an empty set, then two members +/// with one numeric value (`["1", "1"]` and `["1", "1.0"]`). +fn number_set_fault(members: &[Value]) -> Option { + let texts = set_members(members); + if let Some(fault) = texts .iter() - .try_for_each(|text| validate_number_text(text))?; + .find_map(|text| validate_number_text(text).err()) + { + return Some(ValueFault::Number(fault)); + } if members.is_empty() { - return Err(invalid_parameter("An number set may not be empty")); + return Some(ValueFault::NumberSetEmpty); } let mut normalized: Vec = texts .iter() @@ -426,114 +658,148 @@ fn validate_number_set(members: &[Value]) -> Result<(), DynamoDbError> { .collect(); normalized.sort_unstable(); normalized.dedup(); - if normalized.len() != texts.len() { - return Err(invalid_parameter("Input collection contains duplicates.")); - } - Ok(()) + (normalized.len() != texts.len()).then_some(ValueFault::NumberSetDuplicates) } -/// An `SS` of the key: `One or more parameter values were invalid: An string -/// set may not be empty`, then `... Input collection ["a", "a"] contains -/// duplicates.` with every member quoted (measured 2026-11-05). -fn validate_string_set(members: &[Value]) -> Result<(), DynamoDbError> { +/// An `SS`: an empty set, then duplicates. +fn string_set_fault(members: &[Value]) -> Option { if members.is_empty() { - return Err(invalid_parameter("An string set may not be empty")); + return Some(ValueFault::StringSetEmpty); } - let texts: Vec<&str> = members.iter().filter_map(Value::as_str).collect(); - let mut distinct = texts.clone(); - distinct.sort_unstable(); - distinct.dedup(); - if distinct.len() != texts.len() { - let listed: Vec = texts - .iter() - .map(|text| serde_json::to_string(text).unwrap_or_default()) - .collect(); - return Err(invalid_parameter(&format!( - "Input collection [{}] contains duplicates.", - listed.join(", ") - ))); + let texts = set_members(members); + if has_duplicates(&texts) { + return Some(ValueFault::StringSetDuplicates(texts)); } - Ok(()) + None } -/// A `BS` of the key: `One or more parameter values were invalid: Binary sets -/// should not be empty` (measured 2026-11-05); duplicates answer the -/// deserializer's text for a binary set, which is not measured for the key. -fn validate_binary_set(members: &[Value]) -> Result<(), DynamoDbError> { +/// A `BS`: an empty set, then duplicates. +fn binary_set_fault(members: &[Value]) -> Option { if members.is_empty() { - return Err(invalid_parameter("Binary sets should not be empty")); + return Some(ValueFault::BinarySetEmpty); + } + let texts = set_members(members); + if has_duplicates(&texts) { + return Some(ValueFault::BinarySetDuplicates(texts)); } - let texts: Vec<&str> = members.iter().filter_map(Value::as_str).collect(); - let mut distinct = texts.clone(); + None +} + +/// Whether two members of a set are equal. +fn has_duplicates(texts: &[String]) -> bool { + let mut distinct: Vec<&String> = texts.iter().collect(); distinct.sort_unstable(); distinct.dedup(); - if distinct.len() != texts.len() { - return Err(invalid_parameter(&format!( - "Input collection [{}]of type BS contains duplicates.", - texts.join(", ") - ))); - } - Ok(()) + distinct.len() != texts.len() } -/// The `ExpressionAttributeValues` of a `Scan` body whose values do not all -/// deserialize: an empty value, an unknown type descriptor, two descriptors, an -/// empty or duplicate set. The service reports such a value after the key's -/// value stage and ahead of the table lookup (measured 2026-11-05: `{":a": {}}` -/// read by the filter answers `ExpressionAttributeValues contains invalid -/// value: Supplied AttributeValue is empty, must contain exactly one of the -/// supported datatypes for key :a` with no key and with a valid key, and the -/// key's number or set text beside a key with such a fault). [`Self::take`] -/// puts a readable stand-in in the body for each such value, so that the typed -/// deserialization and the expression checks run on the other values, and -/// keeps the deserializer's fault for the map as the request wrote it; -/// [`Self::validate`] reports it. The request is refused before the scan reads -/// the stand-in. A value of the wrong JSON shape stays in the body. +/// The `ExpressionAttributeValues` of a `Scan` body whose entries do not all +/// deserialize: in order U an entry that is not one AttributeValue (empty, an +/// unknown or lower-case type descriptor, two descriptors, at the top or inside +/// an `L` or `M`), in either order an empty or duplicate set. The service +/// reports such an entry at its own stage: in order R the set faults ahead of +/// the no-expression check (2026-11-05, `{":e": {"NS": []}}` with no +/// expression answers `One or more parameter values were invalid: An number +/// set may not be empty`; `{":a": {"SS": ["a", "a"]}}` read by the filter +/// answers `... Input collection ["a", "a"] contains duplicates.`); in order U +/// the unreadable entries after the key's faults and ahead of the nesting +/// check, read by an expression or not (2026-10-31 B4-7 and 2026-11-05): +/// `ExpressionAttributeValues contains invalid value: Supplied AttributeValue +/// is empty, must contain exactly one of the supported datatypes for key :e` +/// for `{}` and for `{"X": "1"}`, and `... has more than one datatypes set, +/// ... for key :a` for two descriptors. A set fault in order U is not measured +/// and is reported with the unreadable entries. Which entry is named when +/// several are faulty is not measured: the first in sorted order. +/// +/// [`Self::take`] puts a readable stand-in in the body for each such entry, so +/// that the typed deserialization and the expression checks run on the other +/// values, and keeps the entry's fault; [`Self::validate`] reports it. The +/// request is refused before the scan reads the stand-in. An entry of the wrong +/// JSON shape stays in the body. pub(crate) struct ExpressionValuesBody { - deferred: Option, + /// The faults of the entries taken out, in sorted order of their names. + faults: Vec, } impl ExpressionValuesBody { - /// Replaces each value the deserializer refuses for a reason that is not a - /// shape fault with a readable stand-in and keeps the deserializer's fault. - pub(crate) fn take(body: &mut Value) -> Self { + /// Replaces each entry the checks here classify with a readable stand-in + /// and keeps its fault for `order`. + pub(crate) fn take(body: &mut Value, order: CheckOrder) -> Self { let Some(Value::Object(entries)) = body.get_mut("ExpressionAttributeValues") else { - return Self { deferred: None }; + return Self { faults: Vec::new() }; }; - let raw = Value::Object(entries.clone()); - let mut substituted = false; - for value in entries.values_mut() { - if let Err(e) = serde_json::from_value::(value.clone()) - && !is_shape_fault(&crate::deserialize_error(e)) - { + let mut names: Vec = entries.keys().cloned().collect(); + names.sort_unstable(); + let mut faults = Vec::new(); + for name in names { + let Some(value) = entries.get_mut(&name) else { + continue; + }; + let fault = match shape_fault(value) { + Some(text) => Some(DynamoDbError::ValidationException(format!( + "ExpressionAttributeValues contains invalid value: {text} for key {name}" + ))), + None => value_fault(value).map(|fault| fault.text(order, Place::ExpressionValue)), + }; + if let Some(fault) = fault { *value = serde_json::json!({"S": ""}); - substituted = true; + faults.push(fault); } } - if !substituted { - return Self { deferred: None }; - } - // The deserializer's own wording and key order for the map as written. - let probe = serde_json::json!({"TableName": "t", "ExpressionAttributeValues": raw}); - let deferred = serde_json::from_value::(probe) - .err() - .map(crate::deserialize_error); - Self { deferred } + Self { faults } } - /// The deferred fault, at the stage after the key's value checks. + /// The first entry's fault, at the stage of `order`. /// /// # Errors /// - /// Returns the deserializer's fault for the first value it refused. + /// Returns the fault of the first entry taken out, in sorted order. pub(crate) fn validate(&self) -> Result<(), DynamoDbError> { - match &self.deferred { + match self.faults.first() { Some(fault) => Err(fault.clone()), None => Ok(()), } } } +/// The `Select` field of a `Scan` body in order R, where the service checks +/// the enum after the no-expression check and the nesting check and ahead of +/// the filter syntax (2026-11-05): [`Self::take`] takes an invalid value out of +/// the body, so that the typed deserialization does not report it first, and +/// keeps the clause; [`Self::validate`] reports it. In order U the enum is a +/// request-time check ([`validate_scan_request_body`]) and nothing is taken. +pub(crate) struct SelectBody { + clause: Option, +} + +impl SelectBody { + /// Takes an invalid `Select` string out of `body` in order R. + pub(crate) fn take(body: &mut Value, order: CheckOrder) -> Self { + if order == CheckOrder::Unreadable { + return Self { clause: None }; + } + let clause = validate_select_enum(body).err(); + if clause.is_some() + && let Some(fields) = body.as_object_mut() + { + fields.remove("Select"); + } + Self { clause } + } + + /// The enum clause, at its stage. + /// + /// # Errors + /// + /// Returns `DynamoDbError::ValidationException` with the clause. + pub(crate) fn validate(&self) -> Result<(), DynamoDbError> { + match &self.clause { + Some(clause) => Err(clause.clone()), + None => Ok(()), + } + } +} + /// The constraints the service aggregates into one message: the length of /// `AttributesToGet` and the ranges of `Limit`, `Segment` and `TotalSegments`, /// every fault in one message, in one of the service's two forms (measured @@ -546,9 +812,9 @@ impl ExpressionValuesBody { /// answers the length clause, then the `Limit` clause, measured 2026-10-30 /// 12:40 UTC; the length clause beside a `Segment` or `TotalSegments` fault /// alone is not measured). When any of the three is a float or outside int32, -/// or a value in `ExpressionAttributeValues`, `ExclusiveStartKey` or a -/// `ScanFilter` `AttributeValueList` is not one AttributeValue (see -/// [`has_unreadable_attribute_value`]), the older form: `Value '' at +/// or the request follows order U (a value in `ExpressionAttributeValues`, +/// `ExclusiveStartKey` or a `ScanFilter` `AttributeValueList` is not one +/// AttributeValue, see [`CheckOrder`]), the older form: `Value '' at /// '' ...` with the value as [`saturated_int32`] reads it and the field /// in lower camel case, in the reverse of the newer form's order: /// `totalSegments`, `segment`, `limit` (measured 2026-10-30 12:40 UTC with a @@ -556,7 +822,7 @@ impl ExpressionValuesBody { /// and `totalSegments` then `limit`). The older form of the length clause is /// not measured: the code echoes the empty list, `Value '[]' at /// 'attributesToGet' ...`, and lists it last. -fn validate_constraints(body: &Value) -> Result<(), DynamoDbError> { +fn validate_constraints(body: &Value, order: CheckOrder) -> Result<(), DynamoDbError> { let numbers: Vec<(&RangeField, &serde_json::Number)> = RANGE_FIELDS .iter() .filter_map(|range| { @@ -565,7 +831,7 @@ fn validate_constraints(body: &Value) -> Result<(), DynamoDbError> { .map(|number| (range, number)) }) .collect(); - let older_form = has_unreadable_attribute_value(body, SCAN_CONDITIONS) + let older_form = order == CheckOrder::Unreadable || numbers.iter().any(|(_, number)| { number .as_i64() diff --git a/crates/engine/src/scan_request/range_tests.rs b/crates/engine/src/scan_request/range_tests.rs index 0554cb1d5..5b44281be 100644 --- a/crates/engine/src/scan_request/range_tests.rs +++ b/crates/engine/src/scan_request/range_tests.rs @@ -11,7 +11,7 @@ use extenddb_core::error::DynamoDbError; use serde_json::Value; -use super::validate_scan_request_body; +use super::{CheckOrder, validate_scan_request_body}; const LIMIT: &str = "Value at 'Limit' failed to satisfy constraint: Member must have value greater than or equal to 1"; const SEGMENT_LOW: &str = "Value at 'Segment' failed to satisfy constraint: Member must have value greater than or equal to 0"; @@ -20,7 +20,7 @@ const TOTAL_LOW: &str = "Value at 'TotalSegments' failed to satisfy constraint: const TOTAL_HIGH: &str = "Value at 'TotalSegments' failed to satisfy constraint: Member must have value less than or equal to 1000000"; fn check(body: &Value) -> Option { - match validate_scan_request_body(body) { + match validate_scan_request_body(body, CheckOrder::of(body)) { Ok(()) => None, Err(DynamoDbError::ValidationException(msg)) => Some(msg), Err(other) => panic!("unexpected error {other:?}"), diff --git a/crates/engine/src/scan_request/request_tests.rs b/crates/engine/src/scan_request/request_tests.rs index 30e801146..466a188a8 100644 --- a/crates/engine/src/scan_request/request_tests.rs +++ b/crates/engine/src/scan_request/request_tests.rs @@ -2,35 +2,51 @@ // SPDX-License-Identifier: Apache-2.0 //! The `ScanFilter` in the parameter mix, the `AttributesToGet` length, the -//! numbers in `ExclusiveStartKey` and the nesting depth of -//! `ExpressionAttributeValues`. +//! nesting depth of `ExpressionAttributeValues`, the two check orders, and the +//! `ExclusiveStartKey` and `ExpressionAttributeValues` checks of each order. //! //! Every row is a request sent to the service (us-east-1, each request twice, //! a Scan of a table the probe created) with the answer it gave: on 2026-10-25 //! at 10:38 and 10:55 UTC, on 2026-10-30 at 12:40 UTC, on 2026-10-31 between //! 23:54 and 23:56 UTC and on 2026-11-05 where a test says so. The tests of -//! `StartKeyBody::take` and `ExpressionValuesBody::take` check which values -//! they hold back, with no service answer, and the test with faults in two key -//! members accepts either member's fault. +//! `StartKeyBody::take`, `ExpressionValuesBody::take` and `SelectBody::take` +//! check which values they hold back, with no service answer; the test of the +//! member order checks the fitted rule's buckets; the rows with two value +//! faults in two key members accept either member's fault. use extenddb_core::error::DynamoDbError; use serde_json::{Value, json}; -use super::{StartKeyBody, validate_scan_request_body}; +use super::{ + CheckOrder, ExpressionValuesBody, SelectBody, StartKeyBody, fitted_member_order, + java_hash_bucket, start_key_member_order, validate_scan_request_body, +}; const MIX: &str = "Can not use both expression and non-expression parameters in the same request: Non-expression parameters: {ScanFilter} Expression parameters: {FilterExpression}"; const LENGTH: &str = "1 validation error detected: Value at 'AttributesToGet' failed to satisfy constraint: Member must have length greater than or equal to 1"; const NUMERIC_BOB: &str = "The parameter cannot be converted to a numeric value: bob"; const NESTING: &str = "Nesting Levels have exceeded supported limits: Attributes in the item have nested levels beyond supported limit"; +const SELECT_BAD: &str = "1 validation error detected: Value 'BAD' at 'select' failed to satisfy constraint: Member must satisfy enum value set: [SPECIFIC_ATTRIBUTES, COUNT, ALL_ATTRIBUTES, ALL_PROJECTED_ATTRIBUTES]"; +const KEY_PREFIX: &str = "The provided starting key is invalid: "; +const EMPTY_VALUE: &str = + "Supplied AttributeValue is empty, must contain exactly one of the supported datatypes"; +const MORE_THAN_ONE: &str = "Supplied AttributeValue has more than one datatypes set, must contain exactly one of the supported datatypes"; +const INVALID: &str = "One or more parameter values were invalid: "; -fn check(body: &Value) -> Option { - match validate_scan_request_body(body) { +/// The message of a `ValidationException`, `None` for `Ok`. +fn validation_message(result: Result<(), DynamoDbError>) -> Option { + match result { Ok(()) => None, Err(DynamoDbError::ValidationException(msg)) => Some(msg), Err(other) => panic!("unexpected error {other:?}"), } } +/// The request-time checks on `body`, in the order of the body's class. +fn check(body: &Value) -> Option { + validation_message(validate_scan_request_body(body, CheckOrder::of(body))) +} + /// A Scan body with the fields of `extra` added to the table name. fn scan(extra: Value) -> Value { let mut body = json!({"TableName": "t"}); @@ -68,6 +84,34 @@ fn with_unused_value(value: Value) -> Value { })) } +/// A Scan body with `key` as its `ExclusiveStartKey`. +fn with_key(key: Value) -> Value { + scan(json!({"ExclusiveStartKey": key})) +} + +/// The key's check on `body` in the order of the body's class, as the handler +/// makes it: `StartKeyBody::take` on a copy of the body with the members in +/// `request_order` (the `json!` macro sorts them), then `validate`. +fn check_key_in(body: &Value, request_order: &[&str]) -> Option { + let mut body = body.clone(); + let order = CheckOrder::of(&body); + let names: Vec = request_order + .iter() + .map(|name| (*name).to_owned()) + .collect(); + let request_order = if names.is_empty() { + None + } else { + Some(names.as_slice()) + }; + validation_message(StartKeyBody::take(&mut body, request_order).validate(order)) +} + +/// [`check_key_in`] with the parsed body's member order. +fn check_key(body: &Value) -> Option { + check_key_in(body, &[]) +} + #[test] fn an_empty_scan_filter_counts_as_present_for_the_parameter_mix() { let mix = with(scan(filter("1")), json!({"ScanFilter": {}})); @@ -110,65 +154,47 @@ fn an_empty_attributes_to_get_joins_the_range_faults() { ); } -/// A Scan body with `key` as its `ExclusiveStartKey`. -fn with_key(key: Value) -> Value { - scan(json!({"ExclusiveStartKey": key})) -} - -/// The key's shape stage on `body`, as the handler makes it after the request -/// parameter checks: `StartKeyBody::take` on a copy of the body, then -/// `validate_shape`. -fn check_key_shape(body: &Value) -> Option { - let mut body = body.clone(); - match StartKeyBody::take(&mut body).validate_shape() { - Ok(()) => None, - Err(DynamoDbError::ValidationException(msg)) => Some(msg), - Err(other) => panic!("unexpected error {other:?}"), - } -} - -/// The key's value stage on `body`, as the handler makes it after the -/// expression checks: `StartKeyBody::take` on a copy of the body, then -/// `validate_values`, for a key whose shape stage passes. -fn check_key_values(body: &Value) -> Option { - let mut body = body.clone(); - let key = StartKeyBody::take(&mut body); - assert!(key.validate_shape().is_ok(), "{body}"); - match key.validate_values() { - Ok(()) => None, - Err(DynamoDbError::ValidationException(msg)) => Some(msg), - Err(other) => panic!("unexpected error {other:?}"), +/// The order flag reads `ExpressionAttributeValues`, `ExclusiveStartKey` and +/// the `ScanFilter` lists, at any depth of `L` and `M`; a set fault, a number +/// fault and a readable value of the wrong type for the key leave the request +/// in order R. +#[test] +fn the_order_follows_an_unreadable_value_anywhere() { + for body in [ + with_key(json!({"pk": {}, "sk": {"S": "x"}})), + with_key(json!({"pk": {"X": "1"}, "sk": {"S": "x"}})), + with_key(json!({"pk": {"s": "x"}, "sk": {"S": "x"}})), + with_key(json!({"pk": {"S": "x", "N": "1"}, "sk": {"S": "x"}})), + with_key(json!({"pk": {"L": [{"M": {"k": {}}}]}, "sk": {"S": "x"}})), + scan(json!({"ExpressionAttributeValues": {":e": {}}})), + scan( + json!({"FilterExpression": "a = :a", "ExpressionAttributeValues": {":a": {"X": "1"}}}), + ), + scan( + json!({"ScanFilter": {"a": {"ComparisonOperator": "EQ", "AttributeValueList": [{}]}}}), + ), + ] { + assert_eq!(CheckOrder::of(&body), CheckOrder::Unreadable, "{body}"); } -} - -/// The message of a `ValidationException`, `None` for `Ok`. -fn validation_message(result: Result<(), DynamoDbError>) -> Option { - match result { - Ok(()) => None, - Err(DynamoDbError::ValidationException(msg)) => Some(msg), - Err(other) => panic!("unexpected error {other:?}"), + for body in [ + scan(json!({})), + with_key(json!({"pk": {"N": "bob"}, "sk": {"NS": []}})), + with_key(json!({"pk": {"SS": ["a", "a"]}, "sk": {"S": "x"}})), + scan(json!({"ExpressionAttributeValues": {":e": {"NS": []}}})), + scan(json!({"ExpressionAttributeValues": {":n": {"N": "bob"}}})), + with_key(json!({"pk": {"N": "1"}, "sk": {"S": "x"}})), + ] { + assert_eq!(CheckOrder::of(&body), CheckOrder::Readable, "{body}"); } } -/// Both stages in the handler's order: the first fault of either. -fn check_key(body: &Value) -> Option { - check_key_shape(body).or_else(|| check_key_values(body)) -} - -const KEY_PREFIX: &str = "The provided starting key is invalid: "; -const EMPTY_VALUE: &str = - "Supplied AttributeValue is empty, must contain exactly one of the supported datatypes"; -const INVALID: &str = "One or more parameter values were invalid: "; - #[test] fn a_malformed_number_in_the_exclusive_start_key_is_reported_with_the_number_text() { let key = |number: &str| json!({"pk": {"N": number}, "sk": {"S": "x"}}); let bad = with(with_key(key("bob")), filter("1")); - // The request-time checks and the shape stage leave the number to the - // value stage. + // The request-time checks leave the number to the key's stage. assert_eq!(check(&bad), None); - assert_eq!(check_key_shape(&bad), None); - assert_eq!(check_key_values(&bad).as_deref(), Some(NUMERIC_BOB)); + assert_eq!(check_key(&bad).as_deref(), Some(NUMERIC_BOB)); assert_eq!(check_key(&with(with_key(key("1")), filter("1"))), None); // Measured 2026-10-30 12:40 UTC: with no filter, the same text. assert_eq!( @@ -178,11 +204,13 @@ fn a_malformed_number_in_the_exclusive_start_key_is_reported_with_the_number_tex // Measured 2026-10-31 between 23:54 and 23:56 UTC: a number as an NS // member, as an L member, and in the L of an attribute that is not a key // attribute; the number texts for a leading space, an empty text and 39 - // digits (on an N hash key). + // digits (on an N hash key). Measured 2026-11-05: `{"pk": {"N": "bob"}}` + // alone, with no `sk`, on the `pk`, `sk` table. for key in [ json!({"pk": {"NS": ["bob"]}, "sk": {"S": "x"}}), json!({"pk": {"L": [{"N": "bob"}]}, "sk": {"S": "x"}}), json!({"pk": {"S": "p0"}, "sk": {"S": "x"}, "z": {"L": [{"N": "bob"}]}}), + json!({"pk": {"N": "bob"}}), ] { assert_eq!( check_key(&with_key(key.clone())).as_deref(), @@ -207,10 +235,11 @@ fn a_malformed_number_in_the_exclusive_start_key_is_reported_with_the_number_tex } /// Measured 2026-10-31 between 23:54 and 23:56 UTC and 2026-11-05 with `{"pk": -/// {"N": "bob"}, "sk": {"S": "x"}}` as the key: each request-time check answers -/// its own text ahead of the key's number, in the newer form; beside a key -/// value that is empty or carries an unknown type descriptor the constraint -/// clauses take the older form (2026-11-05). The request parameter checks, the +/// {"N": "bob"}, "sk": {"S": "x"}}` as the key: each request-time check of +/// order R answers its own text ahead of the key's number, in the newer form; +/// beside a key value that is empty or carries an unknown type descriptor the +/// constraint clauses take the older form and the `Select` enum joins the +/// request-time checks (2026-11-05). The request parameter checks, the /// expression checks and the Segment pairing, which the handler makes after /// deserializing, come ahead of the key's number too; the wire tests send /// those rows. @@ -230,10 +259,6 @@ fn every_request_time_check_comes_ahead_of_the_number_in_the_exclusive_start_key json!({"Segment": -1, "TotalSegments": 2}), "1 validation error detected: Value at 'Segment' failed to satisfy constraint: Member must have value greater than or equal to 0", ), - ( - json!({"Select": "BAD"}), - "1 validation error detected: Value 'BAD' at 'select' failed to satisfy constraint: Member must satisfy enum value set: [SPECIFIC_ATTRIBUTES, COUNT, ALL_ATTRIBUTES, ALL_PROJECTED_ATTRIBUTES]", - ), (with(filter("1"), json!({"ScanFilter": {}})), MIX), (json!({"AttributesToGet": []}), LENGTH), ( @@ -256,7 +281,14 @@ fn every_request_time_check_comes_ahead_of_the_number_in_the_exclusive_start_key let body = with(with_key(bob.clone()), extra); assert_eq!(check(&body).as_deref(), Some(text), "{body}"); } - // Beside a key value the shape stage refuses, the older form (2026-11-05). + // In order R the Select enum is not a request-time check: SelectBody + // holds it back for the handler (measured 2026-11-05: after the + // no-expression check and the nesting check). + let select = with(with_key(bob), json!({"Select": "BAD"})); + assert_eq!(check(&select), None); + assert_eq!(check_select(&select).as_deref(), Some(SELECT_BAD)); + // Beside a key value the shape stage refuses, the older form, and the + // Select enum at request time (2026-11-05). for key in [ json!({"pk": {}, "sk": {"S": "x"}}), json!({"pk": {"X": "1"}, "sk": {"S": "x"}}), @@ -268,15 +300,57 @@ fn every_request_time_check_comes_ahead_of_the_number_in_the_exclusive_start_key ), "{key}" ); + assert_eq!( + check(&with(with_key(key.clone()), json!({"Select": "BAD"}))).as_deref(), + Some(SELECT_BAD), + "{key}" + ); + } + // An unused empty expression value beside Select BAD: the Select clause + // (2026-11-05, order U). + let unused_empty = with(with_unused_value(json!({})), json!({"Select": "BAD"})); + assert_eq!(check(&unused_empty).as_deref(), Some(SELECT_BAD)); +} + +/// `SelectBody::take` on a copy of `body`, then `validate`. +fn check_select(body: &Value) -> Option { + let mut body = body.clone(); + let order = CheckOrder::of(&body); + validation_message(SelectBody::take(&mut body, order).validate()) +} + +/// `SelectBody::take` holds an invalid `Select` back in order R, so that the +/// typed deserialization does not report it, and leaves it in order U, where +/// the request-time checks report it; no service answer is asserted here. +#[test] +fn an_invalid_select_is_held_back_in_order_r_only() { + let mut body = scan(json!({"Select": "BAD"})); + let select = SelectBody::take(&mut body, CheckOrder::Readable); + assert!(body.get("Select").is_none()); + assert_eq!( + validation_message(select.validate()).as_deref(), + Some(SELECT_BAD) + ); + assert!(serde_json::from_value::(body).is_ok()); + let mut body = scan(json!({"Select": "BAD", "ExclusiveStartKey": {"pk": {}}})); + let select = SelectBody::take(&mut body, CheckOrder::Unreadable); + assert!(body.get("Select").is_some()); + assert!(select.validate().is_ok()); + // A valid Select and a Select of the wrong JSON type stay as written. + for value in [json!("COUNT"), json!(5)] { + let mut body = scan(json!({"Select": value})); + let select = SelectBody::take(&mut body, CheckOrder::Readable); + assert_eq!(body["Select"], value); + assert!(select.validate().is_ok()); } } /// Measured 2026-11-05: a key value that is empty, carries an unknown or /// lower-case type descriptor, or carries two, at its top or inside an `L` or /// `M`, in a key member or in a member that is not a key attribute, answers the -/// service's text with the key prefix at the shape stage. +/// service's text with the key prefix at the key's stage of order U. #[test] -fn a_shape_fault_in_a_key_value_answers_the_prefixed_text_at_the_shape_stage() { +fn a_shape_fault_in_a_key_value_answers_the_prefixed_text() { let empty = format!("{KEY_PREFIX}{EMPTY_VALUE}"); for key in [ json!({"pk": {}, "sk": {"S": "x"}}), @@ -289,30 +363,26 @@ fn a_shape_fault_in_a_key_value_answers_the_prefixed_text_at_the_shape_stage() { ] { let body = with_key(key.clone()); assert_eq!(check(&body), None, "{key}"); - assert_eq!( - check_key_shape(&body).as_deref(), - Some(empty.as_str()), - "{key}" - ); + assert_eq!(check_key(&body).as_deref(), Some(empty.as_str()), "{key}"); } let two = with_key(json!({"pk": {"S": "x", "N": "1"}, "sk": {"S": "x"}})); assert_eq!( - check_key_shape(&two).as_deref(), - Some( - "The provided starting key is invalid: Supplied AttributeValue has more than one datatypes set, must contain exactly one of the supported datatypes" - ) - ); - // A readable key passes the shape stage. - assert_eq!( - check_key_shape(&with_key(json!({"pk": {"N": "bob"}, "sk": {"NS": []}}))), - None + check_key(&two), + Some(format!("{KEY_PREFIX}{MORE_THAN_ONE}")) ); + // Inside one member a shape fault comes ahead of a number (2026-11-05, + // `{"M": {"k": {"N": "bob"}, "j": {}}}` in a non-key member). + let inside = with_key(json!({ + "pk": {"S": "x"}, "sk": {"S": "x"}, "z": {"M": {"k": {"N": "bob"}, "j": {}}}, + })); + assert_eq!(check_key(&inside).as_deref(), Some(empty.as_str())); } -/// Measured 2026-11-05: the set faults of a key answer the service's texts with -/// no prefix at the value stage, after every number in the key. +/// Measured 2026-11-05: in order R the set faults of a key answer the service's +/// texts with no prefix, the members JSON-quoted, after every number in the +/// key. #[test] -fn a_set_fault_in_a_key_value_answers_the_service_text_without_a_prefix() { +fn a_set_fault_in_a_key_value_answers_the_order_r_text_without_a_prefix() { for (key, text) in [ (json!({"NS": []}), "An number set may not be empty"), (json!({"SS": []}), "An string set may not be empty"), @@ -329,54 +399,234 @@ fn a_set_fault_in_a_key_value_answers_the_service_text_without_a_prefix() { json!({"SS": ["a", "a"]}), r#"Input collection ["a", "a"] contains duplicates."#, ), + ( + json!({"SS": ["a\"b", "a\"b"]}), + r#"Input collection ["a\"b", "a\"b"] contains duplicates."#, + ), + ( + json!({"BS": ["YQ==", "YQ=="]}), + r#"Input collection ["YQ==","YQ=="]of type BS contains duplicates."#, + ), ] { let body = with_key(json!({"pk": key, "sk": {"S": "x"}})); - assert_eq!( - check_key_values(&body), - Some(format!("{INVALID}{text}")), - "{key}" - ); + assert_eq!(CheckOrder::of(&body), CheckOrder::Readable, "{key}"); + assert_eq!(check_key(&body), Some(format!("{INVALID}{text}")), "{key}"); + } + // A number in a set is reported with the number text, ahead of a + // duplicate in the same set (2026-11-05, `["bob", "1", "1"]` and `["1", + // "1", "bob"]`). + for members in [ + json!(["bob"]), + json!(["bob", "1", "1"]), + json!(["1", "1", "bob"]), + ] { + let bob = with_key(json!({"pk": {"S": "x"}, "sk": {"S": "x"}, "z": {"NS": members}})); + assert_eq!(check_key(&bob).as_deref(), Some(NUMERIC_BOB), "{members}"); } - // A number in a set is reported with the number text. - let bob = with_key(json!({"pk": {"NS": ["bob"]}, "sk": {"S": "x"}})); - assert_eq!(check_key_values(&bob).as_deref(), Some(NUMERIC_BOB)); // A set with distinct numbers passes. assert_eq!( - check_key_values(&with_key( + check_key(&with_key( json!({"pk": {"NS": ["1", "2"]}, "sk": {"S": "x"}}) )), None ); } -/// Faults in two members of one key. The service's choice follows the attribute -/// names and, on some tables, the request's member order (measured 2026-11-05 -/// on three tables): `{"pk": {}, "sk": {"N": "bob"}}` answered `bob` on -/// 2026-10-31 and `{"a": {}, "b": {"N": "bob"}}` the empty value. The code -/// reports a shape fault in any member ahead of a value fault in any member, -/// then the members in sorted order; this test accepts either member's fault. +/// Measured 2026-11-05: in order U (here, the other member of the key is +/// unreadable and the set member comes first in the fitted order) the set texts +/// have no JSON quotes, the `NS` duplicate text is bare, and the empty `SS` +/// text carries the key prefix. The key's number keeps its text. #[test] -fn two_faulty_members_answer_one_of_the_two_faults() { +fn a_set_fault_in_a_key_value_answers_the_order_u_text() { + for (sk, text) in [ + ( + json!({"NS": []}), + format!("{INVALID}An number set may not be empty"), + ), + ( + json!({"SS": []}), + format!("{KEY_PREFIX}{INVALID}An string set may not be empty"), + ), + ( + json!({"NS": ["1", "1"]}), + "Input collection contains duplicates".to_owned(), + ), + ( + json!({"SS": ["a", "a"]}), + format!("{INVALID}Input collection [a, a] contains duplicates."), + ), + (json!({"N": "bob"}), NUMERIC_BOB.to_owned()), + ] { + for pk in [json!({}), json!({"X": "1"}), json!({"S": "x", "N": "1"})] { + let body = with_key(json!({"pk": pk, "sk": sk})); + assert_eq!(CheckOrder::of(&body), CheckOrder::Unreadable); + assert_eq!(check_key(&body).as_deref(), Some(text.as_str()), "{body}"); + } + } + // Not measured in order U, and reported without the key prefix and with + // the members unquoted: an empty BS and a BS with duplicates. + let bs_empty = with_key(json!({"pk": {}, "sk": {"BS": []}})); + assert_eq!( + check_key(&bs_empty), + Some(format!("{INVALID}Binary sets should not be empty")) + ); + let bs_dup = with_key(json!({"pk": {}, "sk": {"BS": ["YQ==", "YQ=="]}})); + assert_eq!( + check_key(&bs_dup), + Some(format!( + "{INVALID}Input collection [YQ==, YQ==]of type BS contains duplicates." + )) + ); +} + +/// The fitted member order: `String.hashCode()` of the name, spread and masked +/// to four buckets, buckets ascending, the request's order inside a bucket +/// (measured 2026-10-31, 2026-11-04 and 2026-11-05 on five tables). +#[test] +fn the_fitted_member_order_follows_the_hash_buckets_then_the_request() { + for (name, bucket) in [ + ("sk", 0), + ("a", 1), + ("b", 2), + ("pk", 3), + ("gpk", 3), + ("id", 3), + ("ts", 3), + ] { + assert_eq!(java_hash_bucket(name), bucket, "{name}"); + } + let order = |names: &[&str]| fitted_member_order(names.iter().copied()); + // The pk, sk table: sk first in both request orders. + assert_eq!(order(&["pk", "sk"]), ["sk", "pk"]); + assert_eq!(order(&["sk", "pk"]), ["sk", "pk"]); + // HASH a, RANGE b: a first; HASH pk, RANGE a: a first. + assert_eq!(order(&["b", "a"]), ["a", "b"]); + assert_eq!(order(&["pk", "a"]), ["a", "pk"]); + assert_eq!(order(&["a", "pk"]), ["a", "pk"]); + // ts and id share a bucket: the request's order. + assert_eq!(order(&["ts", "id"]), ["ts", "id"]); + assert_eq!(order(&["id", "ts"]), ["id", "ts"]); + // The GSI key of three members: sk, then gpk and pk in the request's order. + assert_eq!(order(&["gpk", "pk", "sk"]), ["sk", "gpk", "pk"]); + assert_eq!(order(&["pk", "sk", "gpk"]), ["sk", "pk", "gpk"]); +} + +/// `start_key_member_order` reads the key's member names from the request +/// text in document order; a repeated name keeps its first position; a body +/// with no key, a key that is not an object or text that does not parse gives +/// `None`. +#[test] +fn the_request_order_of_the_key_members_is_read_from_the_raw_body() { + let raw = br#"{"TableName": "t", "ExclusiveStartKey": {"ts": {"N": "bob"}, "id": {}}}"#; + assert_eq!( + start_key_member_order(raw), + Some(vec!["ts".to_owned(), "id".to_owned()]) + ); + let repeated = + br#"{"ExclusiveStartKey": {"b": {}, "a": {}, "b": {"S": "x"}}, "TableName": "t"}"#; + assert_eq!( + start_key_member_order(repeated), + Some(vec!["b".to_owned(), "a".to_owned()]) + ); + assert_eq!(start_key_member_order(br#"{"TableName": "t"}"#), None); + assert_eq!( + start_key_member_order(br#"{"TableName": "t", "ExclusiveStartKey": "x"}"#), + None + ); + assert_eq!(start_key_member_order(b"not json"), None); + assert_eq!(start_key_member_order(b""), None); +} + +/// Faults in two members of one key: the first faulty member in the fitted +/// order is reported. Measured rows: `{"pk": {}, "sk": {"N": "bob"}}` and the +/// reversed request order answer `bob` (2026-10-31); `{"pk": {"X": "1"}, "sk": +/// {"N": "bob"}}` answers `bob`; `{"pk": {}, "sk": {"NS": []}}` answers the +/// empty NS text and `{"pk": {"S": "x", "N": "1"}, "sk": {"SS": []}}` the +/// prefixed SS text (2026-11-05); on HASH `a`, RANGE `b` `{"a": {}, "b": {"N": +/// "bob"}}` answers the empty value and `{"a": {"N": "bob"}, "b": {}}` `bob`; +/// on HASH `pk`, RANGE `a` `{"pk": {}, "a": {"N": "bob"}}` answers `bob` in +/// both request orders; on HASH `id`, RANGE `ts` the member first in the +/// request; on the GSI `{"gpk": {}, "pk": {"N": "bob"}, "sk": {"S": "x"}}` +/// the empty value. With two value faults the service's two sends disagreed on +/// 8 of 22 rows: those rows accept either member's fault. +#[test] +fn two_faulty_members_answer_the_first_in_the_fitted_order() { let empty = format!("{KEY_PREFIX}{EMPTY_VALUE}"); let ns_empty = format!("{INVALID}An number set may not be empty"); - let fred = NUMERIC_BOB.replace("bob", "fred"); - for (key, either) in [ + let ss_empty = format!("{KEY_PREFIX}{INVALID}An string set may not be empty"); + for (key, request_order, text) in [ ( json!({"pk": {}, "sk": {"N": "bob"}}), - [empty.as_str(), NUMERIC_BOB], + ["pk", "sk"], + NUMERIC_BOB, ), ( json!({"sk": {"N": "bob"}, "pk": {}}), - [empty.as_str(), NUMERIC_BOB], + ["sk", "pk"], + NUMERIC_BOB, ), ( json!({"pk": {"X": "1"}, "sk": {"N": "bob"}}), - [empty.as_str(), NUMERIC_BOB], + ["pk", "sk"], + NUMERIC_BOB, ), ( json!({"pk": {"N": "bob"}, "sk": {}}), - [empty.as_str(), NUMERIC_BOB], + ["pk", "sk"], + empty.as_str(), + ), + ( + json!({"pk": {}, "sk": {"NS": []}}), + ["pk", "sk"], + ns_empty.as_str(), + ), + ( + json!({"pk": {"S": "x", "N": "1"}, "sk": {"SS": []}}), + ["pk", "sk"], + ss_empty.as_str(), + ), + ( + json!({"a": {}, "b": {"N": "bob"}}), + ["a", "b"], + empty.as_str(), + ), + (json!({"a": {"N": "bob"}, "b": {}}), ["a", "b"], NUMERIC_BOB), + ( + json!({"pk": {}, "a": {"N": "bob"}}), + ["pk", "a"], + NUMERIC_BOB, + ), + ( + json!({"a": {"N": "bob"}, "pk": {}}), + ["a", "pk"], + NUMERIC_BOB, ), + ( + json!({"id": {}, "ts": {"N": "bob"}}), + ["id", "ts"], + empty.as_str(), + ), + ( + json!({"ts": {"N": "bob"}, "id": {}}), + ["ts", "id"], + NUMERIC_BOB, + ), + ] { + let body = with_key(key.clone()); + assert_eq!( + check_key_in(&body, &request_order).as_deref(), + Some(text), + "{key} in the order {request_order:?}" + ); + } + let gsi = with_key(json!({"gpk": {}, "pk": {"N": "bob"}, "sk": {"S": "x"}})); + assert_eq!( + check_key_in(&gsi, &["gpk", "pk", "sk"]).as_deref(), + Some(empty.as_str()) + ); + // Two value faults: either member's fault. + let fred = NUMERIC_BOB.replace("bob", "fred"); + for (key, either) in [ ( json!({"pk": {"N": "bob"}, "sk": {"N": "fred"}}), [NUMERIC_BOB, fred.as_str()], @@ -385,12 +635,8 @@ fn two_faulty_members_answer_one_of_the_two_faults() { json!({"pk": {"NS": []}, "sk": {"N": "bob"}}), [ns_empty.as_str(), NUMERIC_BOB], ), - ( - json!({"pk": {}, "sk": {"NS": []}}), - [empty.as_str(), ns_empty.as_str()], - ), ] { - let answer = check_key(&with_key(key.clone())); + let answer = check_key_in(&with_key(key.clone()), &["pk", "sk"]); assert!( either.contains(&answer.as_deref().unwrap_or_default()), "{key}: {answer:?}" @@ -406,10 +652,10 @@ fn two_faulty_members_answer_one_of_the_two_faults() { fn a_key_of_the_wrong_shape_stays_in_the_body_and_an_unreadable_value_is_held_back() { for key in [json!("x"), json!({"pk": "x"}), json!({"pk": {"N": 5}})] { let mut body = with_key(key.clone()); - let start_key = StartKeyBody::take(&mut body); + let order = CheckOrder::of(&body); + let start_key = StartKeyBody::take(&mut body, None); assert!(body.get("ExclusiveStartKey").is_some(), "{key}"); - assert!(start_key.validate_shape().is_ok(), "{key}"); - assert!(start_key.validate_values().is_ok(), "{key}"); + assert!(start_key.validate(order).is_ok(), "{key}"); let shape = serde_json::from_value::(body) .map_err(crate::deserialize_error); assert!( @@ -425,65 +671,91 @@ fn a_key_of_the_wrong_shape_stays_in_the_body_and_an_unreadable_value_is_held_ba json!({"pk": {"SS": ["a", "a"]}}), ] { let mut body = with_key(key.clone()); - let start_key = StartKeyBody::take(&mut body); + let order = CheckOrder::of(&body); + let start_key = StartKeyBody::take(&mut body, None); assert!(body.get("ExclusiveStartKey").is_none(), "{key}"); - assert!( - start_key - .validate_shape() - .and(start_key.validate_values()) - .is_err(), - "{key}" - ); + assert!(start_key.validate(order).is_err(), "{key}"); assert!( serde_json::from_value::(body).is_ok(), "{key}" ); } - // A readable key stays in the body and passes both stages. + // A readable key stays in the body and passes. let mut body = with_key(json!({"pk": {"S": "p0"}, "sk": {"S": "x"}})); - let start_key = StartKeyBody::take(&mut body); + let start_key = StartKeyBody::take(&mut body, None); assert!(body.get("ExclusiveStartKey").is_some()); - assert!(start_key.validate_shape().is_ok()); - assert!(start_key.validate_values().is_ok()); + assert!(start_key.validate(CheckOrder::Readable).is_ok()); // No key at all. let mut body = scan(json!({})); - let start_key = StartKeyBody::take(&mut body); - assert!( - start_key - .validate_shape() - .and(start_key.validate_values()) - .is_ok() + let start_key = StartKeyBody::take(&mut body, None); + assert!(start_key.validate(CheckOrder::Readable).is_ok()); + // A request order that does not name the key's members is ignored. + let mut body = with_key(json!({"pk": {}, "sk": {"N": "bob"}})); + let names = ["a".to_owned(), "b".to_owned()]; + let start_key = StartKeyBody::take(&mut body, Some(&names)); + assert_eq!( + validation_message(start_key.validate(CheckOrder::Unreadable)).as_deref(), + Some(NUMERIC_BOB) ); } -/// `ExpressionValuesBody::take` puts a readable stand-in in the body for a -/// value the deserializer refuses for its content and keeps the deserializer's -/// fault for the map as written (measured 2026-11-05: the fault comes after the -/// key's value stage); a value of the wrong JSON shape stays in the body. +/// `ExpressionValuesBody::take` puts a readable stand-in in the body for an +/// entry the checks classify and keeps the fault with the service's text: in +/// order U the `ExpressionAttributeValues contains invalid value: ... for key +/// :a` text for an empty value, an unknown descriptor and two descriptors +/// (2026-10-31 B4-7, 2026-11-05); in order R the set texts with the members +/// JSON-quoted (2026-11-05). An entry of the wrong JSON shape stays in the +/// body. #[test] -fn an_unreadable_expression_value_is_held_back_and_a_stand_in_takes_its_place() { - let mut body = scan(json!({ - "FilterExpression": "a = :a", - "ExpressionAttributeValues": {":a": {}}, - })); - let values = super::ExpressionValuesBody::take(&mut body); - assert_eq!(body["ExpressionAttributeValues"], json!({":a": {"S": ""}})); +fn a_faulty_expression_value_is_held_back_and_a_stand_in_takes_its_place() { + let wrapper = + |text: &str| format!("ExpressionAttributeValues contains invalid value: {text} for key :a"); + for (value, text) in [ + (json!({}), wrapper(EMPTY_VALUE)), + (json!({"X": "1"}), wrapper(EMPTY_VALUE)), + (json!({"S": "x", "N": "1"}), wrapper(MORE_THAN_ONE)), + (json!({"L": [{}]}), wrapper(EMPTY_VALUE)), + ] { + let mut body = scan(json!({ + "FilterExpression": "a = :a", + "ExpressionAttributeValues": {":a": value}, + })); + let order = CheckOrder::of(&body); + assert_eq!(order, CheckOrder::Unreadable, "{value}"); + let values = ExpressionValuesBody::take(&mut body, order); + assert_eq!(body["ExpressionAttributeValues"], json!({":a": {"S": ""}})); + assert_eq!(validation_message(values.validate()), Some(text), "{value}"); + } + for (value, text) in [ + (json!({"NS": []}), "An number set may not be empty"), + ( + json!({"SS": ["a", "a"]}), + r#"Input collection ["a", "a"] contains duplicates."#, + ), + ] { + let mut body = scan(json!({"ExpressionAttributeValues": {":a": value}})); + let order = CheckOrder::of(&body); + assert_eq!(order, CheckOrder::Readable, "{value}"); + let values = ExpressionValuesBody::take(&mut body, order); + assert_eq!(body["ExpressionAttributeValues"], json!({":a": {"S": ""}})); + assert_eq!( + validation_message(values.validate()), + Some(format!("{INVALID}{text}")), + "{value}" + ); + } + // Two faulty entries: the first in sorted order is reported. + let mut body = + scan(json!({"ExpressionAttributeValues": {":b": {}, ":a": {"S": "x", "N": "1"}}})); + let values = ExpressionValuesBody::take(&mut body, CheckOrder::Unreadable); assert_eq!( - validation_message(values.validate()).as_deref(), - Some( - "ExpressionAttributeValues contains invalid value: Supplied AttributeValue is empty, must contain exactly one of the supported datatypes for key :a" - ) - ); - let mut body = scan(json!({"ExpressionAttributeValues": {":a": {"X": "1"}}})); - let values = super::ExpressionValuesBody::take(&mut body); - assert_eq!(body["ExpressionAttributeValues"], json!({":a": {"S": ""}})); - assert!( - matches!(values.validate(), Err(DynamoDbError::SerializationException(msg)) if msg.contains("unknown AttributeValue type descriptor: X")) + validation_message(values.validate()), + Some(wrapper(MORE_THAN_ONE)) ); // A readable value and a value of the wrong JSON shape stay as written. for value in [json!({"S": "x"}), json!({"N": 5})] { let mut body = scan(json!({"ExpressionAttributeValues": {":a": value}})); - let values = super::ExpressionValuesBody::take(&mut body); + let values = ExpressionValuesBody::take(&mut body, CheckOrder::Readable); assert_eq!(body["ExpressionAttributeValues"], json!({":a": value})); assert!(values.validate().is_ok()); } @@ -504,6 +776,23 @@ fn a_value_nested_32_levels_deep_is_refused_and_31_levels_pass() { json!({"Select": "BAD"}), ); assert_eq!(check(&select).as_deref(), Some(NESTING)); + // Measured 2026-10-31 (B4-5a): with no expression at all, the nesting text + // ahead of the no-expression check. + let no_expression = scan(json!({"ExpressionAttributeValues": {":d": nested(&one, 32)}})); + assert_eq!(check(&no_expression).as_deref(), Some(NESTING)); + // In order U the nesting check leaves request time: beside an unused + // empty value (B4-7) or a key shape fault (2026-11-05) the request-time + // checks pass and the handler reports those first. + let beside_empty = scan(json!({ + "FilterExpression": "a = :p", + "ExpressionAttributeValues": {":p": {"S": "x"}, ":d": nested(&one, 32), ":e": {}}, + })); + assert_eq!(check(&beside_empty), None); + let beside_key = with( + with_unused_value(nested(&one, 32)), + json!({"ExclusiveStartKey": {"pk": {}, "sk": {"S": "x"}}}), + ); + assert_eq!(check(&beside_key), None); } #[test] @@ -570,4 +859,16 @@ fn combinations_measured_on_2026_10_31_follow_the_order_of_the_checks() { |levels: usize| (0..levels).fold(json!({"N": "1"}), |inner, _| json!({"M": {"k": inner}})); assert_eq!(check(&with_unused_value(map(32))).as_deref(), Some(NESTING)); assert_eq!(check(&with_unused_value(map(31))), None); + // The ReturnConsumedCapacity enum comes ahead of the parameter mix + // (2026-11-05). + let rcc_mix = with( + with(scan(filter("1")), json!({"ScanFilter": {}})), + json!({"ReturnConsumedCapacity": "BAD"}), + ); + assert_eq!( + check(&rcc_mix).as_deref(), + Some( + "1 validation error detected: Value 'BAD' at 'returnConsumedCapacity' failed to satisfy constraint: Member must satisfy enum value set: [INDEXES, TOTAL, NONE]" + ) + ); } diff --git a/crates/engine/src/scan_request/tests.rs b/crates/engine/src/scan_request/tests.rs index 91005eb22..a5ba5eb41 100644 --- a/crates/engine/src/scan_request/tests.rs +++ b/crates/engine/src/scan_request/tests.rs @@ -9,7 +9,7 @@ use extenddb_core::error::DynamoDbError; use serde_json::{Value, json}; -use super::validate_scan_request_body; +use super::{CheckOrder, validate_scan_request_body}; const NUMERIC_BOB: &str = "The parameter cannot be converted to a numeric value: bob"; const LIMIT_RANGE: &str = "Value at 'Limit' failed to satisfy constraint: Member must have value greater than or equal to 1"; @@ -25,7 +25,7 @@ fn message(result: Result<(), DynamoDbError>) -> Option { } fn check(body: &Value) -> Option { - message(validate_scan_request_body(body)) + message(validate_scan_request_body(body, CheckOrder::of(body))) } /// A Scan whose filter reads `:n = {"N": "bob"}`. @@ -204,12 +204,11 @@ fn parameter_mix_beats_the_enum_and_the_numbers() { Non-expression parameters: {ScanFilter} Expression parameters: {FilterExpression}" ) ); + // The ReturnConsumedCapacity enum comes ahead of the mix (measured + // 2026-11-05). assert_eq!( check(&with(mix, "ReturnConsumedCapacity", json!("NOPE"))).as_deref(), - Some( - "Can not use both expression and non-expression parameters in the same request: \ - Non-expression parameters: {ScanFilter} Expression parameters: {FilterExpression}" - ) + Some(RCC_ENUM) ); let atg = with( with(bad_number(), "AttributesToGet", json!(["a"])), @@ -272,9 +271,18 @@ fn enum_checks_sit_on_either_side_of_the_numbers() { json!("NOPE"), ); assert_eq!(check(&both).as_deref(), Some(RCC_ENUM)); - // With the numbers valid, an invalid Select is reported. + // With every value readable (order R) the Select enum is not a + // request-time check: the handler reports it after the no-expression check + // and the nesting check (measured 2026-11-05); beside an unreadable value + // (order U) it is one. + assert_eq!(check(&json!({"TableName": "t", "Select": "NOPE"})), None); assert_eq!( - check(&json!({"TableName": "t", "Select": "NOPE"})).as_deref(), + check(&json!({ + "TableName": "t", + "Select": "NOPE", + "ExclusiveStartKey": {"pk": {}, "sk": {"S": "x"}}, + })) + .as_deref(), Some(SELECT_ENUM) ); // A valid request passes. @@ -308,7 +316,7 @@ fn a_shape_fault_anywhere_in_the_body_is_reported_first() { for body in shape_faults { assert!( matches!( - validate_scan_request_body(&body), + validate_scan_request_body(&body, CheckOrder::of(&body)), Err(DynamoDbError::SerializationException(_)) ), "{body}" diff --git a/crates/server/src/handler.rs b/crates/server/src/handler.rs index ef3765bda..d9e139aa1 100755 --- a/crates/server/src/handler.rs +++ b/crates/server/src/handler.rs @@ -225,7 +225,7 @@ pub(crate) async fn handle_request( // --- Dispatch segment --- let dispatch_start = std::time::Instant::now(); let dispatch_result = - extenddb_engine::dispatch(&operation, input, &ctx, &state.server_addr).await; + extenddb_engine::dispatch(&operation, input, &body, &ctx, &state.server_addr).await; #[allow(clippy::cast_precision_loss)] let dispatch_us = dispatch_start.elapsed().as_micros() as f64; // P120c: Record storage query metrics for the request's dispatch phase. diff --git a/tests/python/test_scan_request_validation.py b/tests/python/test_scan_request_validation.py index b47d63e4d..ed1b12579 100644 --- a/tests/python/test_scan_request_validation.py +++ b/tests/python/test_scan_request_validation.py @@ -16,14 +16,22 @@ nesting text, 31 reaches the unused-value check. A malformed number at the bottom is reported first, at the depths 31 to 60 that were measured. -The faults of ``ExclusiveStartKey`` come at two places (measured on -2026-11-05, each request twice): a value that is empty, carries an unknown type -descriptor or two descriptors is reported after the constraint clauses and the -request parameter checks, with the prefix ``The provided starting key is -invalid: ``, and ahead of the expression checks; a number or set fault is -reported after the expression and request checks, with no prefix, and ahead of -the table lookup. With faults in two members the service's choice follows the -attribute names and the request's order, so those tests accept either fault. +The service follows one of two check orders (measured 2026-10-31 and +2026-11-05, each request twice; MEASURED-scan-r3 section 6). A request that +holds a value that is not one AttributeValue (empty, an unknown or lower-case +type descriptor, two descriptors) in ``ExpressionAttributeValues``, +``ExclusiveStartKey`` or a ``ScanFilter`` list follows order U: the constraint +clauses in the older form with the ``Select`` enum among them, the request +parameter checks, the key's faults (shape and value together, the first faulty +member in the fitted member order, a shape fault with the prefix ``The provided +starting key is invalid: ``), the unreadable ``ExpressionAttributeValues`` +entries, the nesting depth, the expression checks, the table. Every other +request follows order R: the constraint clauses in the newer form, the parameter +mix, the nesting depth, a set fault in ``ExpressionAttributeValues``, the request +parameter checks, the ``Select`` enum, the expression checks, the key's number +and set faults, the table. The set texts quote their members in order R and not +in order U. With two value faults in two key members the service's own sends +disagreed, so those tests accept either fault. """ from __future__ import annotations @@ -186,22 +194,49 @@ NS_EMPTY), ] -# Measured 2026-11-05: faults in two members of one key. The service's choice -# follows the attribute names and, on some tables, the request's member order: -# {"pk": {}, "sk": {"N": "bob"}} answered bob on 2026-10-31, and {"a": {}, "b": -# {"N": "bob"}} the empty value. Each row accepts either member's fault. +# Measured 2026-10-31 and 2026-11-05: faults in two members of one key, one of +# them a shape fault. The service names the first faulty member in the fitted +# member order (MEASURED-scan-r3 section 6.3: sk ahead of pk and of z in both +# request orders; the set texts of order U where the set member comes first). TWO_MEMBER_FAULTS = [ - ("empty-pk-then-bob", '{"pk": {}, "sk": {"N": "bob"}}', (EMPTY_KEY_VALUE, NUMERIC_BOB)), - ("bob-then-empty-pk", '{"sk": {"N": "bob"}, "pk": {}}', (EMPTY_KEY_VALUE, NUMERIC_BOB)), - ("type-X-pk-then-bob", '{"pk": {"X": "1"}, "sk": {"N": "bob"}}', - (EMPTY_KEY_VALUE, NUMERIC_BOB)), - ("bob-then-empty-sk", '{"pk": {"N": "bob"}, "sk": {}}', (EMPTY_KEY_VALUE, NUMERIC_BOB)), - ("two-descriptors-then-bob", '{"pk": {"S": "x", "N": "1"}, "sk": {"N": "bob"}}', - (TWO_DESCRIPTORS, NUMERIC_BOB)), + ("empty-pk-then-bob", '{"pk": {}, "sk": {"N": "bob"}}', NUMERIC_BOB), + ("bob-then-empty-pk", '{"sk": {"N": "bob"}, "pk": {}}', NUMERIC_BOB), + ("type-X-pk-then-bob", '{"pk": {"X": "1"}, "sk": {"N": "bob"}}', NUMERIC_BOB), + ("bob-then-empty-sk", '{"pk": {"N": "bob"}, "sk": {}}', EMPTY_KEY_VALUE), + ("two-descriptors-then-bob", '{"pk": {"S": "x", "N": "1"}, "sk": {"N": "bob"}}', NUMERIC_BOB), + ("empty-then-NS-empty", '{"pk": {}, "sk": {"NS": []}}', NS_EMPTY), + ("NS-empty-then-empty", '{"pk": {"NS": []}, "sk": {}}', EMPTY_KEY_VALUE), + ("empty-then-NS-bob", '{"pk": {}, "sk": {"NS": ["1", "bob"]}}', NUMERIC_BOB), + ("empty-in-an-L-then-bob", '{"pk": {"L": [{}]}, "sk": {"N": "bob"}}', NUMERIC_BOB), + ("type-X-then-bob-in-z", '{"pk": {"X": "1"}, "sk": {"S": "x"}, "z": {"N": "bob"}}', + NUMERIC_BOB), + ("two-descriptors-then-SS-empty", '{"pk": {"S": "x", "N": "1"}, "sk": {"SS": []}}', + KEY_PREFIX + SS_EMPTY), + ("empty-then-NS-duplicates", '{"pk": {}, "sk": {"NS": ["1", "1"]}}', + "Input collection contains duplicates"), + ("empty-then-SS-duplicates", '{"pk": {}, "sk": {"SS": ["a", "a"]}}', + INVALID + "Input collection [a, a] contains duplicates."), +] + +# Two value faults in two members: the service's two sends disagreed on 8 of +# 22 such rows (2026-11-05), so each row accepts either member's fault. +TWO_VALUE_FAULTS = [ ("NS-empty-then-bob", '{"pk": {"NS": []}, "sk": {"N": "bob"}}', (NS_EMPTY, NUMERIC_BOB)), - ("empty-then-NS-empty", '{"pk": {}, "sk": {"NS": []}}', (EMPTY_KEY_VALUE, NS_EMPTY)), - ("NS-empty-then-empty", '{"pk": {"NS": []}, "sk": {}}', (EMPTY_KEY_VALUE, NS_EMPTY)), - ("empty-then-NS-bob", '{"pk": {}, "sk": {"NS": ["1", "bob"]}}', (EMPTY_KEY_VALUE, NUMERIC_BOB)), + ("SS-duplicates-then-bob", '{"pk": {"SS": ["a", "a"]}, "sk": {"N": "bob"}}', + (SS_DUP, NUMERIC_BOB)), +] + +# Measured 2026-11-05 (R5, MEASURED-scan-r3 section 6.3) on tables with other +# key names: (id, table fixture, key, expected). HASH a, RANGE b: a first; HASH +# pk, RANGE a: a first in both request orders; HASH id, RANGE ts: the member +# first in the request; the GSI key of three members: gpk ahead of pk. +MEMBER_ORDER_ROWS = [ + ("ab-empty-and-bob", "ab_table", '{"a": {}, "b": {"N": "bob"}}', EMPTY_KEY_VALUE), + ("ab-bob-and-empty", "ab_table", '{"a": {"N": "bob"}, "b": {}}', NUMERIC_BOB), + ("pa-empty-and-bob", "pa_table", '{"pk": {}, "a": {"N": "bob"}}', NUMERIC_BOB), + ("pa-bob-and-empty-reversed", "pa_table", '{"a": {"N": "bob"}, "pk": {}}', NUMERIC_BOB), + ("it-empty-and-bob", "it_table", '{"id": {}, "ts": {"N": "bob"}}', EMPTY_KEY_VALUE), + ("it-bob-and-empty-reversed", "it_table", '{"ts": {"N": "bob"}, "id": {}}', NUMERIC_BOB), ] # Measured 2026-11-05 (MEASURED-scan-r3 section 2): beside each class of key @@ -223,8 +258,8 @@ # prefix of the text. ("ExpressionAttributeValues-with-no-expression", '"ExpressionAttributeValues": {":a": {"S": "x"}}', "parameter"), - # An expression check (S3): ahead of a number or set fault, after a shape - # fault. + # An expression check: ahead of a number or set fault (order R), after the + # key's faults (order U). ("unused-value", '"FilterExpression": "a = :p", "ExpressionAttributeValues": {":p": {"S": "x"}, ":u": {"S": "y"}}', "expression"), @@ -243,7 +278,6 @@ "Attempting to store more than 38 significant digits in a Number"), ] - def nest(leaf: dict, levels: int) -> dict: """``leaf`` inside ``levels`` lists.""" value = leaf @@ -252,6 +286,185 @@ def nest(leaf: dict, levels: int) -> dict: return value +EAV_VALUE = "ExpressionAttributeValues contains invalid value: " +EAV_EMPTY_E = (EAV_VALUE + "Supplied AttributeValue is empty, must contain exactly one of the " + "supported datatypes for key :e") +EAV_EMPTY_A = EAV_EMPTY_E.replace(":e", ":a") +EAV_TWO_A = (EAV_VALUE + "Supplied AttributeValue has more than one datatypes set, must contain " + "exactly one of the supported datatypes for key :a") +EAN_NO_EXPRESSION = "ExpressionAttributeNames can only be specified when using expressions" +CONDOP = "ConditionalOperator can only be used when Filter or Expected has two or more elements" +UNDEFINED_ZZ = ("Invalid FilterExpression: An expression attribute value used in expression " + "is not defined; attribute value: :zz") +BEGINS_N = ("Invalid FilterExpression: Incorrect operand type for operator or function; " + "operator or function: begins_with, operand type: N") +SEGMENT_ALONE = ("The TotalSegments parameter is required but was not present in the request " + "when Segment parameter is present") +EMPTY_KEY = '"ExclusiveStartKey": {"pk": {}, "sk": {"S": "x"}}' +FILTER_P = '"FilterExpression": "a = :p"' +NEST32_S = json.dumps(nest({"S": "x"}, 32)) +NEST32_N = json.dumps(nest({"N": "1"}, 32)) + +# Measured 2026-10-31 (B4-7) and 2026-11-05 (R5, MEASURED-scan-r3 section 6): +# representative rows of each order. An expected text given as ("startswith", +# prefix) asserts the check's place where the text predates this branch (#722, +# #838, #839). "4th" names the rows that fail on 45705bf27. +TWO_ORDERS = [ + # Order U: an unreadable ExpressionAttributeValues entry, read or unused, + # after the Select enum, the request parameter checks and the key's faults, + # ahead of the nesting check, the unused values and the table. + ("B4-7", FILTER_P + ', "ExpressionAttributeValues": {":p": {"S": "x"}, ":d": ' + + NEST32_N + ', ":e": {}}', EAV_EMPTY_E), + ("B4-7-flat", FILTER_P + ', "ExpressionAttributeValues": {":p": {"S": "x"}, ":e": {}}', + EAV_EMPTY_E), + ("B4-7-X", FILTER_P + ', "ExpressionAttributeValues": {":p": {"S": "x"}, ":e": {"X": "1"}}', + EAV_EMPTY_E), + ("eav-empty-select-bad", FILTER_P + ', "ExpressionAttributeValues": {":p": {"S": "x"}, ' + '":e": {}}, "Select": "BAD"', SELECT_ENUM), + ("eav-empty-unused-and-key-empty", FILTER_P + ', "ExpressionAttributeValues": ' + '{":p": {"S": "x"}, ":e": {}}, ' + EMPTY_KEY, EMPTY_KEY_VALUE), + ("eav-empty-read-plus-unused", + '"FilterExpression": "a = :a", "ExpressionAttributeValues": {":a": {}, ":u": {"S": "x"}}', + EAV_EMPTY_A), + ("eav-read-X", '"FilterExpression": "a = :a", "ExpressionAttributeValues": {":a": {"X": "1"}}', + EAV_EMPTY_A), + ("eav-read-two", + '"FilterExpression": "a = :a", "ExpressionAttributeValues": {":a": {"S": "x", "N": "1"}}', + EAV_TWO_A), + ("eav-read-empty-begins", + '"FilterExpression": "begins_with(a, :a)", "ExpressionAttributeValues": {":a": {}}', + EAV_EMPTY_A), + ("B4-7-noexpr-empty", '"ExpressionAttributeValues": {":e": {}}', + ("startswith", EAV_NO_EXPRESSION)), + ("B4-7-noexpr-two", '"ExpressionAttributeValues": {":e": {"S": "x", "N": "1"}}', + ("startswith", EAV_NO_EXPRESSION)), + # Order U: the nesting check after the key's faults (the three b.*.nesting + # rows of R3 and the nest-*-key-empty rows of R5). + ("nesting-beside-empty-key", FILTER_P + ', "ExpressionAttributeValues": {":p": ' + + NEST32_S + '}, ' + EMPTY_KEY, EMPTY_KEY_VALUE), + ("nesting-beside-type-X-key", FILTER_P + ', "ExpressionAttributeValues": {":p": ' + + NEST32_S + '}, "ExclusiveStartKey": {"pk": {"X": "1"}, "sk": {"S": "x"}}', + EMPTY_KEY_VALUE), + ("nesting-beside-two-descriptors-key", FILTER_P + ', "ExpressionAttributeValues": {":p": ' + + NEST32_S + '}, "ExclusiveStartKey": {"pk": {"S": "x", "N": "1"}, "sk": {"S": "x"}}', + TWO_DESCRIPTORS), + ("nest-read-N-key-empty", FILTER_P + ', "ExpressionAttributeValues": {":p": ' + + NEST32_N + '}, ' + EMPTY_KEY, EMPTY_KEY_VALUE), + ("nest-unused-S-key-empty", FILTER_P + ', "ExpressionAttributeValues": {":p": {"S": "x"}, ' + '":d": ' + NEST32_S + '}, ' + EMPTY_KEY, EMPTY_KEY_VALUE), + ("nest-unused-N-key-empty", FILTER_P + ', "ExpressionAttributeValues": {":p": {"S": "x"}, ' + '":d": ' + NEST32_N + '}, ' + EMPTY_KEY, EMPTY_KEY_VALUE), + # Order U: the parameter mix and the other request parameter checks ahead of + # the key's shape fault (RS-f Minor-1; arity-key-empty-and-eav-noexpr). + ("mix-beside-empty-key", '"ScanFilter": {"a": {"ComparisonOperator": "EQ", ' + '"AttributeValueList": [{"N": "1"}]}}, "FilterExpression": "a = :p", ' + '"ExpressionAttributeValues": {":p": {"N": "1"}}, ' + EMPTY_KEY, MIX), + ("arity-key-empty-and-ean-noexpr", '"ScanFilter": {"a": {"ComparisonOperator": "EQ", ' + '"AttributeValueList": []}}, "ExpressionAttributeNames": {"#a": "a"}, ' + EMPTY_KEY, + EAN_NO_EXPRESSION), + # Order R: the nesting check ahead of the Select enum, the filter syntax, + # the no-expression check and the key's number. + ("nest-unused-S-select-bad", FILTER_P + ', "ExpressionAttributeValues": {":p": {"S": "x"}, ' + '":d": ' + NEST32_S + '}, "Select": "BAD"', NESTING), + ("nest-read-S-filter-syntax", '"FilterExpression": "a = = :p", "ExpressionAttributeValues": ' + '{":p": ' + NEST32_S + '}', NESTING), + ("nest-read-S-eav-noexpr-sibling", '"ProjectionExpression": "a", ' + '"ExpressionAttributeValues": {":p": ' + NEST32_S + '}', NESTING), + ("nest-read-S-key-bob", FILTER_P + ', "ExpressionAttributeValues": {":p": ' + NEST32_S + + '}, ' + BOB_KEY, NESTING), + ("B4-5a", '"ExpressionAttributeValues": {":d": ' + NEST32_N + '}', NESTING), + # Order R: a set fault in ExpressionAttributeValues ahead of the + # no-expression check; the set texts with the members JSON-quoted. + ("B4-7-noexpr-NSe", '"ExpressionAttributeValues": {":e": {"NS": []}}', NS_EMPTY), + ("eav-read-NSe", '"FilterExpression": "a = :a", "ExpressionAttributeValues": {":a": {"NS": []}}', + NS_EMPTY), + ("eav-read-SSdup", + '"FilterExpression": "a = :a", "ExpressionAttributeValues": {":a": {"SS": ["a", "a"]}}', + SS_DUP), + # Order R: the constraint enums and the request parameter checks among + # themselves, and the Select enum after the no-expression check. + ("mix-rcc-bad", '"ScanFilter": {"a": {"ComparisonOperator": "EQ", "AttributeValueList": ' + '[{"N": "1"}]}}, "FilterExpression": "a = :p", "ExpressionAttributeValues": ' + '{":p": {"N": "1"}}, "ReturnConsumedCapacity": "BAD"', RCC_ENUM), + ("rcc-bad-select-bad", '"ReturnConsumedCapacity": "BAD", "Select": "BAD"', RCC_ENUM), + ("mix-select-bad", '"ScanFilter": {"a": {"ComparisonOperator": "EQ", "AttributeValueList": ' + '[{"N": "1"}]}}, "FilterExpression": "a = :p", "ExpressionAttributeValues": ' + '{":p": {"N": "1"}}, "Select": "BAD"', MIX), + ("select-bad-eav-noexpr", '"ExpressionAttributeValues": {":p": {"S": "x"}}, "Select": "BAD"', + ("startswith", EAV_NO_EXPRESSION)), + ("eav-noexpr-ean-noexpr", '"ExpressionAttributeValues": {":p": {"S": "x"}}, ' + '"ExpressionAttributeNames": {"#a": "a"}', EAN_NO_EXPRESSION), + ("ean-noexpr-condop", '"ExpressionAttributeNames": {"#a": "a"}, "ConditionalOperator": "AND"', + EAN_NO_EXPRESSION), + ("condop-arity", '"ConditionalOperator": "AND", "ScanFilter": {"a": {"ComparisonOperator": ' + '"EQ", "AttributeValueList": []}}', CONDOP), + # Order R: the expression checks among themselves (section 6 step 5). + ("filter-syntax-segment-alone", '"FilterExpression": "a = = :p", ' + '"ExpressionAttributeValues": {":p": {"N": "1"}}, "Segment": 0', + ("startswith", "Invalid FilterExpression:")), + ("segment-alone-unused", FILTER_P + ', "ExpressionAttributeValues": {":p": {"S": "x"}, ' + '":u": {"S": "y"}}, "Segment": 0', UNUSED_U), + ("undefined-unused", '"FilterExpression": "a = :zz", "ExpressionAttributeValues": ' + '{":u": {"S": "y"}}', UNDEFINED_ZZ), + ("undefined-segment-alone", '"FilterExpression": "a = :zz", "Segment": 0', UNDEFINED_ZZ), + ("undefined-select-specific", '"FilterExpression": "a = :zz", "Select": "SPECIFIC_ATTRIBUTES"', + UNDEFINED_ZZ), + ("begins-n-segment-alone", '"FilterExpression": "begins_with(a, :p)", ' + '"ExpressionAttributeValues": {":p": {"N": "1"}}, "Segment": 0', BEGINS_N), + ("begins-n-unused", '"FilterExpression": "begins_with(a, :p)", ' + '"ExpressionAttributeValues": {":p": {"N": "1"}, ":u": {"S": "y"}}', BEGINS_N), + ("projection-syntax-filter-syntax", '"FilterExpression": "a = = :p", ' + '"ExpressionAttributeValues": {":p": {"N": "1"}}, "ProjectionExpression": "a,,b"', + ("startswith", "Invalid FilterExpression:")), + ("select-specific-filter-syntax", '"FilterExpression": "a = = :p", ' + '"ExpressionAttributeValues": {":p": {"N": "1"}}, "Select": "SPECIFIC_ATTRIBUTES"', + ("startswith", "Invalid FilterExpression:")), + ("allproj-noindex-filter-syntax", '"Select": "ALL_PROJECTED_ATTRIBUTES", ' + '"FilterExpression": "a = = :p", "ExpressionAttributeValues": {":p": {"N": "1"}}', + ("startswith", "Invalid FilterExpression:")), + ("select-count-pe-unused", '"Select": "COUNT", "ProjectionExpression": "a", ' + FILTER_P + + ', "ExpressionAttributeValues": {":p": {"S": "x"}, ":u": {"S": "y"}}', UNUSED_U), + ("segment-ge-total-unused", FILTER_P + ', "ExpressionAttributeValues": {":p": {"S": "x"}, ' + '":u": {"S": "y"}}, "Segment": 2, "TotalSegments": 2', UNUSED_U), + ("select-count-atg-segment-alone", '"Select": "COUNT", "AttributesToGet": ["a"], "Segment": 0', + ("startswith", "Cannot specify the ")), + ("select-count-atg-unused", '"Select": "COUNT", "AttributesToGet": ["a"], ' + '"ExpressionAttributeValues": {":u": {"S": "y"}}', ("startswith", EAV_NO_EXPRESSION)), + # Order R: the key's value faults after the expression checks, their texts. + ("key-bob-eav-bob", '"FilterExpression": "a = :n", "ExpressionAttributeValues": ' + '{":n": {"N": "fred"}}, ' + BOB_KEY, NUMERIC_FRED), + ("key-bob-and-NSe-one-member", + '"ExclusiveStartKey": {"pk": {"S": "x"}, "sk": {"S": "x"}, "z": {"NS": ["bob", "1", "1"]}}', + NUMERIC_BOB), + ("key-NS-dup-and-bob", + '"ExclusiveStartKey": {"pk": {"S": "x"}, "sk": {"S": "x"}, "z": {"NS": ["1", "1", "bob"]}}', + NUMERIC_BOB), + ("key-BS-dup", + '"ExclusiveStartKey": {"pk": {"S": "x"}, "sk": {"S": "x"}, "z": {"BS": ["YQ==", "YQ=="]}}', + INVALID + 'Input collection ["YQ==","YQ=="]of type BS contains duplicates.'), + ("key-SS-dup-quotes", + '"ExclusiveStartKey": {"pk": {"S": "x"}, "sk": {"S": "x"}, "z": {"SS": ["a\\"b", "a\\"b"]}}', + INVALID + 'Input collection ["a\\"b", "a\\"b"] contains duplicates.'), + ("key-bob-schema-mismatch", '"ExclusiveStartKey": {"pk": {"N": "bob"}}', NUMERIC_BOB), + # Order U: the key's set faults beside an unreadable expression value, the + # order-U texts (the NS duplicate text bare, the SS members unquoted). + ("key-bob-eav-empty-read-and-NSe", '"FilterExpression": "a = :a", ' + '"ExpressionAttributeValues": {":a": {}}, "ExclusiveStartKey": {"pk": {"NS": []}, ' + '"sk": {"S": "x"}}', NS_EMPTY), + ("key-NS-dup-beside-eav-empty", '"FilterExpression": "a = :a", ' + '"ExpressionAttributeValues": {":a": {}}, "ExclusiveStartKey": {"pk": {"NS": ["1", "1"]}, ' + '"sk": {"S": "x"}}', "Input collection contains duplicates"), + ("key-SS-dup-beside-eav-empty", '"FilterExpression": "a = :a", ' + '"ExpressionAttributeValues": {":a": {}}, "ExclusiveStartKey": {"pk": {"SS": ["a", "a"]}, ' + '"sk": {"S": "x"}}', INVALID + "Input collection [a, a] contains duplicates."), + # Order U: the key's shape fault ahead of the table's index rules. + ("key-empty-and-consistent-gsi", '"IndexName": "gsi", "ConsistentRead": true, ' + '"ExclusiveStartKey": {"gpk": {"S": "g"}, "pk": {"S": "p"}, "sk": {}}', EMPTY_KEY_VALUE), + ("key-shape-M-nested-N-bob", '"ExclusiveStartKey": {"pk": {"S": "x"}, "sk": {"S": "x"}, ' + '"z": {"M": {"k": {"N": "bob"}, "j": {}}}}', EMPTY_KEY_VALUE), +] + + def unused_value(value: dict) -> str: """A filter that reads ``:p`` and ``value`` under ``:d``, which nothing reads.""" return ('"FilterExpression": "a = :p", "ExpressionAttributeValues": ' @@ -336,6 +549,47 @@ def n_table(dynamodb_client): wait_for_deleted(dynamodb_client, name) +def _two_member_table(dynamodb_client, prefix: str, hash_key: str, range_key: str): + name = unique_name(prefix) + dynamodb_client.create_table( + TableName=name, + AttributeDefinitions=[{"AttributeName": hash_key, "AttributeType": "S"}, + {"AttributeName": range_key, "AttributeType": "S"}], + KeySchema=[{"AttributeName": hash_key, "KeyType": "HASH"}, + {"AttributeName": range_key, "KeyType": "RANGE"}], + BillingMode="PAY_PER_REQUEST", + ) + wait_for_active(dynamodb_client, name) + return name + + +@pytest.fixture(scope="module") +def ab_table(dynamodb_client): + """HASH a S, RANGE b S, no items.""" + name = _two_member_table(dynamodb_client, "scanreqab", "a", "b") + yield name + dynamodb_client.delete_table(TableName=name) + wait_for_deleted(dynamodb_client, name) + + +@pytest.fixture(scope="module") +def pa_table(dynamodb_client): + """HASH pk S, RANGE a S, no items.""" + name = _two_member_table(dynamodb_client, "scanreqpa", "pk", "a") + yield name + dynamodb_client.delete_table(TableName=name) + wait_for_deleted(dynamodb_client, name) + + +@pytest.fixture(scope="module") +def it_table(dynamodb_client): + """HASH id S, RANGE ts S, no items.""" + name = _two_member_table(dynamodb_client, "scanreqit", "id", "ts") + yield name + dynamodb_client.delete_table(TableName=name) + wait_for_deleted(dynamodb_client, name) + + class TestRequestChecks: """The parameter mix, the length constraint and the numbers in the key.""" @@ -391,14 +645,33 @@ def test_each_key_alone(self, dynamodb_client, table, key, expected): fields = '"ExclusiveStartKey": ' + key assert _validation_message(*_scan(dynamodb_client, table, fields)) == expected - @pytest.mark.parametrize("key, either", [r[1:] for r in TWO_MEMBER_FAULTS], + @pytest.mark.parametrize("key, expected", [r[1:] for r in TWO_MEMBER_FAULTS], ids=[r[0] for r in TWO_MEMBER_FAULTS]) - def test_two_faulty_members_answer_one_of_the_two_faults( - self, dynamodb_client, table, key, either): + def test_two_faulty_members_answer_the_first_in_the_fitted_order( + self, dynamodb_client, table, key, expected): + fields = '"ExclusiveStartKey": ' + key + assert _validation_message(*_scan(dynamodb_client, table, fields)) == expected + + @pytest.mark.parametrize("key, either", [r[1:] for r in TWO_VALUE_FAULTS], + ids=[r[0] for r in TWO_VALUE_FAULTS]) + def test_two_value_faults_answer_one_of_the_two(self, dynamodb_client, table, key, either): fields = '"ExclusiveStartKey": ' + key message = _validation_message(*_scan(dynamodb_client, table, fields)) assert message in either, message + @pytest.mark.parametrize("fixture, key, expected", [r[1:] for r in MEMBER_ORDER_ROWS], + ids=[r[0] for r in MEMBER_ORDER_ROWS]) + def test_the_member_order_on_other_tables(self, dynamodb_client, request, fixture, key, + expected): + name = request.getfixturevalue(fixture) + fields = '"ExclusiveStartKey": ' + key + assert _validation_message(*_scan(dynamodb_client, name, fields)) == expected + + def test_the_member_order_on_a_gsi_key_of_three_members(self, dynamodb_client, table): + fields = ('"IndexName": "gsi", "ExclusiveStartKey": {"gpk": {}, "pk": {"N": "bob"}, ' + '"sk": {"S": "x"}}') + assert _validation_message(*_scan(dynamodb_client, table, fields)) == EMPTY_KEY_VALUE + def test_two_number_faults_name_one_of_them(self, dynamodb_client, table): # The service named bob twice on 2026-10-31 (23:54 to 23:56 UTC) and # fred on 2026-11-04 (23:45 UTC) for this key. @@ -470,6 +743,32 @@ def test_the_number_texts_on_an_n_hash_key(self, dynamodb_client, n_table, field assert _validation_message(*_scan(dynamodb_client, n_table, fields)) == expected +class TestTwoOrders: + """Representative rows of each order (measured 2026-10-31 and 2026-11-05).""" + + @pytest.mark.parametrize("fields, expected", [r[1:] for r in TWO_ORDERS], + ids=[r[0] for r in TWO_ORDERS]) + def test_rows(self, dynamodb_client, table, fields, expected): + message = _validation_message(*_scan(dynamodb_client, table, fields)) + if isinstance(expected, tuple): + # The text predates this branch (#722, #838, #839): the check's + # place is asserted. + assert message.startswith(expected[1]), message + else: + assert message == expected + + def test_an_unreadable_expression_value_on_a_missing_table(self, dynamodb_client): + # Measured 2026-11-05: ahead of ResourceNotFoundException. + fields = '"FilterExpression": "a = :a", "ExpressionAttributeValues": {":a": {}}' + status, payload = _scan(dynamodb_client, unique_name("nosuch"), fields) + assert payload.get("message") == EAV_EMPTY_A, payload + + def test_bob_in_the_key_of_an_n_table(self, dynamodb_client, n_table): + # Measured 2026-11-05. + fields = '"ExclusiveStartKey": {"pk": {"N": "bob"}}' + assert _validation_message(*_scan(dynamodb_client, n_table, fields)) == NUMERIC_BOB + + class TestNestingDepth: """``ExpressionAttributeValues`` nested in lists.""" diff --git a/tests/test_scan_number_values.py b/tests/test_scan_number_values.py index 89ab066aa..a4fd3d3b7 100644 --- a/tests/test_scan_number_values.py +++ b/tests/test_scan_number_values.py @@ -258,14 +258,13 @@ def test_parameter_mix(self, dynamodb_client, table, extra, non_expression, expr f"Non-expression parameters: {{{non_expression}}} Expression parameters: {{{expression}}}", ) - def test_parameter_mix_beats_return_consumed_capacity(self, dynamodb_client_no_validation, table): + def test_return_consumed_capacity_beats_parameter_mix(self, dynamodb_client_no_validation, table): + # Measured 2026-11-05: the ReturnConsumedCapacity enum comes ahead of + # the parameter mix. assert _scan_error( dynamodb_client_no_validation, TableName=table, ScanFilter=SCAN_FILTER, ReturnConsumedCapacity="NOPE", **OK_FILTER, - ) == ("ValidationException", ( - "Can not use both expression and non-expression parameters in the same request: " - "Non-expression parameters: {ScanFilter} Expression parameters: {FilterExpression}" - )) + ) == ("ValidationException", RCC_ENUM) def test_limit_range_beats_parameter_mix(self, dynamodb_client_no_validation, table): assert _scan_error( From 71c75438d8697ed4f732fdbeef139c337605e88f Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Thu, 5 Nov 2026 05:50:53 +0000 Subject: [PATCH 6/9] fix(scan): check ScanFilter list values and order the value maps as the service does Measured against DynamoDB (us-east-1) on 2026-11-05 at 05:31, 06:20 and 07:38 UTC, each request twice. - In order U a ScanFilter list value that is not one AttributeValue answers Supplied AttributeValue is empty, must contain exactly one of the supported datatypes. The text has no prefix. {} and an unknown descriptor answer alike. Two descriptors answer the more-than-one text. The step comes after the no-expression checks. It comes ahead of ConditionalOperator, the lists' value counts, the key's faults and the table. The conditions are visited in the order of the key's members and each list in its order. The handler holds such a value back from the typed deserialization, which reported it with the deserializer's text. - In order U the ExpressionAttributeValues entries are visited in the order the key's members are visited. A malformed number, a set fault and a value that is not one AttributeValue are reported at the map's step, after the key's faults. Each takes the form ExpressionAttributeValues contains invalid value: for key :a. The number text is The parameter cannot be converted to a numeric value: bob. It applies at the top of the entry and inside its L or NS. The set texts are those of order U. Inside one entry the first fault in the value's element order is named. A number ahead of an empty value answers the number. - In order R the request-time number check visits the entries in the same order as the key's members. Two malformed numbers answer the one in the lower bucket, not the lower name. The set faults are checked at request time too, after the numbers and ahead of the nesting depth, in sorted name order on every backend. The service answered each of two set faults in one of its two sends. - The visiting order of the key's members and of the map's entries is the iteration order of a Java HashMap. The map starts with four buckets and doubles when it holds more than three quarters of its size. Up to three names get four buckets, four to six names get eight, seven to twelve get sixteen. A key of four members on a GSI with a range key names gpk and pk ahead of gsk, which four buckets would reverse. Keys of five and seven members and maps of four and seven entries follow the same rule. Names outside ASCII hash on their UTF-16 code units, a supplementary-plane name on its two. Two names in one bucket keep the request's order. The handler reads the request's order of the key members, the map's entries and the ScanFilter conditions. It reads them from the request text. - An empty BS in the key carries the key prefix in order U, as the empty SS does. It does so beside an unreadable key member or expression value, inside an L, and in a member that is not a key attribute. The empty NS does not. - In order R the request-time number and nesting checks read the entries that deserialize one by one. A set fault in one entry no longer skips the numbers of the others. A malformed number inside an entry the deserializer refuses is reported after the nesting depth of another entry. Left to PR #507 (#564 item 3): a malformed number in a ScanFilter list. The service answers The parameter cannot be converted to a numeric value: bob, ahead of a number in the key (2026-11-05). This commit does not check the lists' numbers. The ComparisonOperator enum clause the service answers in order U is a legacy-condition check too and is not made here. Not measured, and kept in the code's order: an empty value ahead of a malformed number inside one ExpressionAttributeValues entry. The code names the empty value, the element order. A set fault in a ScanFilter list value in order U. A set fault beside a malformed number in another entry of ExpressionAttributeValues in order R, where the service's two sends disagreed. Nine or more names in one hash bucket, where Java's HashMap doubles a table under 64 buckets early and the code does not. Tests: request_tests.rs covers the ScanFilter step of order U and the map's order and invalid-value form in order U. It covers the two orders of the request-time number and set checks and the element order inside one entry. It covers the table's doubling and the request order read from the text. The wire file sends 69 of the 73 rows of 2026-11-05, two of them with the nested value written out. It has 230 tests. The two ScanFilter number rows are marked xfail for #507. Three rows accept either of two service answers. Thirty-nine of the others fail on the previous commit. --- crates/engine/src/query_key_values.rs | 2 +- crates/engine/src/scan.rs | 70 ++- crates/engine/src/scan_request.rs | 585 ++++++++++++++---- crates/engine/src/scan_request/range_tests.rs | 2 +- .../engine/src/scan_request/request_tests.rs | 441 ++++++++++++- crates/engine/src/scan_request/tests.rs | 4 +- tests/python/test_scan_request_validation.py | 309 ++++++++- 7 files changed, 1211 insertions(+), 202 deletions(-) diff --git a/crates/engine/src/query_key_values.rs b/crates/engine/src/query_key_values.rs index 6a10523ed..e3c324b2b 100644 --- a/crates/engine/src/query_key_values.rs +++ b/crates/engine/src/query_key_values.rs @@ -322,7 +322,7 @@ pub fn validate_expression_attribute_value_numbers( /// Every number inside one attribute value (`N`, `NS` members, and the values /// nested in `L` and `M`), with Unicode decimal digits folded for the check. -fn validate_value_numbers(value: &AttributeValue) -> Result<(), DynamoDbError> { +pub(crate) fn validate_value_numbers(value: &AttributeValue) -> Result<(), DynamoDbError> { match value { AttributeValue::N(n) => validate_number_text(n), AttributeValue::NS(set) => set.iter().try_for_each(|n| validate_number_text(n)), diff --git a/crates/engine/src/scan.rs b/crates/engine/src/scan.rs index 9642476c1..ecd0bd549 100755 --- a/crates/engine/src/scan.rs +++ b/crates/engine/src/scan.rs @@ -25,7 +25,9 @@ use crate::index_helpers::{ }; use crate::legacy_filter::desugar_filter; use crate::read_helpers::{PostRead, read_page}; -use crate::scan_request::{CheckOrder, ExpressionValuesBody, SelectBody, StartKeyBody}; +use crate::scan_request::{ + CheckOrder, ExpressionValuesBody, ScanFilterBody, SelectBody, StartKeyBody, member_order, +}; use crate::serialize_output; use crate::{DispatchMetrics, DispatchResult}; @@ -33,8 +35,10 @@ use crate::{DispatchMetrics, DispatchResult}; /// /// Reads all items from the table (or segment), applies `FilterExpression` /// post-read, applies `ProjectionExpression`, and enforces the 1 MB limit. -/// `raw_body` is the request text `body` was parsed from; the checks on the -/// `ExclusiveStartKey` read its member order from it (see `scan_request`). +/// `raw_body` is the request text `body` was parsed from; the checks read the +/// request's order of the `ExclusiveStartKey` members, the +/// `ExpressionAttributeValues` entries and the `ScanFilter` conditions from it +/// (see `scan_request::member_order`). /// /// # Errors /// @@ -51,10 +55,14 @@ pub async fn handle_scan( // - request-time, on the raw body: the AttributesToGet length and the // Limit, Segment and TotalSegments ranges in one message, the // ReturnConsumedCapacity enum, the expression and non-expression - // parameter mix, every number in ExpressionAttributeValues, the nesting - // depth of ExpressionAttributeValues; + // parameter mix, every number in the ExpressionAttributeValues entries + // that deserialize (the entries in the hash order of the key), the + // first empty or duplicate set in ExpressionAttributeValues (the + // entries in sorted name order), the nesting depth of the entries that + // deserialize (the numbers of the ScanFilter lists are #507's check); // - a field of the wrong JSON type anywhere in the body; - // - an empty or duplicate set in ExpressionAttributeValues; + // - a malformed number inside an ExpressionAttributeValues entry the + // deserializer refuses; // - the request parameter checks: ExpressionAttributeNames and // ExpressionAttributeValues with no expression, ConditionalOperator with // no ScanFilter, a ScanFilter condition's value count; @@ -67,19 +75,28 @@ pub async fn handle_scan( // - the table lookup, the index resolution and the index rules, the key // schema. // Order U differs in these places: the constraint clauses take the older - // form and the Select enum joins them at request time; the nesting check - // leaves request time; after the request parameter checks come the + // form and the Select enum joins them at request time; the numbers of + // ExpressionAttributeValues and the nesting check leave request time; the + // ScanFilter list values (a value that is not one AttributeValue) are + // checked after the no-expression checks and ahead of + // ConditionalOperator; after the request parameter checks come the // ExclusiveStartKey's faults, shape and value together, then the - // ExpressionAttributeValues entries the deserializer refuses, then the - // nesting depth, then the expression and request checks and the table. + // faulty ExpressionAttributeValues entries in the same member order (an + // unreadable entry, a malformed number, a set fault, each in the + // invalid-value form), then the nesting depth, then the expression and + // request checks and the table. let order = CheckOrder::of(&body); - crate::scan_request::validate_scan_request_body(&body, order)?; - // A key, an ExpressionAttributeValues entry or a Select value that does - // not deserialize is held back here, so that the typed deserialization - // does not report it ahead of the checks the service makes first. - let key_order = crate::scan_request::start_key_member_order(raw_body); + let values_order = member_order(raw_body, "ExpressionAttributeValues"); + crate::scan_request::validate_scan_request_body(&body, order, values_order.as_deref())?; + // A key, an ExpressionAttributeValues entry, a ScanFilter list value (order + // U) or a Select value (order R) that the service reports later is held + // back here, so that the typed deserialization does not report it ahead of + // the checks the service makes first. + let key_order = member_order(raw_body, "ExclusiveStartKey"); let start_key = StartKeyBody::take(&mut body, key_order.as_deref()); - let expression_values = ExpressionValuesBody::take(&mut body, order); + let expression_values = ExpressionValuesBody::take(&mut body, order, values_order.as_deref()); + let filter_order = member_order(raw_body, "ScanFilter"); + let scan_filter_values = ScanFilterBody::take(&mut body, order, filter_order.as_deref()); let select = SelectBody::take(&mut body, order); let input: ScanInput = serde_json::from_value(body).map_err(crate::deserialize_error)?; @@ -90,8 +107,9 @@ pub async fn handle_scan( input.expression_attribute_values.as_ref(), ); - // Order R: an empty or duplicate set in ExpressionAttributeValues, ahead - // of the no-expression check (measured 2026-11-05). + // Order R: a malformed number inside an ExpressionAttributeValues entry + // the deserializer refuses, ahead of the no-expression check; the set + // faults are request-time checks (measured 2026-11-05). if order == CheckOrder::Readable { expression_values.validate()?; } @@ -112,6 +130,12 @@ pub async fn handle_scan( &[ExpressionKind::Filter], )?; + // Order U: a ScanFilter list value that is not one AttributeValue, after + // the no-expression checks and ahead of ConditionalOperator, the value + // count and the key's faults (measured 2026-11-05; see + // `scan_request::ScanFilterBody`). + scan_filter_values.validate()?; + // ConditionalOperator requires a ScanFilter with two or more conditions. extenddb_core::validation::validate_conditional_operator_usage( has_cond_op, @@ -132,10 +156,12 @@ pub async fn handle_scan( match order { CheckOrder::Unreadable => { - // The key's faults, shape and value together, then the - // ExpressionAttributeValues entries the deserializer refuses, then - // the nesting depth (measured 2026-10-31 and 2026-11-05; see - // `scan_request::StartKeyBody` and `ExpressionValuesBody`). + // The key's faults, shape and value together, then the faulty + // ExpressionAttributeValues entries in the fitted order (an + // unreadable entry, a malformed number in the invalid-value form, + // a set fault), then the nesting depth (measured 2026-10-31 and + // 2026-11-05; see `scan_request::StartKeyBody` and + // `ExpressionValuesBody`). start_key.validate(order)?; expression_values.validate()?; if let Some(ref values) = input.expression_attribute_values { diff --git a/crates/engine/src/scan_request.rs b/crates/engine/src/scan_request.rs index daf7e1fac..b938736eb 100644 --- a/crates/engine/src/scan_request.rs +++ b/crates/engine/src/scan_request.rs @@ -27,16 +27,33 @@ //! object counts as present whether or not it is empty; //! 3. every number in `ExpressionAttributeValues`, `NS` members and values //! nested in `L` and `M` included, whether an expression reads the value or -//! not; a Unicode decimal digit counts as the digit it stands for; -//! 4. the nesting depth of `ExpressionAttributeValues`: a value nested 32 +//! not; a Unicode decimal digit counts as the digit it stands for. The +//! entries are visited in the hash order of [`fitted_member_order`] +//! (2026-11-05: `{":n": {"N": "fred"}, ":a": {"N": "bob"}}` answers `fred`, +//! which sorted order would not). An entry the deserializer refuses (a set +//! fault anywhere in it) is skipped here: its set fault is reported at step +//! 4, and a malformed number inside it after the typed deserialization +//! (2026-11-05: `{"NS": ["bob", "1", "1"]}` beside a value nested 32 levels +//! answers the nesting text; `{"L": [{"N": "bob"}, {"NS": []}]}` beside it +//! answered the number in one send and the nesting text in the other). The +//! numbers of the `ScanFilter` lists are #507's check (#564 item 3, measured +//! 2026-11-05: the number text ahead of a number in the key) and are not +//! checked on this branch; +//! 4. an empty or duplicate set in `ExpressionAttributeValues`, the entries in +//! sorted name order whatever order the backend's parsed body keeps +//! (2026-11-05: `{":e": {"NS": []}}` with no expression answers the set text +//! ahead of the no-expression check; beside a value nested 32 levels in +//! another entry the empty `NS` text comes ahead of the nesting text in both +//! sends and the `SS` duplicates text did in one of two; `{":n": {"NS": []}, +//! ":a": {"SS": []}}` answered the `SS` text in one send and the `NS` text +//! in the other, so the sorted order matches one of the two), see +//! [`expression_values_set_fault`] and [`ExpressionValuesBody`]; +//! 5. the nesting depth of `ExpressionAttributeValues`: a value nested 32 //! levels deep answers `Nesting Levels have exceeded supported limits: //! Attributes in the item have nested levels beyond supported limit`, 31 //! levels pass, read or unused, with or without an expression (2026-10-30 //! and 2026-10-31: ahead of the no-expression check, B4-5a; 2026-11-05: //! ahead of the filter syntax, the `Select` enum and the key's number); -//! 5. an empty or duplicate set in `ExpressionAttributeValues` (2026-11-05: -//! `{":e": {"NS": []}}` with no expression answers the set text ahead of -//! the no-expression check), see [`ExpressionValuesBody`]; //! 6. the request parameter checks: `ExpressionAttributeNames` with no //! expression, then `ExpressionAttributeValues` with no expression, then //! `ConditionalOperator` with no `ScanFilter`, then a `ScanFilter` @@ -48,8 +65,8 @@ //! three `Select` rules, the unused values, `COUNT` with `AttributesToGet`, //! the `Segment` pairing (2026-11-05, where two were sent together: the //! filter syntax first; an undefined value and an operand type ahead of the -//! unused values, the `Segment` pairing and `SPECIFIC_ATTRIBUTES`; `COUNT` -//! with `ProjectionExpression` ahead of the unused values; the unused +//! unused values, the `Segment` pairing and `SPECIFIC_ATTRIBUTES`; the +//! unused values ahead of `COUNT` with `ProjectionExpression`; the unused //! values and `COUNT` with `AttributesToGet` ahead of the `Segment` //! pairing); //! 9. the `ExclusiveStartKey` value faults: a number that does not parse, an @@ -63,43 +80,71 @@ //! camel case, `totalSegments`, `segment`, `limit`), then the //! `ReturnConsumedCapacity` enum, then the `Select` enum (2026-11-05: ahead //! of the key's shape fault and of an unreadable expression value); -//! 2. the parameter mix, then the request parameter checks of order R step 6, -//! with the mix first (2026-11-05: each ahead of the key's faults); +//! 2. the parameter mix, then `ExpressionAttributeNames` and +//! `ExpressionAttributeValues` with no expression, then the `ScanFilter` +//! list values (see [`ScanFilterBody`]: a value that is not one +//! AttributeValue answers `Supplied AttributeValue is empty, must contain +//! exactly one of the supported datatypes` with no prefix, for `{}` and for +//! an unknown descriptor alike, 2026-11-05), then `ConditionalOperator` +//! with no `ScanFilter` and a `ScanFilter` condition's value count +//! (2026-11-05: each ahead of the key's faults). The numbers of +//! `ExpressionAttributeValues` are not checked at request time in this +//! order; //! 3. the `ExclusiveStartKey` faults, shape and value together, the first //! faulty member in the member order of [`StartKeyBody`]: a shape fault //! with the prefix `The provided starting key is invalid: `, a number with //! the value check's text, a set fault with the texts of order U (the -//! members unquoted, the `NS` duplicate text bare, the empty `SS` text with -//! the key prefix); -//! 4. the `ExpressionAttributeValues` entries the deserializer refuses, read by -//! an expression or not: `ExpressionAttributeValues contains invalid value: -//! Supplied AttributeValue is empty, must contain exactly one of the -//! supported datatypes for key :e` for an empty value and for an unknown +//! members unquoted, the `NS` duplicate text bare, the empty `SS` and `BS` +//! texts with the key prefix); +//! 4. the `ExpressionAttributeValues` entries, in the same member order +//! ([`fitted_member_order`]), the first faulty one named, every fault in +//! the form `ExpressionAttributeValues contains invalid value: for +//! key :e`: an entry that is not one AttributeValue, read by an expression +//! or not, `Supplied AttributeValue is empty, must contain exactly one of +//! the supported datatypes` for an empty value and for an unknown //! descriptor, the more-than-one text for two descriptors (2026-10-31 B4-7 //! and 2026-11-05: ahead of the nesting check and after the key's faults); +//! a malformed number, `The parameter cannot be converted to a numeric +//! value: bob`, at the top of the entry or inside its `L` or `NS`, and the +//! 38-digit text (2026-11-05: after the key's faults); a set fault with the +//! texts of order U, `One or more parameter values were invalid: An number +//! set may not be empty` and `... Input collection [a, a] contains +//! duplicates.` (2026-11-05). Inside one entry the first fault in the +//! value's own element order is named, a number or a shape fault +//! (2026-11-05: `{"L": [{"N": "bob"}, {}]}` answers the number); //! 5. the nesting depth, then the expression and request checks of order R //! step 8, then the table. //! //! Pairs never sent together keep the code's order: in order R the nesting //! check against `ExpressionAttributeNames` with no expression, -//! `ConditionalOperator` and the `ScanFilter` value count, a set fault in -//! `ExpressionAttributeValues` against the nesting check and against the -//! `ExpressionAttributeNames` check, the `ProjectionExpression` syntax against -//! an undefined value, an operand type and the unused values; in order U the -//! `Select` enum against the mix, a malformed `ExpressionAttributeValues` -//! number against the key's faults, a set fault in `ExpressionAttributeValues` -//! against the key's faults (reported with the unreadable entries, step 4), an -//! empty `BS` or a `BS` with duplicates in the key (no key prefix, the members -//! unquoted), an empty `SS` in the key beside an unreadable expression value -//! (the key prefix, as beside an unreadable key member). Which entry is named -//! when several `ExpressionAttributeValues` entries are faulty is not measured: -//! the first in sorted order. +//! `ConditionalOperator` and the `ScanFilter` value count; a set fault in +//! `ExpressionAttributeValues` against a malformed number in another entry +//! (the number first, at request time; the service's two sends disagreed, +//! 2026-11-05); which of two set faults in `ExpressionAttributeValues` is +//! named (sorted name order; the service's two sends disagreed, 2026-11-05); +//! the `ProjectionExpression` syntax against an undefined value, an operand +//! type and the unused values. In order U the `Select` enum against the mix +//! (the enum first; the mix beside `Select: "BAD"` and an empty entry answers +//! the `Select` clause, 2026-11-05); an empty value ahead of a malformed +//! number inside one `ExpressionAttributeValues` entry (the empty value, the +//! element order; the other way round is measured); a set fault in a +//! `ScanFilter` list value in order U (at the list's step, the deserializer's +//! text in order R, 2026-11-05); a `BS` with duplicates in the key (no key +//! prefix, the members unquoted); nine or more names in one hash bucket (see +//! [`fitted_member_order`]). The service refuses a member or entry name written +//! twice as a `SerializationException` (`Duplicate key found in payload`, +//! 2026-11-05, #630); the parser here keeps the last value. //! //! A field of the wrong JSON type anywhere in the body is reported ahead of //! all of them, as a `SerializationException` (`"Limit": "5"` with `Select: //! "BAD"`, 2026-11-05; the service's texts differ from the deserializer's). The //! service reports a key shape fault ahead of `"ConsistentRead": "yes"` -//! (2026-11-05); the code reads the whole body first and does not. +//! (2026-11-05); the code reads the whole body first and does not. In order U +//! the service reports an invalid `ScanFilter` `ComparisonOperator` as a +//! constraint clause (`Value 'BAD' at 'scanFilter.a.member.comparisonOperator' +//! failed to satisfy constraint: Member must satisfy enum value set: [...]`, +//! 2026-11-05) ahead of the list's values; the legacy condition checks are +//! #507's and the clause is not made here. //! //! The JSON parser accepts 128 levels of nesting, which is 62 `L` levels; a //! body nested deeper is a `SerializationException` from the parser, ahead of @@ -118,7 +163,7 @@ use serde_json::Value; use crate::query_key_values::{ SCAN_CONDITIONS, TYPE_DESCRIPTORS, has_unreadable_attribute_value, is_shape_fault, non_empty, - present, shape_fault_first, validate_expression_attribute_value_numbers, validate_number_text, + present, shape_fault_first, validate_number_text, validate_value_numbers, }; /// Which of the service's two check orders a `Scan` request follows (see the @@ -171,13 +216,21 @@ impl CheckOrder { pub(crate) fn validate_scan_request_body( body: &Value, order: CheckOrder, + values_order: Option<&[String]>, ) -> Result<(), DynamoDbError> { - scan_request_checks(body, order).map_err(|check| shape_fault_first::(body, check)) + scan_request_checks(body, order, values_order) + .map_err(|check| shape_fault_first::(body, check)) } /// The `Scan` checks in the service's order; see -/// [`validate_scan_request_body`]. -fn scan_request_checks(body: &Value, order: CheckOrder) -> Result<(), DynamoDbError> { +/// [`validate_scan_request_body`]. `values_order` is the +/// `ExpressionAttributeValues` entry names as the request wrote them +/// ([`member_order`]), for the hash order of the number check. +fn scan_request_checks( + body: &Value, + order: CheckOrder, + values_order: Option<&[String]>, +) -> Result<(), DynamoDbError> { validate_constraints(body, order)?; crate::validate_enum_fields( @@ -210,10 +263,72 @@ fn scan_request_checks(body: &Value, order: CheckOrder) -> Result<(), DynamoDbEr ], )?; - if let Some(values) = attribute_value_map(body, "ExpressionAttributeValues") { - validate_expression_attribute_value_numbers(&values)?; - if order == CheckOrder::Readable { - extenddb_core::validation::validate_attribute_values_nesting_depth(values.values())?; + if order == CheckOrder::Readable { + // Every number in ExpressionAttributeValues, the entries in the hash + // order the key and the order-U step use (an entry the deserializer + // refuses, one with a set fault, is skipped here), then the set faults + // in sorted name order, then the nesting depth of the entries that + // deserialize. In order U the numbers and the set faults are reported + // at the map's own step (see the module doc). The numbers of the + // ScanFilter lists are #507's check (#564 item 3). + let values = readable_attribute_values(body, "ExpressionAttributeValues"); + for name in fitted_order_of(body, "ExpressionAttributeValues", values_order) { + if let Some(value) = values.get(&name) { + validate_value_numbers(value)?; + } + } + expression_values_set_fault(body)?; + extenddb_core::validation::validate_attribute_values_nesting_depth(values.values())?; + } + Ok(()) +} + +/// The entries of the object `field` of the body that deserialize as one +/// attribute value each; the others are left to [`ExpressionValuesBody`]. +fn readable_attribute_values(body: &Value, field: &str) -> HashMap { + body.get(field) + .and_then(Value::as_object) + .map(|entries| { + entries + .iter() + .filter_map(|(name, value)| { + serde_json::from_value::(value.clone()) + .ok() + .map(|value| (name.clone(), value)) + }) + .collect() + }) + .unwrap_or_default() +} + +/// The first empty or duplicate set among the `ExpressionAttributeValues` +/// entries, in sorted name order whatever order the backend's parsed body +/// keeps, with the texts of order R (see [`ValueFault::text`]). Measured +/// 2026-11-05: `{":d": <32 levels>, ":e": {"NS": []}}` answers the empty-set +/// text ahead of the nesting text in both sends, and `{":d": <32 levels>, ":s": +/// {"SS": ["a", "a"]}}` the duplicates text in one of two; beside a malformed +/// number in another entry the service's two sends disagreed, and the number +/// comes first here. With two set faults, `{":n": {"NS": []}, ":a": {"SS": +/// []}}`, the service answered the `SS` text in one send and the `NS` text in +/// the other; the sorted order names `:a`. An entry whose first fault, depth +/// first, is a malformed number is left to [`ExpressionValuesBody`]. +/// +/// # Errors +/// +/// Returns `DynamoDbError::ValidationException` with the set text. +fn expression_values_set_fault(body: &Value) -> Result<(), DynamoDbError> { + let Some(entries) = body + .get("ExpressionAttributeValues") + .and_then(Value::as_object) + else { + return Ok(()); + }; + let mut names: Vec<&String> = entries.keys().collect(); + names.sort_unstable(); + for name in names { + match value_fault(&entries[name.as_str()]) { + None | Some(ValueFault::Number(_)) => {} + Some(fault) => return Err(fault.text(CheckOrder::Readable, Place::ExpressionValue)), } } Ok(()) @@ -242,12 +357,6 @@ fn validate_select_enum(body: &Value) -> Result<(), DynamoDbError> { ) } -/// `field` of the body read as a map of attribute values, or `None` when the -/// field is absent or is not one; [`shape_fault_first`] reports the shape. -fn attribute_value_map(body: &Value, field: &str) -> Option> { - serde_json::from_value(body.get(field)?.clone()).ok() -} - /// The service's prefix on a shape fault in an `ExclusiveStartKey` value /// (measured 2026-10-31 and 2026-11-05: `{"pk": {}, "sk": {"S": "x"}}` answers /// `The provided starting key is invalid: Supplied AttributeValue is empty, @@ -269,12 +378,14 @@ const MORE_THAN_ONE_DESCRIPTOR: &str = "Supplied AttributeValue has more than on /// The prefix of the service's set texts. const INVALID_PARAMETER_VALUES: &str = "One or more parameter values were invalid: "; -/// The member names of the body's `ExclusiveStartKey` object in the order the -/// request wrote them, read from the raw request text, since the parsed body -/// sorts object members on every build but the MongoDB one. `None` when the -/// text does not parse, has no key, or the key is not an object; a name that -/// appears twice keeps its first position. -pub(crate) fn start_key_member_order(raw_body: &[u8]) -> Option> { +/// The member names of the body's object `field` (`ExclusiveStartKey`, +/// `ExpressionAttributeValues` or `ScanFilter`) in the order the request wrote +/// them, read from the raw request text, since the parsed body sorts object +/// members on every build but the MongoDB one. `None` when the text does not +/// parse, has no such field, or the field is not an object; a name that +/// appears twice keeps its first position; a field that appears twice is read +/// at its first position. +pub(crate) fn member_order(raw_body: &[u8], field: &str) -> Option> { /// The member names of one JSON object, in document order. struct MemberNames(Vec); @@ -308,41 +419,79 @@ pub(crate) fn start_key_member_order(raw_body: &[u8]) -> Option> { } } - /// The body with only the key's member names read. - #[derive(Deserialize)] - struct KeyNames { - #[serde(rename = "ExclusiveStartKey", default)] - key: Option, + /// The top-level object, with only the member names of `field` read. + struct FieldNames<'f>(&'f str); + + impl<'de> serde::de::Visitor<'de> for FieldNames<'_> { + type Value = Option>; + + fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("an object") + } + + fn visit_map>( + self, + mut map: A, + ) -> Result { + let mut found: Option> = None; + while let Some(name) = map.next_key::()? { + if name == self.0 && found.is_none() { + found = Some(map.next_value::()?.0); + } else { + map.next_value::()?; + } + } + Ok(found) + } } - serde_json::from_slice::(raw_body) - .ok()? - .key - .map(|names| names.0) + let mut deserializer = serde_json::Deserializer::from_slice(raw_body); + serde::Deserializer::deserialize_map(&mut deserializer, FieldNames(field)) + .ok() + .flatten() } /// The order in which the service visits the members of an `ExclusiveStartKey` -/// with faults in more than one member, fitted to the measured rows: the order -/// of a Java `HashMap` of capacity 4 over the member names, that is -/// `String.hashCode()` of the name, spread as `h ^ (h >>> 16)`, masked to four -/// buckets, the buckets in ascending order and the request's order inside a -/// bucket. `sk` falls in bucket 0, `a` in 1, `b` in 2, `pk`, `gpk`, `id` and -/// `ts` in 3. +/// with faults in more than one member, and the entries of +/// `ExpressionAttributeValues` in order U, fitted to the measured rows: the +/// iteration order of a Java `HashMap` over the names, that is +/// `String.hashCode()` of the name, spread as `h ^ (h >>> 16)`, masked to the +/// table's buckets, the buckets in ascending order and the request's order +/// inside a bucket. The table has four buckets for up to three names and +/// doubles when it holds more than three quarters of its size +/// ([`hash_table_mask`]): eight buckets for four to six names. With four +/// buckets `sk` falls in bucket 0, `a` in 1, `b` in 2, `pk`, `gpk`, `id` and +/// `ts` in 3; with eight, `sk` in 0, `gpk` and `pk` in 3, `gsk` in 6. /// -/// Measured (2026-10-31, 2026-11-04 and 2026-11-05, five tables, every row +/// Measured (2026-10-31, 2026-11-04 and 2026-11-05, six tables, every row /// with a shape fault in one of two faulty members, both request orders): on /// the `pk`, `sk` table the service names `sk`; on HASH `a`, RANGE `b` it /// names `a`; on HASH `pk`, RANGE `a` it names `a`; on `ts`, `id` and on `id`, -/// `ts` it names the member first in the request; on the GSI `{"gpk": {}, -/// "pk": {"N": "bob"}, "sk": {"S": "x"}}` it names `gpk`. The rule is fitted -/// to those rows, not documented by the service. With two value faults and no -/// shape fault the service's own two sends disagreed on 8 of 22 rows, so no -/// order is pinned there. Keys of four members (an index scan with a range key -/// on the index and on the table) are not measured; the same rule applies. +/// `ts` it names the member first in the request; on HASH `ä`, RANGE `é` it +/// names `ä`; on the GSI `{"gpk": {}, "pk": {"N": "bob"}, "sk": {"S": "x"}}` +/// it names `gpk`; on a GSI with a range key, where the key has four members, +/// it names `gpk` ahead of `gsk` and `pk` ahead of `gsk`, which four buckets +/// would reverse. In `ExpressionAttributeValues` (order U, 2026-11-05) it +/// names `:n` ahead of `:e`, `:b` ahead of `:p` and `:a`, and `:a` ahead of +/// `:e` when `:a` comes first in the request. The rule is fitted to those +/// rows. The service does not document it. With two value faults and no shape +/// fault the service's own two sends disagreed on 19 of 40 matrix cells, so no +/// order is pinned there. Measured 2026-11-05 as well: a key of five members +/// and one of seven on the same GSI (eight and sixteen buckets), four and +/// seven `ExpressionAttributeValues` entries, two `ScanFilter` conditions, a +/// key name outside the Basic Multilingual Plane (`𝄞`, hashed as its two +/// UTF-16 code units) and the request's order inside one bucket for the +/// entries (`:e` ahead of `:a` when `:e` comes first); each as the rule gives. +/// Java's `HashMap` also doubles a table under 64 buckets when one bucket +/// reaches nine names; the model sizes the table by the count of names alone +/// and departs from the JDK there (not measured). fn fitted_member_order<'a>(names_in_request_order: impl Iterator) -> Vec { - let mut names: Vec<(u32, usize, &str)> = names_in_request_order + let names: Vec<&str> = names_in_request_order.collect(); + let mask = hash_table_mask(names.len()); + let mut names: Vec<(u32, usize, &str)> = names + .into_iter() .enumerate() - .map(|(position, name)| (java_hash_bucket(name), position, name)) + .map(|(position, name)| (java_hash_bucket(name, mask), position, name)) .collect(); names.sort_unstable(); names @@ -351,14 +500,45 @@ fn fitted_member_order<'a>(names_in_request_order: impl Iterator .collect() } -/// The bucket of `name` in a Java `HashMap` of capacity 4: +/// The bucket mask of a Java `HashMap` that starts with four buckets and +/// doubles whenever it holds more than three quarters of its size: 3 for up to +/// three names, 7 for four to six, 15 for seven to twelve, and so on. +fn hash_table_mask(names: usize) -> u32 { + let mut buckets: usize = 4; + while names > buckets * 3 / 4 { + buckets *= 2; + } + u32::try_from(buckets - 1).unwrap_or(u32::MAX) +} + +/// The bucket of `name` in a Java `HashMap` whose bucket mask is `mask`: /// `String.hashCode()` over the UTF-16 code units, spread as `h ^ (h >>> 16)`, -/// masked with 3. The arithmetic wraps as Java's `int` does. -fn java_hash_bucket(name: &str) -> u32 { +/// masked. The arithmetic wraps as Java's `int` does. +fn java_hash_bucket(name: &str, mask: u32) -> u32 { let hash = name.encode_utf16().fold(0u32, |h, unit| { h.wrapping_mul(31).wrapping_add(u32::from(unit)) }); - (hash ^ (hash >> 16)) & 3 + (hash ^ (hash >> 16)) & mask +} + +/// The names of the object `field` of the body in the fitted order +/// ([`fitted_member_order`]): the request's order when `request_order` names +/// exactly the object's members, else the parsed body's order. +fn fitted_order_of(body: &Value, field: &str, request_order: Option<&[String]>) -> Vec { + let Some(members) = body.get(field).and_then(Value::as_object) else { + return Vec::new(); + }; + let parsed: Vec<&str> = members.keys().map(String::as_str).collect(); + let in_request: Vec<&str> = match request_order { + Some(names) + if names.len() == parsed.len() + && names.iter().all(|name| members.contains_key(name)) => + { + names.iter().map(String::as_str).collect() + } + _ => parsed, + }; + fitted_member_order(in_request.into_iter()) } /// The `ExclusiveStartKey` of a `Scan` body as the request wrote it, with the @@ -398,10 +578,11 @@ pub(crate) struct StartKeyBody { impl StartKeyBody { /// Reads the key from `body` and takes it out of the body when it does not /// deserialize for a reason that is not a shape fault. `request_order` is - /// the key's member names as the request wrote them - /// ([`start_key_member_order`]); when it is `None` or does not name the - /// key's members, the parsed body's order stands in for it. + /// the key's member names as the request wrote them ([`member_order`]); + /// when it is `None` or does not name the key's members, the parsed body's + /// order stands in for it. pub(crate) fn take(body: &mut Value, request_order: Option<&[String]>) -> Self { + let members = fitted_order_of(body, "ExclusiveStartKey", request_order); let raw = body.get("ExclusiveStartKey").cloned(); let deferred = match raw .as_ref() @@ -425,22 +606,6 @@ impl StartKeyBody { { fields.remove("ExclusiveStartKey"); } - let members = match raw.as_ref().and_then(Value::as_object) { - Some(key) => { - let parsed: Vec<&str> = key.keys().map(String::as_str).collect(); - let in_request: Vec<&str> = match request_order { - Some(names) - if names.len() == parsed.len() - && names.iter().all(|name| key.contains_key(name)) => - { - names.iter().map(String::as_str).collect() - } - _ => parsed, - }; - fitted_member_order(in_request.into_iter()) - } - None => Vec::new(), - }; Self { raw, members, @@ -535,8 +700,9 @@ impl ValueFault { /// set may not be empty` (the service's two spaces), both orders; /// - an empty `SS`: `... An string set may not be empty`, with the key /// prefix for a key member in order U; - /// - an empty `BS`: `... Binary sets should not be empty` (order R; order - /// U not measured); + /// - an empty `BS`: `... Binary sets should not be empty`, with the key + /// prefix for a key member in order U (2026-11-05, `{"pk": {}, "sk": + /// {"BS": []}}`); an empty `NS` carries no prefix in either order; /// - an `NS` with duplicates: `... Input collection contains duplicates.` /// in order R, `Input collection contains duplicates` in order U; /// - an `SS` with duplicates: `... Input collection ["a", "a"] contains @@ -560,18 +726,16 @@ impl ValueFault { match self { Self::Number(fault) => fault.clone(), Self::NumberSetEmpty => invalid_parameter("An number set may not be empty"), - Self::StringSetEmpty => { - let text = invalid_parameter("An string set may not be empty"); - match (order, place, text) { - ( - CheckOrder::Unreadable, - Place::Key, - DynamoDbError::ValidationException(msg), - ) => DynamoDbError::ValidationException(format!("{START_KEY_PREFIX}{msg}")), - (_, _, text) => text, - } - } - Self::BinarySetEmpty => invalid_parameter("Binary sets should not be empty"), + Self::StringSetEmpty => key_prefixed_in_order_u( + invalid_parameter("An string set may not be empty"), + order, + place, + ), + Self::BinarySetEmpty => key_prefixed_in_order_u( + invalid_parameter("Binary sets should not be empty"), + order, + place, + ), Self::NumberSetDuplicates => match order { CheckOrder::Readable => invalid_parameter("Input collection contains duplicates."), CheckOrder::Unreadable => DynamoDbError::ValidationException( @@ -596,6 +760,18 @@ impl ValueFault { } } +/// `text` with the key prefix when it is a key member's fault in order U +/// (measured 2026-11-05 for an empty `SS` and an empty `BS` beside an +/// unreadable key member), else as it is. +fn key_prefixed_in_order_u(text: DynamoDbError, order: CheckOrder, place: Place) -> DynamoDbError { + match (order, place, text) { + (CheckOrder::Unreadable, Place::Key, DynamoDbError::ValidationException(msg)) => { + DynamoDbError::ValidationException(format!("{START_KEY_PREFIX}{msg}")) + } + (_, _, text) => text, + } +} + /// The first value fault in one raw AttributeValue, depth first: an `N` text, /// the members of an `NS` (every number first, then an empty set, then /// duplicates by numeric value: `["1", "1.0"]` has a duplicate), an empty `SS` @@ -619,6 +795,37 @@ fn value_fault(value: &Value) -> Option { } } +/// A shape fault ([`shape_fault`]) or a value fault ([`value_fault`]) in one +/// raw AttributeValue. +enum Fault { + Shape(&'static str), + Value(ValueFault), +} + +/// The first fault of either kind in one raw AttributeValue, depth first in +/// the value's own element order: at each value, a shape fault of the value +/// itself (empty, an unknown descriptor, two descriptors), else the members of +/// an `L` or `M` in turn, else the value's number or set fault. Measured +/// 2026-11-05 (order U, `ExpressionAttributeValues`): `{"L": [{"N": "bob"}, +/// {}]}` answers the number, which [`shape_fault`] alone would pass over. The +/// empty value ahead of the number is not measured. +fn first_fault(value: &Value) -> Option { + let fields = value.as_object()?; + let mut fields = fields.iter(); + match (fields.next(), fields.next()) { + (None, _) => Some(Fault::Shape(EMPTY_VALUE)), + (Some(_), Some(_)) => Some(Fault::Shape(MORE_THAN_ONE_DESCRIPTOR)), + (Some((descriptor, inner)), None) => match (descriptor.as_str(), inner) { + ("L", Value::Array(members)) => members.iter().find_map(first_fault), + ("M", Value::Object(members)) => members.values().find_map(first_fault), + (descriptor, _) if TYPE_DESCRIPTORS.contains(&descriptor) => { + value_fault(value).map(Fault::Value) + } + _ => Some(Fault::Shape(EMPTY_VALUE)), + }, + } +} + /// A `ValidationException` with the service's prefix for an invalid parameter /// value. fn invalid_parameter(text: &str) -> DynamoDbError { @@ -693,23 +900,36 @@ fn has_duplicates(texts: &[String]) -> bool { distinct.len() != texts.len() } -/// The `ExpressionAttributeValues` of a `Scan` body whose entries do not all -/// deserialize: in order U an entry that is not one AttributeValue (empty, an -/// unknown or lower-case type descriptor, two descriptors, at the top or inside -/// an `L` or `M`), in either order an empty or duplicate set. The service -/// reports such an entry at its own stage: in order R the set faults ahead of -/// the no-expression check (2026-11-05, `{":e": {"NS": []}}` with no -/// expression answers `One or more parameter values were invalid: An number -/// set may not be empty`; `{":a": {"SS": ["a", "a"]}}` read by the filter -/// answers `... Input collection ["a", "a"] contains duplicates.`); in order U -/// the unreadable entries after the key's faults and ahead of the nesting -/// check, read by an expression or not (2026-10-31 B4-7 and 2026-11-05): -/// `ExpressionAttributeValues contains invalid value: Supplied AttributeValue -/// is empty, must contain exactly one of the supported datatypes for key :e` -/// for `{}` and for `{"X": "1"}`, and `... has more than one datatypes set, -/// ... for key :a` for two descriptors. A set fault in order U is not measured -/// and is reported with the unreadable entries. Which entry is named when -/// several are faulty is not measured: the first in sorted order. +/// The `ExpressionAttributeValues` of a `Scan` body whose entries the handler +/// reports at the map's own step rather than at request time. In order R the +/// numbers, the set faults and the nesting depth are request-time checks +/// ([`scan_request_checks`]); an empty or duplicate set is reported there ahead +/// of the no-expression check (2026-11-05, `{":e": {"NS": []}}` with no +/// expression answers `One or more parameter values were invalid: An number set +/// may not be empty`; `{":a": {"SS": ["a", "a"]}}` read by the filter answers +/// `... Input collection ["a", "a"] contains duplicates.`), and here only an +/// entry the request-time checks did not reach is held back, with the same +/// texts: a malformed number inside an entry the deserializer refuses for a set +/// fault (2026-11-05: `{"NS": ["bob", "1", "1"]}` beside a value nested 32 +/// levels answers the nesting text, so the number is reported after it). In +/// order U, after the key's faults and ahead of the nesting check, read by an +/// expression or not (2026-10-31 B4-7 and 2026-11-05), every fault in the form +/// `ExpressionAttributeValues contains invalid value: for key :e`: an +/// entry that is not one AttributeValue (empty, an unknown or lower-case type +/// descriptor, two descriptors, at the top or inside an `L` or `M`), `Supplied +/// AttributeValue is empty, must contain exactly one of the supported datatypes` +/// for `{}` and for `{"X": "1"}`, `... has more than one datatypes set, ...` +/// for two descriptors; a malformed number, `The parameter cannot be converted +/// to a numeric value: bob`, at the top or inside an `L` or `NS`, and the +/// 38-digit text; a set fault with the set texts of order U (`One or more +/// parameter values were invalid: An number set may not be empty`, `... Input +/// collection [a, a] contains duplicates.`). Inside one entry the first fault +/// in the value's element order is named ([`first_fault`]). The entries are +/// visited in the fitted order ([`fitted_member_order`]), and the first faulty +/// one is named (2026-11-05: `:n` ahead of `:e`, `:b` ahead of `:p` and `:a`, +/// `:a` ahead of `:e` when `:a` comes first in the request and `:e` ahead of +/// `:a` when `:e` does, `:s` ahead of `:e`; with four and seven entries the +/// eight and sixteen buckets). /// /// [`Self::take`] puts a readable stand-in in the body for each such entry, so /// that the typed deserialization and the expression checks run on the other @@ -717,29 +937,48 @@ fn has_duplicates(texts: &[String]) -> bool { /// request is refused before the scan reads the stand-in. An entry of the wrong /// JSON shape stays in the body. pub(crate) struct ExpressionValuesBody { - /// The faults of the entries taken out, in sorted order of their names. + /// The faults of the entries taken out, in the fitted order of their names. faults: Vec, } impl ExpressionValuesBody { /// Replaces each entry the checks here classify with a readable stand-in - /// and keeps its fault for `order`. - pub(crate) fn take(body: &mut Value, order: CheckOrder) -> Self { + /// and keeps its fault for `order`. `request_order` is the entry names as + /// the request wrote them ([`member_order`]). + pub(crate) fn take( + body: &mut Value, + order: CheckOrder, + request_order: Option<&[String]>, + ) -> Self { + let names = fitted_order_of(body, "ExpressionAttributeValues", request_order); let Some(Value::Object(entries)) = body.get_mut("ExpressionAttributeValues") else { return Self { faults: Vec::new() }; }; - let mut names: Vec = entries.keys().cloned().collect(); - names.sort_unstable(); let mut faults = Vec::new(); for name in names { let Some(value) = entries.get_mut(&name) else { continue; }; - let fault = match shape_fault(value) { - Some(text) => Some(DynamoDbError::ValidationException(format!( - "ExpressionAttributeValues contains invalid value: {text} for key {name}" - ))), - None => value_fault(value).map(|fault| fault.text(order, Place::ExpressionValue)), + let fault = match (order, first_fault(value)) { + (_, None) => None, + (_, Some(Fault::Shape(text))) => Some(invalid_expression_value(text, &name)), + // In order R a number is reported here only when the request-time + // check skipped its entry (a set fault below it), with the plain + // text; the set faults were reported at request time. + (CheckOrder::Readable, Some(Fault::Value(ValueFault::Number(fault)))) => { + Some(fault) + } + (CheckOrder::Readable, Some(Fault::Value(fault))) => { + Some(fault.text(order, Place::ExpressionValue)) + } + (CheckOrder::Unreadable, Some(Fault::Value(fault))) => { + match fault.text(order, Place::ExpressionValue) { + DynamoDbError::ValidationException(msg) => { + Some(invalid_expression_value(&msg, &name)) + } + other => Some(other), + } + } }; if let Some(fault) = fault { *value = serde_json::json!({"S": ""}); @@ -753,7 +992,79 @@ impl ExpressionValuesBody { /// /// # Errors /// - /// Returns the fault of the first entry taken out, in sorted order. + /// Returns the fault of the first entry taken out, in the fitted order. + pub(crate) fn validate(&self) -> Result<(), DynamoDbError> { + match self.faults.first() { + Some(fault) => Err(fault.clone()), + None => Ok(()), + } + } +} + +/// The service's text for a faulty `ExpressionAttributeValues` entry at the +/// map's own step: `ExpressionAttributeValues contains invalid value: +/// for key `. +fn invalid_expression_value(text: &str, name: &str) -> DynamoDbError { + DynamoDbError::ValidationException(format!( + "ExpressionAttributeValues contains invalid value: {text} for key {name}" + )) +} + +/// The `ScanFilter` lists of a `Scan` body in order U, whose values the service +/// checks after the no-expression checks and ahead of `ConditionalOperator`, +/// the lists' value counts, the key's faults and the table (2026-11-05): a +/// value that is not one AttributeValue answers `Supplied AttributeValue is +/// empty, must contain exactly one of the supported datatypes` for `{}` and +/// for `{"X": "1"}`, with no prefix. A malformed number in a list is #507's +/// check (#564 item 3) and stays in the body. Two descriptors in a list value, +/// a set fault in a list value, and which value is named when several are +/// faulty (the conditions in the fitted order, the list in its order) are not +/// measured. [`Self::take`] puts a readable stand-in in the body for each such +/// value and keeps the fault; [`Self::validate`] reports it. In order R nothing +/// is taken. +pub(crate) struct ScanFilterBody { + faults: Vec, +} + +impl ScanFilterBody { + /// Replaces each list value that is not one AttributeValue with a readable + /// stand-in in order U and keeps its fault. `request_order` is the + /// condition names as the request wrote them ([`member_order`]). + pub(crate) fn take( + body: &mut Value, + order: CheckOrder, + request_order: Option<&[String]>, + ) -> Self { + if order == CheckOrder::Readable { + return Self { faults: Vec::new() }; + } + let names = fitted_order_of(body, "ScanFilter", request_order); + let Some(Value::Object(conditions)) = body.get_mut("ScanFilter") else { + return Self { faults: Vec::new() }; + }; + let mut faults = Vec::new(); + for name in names { + let Some(Value::Array(values)) = conditions + .get_mut(&name) + .and_then(|condition| condition.get_mut("AttributeValueList")) + else { + continue; + }; + for value in values.iter_mut() { + if let Some(text) = shape_fault(value) { + *value = serde_json::json!({"S": ""}); + faults.push(DynamoDbError::ValidationException(text.to_owned())); + } + } + } + Self { faults } + } + + /// The first list value's fault, at the step of order U. + /// + /// # Errors + /// + /// Returns the fault of the first value taken out. pub(crate) fn validate(&self) -> Result<(), DynamoDbError> { match self.faults.first() { Some(fault) => Err(fault.clone()), diff --git a/crates/engine/src/scan_request/range_tests.rs b/crates/engine/src/scan_request/range_tests.rs index 5b44281be..3ef407f8d 100644 --- a/crates/engine/src/scan_request/range_tests.rs +++ b/crates/engine/src/scan_request/range_tests.rs @@ -20,7 +20,7 @@ const TOTAL_LOW: &str = "Value at 'TotalSegments' failed to satisfy constraint: const TOTAL_HIGH: &str = "Value at 'TotalSegments' failed to satisfy constraint: Member must have value less than or equal to 1000000"; fn check(body: &Value) -> Option { - match validate_scan_request_body(body, CheckOrder::of(body)) { + match validate_scan_request_body(body, CheckOrder::of(body), None) { Ok(()) => None, Err(DynamoDbError::ValidationException(msg)) => Some(msg), Err(other) => panic!("unexpected error {other:?}"), diff --git a/crates/engine/src/scan_request/request_tests.rs b/crates/engine/src/scan_request/request_tests.rs index 466a188a8..b2692d9c6 100644 --- a/crates/engine/src/scan_request/request_tests.rs +++ b/crates/engine/src/scan_request/request_tests.rs @@ -9,17 +9,19 @@ //! a Scan of a table the probe created) with the answer it gave: on 2026-10-25 //! at 10:38 and 10:55 UTC, on 2026-10-30 at 12:40 UTC, on 2026-10-31 between //! 23:54 and 23:56 UTC and on 2026-11-05 where a test says so. The tests of -//! `StartKeyBody::take`, `ExpressionValuesBody::take` and `SelectBody::take` -//! check which values they hold back, with no service answer; the test of the -//! member order checks the fitted rule's buckets; the rows with two value -//! faults in two key members accept either member's fault. +//! `StartKeyBody::take` and `SelectBody::take` check which values they hold +//! back, with no service answer; the tests of `ExpressionValuesBody::take` and +//! `ScanFilterBody::take` also check the measured texts they keep; the tests of +//! the member order check the fitted rule's buckets and masks; the rows with +//! two value faults in two key members accept either member's fault. use extenddb_core::error::DynamoDbError; use serde_json::{Value, json}; use super::{ - CheckOrder, ExpressionValuesBody, SelectBody, StartKeyBody, fitted_member_order, - java_hash_bucket, start_key_member_order, validate_scan_request_body, + CheckOrder, ExpressionValuesBody, ScanFilterBody, SelectBody, StartKeyBody, + fitted_member_order, hash_table_mask, java_hash_bucket, member_order, + validate_scan_request_body, }; const MIX: &str = "Can not use both expression and non-expression parameters in the same request: Non-expression parameters: {ScanFilter} Expression parameters: {FilterExpression}"; @@ -44,7 +46,7 @@ fn validation_message(result: Result<(), DynamoDbError>) -> Option { /// The request-time checks on `body`, in the order of the body's class. fn check(body: &Value) -> Option { - validation_message(validate_scan_request_body(body, CheckOrder::of(body))) + validation_message(validate_scan_request_body(body, CheckOrder::of(body), None)) } /// A Scan body with the fields of `extra` added to the table name. @@ -463,12 +465,15 @@ fn a_set_fault_in_a_key_value_answers_the_order_u_text() { assert_eq!(check_key(&body).as_deref(), Some(text.as_str()), "{body}"); } } - // Not measured in order U, and reported without the key prefix and with - // the members unquoted: an empty BS and a BS with duplicates. + // Measured 2026-11-05 beside an unreadable key member: an empty BS with + // the key prefix, as the empty SS; a BS with duplicates with the members + // unquoted and no prefix. let bs_empty = with_key(json!({"pk": {}, "sk": {"BS": []}})); assert_eq!( check_key(&bs_empty), - Some(format!("{INVALID}Binary sets should not be empty")) + Some(format!( + "{KEY_PREFIX}{INVALID}Binary sets should not be empty" + )) ); let bs_dup = with_key(json!({"pk": {}, "sk": {"BS": ["YQ==", "YQ=="]}})); assert_eq!( @@ -492,8 +497,31 @@ fn the_fitted_member_order_follows_the_hash_buckets_then_the_request() { ("gpk", 3), ("id", 3), ("ts", 3), + ("\u{e4}", 0), + ("\u{e9}", 1), + (":n", 0), + (":b", 0), + (":p", 2), + (":a", 3), + (":e", 3), + ] { + assert_eq!(java_hash_bucket(name, 3), bucket, "{name}"); + } + // The table doubles when it holds more than three quarters of its size. + for (names, mask) in [ + (0, 3), + (1, 3), + (3, 3), + (4, 7), + (6, 7), + (7, 15), + (12, 15), + (13, 31), ] { - assert_eq!(java_hash_bucket(name), bucket, "{name}"); + assert_eq!(hash_table_mask(names), mask, "{names} names"); + } + for (name, bucket) in [("sk", 0), ("gpk", 3), ("pk", 3), ("gsk", 6)] { + assert_eq!(java_hash_bucket(name, 7), bucket, "{name}"); } let order = |names: &[&str]| fitted_member_order(names.iter().copied()); // The pk, sk table: sk first in both request orders. @@ -509,32 +537,65 @@ fn the_fitted_member_order_follows_the_hash_buckets_then_the_request() { // The GSI key of three members: sk, then gpk and pk in the request's order. assert_eq!(order(&["gpk", "pk", "sk"]), ["sk", "gpk", "pk"]); assert_eq!(order(&["pk", "sk", "gpk"]), ["sk", "pk", "gpk"]); + // Four members (measured 2026-11-05 on a GSI with a range key): eight + // buckets, so gpk and pk (3) come ahead of gsk (6) in both request orders; + // with four buckets gsk (2) would come ahead of them. + assert_eq!( + order(&["gpk", "gsk", "pk", "sk"]), + ["sk", "gpk", "pk", "gsk"] + ); + assert_eq!( + order(&["sk", "pk", "gsk", "gpk"]), + ["sk", "pk", "gpk", "gsk"] + ); + // HASH \u{e4}, RANGE \u{e9}: \u{e4} first in both request orders. + assert_eq!(order(&["\u{e4}", "\u{e9}"]), ["\u{e4}", "\u{e9}"]); + assert_eq!(order(&["\u{e9}", "\u{e4}"]), ["\u{e4}", "\u{e9}"]); + // ExpressionAttributeValues entries (order U, 2026-11-05): :n ahead of :e, + // :b ahead of :p and :a, :a and :e in the request's order. + assert_eq!(order(&[":p", ":a", ":e"]), [":p", ":a", ":e"]); + assert_eq!(order(&[":p", ":n", ":e"]), [":n", ":p", ":e"]); + assert_eq!(order(&[":p", ":b", ":a"]), [":b", ":p", ":a"]); + // Seven names: sixteen buckets. + let seven = ["a", "b", "c", "d", "e", "f", "g"]; + assert_eq!(order(&seven).len(), 7); + assert_eq!(hash_table_mask(seven.len()), 15); } -/// `start_key_member_order` reads the key's member names from the request +/// `member_order` reads a top-level object's member names from the request /// text in document order; a repeated name keeps its first position; a body -/// with no key, a key that is not an object or text that does not parse gives -/// `None`. +/// without the field, a field that is not an object or text that does not +/// parse gives `None`. #[test] -fn the_request_order_of_the_key_members_is_read_from_the_raw_body() { - let raw = br#"{"TableName": "t", "ExclusiveStartKey": {"ts": {"N": "bob"}, "id": {}}}"#; +fn the_request_order_of_the_members_is_read_from_the_raw_body() { + let raw = br#"{"TableName": "t", "ExclusiveStartKey": {"ts": {"N": "bob"}, "id": {}}, "ExpressionAttributeValues": {":p": {"S": "x"}, ":n": {"N": "bob"}}, "ScanFilter": {"b": {}, "a": {}}}"#; + let names = |names: &[&str]| Some(names.iter().map(|n| (*n).to_owned()).collect::>()); + assert_eq!(member_order(raw, "ExclusiveStartKey"), names(&["ts", "id"])); assert_eq!( - start_key_member_order(raw), - Some(vec!["ts".to_owned(), "id".to_owned()]) + member_order(raw, "ExpressionAttributeValues"), + names(&[":p", ":n"]) ); + assert_eq!(member_order(raw, "ScanFilter"), names(&["b", "a"])); let repeated = br#"{"ExclusiveStartKey": {"b": {}, "a": {}, "b": {"S": "x"}}, "TableName": "t"}"#; assert_eq!( - start_key_member_order(repeated), - Some(vec!["b".to_owned(), "a".to_owned()]) + member_order(repeated, "ExclusiveStartKey"), + names(&["b", "a"]) + ); + assert_eq!(member_order(raw, "ProjectionExpression"), None); + assert_eq!( + member_order(br#"{"TableName": "t"}"#, "ExclusiveStartKey"), + None ); - assert_eq!(start_key_member_order(br#"{"TableName": "t"}"#), None); assert_eq!( - start_key_member_order(br#"{"TableName": "t", "ExclusiveStartKey": "x"}"#), + member_order( + br#"{"TableName": "t", "ExclusiveStartKey": "x"}"#, + "ExclusiveStartKey" + ), None ); - assert_eq!(start_key_member_order(b"not json"), None); - assert_eq!(start_key_member_order(b""), None); + assert_eq!(member_order(b"not json", "ExclusiveStartKey"), None); + assert_eq!(member_order(b"", "ExclusiveStartKey"), None); } /// Faults in two members of one key: the first faulty member in the fitted @@ -722,7 +783,7 @@ fn a_faulty_expression_value_is_held_back_and_a_stand_in_takes_its_place() { })); let order = CheckOrder::of(&body); assert_eq!(order, CheckOrder::Unreadable, "{value}"); - let values = ExpressionValuesBody::take(&mut body, order); + let values = ExpressionValuesBody::take(&mut body, order, None); assert_eq!(body["ExpressionAttributeValues"], json!({":a": {"S": ""}})); assert_eq!(validation_message(values.validate()), Some(text), "{value}"); } @@ -736,7 +797,7 @@ fn a_faulty_expression_value_is_held_back_and_a_stand_in_takes_its_place() { let mut body = scan(json!({"ExpressionAttributeValues": {":a": value}})); let order = CheckOrder::of(&body); assert_eq!(order, CheckOrder::Readable, "{value}"); - let values = ExpressionValuesBody::take(&mut body, order); + let values = ExpressionValuesBody::take(&mut body, order, None); assert_eq!(body["ExpressionAttributeValues"], json!({":a": {"S": ""}})); assert_eq!( validation_message(values.validate()), @@ -744,23 +805,313 @@ fn a_faulty_expression_value_is_held_back_and_a_stand_in_takes_its_place() { "{value}" ); } - // Two faulty entries: the first in sorted order is reported. + // Two faulty entries: the first in the fitted order is reported (measured + // 2026-11-05: `:b` ahead of `:a`, which sorted order would reverse). let mut body = - scan(json!({"ExpressionAttributeValues": {":b": {}, ":a": {"S": "x", "N": "1"}}})); - let values = ExpressionValuesBody::take(&mut body, CheckOrder::Unreadable); + scan(json!({"ExpressionAttributeValues": {":p": {"S": "x"}, ":b": {}, ":a": {"X": "1"}}})); + let values = ExpressionValuesBody::take(&mut body, CheckOrder::Unreadable, None); assert_eq!( validation_message(values.validate()), - Some(wrapper(MORE_THAN_ONE)) + Some(wrapper(EMPTY_VALUE).replace(":a", ":b")) ); // A readable value and a value of the wrong JSON shape stay as written. for value in [json!({"S": "x"}), json!({"N": 5})] { let mut body = scan(json!({"ExpressionAttributeValues": {":a": value}})); - let values = ExpressionValuesBody::take(&mut body, CheckOrder::Readable); + let values = ExpressionValuesBody::take(&mut body, CheckOrder::Readable, None); assert_eq!(body["ExpressionAttributeValues"], json!({":a": value})); assert!(values.validate().is_ok()); } } +/// Measured 2026-11-05 (order U): a malformed number in `ExpressionAttributeValues` +/// is reported at the map's step in the invalid-value form, the entries visited +/// in the fitted order: `{":p": {"S": "x"}, ":a": {"N": "bob"}, ":e": {}}` +/// names `:a`, the same with `:n` names `:n`, and the request-time checks pass +/// over the number. In order R the number is a request-time check with the +/// plain text. +#[test] +fn a_malformed_expression_value_number_in_order_u_takes_the_invalid_value_form() { + let wrapper = |name: &str| { + format!("ExpressionAttributeValues contains invalid value: {NUMERIC_BOB} for key {name}") + }; + for (name, request_order) in [(":a", [":p", ":a", ":e"]), (":n", [":p", ":n", ":e"])] { + let mut body = scan(json!({ + "FilterExpression": "a = :p", + "ExpressionAttributeValues": {":p": {"S": "x"}, name: {"N": "bob"}, ":e": {}}, + })); + assert_eq!(CheckOrder::of(&body), CheckOrder::Unreadable); + assert_eq!(check(&body), None, "{name}"); + let names: Vec = request_order.iter().map(|n| (*n).to_owned()).collect(); + let values = ExpressionValuesBody::take(&mut body, CheckOrder::Unreadable, Some(&names)); + assert_eq!( + validation_message(values.validate()), + Some(wrapper(name)), + "{name}" + ); + assert_eq!(body["ExpressionAttributeValues"][name], json!({"S": ""})); + } + // With the unreadable value in the key, the number is still held back for + // the map's step, after the key's faults. + let mut body = scan(json!({ + "FilterExpression": "a = :n", + "ExpressionAttributeValues": {":n": {"N": "bob"}}, + "ExclusiveStartKey": {"pk": {}, "sk": {"S": "x"}}, + })); + assert_eq!(check(&body), None); + let values = ExpressionValuesBody::take(&mut body, CheckOrder::Unreadable, None); + assert_eq!(validation_message(values.validate()), Some(wrapper(":n"))); + // Order R: the request-time check, the plain text. + let body = scan(json!({"ExpressionAttributeValues": {":n": {"N": "bob"}}})); + assert_eq!(CheckOrder::of(&body), CheckOrder::Readable); + assert_eq!(check(&body).as_deref(), Some(NUMERIC_BOB)); +} + +/// Measured 2026-11-05 (order R): the request-time number check visits the +/// `ExpressionAttributeValues` entries in the hash order the key and the +/// order-U step use. `{":n": {"N": "fred"}, ":a": {"N": "bob"}}` answers +/// `fred` (`:n` in bucket 0, `:a` in 3), whichever order the request wrote and +/// whatever order the parsed body keeps; sorted name order would answer `bob`. +#[test] +fn the_order_r_number_check_visits_the_entries_in_the_hash_order() { + let numeric = + |text: &str| format!("The parameter cannot be converted to a numeric value: {text}"); + let body = scan(json!({ + "FilterExpression": "a = :a", + "ExpressionAttributeValues": {":n": {"N": "fred"}, ":a": {"N": "bob"}}, + })); + assert_eq!(CheckOrder::of(&body), CheckOrder::Readable); + assert_eq!(check(&body), Some(numeric("fred"))); + for request_order in [[":n", ":a"], [":a", ":n"]] { + let names: Vec = request_order.iter().map(|n| (*n).to_owned()).collect(); + assert_eq!( + validation_message(validate_scan_request_body( + &body, + CheckOrder::Readable, + Some(&names) + )), + Some(numeric("fred")), + "{request_order:?}" + ); + } + // Two names in one bucket: the request's order decides (`:a` and `:e` + // share bucket 3). + let body = scan(json!({ + "FilterExpression": "a = :p", + "ExpressionAttributeValues": {":p": {"S": "x"}, ":a": {"N": "bob"}, ":e": {"N": "fred"}}, + })); + for (request_order, first) in [([":p", ":a", ":e"], "bob"), ([":p", ":e", ":a"], "fred")] { + let names: Vec = request_order.iter().map(|n| (*n).to_owned()).collect(); + assert_eq!( + validation_message(validate_scan_request_body( + &body, + CheckOrder::Readable, + Some(&names) + )), + Some(numeric(first)), + "{request_order:?}" + ); + } +} + +/// Measured 2026-11-05 (order R): with two set faults, `{":n": {"NS": []}, +/// ":a": {"SS": []}}` answered the empty `SS` text in one send and the empty +/// `NS` text in the other. The set pass visits the entries in sorted name +/// order, whatever order the backend's parsed body keeps, so that the answer +/// does not depend on the backend: it names `:a`. A number in an entry is not +/// the set pass's to report (`eavR-NSbobdup-and-nest`: the number inside a +/// refused entry comes after the nesting check). +#[test] +fn the_order_r_set_pass_visits_the_entries_in_sorted_name_order() { + let body = scan(json!({"ExpressionAttributeValues": {":n": {"NS": []}, ":a": {"SS": []}}})); + assert_eq!(CheckOrder::of(&body), CheckOrder::Readable); + assert_eq!( + check(&body), + Some(format!("{INVALID}An string set may not be empty")) + ); + let body = scan(json!({"ExpressionAttributeValues": {":a": {"NS": []}, ":n": {"SS": []}}})); + assert_eq!( + check(&body), + Some(format!("{INVALID}An number set may not be empty")) + ); + // A malformed number inside an entry with a set fault below it: skipped by + // the request-time checks (the nesting check of another entry answers + // first), reported by `ExpressionValuesBody` with the plain text. + let one = json!({"N": "1"}); + let mut body = scan(json!({ + "FilterExpression": "a = :p", + "ExpressionAttributeValues": {":p": {"S": "x"}, ":d": nested(&one, 32), ":a": {"NS": ["bob", "1", "1"]}}, + })); + assert_eq!(check(&body).as_deref(), Some(NESTING)); + body["ExpressionAttributeValues"] + .as_object_mut() + .expect("an object") + .remove(":d"); + assert_eq!(check(&body), None); + let values = ExpressionValuesBody::take(&mut body, CheckOrder::Readable, None); + assert_eq!( + validation_message(values.validate()).as_deref(), + Some(NUMERIC_BOB) + ); +} + +/// Measured 2026-11-05 (order U): a set fault in an `ExpressionAttributeValues` +/// entry takes the invalid-value form at the map's step, with the set texts of +/// order U, in the entries' hash order: `{":p": {"S": "x"}, ":s": {"NS": []}, +/// ":e": {}}` names `:s` (bucket 1, ahead of `:e` in 3); an `SS` with +/// duplicates answers `... Input collection [a, a] contains duplicates. for key +/// :s`. Inside one entry the first fault in the value's element order is named: +/// `{"L": [{"N": "bob"}, {}]}` answers the number (measured), `{"L": [{}, +/// {"N": "bob"}]}` the empty value (not measured, the element order). +#[test] +fn an_order_u_set_fault_in_an_expression_value_takes_the_invalid_value_form() { + let wrapper = |text: &str, name: &str| { + format!("ExpressionAttributeValues contains invalid value: {text} for key {name}") + }; + for (set, text) in [ + ( + json!({"NS": []}), + format!("{INVALID}An number set may not be empty"), + ), + ( + json!({"SS": ["a", "a"]}), + format!("{INVALID}Input collection [a, a] contains duplicates."), + ), + ( + json!({"BS": []}), + format!("{INVALID}Binary sets should not be empty"), + ), + ] { + let mut body = scan(json!({ + "FilterExpression": "a = :p", + "ExpressionAttributeValues": {":p": {"S": "x"}, ":s": set, ":e": {}}, + })); + assert_eq!(CheckOrder::of(&body), CheckOrder::Unreadable); + assert_eq!(check(&body), None, "{set}"); + let values = ExpressionValuesBody::take(&mut body, CheckOrder::Unreadable, None); + assert_eq!( + validation_message(values.validate()), + Some(wrapper(&text, ":s")), + "{set}" + ); + assert_eq!(body["ExpressionAttributeValues"][":s"], json!({"S": ""})); + assert_eq!(body["ExpressionAttributeValues"][":e"], json!({"S": ""})); + } + // Inside one entry: the element order. + for (value, text) in [ + (json!({"L": [{"N": "bob"}, {}]}), NUMERIC_BOB), + (json!({"L": [{}, {"N": "bob"}]}), EMPTY_VALUE), + ( + json!({"L": [{"NS": []}, {}]}), + "One or more parameter values were invalid: An number set may not be empty", + ), + ( + json!({"L": [{"L": [{"N": "bob"}]}, {"S": "x", "N": "1"}]}), + NUMERIC_BOB, + ), + ] { + let mut body = scan(json!({ + "FilterExpression": "a = :p", + "ExpressionAttributeValues": {":p": {"S": "x"}, ":l": value}, + })); + assert_eq!(CheckOrder::of(&body), CheckOrder::Unreadable, "{value}"); + let values = ExpressionValuesBody::take(&mut body, CheckOrder::Unreadable, None); + assert_eq!( + validation_message(values.validate()), + Some(wrapper(text, ":l")), + "{value}" + ); + } +} + +/// Measured 2026-11-05: in order U `ScanFilterBody::take` holds a `ScanFilter` +/// list value back for the step after the no-expression checks: a value that is +/// not one AttributeValue answers `Supplied AttributeValue is empty, must +/// contain exactly one of the supported datatypes` with no prefix, for `{}` and +/// for `{"X": "1"}`. In order R nothing is taken. A malformed number in a list +/// is #507's check (#564 item 3): the request-time checks pass it and `take` +/// leaves it in the body in both orders. +#[test] +fn a_scan_filter_list_value_is_checked_at_its_step() { + let list = |values: Value| { + scan( + json!({"ScanFilter": {"a": {"ComparisonOperator": "EQ", "AttributeValueList": values}}}), + ) + }; + // The numbers of the lists are #507's: not a request-time check here. + let mut bob = list(json!([{"N": "bob"}, {"NS": ["bob"]}])); + assert_eq!(CheckOrder::of(&bob), CheckOrder::Readable); + assert_eq!(check(&bob), None); + let beside_key = with( + list(json!([{"N": "bob"}])), + json!({"ExclusiveStartKey": {"pk": {"N": "fred"}, "sk": {"S": "x"}}}), + ); + assert_eq!(check(&beside_key), None); + assert!( + ScanFilterBody::take(&mut bob, CheckOrder::Readable, None) + .validate() + .is_ok() + ); + assert_eq!( + bob["ScanFilter"]["a"]["AttributeValueList"], + json!([{"N": "bob"}, {"NS": ["bob"]}]) + ); + // Order U: the list value is held back. + for value in [json!({}), json!({"X": "1"}), json!({"L": [{}]})] { + let mut body = list(json!([value])); + assert_eq!(CheckOrder::of(&body), CheckOrder::Unreadable, "{value}"); + assert_eq!(check(&body), None, "{value}"); + let filter = ScanFilterBody::take(&mut body, CheckOrder::Unreadable, None); + assert_eq!( + body["ScanFilter"]["a"]["AttributeValueList"], + json!([{"S": ""}]), + "{value}" + ); + assert_eq!( + validation_message(filter.validate()).as_deref(), + Some(EMPTY_VALUE), + "{value}" + ); + assert!( + serde_json::from_value::(body).is_ok(), + "{value}" + ); + } + let mut two = list(json!([{"S": "x", "N": "1"}])); + let filter = ScanFilterBody::take(&mut two, CheckOrder::Unreadable, None); + assert_eq!( + validation_message(filter.validate()).as_deref(), + Some(MORE_THAN_ONE) + ); + // A number beside an unreadable key member stays in the body in order U + // too (#507's check). + let mut number = with( + list(json!([{"N": "bob"}])), + json!({"ExclusiveStartKey": {"pk": {}, "sk": {"S": "x"}}}), + ); + assert_eq!(check(&number), None); + let filter = ScanFilterBody::take(&mut number, CheckOrder::Unreadable, None); + assert!(filter.validate().is_ok()); + assert_eq!( + number["ScanFilter"]["a"]["AttributeValueList"], + json!([{"N": "bob"}]) + ); + // Order R: nothing is taken, a readable list stays as written. + let mut readable = list(json!([{"N": "1"}])); + let filter = ScanFilterBody::take(&mut readable, CheckOrder::Readable, None); + assert_eq!( + readable["ScanFilter"]["a"]["AttributeValueList"], + json!([{"N": "1"}]) + ); + assert!(filter.validate().is_ok()); + // No ScanFilter at all. + let mut none = scan(json!({})); + assert!( + ScanFilterBody::take(&mut none, CheckOrder::Unreadable, None) + .validate() + .is_ok() + ); +} + #[test] fn a_value_nested_32_levels_deep_is_refused_and_31_levels_pass() { let one = json!({"N": "1"}); @@ -780,6 +1131,34 @@ fn a_value_nested_32_levels_deep_is_refused_and_31_levels_pass() { // ahead of the no-expression check. let no_expression = scan(json!({"ExpressionAttributeValues": {":d": nested(&one, 32)}})); assert_eq!(check(&no_expression).as_deref(), Some(NESTING)); + // Measured 2026-11-05 (eavR-NSe-and-nest, both sends): an empty set in + // another entry answers the set text ahead of the nesting text, with the + // text of order R; a duplicate set too (not measured beside a nested + // value). A number in a third entry comes ahead of both. + let beside_empty_set = scan( + json!({"FilterExpression": "a = :p", "ExpressionAttributeValues": + {":p": {"S": "x"}, ":d": nested(&one, 32), ":e": {"NS": []}}}), + ); + assert_eq!(CheckOrder::of(&beside_empty_set), CheckOrder::Readable); + assert_eq!( + check(&beside_empty_set).as_deref(), + Some("One or more parameter values were invalid: An number set may not be empty") + ); + let beside_duplicates = scan( + json!({"FilterExpression": "a = :p", "ExpressionAttributeValues": + {":p": {"S": "x"}, ":d": nested(&one, 32), ":a": {"SS": ["a", "a"]}}}), + ); + assert_eq!( + check(&beside_duplicates).as_deref(), + Some( + "One or more parameter values were invalid: Input collection [\"a\", \"a\"] contains duplicates." + ) + ); + let number_first = scan( + json!({"FilterExpression": "a = :p", "ExpressionAttributeValues": + {":p": {"S": "x"}, ":d": nested(&one, 32), ":e": {"NS": []}, ":n": {"N": "bob"}}}), + ); + assert_eq!(check(&number_first).as_deref(), Some(NUMERIC_BOB)); // In order U the nesting check leaves request time: beside an unused // empty value (B4-7) or a key shape fault (2026-11-05) the request-time // checks pass and the handler reports those first. diff --git a/crates/engine/src/scan_request/tests.rs b/crates/engine/src/scan_request/tests.rs index a5ba5eb41..d576b4e98 100644 --- a/crates/engine/src/scan_request/tests.rs +++ b/crates/engine/src/scan_request/tests.rs @@ -25,7 +25,7 @@ fn message(result: Result<(), DynamoDbError>) -> Option { } fn check(body: &Value) -> Option { - message(validate_scan_request_body(body, CheckOrder::of(body))) + message(validate_scan_request_body(body, CheckOrder::of(body), None)) } /// A Scan whose filter reads `:n = {"N": "bob"}`. @@ -316,7 +316,7 @@ fn a_shape_fault_anywhere_in_the_body_is_reported_first() { for body in shape_faults { assert!( matches!( - validate_scan_request_body(&body, CheckOrder::of(&body)), + validate_scan_request_body(&body, CheckOrder::of(&body), None), Err(DynamoDbError::SerializationException(_)) ), "{body}" diff --git a/tests/python/test_scan_request_validation.py b/tests/python/test_scan_request_validation.py index ed1b12579..27b3bbb23 100644 --- a/tests/python/test_scan_request_validation.py +++ b/tests/python/test_scan_request_validation.py @@ -17,7 +17,7 @@ bottom is reported first, at the depths 31 to 60 that were measured. The service follows one of two check orders (measured 2026-10-31 and -2026-11-05, each request twice; MEASURED-scan-r3 section 6). A request that +2026-11-05, each request twice). A request that holds a value that is not one AttributeValue (empty, an unknown or lower-case type descriptor, two descriptors) in ``ExpressionAttributeValues``, ``ExclusiveStartKey`` or a ``ScanFilter`` list follows order U: the constraint @@ -32,6 +32,17 @@ and set faults, the table. The set texts quote their members in order R and not in order U. With two value faults in two key members the service's own sends disagreed, so those tests accept either fault. + +A malformed number in a ``ScanFilter`` list answers the number text ahead of +the key's faults (measured 2026-11-05); that check is #564 item 3, which PR +#507 makes, so its two rows are expected to fail here. In order U the +``ExpressionAttributeValues`` entries are visited in the same fitted order as +the key's members (the hash buckets, then the request's order), a malformed +number among them answers ``ExpressionAttributeValues contains invalid value: +... for key :a``, and a ``ScanFilter`` list value that is not one AttributeValue +answers the bare empty-value text after the no-expression checks. A key of four +members is visited with eight buckets (measured 2026-11-05 on a GSI with a +range key). """ from __future__ import annotations @@ -173,7 +184,7 @@ '"ln": {"N": "bob"}}', NUMERIC_BOB), ] -# Measured 2026-11-05: each key alone (the keys of MEASURED-scan-r3 section 4). +# Measured 2026-11-05: each key alone. # A shape fault answers the service's text with the key prefix; a set fault # answers the service's text with no prefix. KEYS_ALONE = [ @@ -196,7 +207,7 @@ # Measured 2026-10-31 and 2026-11-05: faults in two members of one key, one of # them a shape fault. The service names the first faulty member in the fitted -# member order (MEASURED-scan-r3 section 6.3: sk ahead of pk and of z in both +# member order (sk ahead of pk and of z in both # request orders; the set texts of order U where the set member comes first). TWO_MEMBER_FAULTS = [ ("empty-pk-then-bob", '{"pk": {}, "sk": {"N": "bob"}}', NUMERIC_BOB), @@ -226,7 +237,7 @@ (SS_DUP, NUMERIC_BOB)), ] -# Measured 2026-11-05 (R5, MEASURED-scan-r3 section 6.3) on tables with other +# Measured 2026-11-05 on tables with other # key names: (id, table fixture, key, expected). HASH a, RANGE b: a first; HASH # pk, RANGE a: a first in both request orders; HASH id, RANGE ts: the member # first in the request; the GSI key of three members: gpk ahead of pk. @@ -239,7 +250,7 @@ ("it-bob-and-empty-reversed", "it_table", '{"ts": {"N": "bob"}, "id": {}}', NUMERIC_BOB), ] -# Measured 2026-11-05 (MEASURED-scan-r3 section 2): beside each class of key +# Measured 2026-11-05: beside each class of key # fault, a check of each stage. The request parameter checks come ahead of # every key fault; the shape faults come ahead of the expression checks and of # the table lookup; the expression checks come ahead of the number and set @@ -305,7 +316,7 @@ def nest(leaf: dict, levels: int) -> dict: NEST32_S = json.dumps(nest({"S": "x"}, 32)) NEST32_N = json.dumps(nest({"N": "1"}, 32)) -# Measured 2026-10-31 (B4-7) and 2026-11-05 (R5, MEASURED-scan-r3 section 6): +# Measured 2026-10-31 (B4-7) and 2026-11-05: # representative rows of each order. An expected text given as ("startswith", # prefix) asserts the check's place where the text predates this branch (#722, # #838, #839). "4th" names the rows that fail on 45705bf27. @@ -339,7 +350,7 @@ def nest(leaf: dict, levels: int) -> dict: ("B4-7-noexpr-two", '"ExpressionAttributeValues": {":e": {"S": "x", "N": "1"}}', ("startswith", EAV_NO_EXPRESSION)), # Order U: the nesting check after the key's faults (the three b.*.nesting - # rows of R3 and the nest-*-key-empty rows of R5). + # rows of 2026-11-05 and the nest-*-key-empty rows). ("nesting-beside-empty-key", FILTER_P + ', "ExpressionAttributeValues": {":p": ' + NEST32_S + '}, ' + EMPTY_KEY, EMPTY_KEY_VALUE), ("nesting-beside-type-X-key", FILTER_P + ', "ExpressionAttributeValues": {":p": ' @@ -355,7 +366,7 @@ def nest(leaf: dict, levels: int) -> dict: ("nest-unused-N-key-empty", FILTER_P + ', "ExpressionAttributeValues": {":p": {"S": "x"}, ' '":d": ' + NEST32_N + '}, ' + EMPTY_KEY, EMPTY_KEY_VALUE), # Order U: the parameter mix and the other request parameter checks ahead of - # the key's shape fault (RS-f Minor-1; arity-key-empty-and-eav-noexpr). + # the key's shape fault (arity-key-empty-and-eav-noexpr). ("mix-beside-empty-key", '"ScanFilter": {"a": {"ComparisonOperator": "EQ", ' '"AttributeValueList": [{"N": "1"}]}}, "FilterExpression": "a = :p", ' '"ExpressionAttributeValues": {":p": {"N": "1"}}, ' + EMPTY_KEY, MIX), @@ -465,6 +476,215 @@ def nest(leaf: dict, levels: int) -> dict: ] +EMPTY_VALUE_BARE = ("Supplied AttributeValue is empty, must contain exactly one of the " + "supported datatypes") +MORE_THAN_ONE_BARE = ("Supplied AttributeValue has more than one datatypes set, must contain " + "exactly one of the supported datatypes") +EAV_EMPTY_B = EAV_EMPTY_E.replace(":e", ":b") +EAV_EMPTY_P = EAV_EMPTY_E.replace(":e", ":p") +EAV_NUMBER_A = EAV_VALUE + NUMERIC_BOB + " for key :a" +EAV_NUMBER_N = EAV_VALUE + NUMERIC_BOB + " for key :n" +SF_EMPTY = '"ScanFilter": {"a": {"ComparisonOperator": "EQ", "AttributeValueList": [{}]}}' +SF_BOB = '"ScanFilter": {"a": {"ComparisonOperator": "EQ", "AttributeValueList": [{"N": "bob"}]}}' +NONASCII_KEY_A_BOB = '"ExclusiveStartKey": {"\\u00e4": {"N": "bob"}, "\\u00e9": {}}' +NONASCII_KEY_E_EMPTY = '"ExclusiveStartKey": {"\\u00e9": {}, "\\u00e4": {"N": "bob"}}' + +# Measured 2026-11-05 at 05:31, 06:20 and 07:38 UTC: (id, table +# fixture or None for a missing table, the fields, the expected text). A text +# given as ("startswith", prefix) asserts the check's place where the text +# predates this branch (#838); ("either", a, b) accepts both answers where the +# service's two sends differed. +ROWS_2026_11_05 = [ + # 7.1: a malformed number in a ScanFilter list, ahead of the key's number. + # The check is #564 item 3, which PR #507 makes (SCAN_FILTER_NUMBER_ROWS). + ("sfR-N-bob", "table", SF_BOB, NUMERIC_BOB), + ("sfR-N-bob-and-key-bob", "table", + SF_BOB + ', "ExclusiveStartKey": {"pk": {"N": "fred"}, "sk": {"S": "x"}}', NUMERIC_BOB), + # 7.2: order U, the ExpressionAttributeValues entries in the fitted order, + # a number among them in the invalid-value form, after the key's faults. + ("eavU-num-first-sorted", "table", FILTER_P + ', "ExpressionAttributeValues": ' + '{":p": {"S": "x"}, ":a": {"N": "bob"}, ":e": {}}', EAV_NUMBER_A), + ("eavU-num-last-sorted", "table", FILTER_P + ', "ExpressionAttributeValues": ' + '{":p": {"S": "x"}, ":n": {"N": "bob"}, ":e": {}}', EAV_NUMBER_N), + ("eavU-num-and-key-empty", "table", '"FilterExpression": "a = :n", ' + '"ExpressionAttributeValues": {":n": {"N": "bob"}}, ' + EMPTY_KEY, EMPTY_KEY_VALUE), + ("eavU-num-and-key-empty-and-eav-empty", "table", '"FilterExpression": "a = :n", ' + '"ExpressionAttributeValues": {":n": {"N": "bob"}, ":e": {}}, ' + EMPTY_KEY, + EMPTY_KEY_VALUE), + ("eavU-two-faulty-entries", "table", FILTER_P + ', "ExpressionAttributeValues": ' + '{":p": {"S": "x"}, ":b": {}, ":a": {"X": "1"}}', EAV_EMPTY_B), + # 7.3: four members on a GSI with a range key (eight buckets: gpk and pk + # ahead of gsk), and names outside ASCII. + ("key4-gsk-bob-gpk-empty", "t4_table", '"IndexName": "gsi2", "ExclusiveStartKey": ' + '{"gpk": {}, "gsk": {"N": "bob"}, "pk": {"S": "p"}, "sk": {"S": "s"}}', EMPTY_KEY_VALUE), + ("key4-gsk-bob-gpk-empty-reversed", "t4_table", '"IndexName": "gsi2", "ExclusiveStartKey": ' + '{"sk": {"S": "s"}, "pk": {"S": "p"}, "gsk": {"N": "bob"}, "gpk": {}}', EMPTY_KEY_VALUE), + ("key4-pk-bob-gsk-empty", "t4_table", '"IndexName": "gsi2", "ExclusiveStartKey": ' + '{"gpk": {"S": "g"}, "gsk": {}, "pk": {"N": "bob"}, "sk": {"S": "s"}}', NUMERIC_BOB), + ("keyU-nonascii-a-bob-e-empty", "u_table", NONASCII_KEY_A_BOB, NUMERIC_BOB), + ("keyU-nonascii-e-empty-a-bob", "u_table", NONASCII_KEY_E_EMPTY, NUMERIC_BOB), + # 7.4: a ScanFilter list value that is not one AttributeValue, in order U: + # the bare empty-value text for {} and for an unknown descriptor, after the + # no-expression checks and ahead of ConditionalOperator, the value count, + # the key's faults and the table; the Select enum ahead of it. + ("sfU-empty-alone", "table", SF_EMPTY, EMPTY_VALUE_BARE), + ("sfU-X-alone", "table", + '"ScanFilter": {"a": {"ComparisonOperator": "EQ", "AttributeValueList": [{"X": "1"}]}}', + EMPTY_VALUE_BARE), + ("sfU-empty-and-ean-noexpr", "table", SF_EMPTY + ', "ExpressionAttributeNames": {"#a": "a"}', + EAN_NO_EXPRESSION), + ("sfU-empty-and-key-empty", "table", SF_EMPTY + ", " + EMPTY_KEY, EMPTY_VALUE_BARE), + ("sfU-empty-and-key-bob", "table", SF_EMPTY + ", " + BOB_KEY, EMPTY_VALUE_BARE), + ("sfU-empty-arity", "table", + '"ScanFilter": {"a": {"ComparisonOperator": "EQ", "AttributeValueList": [{}, {}]}}', + EMPTY_VALUE_BARE), + ("sfU-empty-condop", "table", '"ConditionalOperator": "AND", ' + SF_EMPTY, EMPTY_VALUE_BARE), + ("sfU-empty-select-bad", "table", SF_EMPTY + ', "Select": "BAD"', SELECT_ENUM), + ("sfU-empty-missing-table", None, SF_EMPTY, EMPTY_VALUE_BARE), + # 7.5 and the other rows the head answers as the service does. + ("keyU-SSe-beside-eav-empty", "table", '"FilterExpression": "a = :a", ' + '"ExpressionAttributeValues": {":a": {}}, "ExclusiveStartKey": {"pk": {"SS": []}, ' + '"sk": {"S": "x"}}', KEY_PREFIX + SS_EMPTY), + ("keyU-BSe-beside-pk-empty", "table", '"ExclusiveStartKey": {"pk": {}, "sk": {"BS": []}}', + KEY_PREFIX + BS_EMPTY), + ("keyU-BSdup-beside-pk-empty", "table", + '"ExclusiveStartKey": {"pk": {}, "sk": {"BS": ["YQ==", "YQ=="]}}', + INVALID + "Input collection [YQ==, YQ==]of type BS contains duplicates."), + ("eavR-NSe-and-num-first-sorted", "table", FILTER_P + ', "ExpressionAttributeValues": ' + '{":p": {"S": "x"}, ":a": {"N": "bob"}, ":e": {"NS": []}}', ("either", NS_EMPTY, NUMERIC_BOB)), + ("eavR-NSe-and-num-last-sorted", "table", FILTER_P + ', "ExpressionAttributeValues": ' + '{":p": {"S": "x"}, ":n": {"N": "bob"}, ":e": {"NS": []}}', ("either", NS_EMPTY, NUMERIC_BOB)), + ("eavR-NSe-and-ean-noexpr", "table", '"ExpressionAttributeNames": {"#a": "a"}, ' + '"ExpressionAttributeValues": {":e": {"NS": []}}', NS_EMPTY), + ("eavR-NSe-and-rcc-bad", "table", '"ReturnConsumedCapacity": "BAD", ' + '"ExpressionAttributeValues": {":e": {"NS": []}}', RCC_ENUM), + ("eavR-NSe-and-select-bad", "table", FILTER_P + ', "ExpressionAttributeValues": ' + '{":p": {"S": "x"}, ":e": {"NS": []}}, "Select": "BAD"', NS_EMPTY), + # Re-measured 2026-11-05 at 06:20 UTC with the nested value written out + # (32 L levels): the empty set ahead of the nesting check in order R, and + # the empty leaf of a nested entry named in order U. + ("eavR-NSe-and-nest", "table", FILTER_P + ', "ExpressionAttributeValues": ' + '{":p": {"S": "x"}, ":d": ' + NEST32_N + ', ":e": {"NS": []}}', NS_EMPTY), + ("eavU-nest32-empty-leaf", "table", FILTER_P + ', "ExpressionAttributeValues": ' + '{":p": {"S": "x"}, ":d": ' + json.dumps(nest({}, 32)) + '}', EAV_EMPTY_E.replace(":e", ":d")), + ("eavU-empty-and-filter-syntax", "table", '"FilterExpression": "a = = :p", ' + '"ExpressionAttributeValues": {":p": {}}', EAV_EMPTY_P), + ("eavU-empty-and-undefined", "table", '"FilterExpression": "a = :zz", ' + '"ExpressionAttributeValues": {":e": {}}', EAV_EMPTY_E), + ("eavU-empty-and-projection-syntax", "table", '"ProjectionExpression": "a,,b", ' + FILTER_P + + ', "ExpressionAttributeValues": {":p": {}}', EAV_EMPTY_P), + ("eavU-empty-and-select-bad-and-mix", "table", '"ScanFilter": {"a": {"ComparisonOperator": ' + '"EQ", "AttributeValueList": [{"N": "1"}]}}, ' + FILTER_P + ', "ExpressionAttributeValues": ' + '{":p": {"N": "1"}, ":e": {}}, "Select": "BAD"', SELECT_ENUM), + ("eavU-empty-and-rcc-bad", "table", '"ReturnConsumedCapacity": "BAD", ' + FILTER_P + + ', "ExpressionAttributeValues": {":p": {"S": "x"}, ":e": {}}', RCC_ENUM), + ("selR-condop-select-bad", "table", '"ConditionalOperator": "AND", "ScanFilter": {"a": ' + '{"ComparisonOperator": "EQ", "AttributeValueList": [{"N": "1"}]}}, "Select": "BAD"', CONDOP), + ("selR-arity-select-bad", "table", '"ScanFilter": {"a": {"ComparisonOperator": "EQ", ' + '"AttributeValueList": []}}, "Select": "BAD"', + ("either", "Invalid number of attribute values for comparison", + INVALID + "Invalid number of argument(s) for the EQ ComparisonOperator")), + ("flag-unknown-field-empty", "table", '"Limit": 0, "Foo": {}', LIMIT), + ("flag-queryfilter-on-scan", "table", '"Limit": 0, "QueryFilter": {"a": {"ComparisonOperator": ' + '"EQ", "AttributeValueList": [{}]}}', LIMIT), + # Measured 2026-11-05 at 07:38 UTC. Order U, an ExpressionAttributeValues + # set fault takes the invalid-value form at the map's step, in the entries' + # hash order (:s in bucket 1 ahead of :e in 3). + ("eavU-NSe-entry", "table", FILTER_P + ', "ExpressionAttributeValues": ' + '{":p": {"S": "x"}, ":s": {"NS": []}, ":e": {}}', EAV_VALUE + NS_EMPTY + " for key :s"), + ("eavU-SSdup-entry", "table", FILTER_P + ', "ExpressionAttributeValues": ' + '{":p": {"S": "x"}, ":s": {"SS": ["a", "a"]}, ":e": {}}', + EAV_VALUE + INVALID + "Input collection [a, a] contains duplicates. for key :s"), + # Inside one entry the first fault in the value's element order is named. + ("eavU-number-then-empty-in-one-L", "table", FILTER_P + ', "ExpressionAttributeValues": ' + '{":p": {"S": "x"}, ":l": {"L": [{"N": "bob"}, {}]}}', EAV_NUMBER_A.replace(":a", ":l")), + # Order U: the entries' hash order with the request's order inside a bucket + # (:e and :a share bucket 3), four and seven entries (eight and sixteen + # buckets), a number below the top of an entry, the 38-digit text. + ("eavU-reqorder-e-then-a", "table", FILTER_P + ', "ExpressionAttributeValues": ' + '{":p": {"S": "x"}, ":e": {}, ":a": {"N": "bob"}}', EAV_EMPTY_E), + ("eavU-four-entries", "table", FILTER_P + ', "ExpressionAttributeValues": ' + '{":p": {"S": "x"}, ":b": {"S": "y"}, ":a": {"N": "bob"}, ":e": {}}', EAV_EMPTY_E), + ("eavU-seven-entries", "table", FILTER_P + ', "ExpressionAttributeValues": ' + '{":p": {"S": "x"}, ":b": {}, ":e": {"S": "e"}, ":a": {"S": "a"}, ":n": {"N": "bob"}, ' + '":d": {"S": "d"}, ":z": {"S": "z"}}', EAV_NUMBER_N), + ("eavU-number-in-L", "table", FILTER_P + ', "ExpressionAttributeValues": ' + '{":p": {"S": "x"}, ":l": {"L": [{"N": "bob"}]}, ":e": {}}', EAV_NUMBER_A.replace(":a", ":l")), + ("eavU-number-in-NS", "table", FILTER_P + ', "ExpressionAttributeValues": ' + '{":p": {"S": "x"}, ":s": {"NS": ["bob"]}, ":e": {}}', EAV_NUMBER_A.replace(":a", ":s")), + ("eavU-39-digits", "table", FILTER_P + ', "ExpressionAttributeValues": {":p": {"S": "x"}, ' + '":a": {"N": "123456789012345678901234567890123456789"}, ":e": {}}', + EAV_VALUE + "Attempting to store more than 38 significant digits in a Number for key :a"), + # Order R, two faulty entries: the numbers in the hash order (:n in bucket 0 + # ahead of :a in 3); the set faults in sorted name order, where the + # service's two sends disagreed (the SS text, then the NS text) and the + # sorted order is pinned so that the answer does not depend on the backend. + ("eavR-two-numbers", "table", '"FilterExpression": "a = :a", "ExpressionAttributeValues": ' + '{":n": {"N": "fred"}, ":a": {"N": "bob"}}', NUMERIC_BOB.replace("bob", "fred")), + ("eavR-two-set-faults", "table", '"ExpressionAttributeValues": {":n": {"NS": []}, ":a": {"SS": []}}', + ("either", SS_EMPTY, NS_EMPTY)), + # Order R: a malformed number inside an entry the deserializer refuses comes + # after the nesting check of another entry; with the number ahead of the + # set fault in one L the service's two sends disagreed. The SS duplicates + # beside a nested value answered the duplicates text and the nesting text. + ("eavR-NSbobdup-and-nest", "table", FILTER_P + ', "ExpressionAttributeValues": ' + '{":p": {"S": "x"}, ":d": ' + NEST32_N + ', ":a": {"NS": ["bob", "1", "1"]}}', NESTING), + ("eavR-L-bob-then-NSe-and-nest", "table", FILTER_P + ', "ExpressionAttributeValues": ' + '{":p": {"S": "x"}, ":d": ' + NEST32_N + ', ":l": {"L": [{"N": "bob"}, {"NS": []}]}}', + ("either", NUMERIC_BOB, NESTING)), + ("eavR-SSdup-and-nest", "table", FILTER_P + ', "ExpressionAttributeValues": ' + '{":p": {"S": "x"}, ":d": ' + NEST32_N + ', ":s": {"SS": ["a", "a"]}}', + ("either", INVALID + 'Input collection ["a", "a"] contains duplicates.', NESTING)), + ("eavR-NSe-and-mix", "table", FILTER_P + ', "ScanFilter": {"a": {"ComparisonOperator": "EQ", ' + '"AttributeValueList": [{"N": "1"}]}}, "ExpressionAttributeValues": {":p": {"S": "x"}, ' + '":e": {"NS": []}}', MIX), + # Keys of five and seven members on the GSI (eight and sixteen buckets), and + # a key name outside the Basic Multilingual Plane. + ("key5-gpk-bob-gsk-empty", "t4_table", '"IndexName": "gsi2", "ExclusiveStartKey": ' + '{"gpk": {"N": "bob"}, "gsk": {}, "pk": {"S": "p"}, "sk": {"S": "s"}, "z": {"S": "z"}}', + NUMERIC_BOB), + ("key7-a-bob-sk-empty", "t4_table", '"IndexName": "gsi2", "ExclusiveStartKey": ' + '{"gpk": {"S": "g"}, "gsk": {"S": "h"}, "pk": {"S": "p"}, "sk": {}, "a": {"N": "bob"}, ' + '"b": {"S": "b"}, "c": {"S": "c"}}', NUMERIC_BOB), + ("keyU-supplementary-first", "v_table", + '"ExclusiveStartKey": {"\\ud834\\udd1e": {}, "a": {"N": "bob"}}', EMPTY_KEY_VALUE), + ("keyU-supplementary-second", "v_table", + '"ExclusiveStartKey": {"a": {"N": "bob"}, "\\ud834\\udd1e": {}}', NUMERIC_BOB), + # Order U, the ScanFilter lists: two descriptors, an empty value inside an + # L, which condition is named (the fitted order: pk ahead of sk, the + # request's order for ts and id in one bucket), which list value (the + # list's order), and ExpressionAttributeValues with no expression first. + ("sfU-two-descriptors", "table", '"ScanFilter": {"a": {"ComparisonOperator": "EQ", ' + '"AttributeValueList": [{"S": "x", "N": "1"}]}}', MORE_THAN_ONE_BARE), + ("sfU-empty-in-L", "table", '"ScanFilter": {"a": {"ComparisonOperator": "EQ", ' + '"AttributeValueList": [{"L": [{}]}]}}', EMPTY_VALUE_BARE), + ("sfU-two-conditions-pk-sk", "table", '"ScanFilter": {"pk": {"ComparisonOperator": "EQ", ' + '"AttributeValueList": [{}]}, "sk": {"ComparisonOperator": "EQ", "AttributeValueList": ' + '[{"S": "x", "N": "1"}]}}', MORE_THAN_ONE_BARE), + ("sfU-two-conditions-same-bucket", "table", '"ScanFilter": {"ts": {"ComparisonOperator": "EQ", ' + '"AttributeValueList": [{}]}, "id": {"ComparisonOperator": "EQ", "AttributeValueList": ' + '[{"S": "x", "N": "1"}]}}', EMPTY_VALUE_BARE), + ("sfU-list-two-values", "table", '"ScanFilter": {"a": {"ComparisonOperator": "BETWEEN", ' + '"AttributeValueList": [{"S": "x", "N": "1"}, {}]}}', MORE_THAN_ONE_BARE), + ("sfU-eav-noexpr-and-list-empty", "table", '"ExpressionAttributeValues": {":a": {"S": "x"}}, ' + '"ScanFilter": {"a": {"ComparisonOperator": "EQ", "AttributeValueList": [{}]}}', + ("startswith", EAV_NO_EXPRESSION)), + # Order R: a set fault in a ScanFilter list value, the deserializer's text. + ("sfR-NSe-in-list", "table", '"ScanFilter": {"a": {"ComparisonOperator": "CONTAINS", ' + '"AttributeValueList": [{"NS": []}]}}', NS_EMPTY), + # Order U: the empty BS of a key member keeps the key prefix beside an + # unreadable expression value, inside an L and in a member that is not a + # key attribute. + ("keyU-BSe-beside-eav-empty", "table", '"FilterExpression": "a = :a", ' + '"ExpressionAttributeValues": {":a": {}}, "ExclusiveStartKey": {"pk": {"BS": []}, ' + '"sk": {"S": "x"}}', KEY_PREFIX + INVALID + "Binary sets should not be empty"), + ("keyU-BSe-in-L-beside-pk-empty", "table", '"ExclusiveStartKey": {"pk": {}, "sk": {"L": ' + '[{"BS": []}]}}', KEY_PREFIX + INVALID + "Binary sets should not be empty"), + ("keyU-BSe-in-nonkey-member", "table", '"ExclusiveStartKey": {"pk": {}, "sk": {"S": "s"}, ' + '"b": {"BS": []}}', KEY_PREFIX + INVALID + "Binary sets should not be empty"), +] + + def unused_value(value: dict) -> str: """A filter that reads ``:p`` and ``value`` under ``:d``, which nothing reads.""" return ('"FilterExpression": "a = :p", "ExpressionAttributeValues": ' @@ -590,6 +810,48 @@ def it_table(dynamodb_client): wait_for_deleted(dynamodb_client, name) +@pytest.fixture(scope="module") +def t4_table(dynamodb_client): + """pk S, sk S, with a GSI ``gsi2`` on gpk S HASH and gsk S RANGE: a four-member index key.""" + name = unique_name("scanreqt4") + dynamodb_client.create_table( + TableName=name, + AttributeDefinitions=[{"AttributeName": n, "AttributeType": "S"} + for n in ("pk", "sk", "gpk", "gsk")], + KeySchema=[{"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"}], + GlobalSecondaryIndexes=[{ + "IndexName": "gsi2", + "KeySchema": [{"AttributeName": "gpk", "KeyType": "HASH"}, + {"AttributeName": "gsk", "KeyType": "RANGE"}], + "Projection": {"ProjectionType": "ALL"}, + }], + BillingMode="PAY_PER_REQUEST", + ) + wait_for_active(dynamodb_client, name) + yield name + dynamodb_client.delete_table(TableName=name) + wait_for_deleted(dynamodb_client, name) + + +@pytest.fixture(scope="module") +def u_table(dynamodb_client): + """HASH \u00e4 S, RANGE \u00e9 S: member names outside ASCII.""" + name = _two_member_table(dynamodb_client, "scanrequ", "\u00e4", "\u00e9") + yield name + dynamodb_client.delete_table(TableName=name) + wait_for_deleted(dynamodb_client, name) + + +@pytest.fixture(scope="module") +def v_table(dynamodb_client): + """HASH \U0001d11e S, RANGE a S: a member name outside the Basic Multilingual Plane.""" + name = _two_member_table(dynamodb_client, "scanreqv", "\U0001d11e", "a") + yield name + dynamodb_client.delete_table(TableName=name) + wait_for_deleted(dynamodb_client, name) + + class TestRequestChecks: """The parameter mix, the length constraint and the numbers in the key.""" @@ -769,6 +1031,37 @@ def test_bob_in_the_key_of_an_n_table(self, dynamodb_client, n_table): assert _validation_message(*_scan(dynamodb_client, n_table, fields)) == NUMERIC_BOB +# The two ScanFilter number rows: the measured answers are PR #507's check +# (#564 item 3, measured 2026-11-05), so they are expected to fail here until +# #507 lands. +SCAN_FILTER_NUMBER_ROWS = {"sfR-N-bob", "sfR-N-bob-and-key-bob"} + + +def _rows_2026_11_05_params(): + return [ + pytest.param(*r[1:], marks=pytest.mark.xfail(reason="#507")) + if r[0] in SCAN_FILTER_NUMBER_ROWS else r[1:] + for r in ROWS_2026_11_05 + ] + + +class TestRowsOf20261105: + """The rows measured 2026-11-05 at 05:31, 06:20 and 07:38 UTC.""" + + @pytest.mark.parametrize("fixture, fields, expected", _rows_2026_11_05_params(), + ids=[r[0] for r in ROWS_2026_11_05]) + def test_rows(self, dynamodb_client, request, fixture, fields, expected): + name = request.getfixturevalue(fixture) if fixture else unique_name("nosuch") + message = _validation_message(*_scan(dynamodb_client, name, fields)) + if isinstance(expected, tuple) and expected[0] == "startswith": + # The text predates this branch (#838): the check's place is asserted. + assert message.startswith(expected[1]), message + elif isinstance(expected, tuple): + assert message in expected[1:], message + else: + assert message == expected + + class TestNestingDepth: """``ExpressionAttributeValues`` nested in lists.""" From a64d846818321e19bba6fe560da39d2de6a4896c Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Fri, 6 Nov 2026 20:21:12 +0000 Subject: [PATCH 7/9] test(backup): pass the raw request body to dispatch in the SQLite backup tests Main adds three dispatch calls to the SQLite backup tests (#700), and this branch adds the raw_body argument to dispatch. The calls now pass an empty raw body, because only Scan reads it and these calls do not send a Scan. --- crates/storage-sqlite/src/backup_catalog.rs | 2 +- crates/storage-sqlite/tests/backup_job_panic.rs | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/storage-sqlite/src/backup_catalog.rs b/crates/storage-sqlite/src/backup_catalog.rs index 562f94b0b..254acbf3a 100644 --- a/crates/storage-sqlite/src/backup_catalog.rs +++ b/crates/storage-sqlite/src/backup_catalog.rs @@ -1397,7 +1397,7 @@ mod write_lock_tests { operation: &str, body: serde_json::Value, ) -> serde_json::Value { - dispatch(operation, body, ctx, "127.0.0.1:0") + dispatch(operation, body, &[], ctx, "127.0.0.1:0") .await .unwrap_or_else(|e| panic!("{operation}: {e:?}")) .body diff --git a/crates/storage-sqlite/tests/backup_job_panic.rs b/crates/storage-sqlite/tests/backup_job_panic.rs index d366de136..281582629 100644 --- a/crates/storage-sqlite/tests/backup_job_panic.rs +++ b/crates/storage-sqlite/tests/backup_job_panic.rs @@ -101,7 +101,7 @@ async fn call( operation: &str, body: serde_json::Value, ) -> serde_json::Value { - dispatch(operation, body, ctx, "127.0.0.1:0") + dispatch(operation, body, &[], ctx, "127.0.0.1:0") .await .unwrap_or_else(|e| panic!("{operation}: {e:?}")) .body @@ -316,6 +316,7 @@ async fn a_job_task_that_panics_leaves_a_row_the_next_cycle_resolves_without_a_r let described = dispatch( "DescribeBackup", json!({"BackupArn": arn_text}), + &[], &ctx, "127.0.0.1:0", ) From 06a4a4000ab09403917ef6b060e07cef0f7b0d6a Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Sat, 7 Nov 2026 01:22:39 +0000 Subject: [PATCH 8/9] test(backup): pass an empty raw body to dispatch in the SQLite restore test Main adds the SQLite restore test (#740), which calls dispatch with four arguments, and this branch adds the raw_body argument to dispatch. The call now passes an empty raw body. Only Scan reads the raw body, for the request order of the members of ExclusiveStartKey, ExpressionAttributeValues and ScanFilter. With an empty raw body, the Scan checks use the order of the parsed body. The order sets which fault the Scan reports first. The Scan calls in this test send no faults. Each one sends TableName and Select, with IndexName for an index and, after the first page, the LastEvaluatedKey of the previous page as ExclusiveStartKey. --- crates/storage-sqlite/tests/restore_job_resume.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/storage-sqlite/tests/restore_job_resume.rs b/crates/storage-sqlite/tests/restore_job_resume.rs index 2b1a5dd81..ebad4cbb5 100644 --- a/crates/storage-sqlite/tests/restore_job_resume.rs +++ b/crates/storage-sqlite/tests/restore_job_resume.rs @@ -62,7 +62,7 @@ async fn call( operation: &str, body: serde_json::Value, ) -> serde_json::Value { - dispatch(operation, body, ctx, "127.0.0.1:0") + dispatch(operation, body, &[], ctx, "127.0.0.1:0") .await .unwrap_or_else(|e| panic!("{operation}: {e:?}")) .body From f30a2cd3d842462cd966133aec51211ca3a62d4f Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Sat, 7 Nov 2026 01:48:00 +0000 Subject: [PATCH 9/9] docs: name the three members whose order Scan reads from raw_body The doc comment of dispatch said that Scan reads the member order of ExclusiveStartKey from raw_body. handle_scan also reads the member order of ExpressionAttributeValues and ScanFilter from it. --- crates/engine/src/lib.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 2ea21f16a..ecd644b84 100755 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -442,8 +442,9 @@ impl OperationContext { /// Dispatch an operation by name. /// -/// `raw_body` is the request text `body` was parsed from; `Scan` reads the -/// member order of its `ExclusiveStartKey` from it. +/// `raw_body` is the request text `body` was parsed from. `Scan` reads the +/// member order of its `ExpressionAttributeValues`, `ExclusiveStartKey` and +/// `ScanFilter` from it. /// /// Returns the JSON response body and sideband metrics for the metrics collector. pub async fn dispatch(