From af8e8a73cc03a1fe0352701ea0692d58c9fc8c6a Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Tue, 3 Nov 2026 02:02:54 +0000 Subject: [PATCH 1/2] fix(server): read request members in sorted order on every build The MongoDB build depends on bson, which turns on serde_json's preserve_order feature, and Cargo enables it for every crate in that binary. A parsed request object there keeps the order the client wrote its members in; the SQLite, PostgreSQL and Cassandra builds sort them by name. The typed input deserializers stop at the first member that fails, so a body with two faults got a different error on the MongoDB build, and which one depended on the client's member order. The data-plane handler now calls Value::sort_all_objects on the parsed body after authentication, which reads the raw bytes. The call sorts every object in the body when preserve_order is on and does nothing otherwise, so every build reads the members in the sorted order the others already used. Amazon DynamoDB answered both orders of each two-fault body in the new test with the same error (us-east-1, 2026-11-03). The test sends each body in both orders, signed, and passes against the service. Fixes #723 --- crates/server/src/handler.rs | 14 +++- tests/test_request_member_order.py | 110 +++++++++++++++++++++++++++++ 2 files changed, 123 insertions(+), 1 deletion(-) create mode 100644 tests/test_request_member_order.py diff --git a/crates/server/src/handler.rs b/crates/server/src/handler.rs index ef3765bda..32355c26f 100755 --- a/crates/server/src/handler.rs +++ b/crates/server/src/handler.rs @@ -77,7 +77,7 @@ pub(crate) async fn handle_request( // --- Pre-auth body validation --- // Real DynamoDB validates request format (empty body, invalid JSON) before // authentication. Malformed requests get 400 regardless of auth state. - let input: Value = match serde_json::from_slice(&body) { + let mut input: Value = match serde_json::from_slice(&body) { Ok(v) => v, Err(e) => { return error_response( @@ -98,6 +98,18 @@ pub(crate) async fn handle_request( #[allow(clippy::cast_precision_loss)] let auth_us = auth_start.elapsed().as_micros() as f64; + // A build with the MongoDB backend gets serde_json's `preserve_order` + // feature through `bson`, so there a parsed object keeps the order the + // client wrote its members in; every other build sorts them by name. The + // typed input deserializers stop at the first member that fails, so on the + // MongoDB build a body with two faults got an error chosen by the client's + // member order. DynamoDB answered each two-fault Query and Scan body + // measured on 2026-11-03 alike in both orders. Sorting gives every build + // the sorted order; without `preserve_order` the call does nothing. It runs + // after authentication, which reads the raw bytes, so an unauthenticated + // caller pays only for the parse. + input.sort_all_objects(); + let account_id: Arc = match &identity { extenddb_auth::AuthIdentity::User { account_id, .. } | extenddb_auth::AuthIdentity::RoleSession { account_id, .. } => { diff --git a/tests/test_request_member_order.py b/tests/test_request_member_order.py new file mode 100644 index 000000000..b431b72ae --- /dev/null +++ b/tests/test_request_member_order.py @@ -0,0 +1,110 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""The order of a request body's members does not change the error it gets. + +Each body below has two faults that ExtendDB's typed input deserializer +catches in different members. A server that reads the members in the order +the client wrote them answers the two orders with different errors; the +MongoDB build did, because ``bson`` turns on ``serde_json``'s +``preserve_order`` feature for the whole binary. Amazon DynamoDB answered +both orders of each body with the same error (us-east-1, 2026-11-03). + +The bodies are posted as written, signed with the credentials the client +signs with, because botocore refuses these values before it sends them. + +Dual-target against Amazon DynamoDB and ExtendDB. +""" + +from __future__ import annotations + +import json +from typing import Any + +import pytest +import requests +from botocore.auth import SigV4Auth +from botocore.awsrequest import AWSRequest + +from conftest import scoped_table + +P = {"S": "p0"} + + +@pytest.fixture(scope="module") +def hash_table(dynamodb_client): + """Hash-only table for the module, deleted on teardown.""" + with scoped_table(dynamodb_client) as name: + yield name + + +def _answer(client, endpoint_url: str | None, operation: str, body: dict[str, Any]): + """POST ``body`` with its members in insertion order and return the HTTP + status, the error code and the message.""" + url = endpoint_url or client.meta.endpoint_url + payload = json.dumps(body).encode("utf-8") + headers = { + "X-Amz-Target": f"DynamoDB_20120810.{operation}", + "Content-Type": "application/x-amz-json-1.0", + } + request = AWSRequest(method="POST", url=url, data=payload, headers=headers) + creds = client._request_signer._credentials # noqa: SLF001 - test helper + SigV4Auth(creds.get_frozen_credentials(), "dynamodb", client.meta.region_name).add_auth( + request + ) + # The certificate `extenddb init` writes is self-signed; the service's is not. + resp = requests.post( + url, data=payload, headers=dict(request.headers), verify=endpoint_url is None, timeout=30 + ) + doc = resp.json() + return ( + resp.status_code, + doc.get("__type", "").rsplit("#", 1)[-1], + doc.get("message", doc.get("Message", "")), + ) + + +@pytest.mark.parametrize( + "limit, value", + [ + pytest.param(0.0, {}, id="float_limit_empty_value"), + pytest.param("0", {}, id="string_limit_empty_value"), + ], +) +def test_query_limit_and_value_faults_answer_alike_in_either_order( + dynamodb_client, endpoint_url, hash_table, limit, value +): + values = {":p": P, ":a": value} + head = {"TableName": hash_table, "KeyConditionExpression": "pk = :p"} + limit_first = _answer( + dynamodb_client, + endpoint_url, + "Query", + {**head, "Limit": limit, "ExpressionAttributeValues": values}, + ) + values_first = _answer( + dynamodb_client, + endpoint_url, + "Query", + {**head, "ExpressionAttributeValues": values, "Limit": limit}, + ) + assert limit_first[0] == 400, limit_first + assert values_first == limit_first + + +def test_two_invalid_values_answer_alike_in_either_order(dynamodb_client, endpoint_url, hash_table): + head = {"TableName": hash_table, "FilterExpression": "pk IN (:p, :a, :b)"} + b_first = _answer( + dynamodb_client, + endpoint_url, + "Scan", + {**head, "ExpressionAttributeValues": {":p": P, ":b": {}, ":a": {"X": "1"}}}, + ) + a_first = _answer( + dynamodb_client, + endpoint_url, + "Scan", + {**head, "ExpressionAttributeValues": {":p": P, ":a": {"X": "1"}, ":b": {}}}, + ) + assert b_first[0] == 400, b_first + assert a_first == b_first From 1ddf02d93507efa59f83da5b40127e0e93c04d95 Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Tue, 3 Nov 2026 02:42:09 +0000 Subject: [PATCH 2/2] fix(server): sort the parsed request beside the parse The first commit sorts the body after authentication. Open pull requests #587, #549 and #547 edit the lines around the end of the authentication segment, so there the call conflicts with all three. This commit moves it to immediately after the parse, where it merges cleanly with the nine open pull requests that edit handler.rs. On a preserve_order build an unauthenticated request now pays for the sort, as it already pays for the parse. Without preserve_order the call does nothing. The comment now says which builds have preserve_order: any build that compiles the MongoDB backend crate, which includes a build of the whole workspace, not only the MongoDB binary. --- crates/server/src/handler.rs | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/crates/server/src/handler.rs b/crates/server/src/handler.rs index 32355c26f..2cd7fa6f8 100755 --- a/crates/server/src/handler.rs +++ b/crates/server/src/handler.rs @@ -88,6 +88,16 @@ pub(crate) async fn handle_request( ); } }; + // Any build that compiles the MongoDB backend crate (the MongoDB binary, + // or a build of the whole workspace) gets serde_json's `preserve_order` + // feature through `bson`, so there a parsed object keeps the order the + // client wrote its members in; other builds sort them by name. The typed + // input deserializers stop at the first member that fails, so on such a + // build a body with two faults got an error chosen by the client's member + // order. DynamoDB answered each two-fault Query and Scan body measured on + // 2026-11-03 alike in both orders. Sorting gives every build the sorted + // order; without `preserve_order` the call does nothing. + input.sort_all_objects(); // --- Auth segment --- let auth_start = std::time::Instant::now(); @@ -98,18 +108,6 @@ pub(crate) async fn handle_request( #[allow(clippy::cast_precision_loss)] let auth_us = auth_start.elapsed().as_micros() as f64; - // A build with the MongoDB backend gets serde_json's `preserve_order` - // feature through `bson`, so there a parsed object keeps the order the - // client wrote its members in; every other build sorts them by name. The - // typed input deserializers stop at the first member that fails, so on the - // MongoDB build a body with two faults got an error chosen by the client's - // member order. DynamoDB answered each two-fault Query and Scan body - // measured on 2026-11-03 alike in both orders. Sorting gives every build - // the sorted order; without `preserve_order` the call does nothing. It runs - // after authentication, which reads the raw bytes, so an unauthenticated - // caller pays only for the parse. - input.sort_all_objects(); - let account_id: Arc = match &identity { extenddb_auth::AuthIdentity::User { account_id, .. } | extenddb_auth::AuthIdentity::RoleSession { account_id, .. } => {