diff --git a/Cargo.lock b/Cargo.lock index c033bcd24..37c0a339e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1772,6 +1772,7 @@ dependencies = [ "extenddb-auth", "extenddb-core", "extenddb-storage", + "extenddb-storage-sqlite", "futures", "hex", "hmac 0.12.1", diff --git a/crates/auth/src/policy/context.rs b/crates/auth/src/policy/context.rs index 85ca0288f..79f1cfc7b 100755 --- a/crates/auth/src/policy/context.rs +++ b/crates/auth/src/policy/context.rs @@ -183,6 +183,10 @@ pub struct RequestParams { pub return_consumed_capacity: Option, /// The enclosing operation for batch/transact sub-operations. pub enclosing_operation: Option, + /// `dynamodb:FullTableScan` for a request that is not a scan + /// (`RequestContext::build` sets it to `true` for one). `None` leaves the + /// key absent; a PartiQL read by key sets `false`. + pub full_table_scan: Option, /// The caller ARN (`aws:PrincipalArn`), when known. pub principal_arn: Option, /// The caller's account id (`aws:PrincipalAccount`), when known. @@ -299,7 +303,8 @@ impl RequestContext { /// Build context for a `DynamoDB` operation. /// /// `principal_tags` and `resource_tags` come from the identity and target - /// table respectively. `is_scan` should be true for Scan operations. + /// table respectively. `is_scan` should be true for Scan operations; for + /// any other request `params.full_table_scan` gives the key. /// `params` carries operation-specific request parameters. #[must_use] pub fn build( @@ -317,7 +322,11 @@ impl RequestContext { select: params.select, return_values: params.return_values, return_consumed_capacity: params.return_consumed_capacity, - full_table_scan: if is_scan { Some(true) } else { None }, + full_table_scan: if is_scan { + Some(true) + } else { + params.full_table_scan + }, enclosing_operation: params.enclosing_operation, principal_arn: params.principal_arn, principal_account: params.principal_account, @@ -997,6 +1006,25 @@ mod tests { ); } + #[test] + fn request_context_full_table_scan_set_by_the_request() { + // A PartiQL read by key carries "false" (measured 2026-11-03); a + // request that sets nothing leaves the key absent unless it is a scan. + let ctx = RequestContext::build( + HashMap::new(), + HashMap::new(), + false, + RequestParams { + full_table_scan: Some(false), + ..Default::default() + }, + ); + assert_eq!( + ctx.resolve_key("dynamodb:FullTableScan"), + Some(vec!["false"]) + ); + } + #[test] fn assume_role_context_external_id() { let ctx = ar_ctx(HashMap::new(), Some("ext-123".to_owned()), HashMap::new()); diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 7e04256bf..75495dda6 100755 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -18,3 +18,9 @@ md-5 = { workspace = true } sha2 = { workspace = true } time = { workspace = true } uuid = { workspace = true } + +[features] +default = ["partiql"] +# PartiQL grammar, AST and query planning. Removable: with this off, the +# `partiql` module is not compiled and nothing else in core references it. +partiql = [] diff --git a/crates/core/src/error/mod.rs b/crates/core/src/error/mod.rs index 071d15278..2e27ccb30 100755 --- a/crates/core/src/error/mod.rs +++ b/crates/core/src/error/mod.rs @@ -57,6 +57,13 @@ pub enum DynamoDbError { LimitExceededException(String), #[error("{0}")] ConditionalCheckFailedException(String, Option), + /// A PartiQL `INSERT` targeted a primary key that already exists. + /// + /// `INSERT` is create-only, so a duplicate full key fails with this error + /// rather than `ConditionalCheckFailedException`. HTTP 400. The message is + /// always `Duplicate primary key exists in table`, measured on the service. + #[error("{0}")] + DuplicateItemException(String), #[error("{message}")] TransactionCanceledException { /// Human-readable message. @@ -157,6 +164,7 @@ impl DynamoDbError { | Self::TableNotFoundException(_) | Self::LimitExceededException(_) | Self::ConditionalCheckFailedException(..) + | Self::DuplicateItemException(_) | Self::TransactionCanceledException { .. } | Self::IdempotentParameterMismatchException(_) | Self::SerializationException(_) @@ -204,6 +212,7 @@ impl DynamoDbError { Self::TableNotFoundException(_) => "TableNotFoundException", Self::LimitExceededException(_) => "LimitExceededException", Self::ConditionalCheckFailedException(..) => "ConditionalCheckFailedException", + Self::DuplicateItemException(_) => "DuplicateItemException", Self::TransactionCanceledException { .. } => "TransactionCanceledException", Self::IdempotentParameterMismatchException(_) => "IdempotentParameterMismatchException", Self::SerializationException(_) => "SerializationException", @@ -280,6 +289,7 @@ impl DynamoDbError { | Self::TableNotFoundException(m) | Self::LimitExceededException(m) | Self::ConditionalCheckFailedException(m, _) + | Self::DuplicateItemException(m) | Self::IdempotentParameterMismatchException(m) | Self::SerializationException(m) | Self::UnknownOperationException(m) diff --git a/crates/core/src/expression/resolver.rs b/crates/core/src/expression/resolver.rs index 699cd4fdf..3c7414d5d 100755 --- a/crates/core/src/expression/resolver.rs +++ b/crates/core/src/expression/resolver.rs @@ -30,6 +30,11 @@ pub struct ExpressionMaps { /// and was desugared into an update expression. The service sizes that /// form against the item limit differently from an expression. pub legacy_attribute_updates: bool, + /// Set by the engine for an update translated from a PartiQL `UPDATE`. + /// The service sizes that statement by its finished item alone, as it + /// sizes a transacted Update, and not by the statement as it sizes + /// `UpdateItem`. + pub finished_item_sizing: bool, /// Pre-parsed `BigDecimal` values for numeric placeholders. /// Populated eagerly by `new()` and refreshed by `pre_parse_numerics()`. parsed_numerics: HashMap, @@ -45,6 +50,7 @@ impl ExpressionMaps { names, values, legacy_attribute_updates: false, + finished_item_sizing: false, parsed_numerics: HashMap::new(), }; maps.pre_parse_numerics(); diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index c0891b01a..35fd2d055 100755 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -13,6 +13,8 @@ pub mod error; pub mod expression; pub mod limits; pub mod metrics; +#[cfg(feature = "partiql")] +pub mod partiql; pub mod serde_helpers; pub mod settings_keys; pub mod throttle; diff --git a/crates/core/src/partiql/ast.rs b/crates/core/src/partiql/ast.rs new file mode 100644 index 000000000..91d231f8d --- /dev/null +++ b/crates/core/src/partiql/ast.rs @@ -0,0 +1,539 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! PartiQL abstract syntax tree. +//! +//! One `Statement` per `ExecuteStatement`. The grammar and every node here is +//! justified by behaviour measured against the live service, cited where it +//! is implemented. The planner (`super::plan`) turns a `Statement` +//! into a plan expressed in terms of the operations the engine already has, so +//! nothing here introduces a new storage concept. + +use crate::types::AttributeValue; + +/// A parsed PartiQL statement. +#[derive(Debug, Clone, PartialEq)] +pub enum Statement { + Select(SelectStatement), + Insert(InsertStatement), + Update(UpdateStatement), + Delete(DeleteStatement), + /// `EXISTS(SELECT ...)`: a condition check on one item, accepted as an + /// `ExecuteTransaction` member. + Exists(SelectStatement), +} + +impl Statement { + /// Whether the statement reads (`SELECT`) or writes. Batch and transaction + /// requests must be all-read or all-write, so this classifies each member. + /// An `EXISTS` condition check counts with the writes, as a + /// `ConditionCheck` does in `TransactWriteItems`. + #[must_use] + pub fn is_read(&self) -> bool { + matches!(self, Statement::Select(_)) + } + + /// How many `?` parameters the statement reads: one past the highest + /// index the parser gave a placeholder, since it numbers them in the + /// order they are written. + #[must_use] + pub fn parameter_count(&self) -> usize { + let mut count = 0; + match self { + Statement::Select(select) | Statement::Exists(select) => { + if let Some(where_clause) = &select.where_clause { + where_parameters(where_clause, &mut count); + } + } + Statement::Insert(insert) => { + for (_, value) in &insert.value { + operand_parameters(value, &mut count); + } + } + Statement::Update(update) => { + for action in &update.sets { + operand_parameters(&action.value, &mut count); + } + for action in &update.set_members { + operand_parameters(&action.members, &mut count); + } + if let Some(where_clause) = &update.where_clause { + where_parameters(where_clause, &mut count); + } + } + Statement::Delete(delete) => where_parameters(&delete.where_clause, &mut count), + } + count + } +} + +fn where_parameters(expr: &WhereExpr, count: &mut usize) { + match expr { + WhereExpr::Compare(left, _, right) + | WhereExpr::BeginsWith(left, right) + | WhereExpr::Contains(left, right) + | WhereExpr::AttributeType(left, right) => { + operand_parameters(left, count); + operand_parameters(right, count); + } + WhereExpr::Between(value, low, high) => { + for operand in [value, low, high] { + operand_parameters(operand, count); + } + } + WhereExpr::In(value, items) => { + operand_parameters(value, count); + for item in items { + operand_parameters(item, count); + } + } + WhereExpr::Function(_, args) => { + for arg in args { + operand_parameters(arg, count); + } + } + WhereExpr::IsType(operand, ..) => operand_parameters(operand, count), + WhereExpr::IsMissing(..) => {} + WhereExpr::And(left, right) | WhereExpr::Or(left, right) => { + where_parameters(left, count); + where_parameters(right, count); + } + WhereExpr::Not(inner) | WhereExpr::ComparedWithBoolean(inner, _) => { + where_parameters(inner, count); + } + } +} + +fn operand_parameters(operand: &Operand, count: &mut usize) { + match operand { + Operand::Parameter(index) => *count = (*count).max(index + 1), + Operand::Arith(left, _, right) => { + operand_parameters(left, count); + operand_parameters(right, count); + } + Operand::Function(_, args) => { + for arg in args { + operand_parameters(arg, count); + } + } + Operand::Literal(_) | Operand::Path(_) => {} + } +} + +/// `SELECT FROM [WHERE ] [ORDER BY [ASC|DESC], ...]`. +#[derive(Debug, Clone, PartialEq)] +pub struct SelectStatement { + pub projection: Projection, + pub from: FromSource, + pub where_clause: Option, + pub order_by: Option, + /// The refusal for a projection item that is a call (`COUNT(*)`), when + /// there is one. The statement parses; the service reports the refusal + /// once it has resolved the target, so a multi-attribute index key is + /// reported first, and in place of a missing table (measured 2026-11-02). + pub projection_error: Option, +} + +/// `INSERT INTO VALUE { ... }`. +#[derive(Debug, Clone, PartialEq)] +pub struct InsertStatement { + pub table: String, + pub value: Vec<(String, Operand)>, +} + +/// `UPDATE
SET ... REMOVE ... WHERE [AND ] [RETURNING ]`, +/// or the FROM-first form `FROM
[WHERE ...] SET ... REMOVE ... [RETURNING ]`. +#[derive(Debug, Clone, PartialEq)] +pub struct UpdateStatement { + pub table: String, + pub sets: Vec, + pub removes: Vec, + /// `SET a = set_add(a, ...)` and `SET a = set_delete(a, ...)` assignments. + pub set_members: Vec, + /// `None` only in the FROM-first form, whose grammar makes WHERE optional; + /// execution refuses it with the full-key message. + pub where_clause: Option, + pub returning: Option, +} + +/// `DELETE FROM
WHERE [AND ] [RETURNING ALL OLD *]`. +#[derive(Debug, Clone, PartialEq)] +pub struct DeleteStatement { + pub table: String, + pub where_clause: WhereExpr, + pub returning: Option, +} + +/// The attributes a `SELECT` returns. +#[derive(Debug, Clone, PartialEq)] +pub enum Projection { + /// `SELECT *`. + All, + /// `SELECT a, b.c, d[0]`. The projected key naming follows the service: + /// a plain or nested attribute keys on its leaf, a list-index path keys on + /// the literal path text. + Paths(Vec), +} + +/// A `FROM` target: a table, optionally an index segment (`"table"."index"`). +#[derive(Debug, Clone, PartialEq)] +pub struct FromSource { + pub table: String, + pub index: Option, +} + +/// A document path: a first attribute then zero or more attribute or index steps. +#[derive(Debug, Clone, PartialEq)] +pub struct Path { + pub root: String, + pub steps: Vec, +} + +#[derive(Debug, Clone, PartialEq)] +pub enum PathStep { + Attribute(String), + Index(usize), +} + +impl Path { + /// The single attribute name when the path has no steps, else `None`. + #[must_use] + pub fn as_simple(&self) -> Option<&str> { + self.steps.is_empty().then_some(self.root.as_str()) + } +} + +/// A `SET` target and its value expression. +#[derive(Debug, Clone, PartialEq)] +pub struct SetAction { + pub path: Path, + pub value: Operand, +} + +/// `SET a = set_add(a, )` or `SET a = set_delete(a, )`: add the +/// members of `` to the set `a` holds, or remove them from it. An add to +/// a missing attribute creates the set, a delete from one leaves it missing, +/// and removing every member removes the attribute. +#[derive(Debug, Clone, PartialEq)] +pub struct SetMembersAction { + pub path: Path, + pub op: SetMembersOp, + /// The members: a set literal, or a `?` parameter that must bind to a set. + pub members: Operand, + /// `line:column:length` of the function's first argument, which the + /// refusal of a value that is not a set names. + pub first_argument_at: String, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SetMembersOp { + Add, + Delete, +} + +impl SetMembersOp { + /// The function name as the service writes it in messages. + #[must_use] + pub fn function_name(self) -> &'static str { + match self { + SetMembersOp::Add => "SET_ADD", + SetMembersOp::Delete => "SET_DELETE", + } + } + + /// The refusal of a second argument that is not a set, positioned at the + /// first argument (`line:column:length`). + #[must_use] + pub fn not_a_set_message(self, first_argument_at: &str) -> String { + format!( + "The second argument to {} must be a value with type SET at {first_argument_at}", + self.function_name() + ) + } +} + +/// An operand in a value or condition position. +#[derive(Debug, Clone, PartialEq)] +pub enum Operand { + /// A literal attribute value (`'s'`, `1`, `true`, `null`, `{...}`, `[...]`, `<<...>>`). + Literal(AttributeValue), + /// A positional `?` parameter, resolved from the request `Parameters` by index. + Parameter(usize), + /// A document path (an attribute reference). + Path(Path), + /// `left + right` or `left - right` (numbers only). + Arith(Box, ArithOp, Box), + /// A function call: `if_not_exists(path, operand)`, `list_append(a, b)`, etc. + Function(String, Vec), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ArithOp { + Add, + Sub, +} + +/// The `WHERE` expression: a boolean tree over comparisons and functions. Used +/// both for `SELECT` planning and as the condition on a write (where any term +/// beyond the key equalities becomes a condition expression on the item). +#[derive(Debug, Clone, PartialEq)] +pub enum WhereExpr { + Compare(Operand, CompareOp, Operand), + Between(Operand, Operand, Operand), + In(Operand, Vec), + BeginsWith(Operand, Operand), + Contains(Operand, Operand), + AttributeType(Operand, Operand), + Function(String, Vec), + IsMissing(Path, bool), + /// `operand IS [NOT] ` for any type but MISSING; the flag is true for + /// `IS NOT`. + IsType(Operand, TypeName, bool), + And(Box, Box), + Or(Box, Box), + Not(Box), + /// ` = true`, `<> true`, `= false` or `<> false`. As a + /// condition it holds where the condition does, or where it does not for + /// `= false` and `<> true` (see [`BooleanComparison::negates`]). It is + /// kept as it was written for the key reading, where a key condition + /// compared with a boolean is read by key and the literal alone decides + /// which items it returns (see `plan`). + ComparedWithBoolean(Box, BooleanComparison), +} + +/// How a condition is compared with a boolean literal. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct BooleanComparison { + /// `=` rather than `<>`. + pub equal: bool, + /// The literal, `true` or `false`. + pub literal: bool, +} + +impl BooleanComparison { + /// Whether the comparison holds where the condition does not: `= false` + /// and `<> true`. + #[must_use] + pub fn negates(self) -> bool { + self.equal != self.literal + } +} + +impl WhereExpr { + /// The condition under a run of `NOT`s and comparisons with a boolean, + /// and whether the run negates it (an odd number of `NOT`s, `= false`s + /// and `<> true`s). A condition is true or false, so two negations + /// cancel: a translation can write the run as one `NOT` or none, which + /// keeps a WHERE of 299 `NOT`s (which runs, measured 2026-11-02) within + /// the classic parser's nesting limit. + #[must_use] + pub fn without_nots(&self) -> (bool, &WhereExpr) { + let mut negated = false; + let mut expr = self; + loop { + expr = match expr { + WhereExpr::Not(inner) => { + negated = !negated; + inner + } + WhereExpr::ComparedWithBoolean(inner, comparison) => { + negated ^= comparison.negates(); + inner + } + _ => return (negated, expr), + }; + } + } + + /// The condition under a run of comparisons with a boolean that keep it + /// (`= true` and `<> false`). + #[must_use] + pub fn without_kept_comparisons(&self) -> &WhereExpr { + let mut expr = self; + while let WhereExpr::ComparedWithBoolean(inner, comparison) = expr + && !comparison.negates() + { + expr = inner; + } + expr + } +} + +/// A type name the service's parser accepts after `IS`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TypeName { + Missing, + Null, + Boolean, + Smallint, + Integer4, + Integer8, + Integer, + Float, + Real, + DoublePrecision, + Decimal, + Numeric, + Timestamp, + Date, + Time, + Character, + CharacterVarying, + String, + Symbol, + Clob, + Blob, + Struct, + Tuple, + List, + Sexp, + Bag, + Any, +} + +impl TypeName { + /// The type named by a one-word type keyword (lower case), aliases + /// included (`bool`, `int`, `int4`, `bigint`, `varchar`, ...). + #[must_use] + pub fn from_keyword(word: &str) -> Option { + Some(match word { + "missing" => Self::Missing, + "null" => Self::Null, + "bool" | "boolean" => Self::Boolean, + "smallint" | "int2" | "integer2" => Self::Smallint, + "int4" | "integer4" => Self::Integer4, + "int8" | "integer8" | "bigint" => Self::Integer8, + "int" | "integer" => Self::Integer, + "float" => Self::Float, + "real" => Self::Real, + "decimal" | "dec" => Self::Decimal, + "numeric" => Self::Numeric, + "timestamp" => Self::Timestamp, + "date" => Self::Date, + "time" => Self::Time, + "character" | "char" => Self::Character, + "varchar" => Self::CharacterVarying, + "string" => Self::String, + "symbol" => Self::Symbol, + "clob" => Self::Clob, + "blob" => Self::Blob, + "struct" => Self::Struct, + "tuple" => Self::Tuple, + "list" => Self::List, + "sexp" => Self::Sexp, + "bag" => Self::Bag, + "any" => Self::Any, + _ => return None, + }) + } + + /// The `attribute_type` code a filter on this type tests for, or `None` + /// for a type the service does not filter on (MISSING and NULL have + /// filters of their own). + #[must_use] + pub fn attribute_type(self) -> Option<&'static str> { + match self { + Self::Boolean => Some("BOOL"), + Self::Numeric => Some("N"), + Self::String => Some("S"), + Self::Blob => Some("B"), + Self::Struct | Self::Tuple => Some("M"), + Self::List => Some("L"), + _ => None, + } + } + + /// The refusal of an `IS` filter on a type with no filter. + #[must_use] + pub fn unsupported_filter_message(self) -> String { + format!( + "Unsupported data type for IS operator: {}. Supported data types are [MISSING, \ + NULL, BOOLEAN, BLOB, STRING, NUMERIC, STRUCT, TUPLE, LIST]", + self.service_name() + ) + } + + /// The name the service uses for the type in its refusals. + #[must_use] + pub fn service_name(self) -> &'static str { + match self { + Self::Missing => "MissingType", + Self::Null => "NullType", + Self::Boolean => "BooleanType", + Self::Smallint => "SmallintType", + Self::Integer4 => "Integer4Type", + Self::Integer8 => "Integer8Type", + Self::Integer => "IntegerType", + Self::Float => "FloatType", + Self::Real => "RealType", + Self::DoublePrecision => "DoublePrecisionType", + Self::Decimal => "DecimalType", + Self::Numeric => "NumericType", + Self::Timestamp => "TimestampType", + Self::Date => "DateType", + Self::Time => "TimeType", + Self::Character => "CharacterType", + Self::CharacterVarying => "CharacterVaryingType", + Self::String => "StringType", + Self::Symbol => "SymbolType", + Self::Clob => "ClobType", + Self::Blob => "BlobType", + Self::Struct => "StructType", + Self::Tuple => "TupleType", + Self::List => "ListType", + Self::Sexp => "SexpType", + Self::Bag => "BagType", + Self::Any => "AnyType", + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CompareOp { + Eq, + Ne, + Lt, + Le, + Gt, + Ge, +} + +/// `ORDER BY [ASC | DESC], ...`: one or more paths, each with its +/// direction. +#[derive(Debug, Clone, PartialEq)] +pub struct OrderBy { + /// The paths in statement order; never empty. + pub terms: Vec, +} + +/// One path of an `ORDER BY` list and its direction. +#[derive(Debug, Clone, PartialEq)] +pub struct OrderTerm { + pub path: Path, + pub descending: bool, + /// The `line:column:length` span of the path's first token, where the + /// refusal of a path with steps names it. + pub at: String, +} + +impl OrderBy { + /// The first term that names the attribute `name` by itself. + #[must_use] + pub fn term_for(&self, name: &str) -> Option<&OrderTerm> { + self.terms + .iter() + .find(|term| term.path.as_simple() == Some(name)) + } +} + +/// The `RETURNING` clause of a write. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Returning { + AllOld, + ModifiedOld, + AllNew, + ModifiedNew, +} + +#[cfg(test)] +mod tests; diff --git a/crates/core/src/partiql/ast/tests.rs b/crates/core/src/partiql/ast/tests.rs new file mode 100644 index 000000000..efb8de41b --- /dev/null +++ b/crates/core/src/partiql/ast/tests.rs @@ -0,0 +1,52 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Unit tests for the statement tree. + +use super::{Statement, WhereExpr}; +use crate::partiql::parse; + +fn count(sql: &str) -> usize { + parse(sql).unwrap().parameter_count() +} + +#[test] +fn the_parameter_count_covers_every_position() { + for (sql, expected) in [ + ("SELECT * FROM \"t\" WHERE pk = 'a'", 0), + ("SELECT * FROM \"t\" WHERE pk = ? AND sk BETWEEN ? AND ?", 3), + ( + "SELECT * FROM \"t\" WHERE pk IN [?, ?] OR begins_with(a, ?)", + 3, + ), + ("INSERT INTO \"t\" VALUE {'pk': ?, 'a': 'x', 'b': ?}", 2), + ( + "UPDATE \"t\" SET a = ?, b = list_append(b, ?) SET ss = set_add(ss, ?) \ + WHERE pk = ? AND NOT c = ?", + 5, + ), + ("DELETE FROM \"t\" WHERE pk = ? AND x <> ?", 2), + ("EXISTS(SELECT * FROM \"t\" WHERE pk = ? AND a = ?)", 2), + // A `?` inside a string is text. + ("UPDATE \"t\" SET a = '?' WHERE pk = 'k'", 0), + ] { + assert_eq!(count(sql), expected, "{sql}"); + } +} + +#[test] +fn a_run_of_nots_comes_off_with_its_parity() { + let condition = |sql: &str| match parse(sql).unwrap() { + Statement::Select(s) => s.where_clause.unwrap(), + other => panic!("expected a select, got {other:?}"), + }; + for (nots, negated) in [(1, true), (2, false), (3, true), (299, true)] { + let sql = format!("SELECT * FROM \"t\" WHERE {}a = 1", "NOT ".repeat(nots)); + let expr = condition(&sql); + let (odd, inner) = expr.without_nots(); + assert_eq!(odd, negated, "{nots}"); + assert!(matches!(inner, WhereExpr::Compare(..)), "{nots}"); + } + let expr = condition("SELECT * FROM \"t\" WHERE a = 1"); + assert_eq!(expr.without_nots(), (false, &expr)); +} diff --git a/crates/core/src/partiql/from_path.rs b/crates/core/src/partiql/from_path.rs new file mode 100644 index 000000000..d459ad573 --- /dev/null +++ b/crates/core/src/partiql/from_path.rs @@ -0,0 +1,206 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Refusals of a statement's FROM path (`"table"` or `"table"."index"`). +//! +//! The service checks the path after the statement has parsed and before it +//! looks the table up (measured 2026-11-02), so the parser records the first +//! refusal and raises it only once the whole statement has parsed. The table +//! and index names are not resolved here: a qualifier naming no index gets the +//! same refusal as one naming a real index. + +use super::Statement; + +/// The refusal of a table or index name outside the service's name rule: +/// 3 to 255 characters, each a letter, a digit, `_`, `-` or `.` (measured +/// 2026-11-02 with 1, 2, 3, 255 and 256 characters and with `bad!name`). +pub const NAME_RULE: &str = "Table name should be within the length [3, 255] and only contain 'a-z', 'A-Z', '0-9', '-', '_', '.'."; + +pub(super) fn breaks_name_rule(name: &str) -> bool { + !(3..=255).contains(&name.len()) + || !name + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'_' | b'-' | b'.')) +} + +/// The refusal of a `SELECT` FROM path, or `None` when the path is accepted. +/// +/// An empty component is reported ahead of a path that is too long, whichever +/// component is empty, and both ahead of an index name outside the name rule +/// (measured 2026-11-02: the rule answers ahead of every other check on an +/// index name). The table name gets the same rule from +/// [`table_name_refusal`] once the statement has parsed. +pub(super) fn refusal(components: &[&str]) -> Option { + if components.iter().any(|c| c.is_empty()) { + Some("Path component cannot be an empty string".to_owned()) + } else if components.len() > 2 { + Some("A path may contain at most 2 components in the FROM clause".to_owned()) + } else if components[1..].iter().any(|c| breaks_name_rule(c)) { + Some(NAME_RULE.to_owned()) + } else { + None + } +} + +/// The refusal of a parsed statement's table name outside the name rule. The +/// service checks it before it looks the table up, for SELECT, INSERT, UPDATE +/// and DELETE alike (measured 2026-11-02: `"ab"` on a table that cannot +/// exist, and `INSERT INTO ""`). +#[must_use] +pub fn table_name_refusal(stmt: &Statement) -> Option<&'static str> { + let table = match stmt { + Statement::Select(s) | Statement::Exists(s) => &s.from.table, + Statement::Insert(i) => &i.table, + Statement::Update(u) => &u.table, + Statement::Delete(d) => &d.table, + }; + breaks_name_rule(table).then_some(NAME_RULE) +} + +/// The refusal of an `UPDATE` or `DELETE` whose target path names an index +/// and is not refused by [`refusal`]. Those statements accept an index +/// qualifier grammatically and refuse it after the parse. +pub(super) const ON_INDEX: &str = "This operation is not supported on an index"; + +/// The refusal of an `INSERT` whose target is a path rather than a table +/// name. It carries the `line:column:length` span of the table name token. +pub(super) fn insert_refusal(table_span: &str) -> String { + format!( + "FROM clause may only contain a single table name in data manipulation statements \ + at {table_span}" + ) +} + +#[cfg(test)] +mod tests { + use crate::partiql::parse; + + fn err(sql: &str) -> String { + parse(sql).unwrap_err().message + } + + const EMPTY: &str = "Path component cannot be an empty string"; + const TOO_LONG: &str = "A path may contain at most 2 components in the FROM clause"; + const ON_INDEX: &str = "This operation is not supported on an index"; + + #[test] + fn select_path_with_three_components_is_refused() { + assert_eq!( + err("SELECT * FROM \"t\".\"i\".\"x\" WHERE pk = 'a'"), + TOO_LONG + ); + assert_eq!(err("SELECT * FROM \"t\".\"i\".x.y"), TOO_LONG); + } + + #[test] + fn select_path_with_an_empty_component_is_refused() { + assert_eq!(err("SELECT * FROM \"t\".\"\" WHERE pk = 'a'"), EMPTY); + assert_eq!(err("SELECT * FROM \"\" WHERE pk = 'a'"), EMPTY); + // The empty component wins over the length, wherever it sits. + assert_eq!(err("SELECT * FROM \"t\".\"i\".\"\""), EMPTY); + } + + #[test] + fn select_path_refusal_waits_for_the_rest_of_the_statement() { + // A syntax error later in the statement is reported first. + assert_eq!( + err("SELECT * FROM \"t\".\"i\".\"x\" WHERE pk = 'a'; SELECT 1"), + "Statement wasn't well formed, can't be processed: Unexpected token after \ + semicolon. (Only one query is allowed.)" + ); + // The path is refused ahead of the LIMIT clause. + assert_eq!(err("SELECT * FROM \"t\".\"i\".\"x\" LIMIT 1"), TOO_LONG); + } + + #[test] + fn select_with_one_qualifier_still_parses() { + assert!(parse("SELECT * FROM \"t\".\"idx\" WHERE pk = 'a'").is_ok()); + assert!(parse("SELECT * FROM \"t\".idx WHERE pk = 'a'").is_ok()); + } + + #[test] + fn an_index_name_outside_the_name_rule_is_refused_first() { + let long = "x".repeat(256); + for index in ["i", "ix", "bad!name", long.as_str()] { + for sql in [ + format!("SELECT * FROM \"t\".\"{index}\" WHERE pk = 'a'"), + format!("UPDATE \"t\".\"{index}\" SET a = 1 WHERE pk = 'a'"), + format!("DELETE FROM \"t\".\"{index}\" WHERE pk = 'a'"), + ] { + assert_eq!(err(&sql), super::NAME_RULE, "{sql}"); + } + } + let longest = "x".repeat(255); + assert!(parse(&format!("SELECT * FROM \"t\".\"{longest}\"")).is_ok()); + assert_eq!( + err(&format!( + "UPDATE \"t\".\"{longest}\" SET a = 1 WHERE pk = 'a'" + )), + ON_INDEX + ); + } + + #[test] + fn a_table_name_outside_the_name_rule_is_found_on_the_parsed_statement() { + let refusal = |sql: &str| super::table_name_refusal(&parse(sql).unwrap()); + assert_eq!(refusal("SELECT * FROM \"ab\""), Some(super::NAME_RULE)); + assert_eq!( + refusal("INSERT INTO \"\" VALUE {'pk': 'a'}"), + Some(super::NAME_RULE) + ); + assert_eq!( + refusal("DELETE FROM \"bad!name\" WHERE pk = 'a'"), + Some(super::NAME_RULE) + ); + assert_eq!(refusal("UPDATE \"a-b.c_1\" SET a = 1 WHERE pk = 'a'"), None); + } + + #[test] + fn update_and_delete_refuse_an_index_qualifier_after_the_parse() { + assert_eq!( + err("UPDATE \"t\".\"idx\" SET a = 1 WHERE pk = 'a' AND sk = 'b'"), + ON_INDEX + ); + assert_eq!( + err("DELETE FROM \"t\".\"idx\" WHERE pk = 'a' AND sk = 'b'"), + ON_INDEX + ); + // The statement must parse first: a missing SET is a syntax error. + assert_eq!( + err("UPDATE \"t\".\"i\" WHERE pk = 'a'"), + "Statement wasn't well formed, can't be processed: Expected data manipulation" + ); + } + + #[test] + fn update_and_delete_paths_take_the_select_path_refusals_first() { + assert_eq!( + err("UPDATE \"t\".\"i\".\"x\" SET a = 1 WHERE pk = 'a'"), + TOO_LONG + ); + assert_eq!(err("DELETE FROM \"t\".\"\" WHERE pk = 'a'"), EMPTY); + assert_eq!(err("UPDATE \"\" SET a = 1 WHERE pk = 'a'"), EMPTY); + } + + #[test] + fn insert_into_a_path_is_refused_at_the_table_name() { + // The table name token starts at column 13 and is 3 characters long. + assert_eq!( + err("INSERT INTO \"t\".\"i\" VALUE {'pk': 'a'}"), + "FROM clause may only contain a single table name in data manipulation \ + statements at 1:13:3" + ); + assert_eq!( + err("INSERT INTO \"t\".\"\".\"x\" VALUE {'pk': 'a'}"), + "FROM clause may only contain a single table name in data manipulation \ + statements at 1:13:3" + ); + } + + #[test] + fn plain_targets_are_unchanged() { + assert!(parse("INSERT INTO \"t\" VALUE {'pk': 'a'}").is_ok()); + assert!(parse("UPDATE \"t\" SET a = 1 WHERE pk = 'a'").is_ok()); + assert!(parse("DELETE FROM \"t\" WHERE pk = 'a'").is_ok()); + } +} diff --git a/crates/core/src/partiql/lexer.rs b/crates/core/src/partiql/lexer.rs new file mode 100644 index 000000000..e606a7f9c --- /dev/null +++ b/crates/core/src/partiql/lexer.rs @@ -0,0 +1,415 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! PartiQL lexer. +//! +//! Quoting rules match the service: single quotes delimit a string literal, +//! double quotes delimit an identifier, backticks delimit an Ion literal. `?` +//! is a positional parameter, `<<` and `>>` bracket a set (bag) literal. `--` +//! starts a comment that runs to the end of the line and `/* ... */` brackets +//! a comment; both read as whitespace. A token carries a 1-based +//! `line:column:length` span so the parser can build the service's exact +//! `... at ` messages. +//! +//! An unquoted word is ASCII letters, digits, `_` and `$`, not starting with a +//! digit, and `@` before a word reads as that word (measured 2026-11-02: `$x` +//! and `@n` are names, `café` does not lex). `%`, `/` and `||` are operators +//! the parser reads and then refuses. +//! +//! A number is digits, an optional dot with optional digits, then an optional +//! exponent that must have digits; it may also start with its decimal point +//! (`.5`). What follows a complete number starts a new token, except a second +//! dot right after the number's own dot, and an exponent with no digits: both +//! fail to lex (measured 2026-11-02 on 38 shapes: `1.2.3` is two numbers, +//! `1..2`, `1e+` and `5.e` do not lex, `1.e3` and `00.5` are numbers). +//! +//! The service reports every lexing failure (a character it does not read such +//! as `#`, an unterminated comment, string or quoted identifier) with nothing +//! after the "wasn't well formed" prefix (measured 2026-11-02), so the parser +//! drops [`LexError::message`]; the message is kept for diagnostics only. + +mod words; + +pub use words::{WordClass, is_function_keyword, word_class}; + +/// A lexical token with its 1-based line, column and length in the source. +#[derive(Debug, Clone, PartialEq)] +pub struct Token { + pub kind: TokenKind, + pub line: usize, + pub column: usize, + pub length: usize, +} + +#[derive(Debug, Clone, PartialEq)] +pub enum TokenKind { + /// A bare identifier or keyword (case-insensitive match happens in the parser). + Ident(String), + /// A double-quoted identifier. + QuotedIdent(String), + /// A single-quoted string literal. + String(String), + /// A numeric literal, kept as text to preserve precision. + Number(String), + /// A backtick-quoted Ion literal, the text between the backticks. + Ion(String), + Param, + Comma, + Dot, + Star, + LParen, + RParen, + LBracket, + RBracket, + LDoubleAngle, + RDoubleAngle, + LBrace, + RBrace, + Colon, + Eq, + Ne, + Lt, + Le, + Gt, + Ge, + Plus, + Minus, + Percent, + Slash, + /// `||` + Concat, + Semicolon, + Eof, +} + +/// A lexing error with the span that triggered it. +#[derive(Debug, Clone, PartialEq)] +pub struct LexError { + pub message: String, + pub line: usize, + pub column: usize, + pub length: usize, +} + +/// Tokenize a PartiQL statement. +/// +/// # Errors +/// +/// Returns a [`LexError`] on an unterminated string or identifier or an +/// unrecognized character. +// Each arm of the loop updates `i`, `line` and `col`, which a helper per arm would need passed in. +#[allow(clippy::too_many_lines)] +pub fn lex(input: &str) -> Result, LexError> { + let chars: Vec = input.chars().collect(); + let mut tokens = Vec::new(); + let mut i = 0; + let mut line = 1; + let mut col = 1; + + let advance = |i: &mut usize, line: &mut usize, col: &mut usize, n: usize, chars: &[char]| { + for _ in 0..n { + if chars.get(*i) == Some(&'\n') { + *line += 1; + *col = 1; + } else { + *col += 1; + } + *i += 1; + } + }; + + while i < chars.len() { + let c = chars[i]; + if c.is_whitespace() { + advance(&mut i, &mut line, &mut col, 1, &chars); + continue; + } + let (start_line, start_col) = (line, col); + // `-- ...` runs to the end of the line. + if c == '-' && chars.get(i + 1) == Some(&'-') { + let mut j = i + 2; + while j < chars.len() && chars[j] != '\n' && chars[j] != '\r' { + j += 1; + } + let len = j - i; + advance(&mut i, &mut line, &mut col, len, &chars); + continue; + } + // `/* ... */`; one with no closing `*/` is a lexing failure. + if c == '/' && chars.get(i + 1) == Some(&'*') { + let mut j = i + 2; + while j + 1 < chars.len() && !(chars[j] == '*' && chars[j + 1] == '/') { + j += 1; + } + if j + 1 >= chars.len() { + return Err(LexError { + message: "unterminated block comment".to_owned(), + line: start_line, + column: start_col, + length: chars.len() - i, + }); + } + let len = j + 2 - i; + advance(&mut i, &mut line, &mut col, len, &chars); + continue; + } + // A number written from its decimal point (`.5`) reads as a number; + // any other `.` is the path dot. + if c == '.' && chars.get(i + 1).is_some_and(char::is_ascii_digit) { + let len = + number_len(&chars, i).ok_or_else(|| malformed_number(start_line, start_col))?; + tokens.push(Token { + kind: TokenKind::Number(chars[i..i + len].iter().collect()), + line: start_line, + column: start_col, + length: len, + }); + advance(&mut i, &mut line, &mut col, len, &chars); + continue; + } + match c { + '\'' => { + // String literal; '' is an escaped single quote. + let mut s = String::new(); + let mut j = i + 1; + loop { + match chars.get(j) { + None => { + return Err(LexError { + message: "unterminated string literal".to_owned(), + line: start_line, + column: start_col, + length: j - i, + }); + } + Some('\'') if chars.get(j + 1) == Some(&'\'') => { + s.push('\''); + j += 2; + } + Some('\'') => { + j += 1; + break; + } + Some(ch) => { + s.push(*ch); + j += 1; + } + } + } + let len = j - i; + tokens.push(Token { + kind: TokenKind::String(s), + line: start_line, + column: start_col, + length: len, + }); + advance(&mut i, &mut line, &mut col, len, &chars); + } + '"' => { + let mut s = String::new(); + let mut j = i + 1; + loop { + match chars.get(j) { + None => { + return Err(LexError { + message: "unterminated quoted identifier".to_owned(), + line: start_line, + column: start_col, + length: j - i, + }); + } + Some('"') if chars.get(j + 1) == Some(&'"') => { + s.push('"'); + j += 2; + } + Some('"') => { + j += 1; + break; + } + Some(ch) => { + s.push(*ch); + j += 1; + } + } + } + let len = j - i; + tokens.push(Token { + kind: TokenKind::QuotedIdent(s), + line: start_line, + column: start_col, + length: len, + }); + advance(&mut i, &mut line, &mut col, len, &chars); + } + '0'..='9' => { + let len = + number_len(&chars, i).ok_or_else(|| malformed_number(start_line, start_col))?; + let text: String = chars[i..i + len].iter().collect(); + tokens.push(Token { + kind: TokenKind::Number(text), + line: start_line, + column: start_col, + length: len, + }); + advance(&mut i, &mut line, &mut col, len, &chars); + } + c if is_word_start(c) => { + let len = word_len(&chars, i); + let text: String = chars[i..i + len].iter().collect(); + tokens.push(Token { + kind: TokenKind::Ident(text), + line: start_line, + column: start_col, + length: len, + }); + advance(&mut i, &mut line, &mut col, len, &chars); + } + // `@name` reads as the name; the span keeps the `@`. + '@' if chars.get(i + 1).copied().is_some_and(is_word_start) => { + let len = 1 + word_len(&chars, i + 1); + let text: String = chars[i + 1..i + len].iter().collect(); + tokens.push(Token { + kind: TokenKind::Ident(text), + line: start_line, + column: start_col, + length: len, + }); + advance(&mut i, &mut line, &mut col, len, &chars); + } + '`' => { + let Some(close) = chars[i + 1..].iter().position(|&ch| ch == '`') else { + return Err(LexError { + message: "unterminated Ion literal".to_owned(), + line: start_line, + column: start_col, + length: chars.len() - i, + }); + }; + let len = close + 2; + tokens.push(Token { + kind: TokenKind::Ion(chars[i + 1..i + 1 + close].iter().collect()), + line: start_line, + column: start_col, + length: len, + }); + advance(&mut i, &mut line, &mut col, len, &chars); + } + _ => { + // Multi- and single-character operators. + let two: String = chars[i..(i + 2).min(chars.len())].iter().collect(); + let (kind, len) = match two.as_str() { + "<<" => (TokenKind::LDoubleAngle, 2), + ">>" => (TokenKind::RDoubleAngle, 2), + "<=" => (TokenKind::Le, 2), + ">=" => (TokenKind::Ge, 2), + "<>" | "!=" => (TokenKind::Ne, 2), + "||" => (TokenKind::Concat, 2), + _ => { + let k = match c { + ',' => TokenKind::Comma, + '.' => TokenKind::Dot, + '*' => TokenKind::Star, + '(' => TokenKind::LParen, + ')' => TokenKind::RParen, + '[' => TokenKind::LBracket, + ']' => TokenKind::RBracket, + '{' => TokenKind::LBrace, + '}' => TokenKind::RBrace, + ':' => TokenKind::Colon, + '=' => TokenKind::Eq, + '<' => TokenKind::Lt, + '>' => TokenKind::Gt, + '+' => TokenKind::Plus, + '-' => TokenKind::Minus, + '%' => TokenKind::Percent, + '/' => TokenKind::Slash, + '?' => TokenKind::Param, + ';' => TokenKind::Semicolon, + other => { + return Err(LexError { + message: format!("unexpected character '{other}'"), + line: start_line, + column: start_col, + length: 1, + }); + } + }; + (k, 1) + } + }; + tokens.push(Token { + kind, + line: start_line, + column: start_col, + length: len, + }); + advance(&mut i, &mut line, &mut col, len, &chars); + } + } + } + tokens.push(Token { + kind: TokenKind::Eof, + line, + column: col, + length: 0, + }); + Ok(tokens) +} + +/// The length of the number literal starting at `start` (a digit, or a dot +/// with a digit after it), or `None` when the text there does not lex: a dot +/// right after the number's own dot (`1..2`), or an exponent with no digits +/// (`1e+`, `1ee5`). See the module notes. The text is handed on as written; +/// converting it to a number is the parser's and engine's job. +fn number_len(chars: &[char], start: usize) -> Option { + let digits_from = |mut j: usize| { + while chars.get(j).is_some_and(char::is_ascii_digit) { + j += 1; + } + j + }; + let mut j = digits_from(start); + if chars.get(j) == Some(&'.') { + let after = digits_from(j + 1); + if after == j + 1 && chars.get(after) == Some(&'.') { + return None; + } + j = after; + } + if matches!(chars.get(j), Some('e' | 'E')) { + let mut k = j + 1; + if matches!(chars.get(k), Some('+' | '-')) { + k += 1; + } + let end = digits_from(k); + if end == k { + return None; + } + j = end; + } + Some(j - start) +} + +fn malformed_number(line: usize, column: usize) -> LexError { + LexError { + message: "malformed number".to_owned(), + line, + column, + length: 1, + } +} + +fn is_word_start(c: char) -> bool { + c.is_ascii_alphabetic() || c == '_' || c == '$' +} + +/// The length of the unquoted word starting at `start`. +fn word_len(chars: &[char], start: usize) -> usize { + chars[start..] + .iter() + .take_while(|c| c.is_ascii_alphanumeric() || **c == '_' || **c == '$') + .count() +} + +#[cfg(test)] +mod tests; diff --git a/crates/core/src/partiql/lexer/tests.rs b/crates/core/src/partiql/lexer/tests.rs new file mode 100644 index 000000000..18ca1c3b4 --- /dev/null +++ b/crates/core/src/partiql/lexer/tests.rs @@ -0,0 +1,174 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Unit tests for the PartiQL lexer. + +use super::*; + +fn kinds(input: &str) -> Vec { + lex(input).unwrap().into_iter().map(|t| t.kind).collect() +} + +#[test] +fn line_comment_reads_as_whitespace() { + assert_eq!( + kinds("a -- note\nb -- trailing"), + vec![ + TokenKind::Ident("a".to_owned()), + TokenKind::Ident("b".to_owned()), + TokenKind::Eof + ] + ); +} + +#[test] +fn block_comment_reads_as_whitespace() { + assert_eq!( + kinds("a /* note\n more */ b /**/"), + vec![ + TokenKind::Ident("a".to_owned()), + TokenKind::Ident("b".to_owned()), + TokenKind::Eof + ] + ); +} + +#[test] +fn a_comment_keeps_later_spans_on_their_line() { + let tokens = lex("-- note\n b").unwrap(); + assert_eq!((tokens[0].line, tokens[0].column), (2, 3)); +} + +#[test] +fn unterminated_block_comment_is_a_lex_error() { + assert!(lex("a /* note").is_err()); + assert!(lex("a /* note *").is_err()); +} + +#[test] +fn hash_is_a_lex_error() { + assert!(lex("a # note").is_err()); +} + +#[test] +fn a_single_minus_stays_an_operator() { + assert_eq!( + kinds("a - 1"), + vec![ + TokenKind::Ident("a".to_owned()), + TokenKind::Minus, + TokenKind::Number("1".to_owned()), + TokenKind::Eof + ] + ); +} + +#[test] +fn a_number_may_start_with_its_decimal_point() { + assert_eq!( + kinds(".73e1 .5"), + vec![ + TokenKind::Number(".73e1".to_owned()), + TokenKind::Number(".5".to_owned()), + TokenKind::Eof + ] + ); +} + +fn numbers(input: &str) -> Vec { + kinds(input) + .into_iter() + .map(|k| match k { + TokenKind::Number(n) => n, + TokenKind::Ident(w) => format!("word {w}"), + TokenKind::Dot => ".".to_owned(), + TokenKind::Eof => "end".to_owned(), + other => format!("{other:?}"), + }) + .collect() +} + +#[test] +fn a_complete_number_ends_where_a_second_dot_or_a_letter_starts() { + assert_eq!(numbers("1.2.3"), ["1.2", ".3", "end"]); + assert_eq!(numbers("1e5.5"), ["1e5", ".5", "end"]); + assert_eq!(numbers("1e-5.2"), ["1e-5", ".2", "end"]); + assert_eq!(numbers("1e5e5"), ["1e5", "word e5", "end"]); + assert_eq!(numbers("1.2e3e4"), ["1.2e3", "word e4", "end"]); + assert_eq!(numbers("1_000"), ["1", "word _000", "end"]); + assert_eq!(numbers("1.2a"), ["1.2", "word a", "end"]); + assert_eq!(numbers("1.2.e3"), ["1.2", ".", "word e3", "end"]); + assert_eq!(numbers("1.2..3"), ["1.2", ".", ".3", "end"]); + assert_eq!(numbers("..5"), [".", ".5", "end"]); +} + +#[test] +fn numbers_the_service_reads() { + for text in ["1.e3", "1.", ".5", "00.5", "1e05", "1E5", "1.5e+3"] { + assert_eq!(numbers(text), [text, "end"], "{text}"); + } +} + +#[test] +fn a_dot_after_the_numbers_own_dot_or_an_exponent_without_digits_does_not_lex() { + for text in [ + "1..2", "5..", "1..", "1...2", "1.5e", "1e+", "5.e", "1ee5", "1e.5", + ] { + assert!(lex(text).is_err(), "{text}"); + } +} + +#[test] +fn a_word_is_ascii_letters_digits_underscore_and_dollar() { + assert_eq!(numbers("$x a$b_1"), ["word $x", "word a$b_1", "end"]); + assert!(lex("café = 1").is_err()); +} + +#[test] +fn an_at_sign_before_a_word_reads_as_the_word() { + let tokens = lex("@n = 1").unwrap(); + assert_eq!(tokens[0].kind, TokenKind::Ident("n".to_owned())); + assert_eq!((tokens[0].column, tokens[0].length), (1, 2)); + assert!(lex("@ n").is_err()); +} + +#[test] +fn percent_slash_and_concatenation_are_operators() { + assert_eq!( + kinds("a % b / c || d"), + vec![ + TokenKind::Ident("a".to_owned()), + TokenKind::Percent, + TokenKind::Ident("b".to_owned()), + TokenKind::Slash, + TokenKind::Ident("c".to_owned()), + TokenKind::Concat, + TokenKind::Ident("d".to_owned()), + TokenKind::Eof + ] + ); + for text in ["a | b", "a ^ b", "a & b", "~a", "!a"] { + assert!(lex(text).is_err(), "{text}"); + } +} + +#[test] +fn a_backtick_literal_is_one_token() { + let tokens = lex("s = `1`").unwrap(); + assert_eq!(tokens[2].kind, TokenKind::Ion("1".to_owned())); + assert_eq!((tokens[2].column, tokens[2].length), (5, 3)); + assert!(lex("s = `1").is_err()); +} + +#[test] +fn a_dot_before_a_name_stays_the_path_dot() { + assert_eq!( + kinds("a.b"), + vec![ + TokenKind::Ident("a".to_owned()), + TokenKind::Dot, + TokenKind::Ident("b".to_owned()), + TokenKind::Eof + ] + ); +} diff --git a/crates/core/src/partiql/lexer/words.rs b/crates/core/src/partiql/lexer/words.rs new file mode 100644 index 000000000..fff901d0f --- /dev/null +++ b/crates/core/src/partiql/lexer/words.rs @@ -0,0 +1,363 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! How the service's lexer classifies an unquoted word, and its lists of reserved words. + +/// How the service's lexer reads an unquoted word. The parser reports a word +/// by this class where a term, a path step or a name should start. Measured +/// 2026-11-02 for 274 words in four positions (a condition, the argument of +/// `attribute_exists`, a SET target and a projection): every word in these +/// lists is refused as a SET target with "Expected identifier for simple +/// path" and after a dot with "Invalid path dot component", and any other +/// word is a name. Where a condition starts, a [`WordClass::Keyword`] is +/// "Unexpected keyword", an operator "Unexpected operator" and a special word +/// "Unexpected term"; the keywords that start a form of their own (`date`, +/// `cast`, `select`, ...) are answered by the parser. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum WordClass { + /// An ordinary identifier: an attribute or function name. + Name, + /// A reserved keyword (`select`, `value`, `inner`, `size`, ...). + Keyword, + /// A keyword that is an operator: `and`, `or`, `not`, `is`, `in`, `like`, + /// `between`, `union`, `except`, `intersect`. `not` also starts a term of + /// its own, which the parser reads. + Operator, + /// `true`, `false`, `null` and `missing`. + Literal, + /// A word with a token type of its own (`as`, `at`, `by`, `asc`, `desc`, + /// `for`, `nulls`, `first`, `last`): never a keyword term, never a name. + Special, +} + +/// Classify an unquoted word, case-insensitively. +#[must_use] +pub fn word_class(word: &str) -> WordClass { + let lower = word.to_ascii_lowercase(); + let lower = lower.as_str(); + if OPERATOR_WORDS.contains(&lower) { + WordClass::Operator + } else if SPECIAL_WORDS.contains(&lower) { + WordClass::Special + } else if matches!(lower, "true" | "false" | "null" | "missing") { + WordClass::Literal + } else if KEYWORDS.binary_search(&lower).is_ok() { + WordClass::Keyword + } else { + WordClass::Name + } +} + +/// Whether an unquoted word is a keyword that names a function and so may be +/// called (`size(x)`, `exists(...)`); any other keyword cannot start a call. +#[must_use] +pub fn is_function_keyword(word: &str) -> bool { + FUNCTION_KEYWORDS.contains(&word.to_ascii_lowercase().as_str()) +} + +const OPERATOR_WORDS: &[&str] = &[ + "and", + "or", + "not", + "is", + "in", + "like", + "between", + "union", + "except", + "intersect", +]; + +const SPECIAL_WORDS: &[&str] = &[ + "as", "at", "by", "asc", "desc", "for", "nulls", "first", "last", +]; + +/// Keywords that start a function call when `(` follows them. +const FUNCTION_KEYWORDS: &[&str] = &[ + "exists", + "count", + "avg", + "max", + "min", + "sum", + "substring", + "char_length", + "character_length", + "octet_length", + "bit_length", + "upper", + "lower", + "size", + "sexp", + "list", + "bag", + "cast", + "can_cast", + "can_lossless_cast", + "nullif", + "coalesce", + "trim", + "extract", + "date_add", + "date_diff", +]; + +/// The reserved keywords, sorted for binary search. Words in +/// [`OPERATOR_WORDS`], [`SPECIAL_WORDS`] and the literals are classified +/// before this list is consulted. +const KEYWORDS: &[&str] = &[ + "absolute", + "action", + "add", + "all", + "allocate", + "alter", + "any", + "are", + "assertion", + "authorization", + "avg", + "bag", + "begin", + "bigint", + "bit", + "bit_length", + "blob", + "bool", + "boolean", + "can_cast", + "can_lossless_cast", + "cascade", + "cascaded", + "case", + "cast", + "catalog", + "char", + "char_length", + "character", + "character_length", + "check", + "clob", + "close", + "coalesce", + "collate", + "collation", + "column", + "commit", + "conflict", + "connect", + "connection", + "constraint", + "constraints", + "continue", + "convert", + "corresponding", + "count", + "create", + "cross", + "current", + "current_date", + "current_time", + "current_timestamp", + "current_user", + "cursor", + "date", + "date_add", + "date_diff", + "deallocate", + "dec", + "decimal", + "declare", + "default", + "deferrable", + "deferred", + "delete", + "describe", + "descriptor", + "diagnostics", + "disconnect", + "distinct", + "do", + "domain", + "double", + "drop", + "else", + "end", + "escape", + "exception", + "exec", + "execute", + "exists", + "external", + "extract", + "fetch", + "float", + "foreign", + "found", + "from", + "full", + "get", + "global", + "go", + "goto", + "grant", + "group", + "having", + "identity", + "immediate", + "index", + "indicator", + "initially", + "inner", + "input", + "insensitive", + "insert", + "int", + "int2", + "int4", + "int8", + "integer", + "integer2", + "integer4", + "integer8", + "interval", + "into", + "isolation", + "join", + "key", + "language", + "left", + "let", + "level", + "limit", + "list", + "local", + "lower", + "match", + "max", + "min", + "modified", + "module", + "names", + "national", + "natural", + "nchar", + "new", + "next", + "no", + "nothing", + "nullif", + "numeric", + "octet_length", + "of", + "offset", + "old", + "on", + "only", + "open", + "option", + "order", + "outer", + "output", + "overlaps", + "pad", + "partial", + "pivot", + "position", + "precision", + "prepare", + "preserve", + "primary", + "prior", + "privileges", + "procedure", + "public", + "read", + "real", + "references", + "relative", + "remove", + "restrict", + "returning", + "revoke", + "right", + "rollback", + "rows", + "schema", + "scroll", + "section", + "select", + "session", + "session_user", + "set", + "sexp", + "size", + "smallint", + "some", + "space", + "sql", + "sqlcode", + "sqlerror", + "sqlstate", + "string", + "struct", + "substring", + "sum", + "symbol", + "system_user", + "table", + "temporary", + "then", + "time", + "timestamp", + "to", + "transaction", + "translate", + "translation", + "trim", + "tuple", + "unique", + "unknown", + "unpivot", + "update", + "upper", + "usage", + "user", + "using", + "value", + "values", + "varchar", + "varying", + "view", + "when", + "whenever", + "where", + "with", + "work", + "write", + "zone", +]; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_keyword_list_is_sorted_for_binary_search() { + assert!(KEYWORDS.windows(2).all(|w| w[0] < w[1])); + } + + #[test] + fn words_are_classified_case_insensitively() { + assert_eq!(word_class("ORDER"), WordClass::Keyword); + assert_eq!(word_class("inner"), WordClass::Keyword); + assert_eq!(word_class("Is"), WordClass::Operator); + assert_eq!(word_class("LIKE"), WordClass::Operator); + assert_eq!(word_class("as"), WordClass::Special); + assert_eq!(word_class("NULL"), WordClass::Literal); + assert_eq!(word_class("pk"), WordClass::Name); + assert_eq!(word_class("attribute_exists"), WordClass::Name); + assert!(is_function_keyword("SIZE")); + assert!(!is_function_keyword("value")); + } +} diff --git a/crates/core/src/partiql/mod.rs b/crates/core/src/partiql/mod.rs new file mode 100644 index 000000000..f78c40c14 --- /dev/null +++ b/crates/core/src/partiql/mod.rs @@ -0,0 +1,36 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! PartiQL grammar, AST and query planning. +//! +//! This module is compiled only with the `partiql` feature. It is a pure +//! front end: it turns a statement string into an [`ast::Statement`] and then +//! (in [`plan`]) into a plan expressed in terms of the operations the engine +//! already has. It introduces no storage concept and nothing else in `core` +//! depends on it, so the feature can be removed by deleting this directory. +//! +//! Every grammar rule and every error string is justified by behaviour measured +//! against the live service, cited where it is implemented. + +pub mod ast; +mod from_path; +pub mod lexer; +pub mod number; +pub mod operand_types; +pub mod params; +pub mod parser; +pub mod plan; + +pub use ast::Statement; +pub use from_path::table_name_refusal; +pub use parser::{ParseError, parse, statement_length_refusal}; + +/// The prefix the service puts on a PartiQL parse failure. The full +/// `ValidationException` message is this prefix followed by a specific reason. +pub const PARSE_ERROR_PREFIX: &str = "Statement wasn't well formed, can't be processed: "; + +/// Build the service's parse-error message from a reason. +#[must_use] +pub fn parse_error(reason: &str) -> String { + format!("{PARSE_ERROR_PREFIX}{reason}") +} diff --git a/crates/core/src/partiql/number.rs b/crates/core/src/partiql/number.rs new file mode 100644 index 000000000..0addeffa8 --- /dev/null +++ b/crates/core/src/partiql/number.rs @@ -0,0 +1,400 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! PartiQL number conversion: number literals in a statement and `N` values in +//! the request `Parameters`. +//! +//! The service reads a PartiQL number the way a Java `BigDecimal` reads a +//! string, then applies the item number rules to the result. Two consequences +//! differ from the classic data-plane number check (`validation::number`), +//! which is why this module exists instead of reusing it (measured 2026-11-02): +//! +//! - A number whose exponent does not fit a 32-bit integer, or whose scale +//! (digits after the point less the exponent) does not, cannot be converted +//! at all: `1e2147483648`, `1e99999999999`, `1e-2147483648` and +//! `1.5e-2147483647`. A parameter answers "The parameter cannot be converted +//! to a numeric value: " and a literal answers the parse-error prefix +//! with nothing after it. +//! - The magnitude check works on precision minus scale in 32-bit arithmetic, +//! which wraps: `1e2147483647` converts and is then reported as an +//! underflow, not an overflow. +//! +//! A literal that converts but breaks a number rule gets the rule's message +//! with the position inside the value appended (`under root`, `under root.x`, +//! `under root[1]`); a parameter gets the bare message. +//! +//! A zero written with a decimal point or an exponent (`0.0`, `.0`, `0e0`) +//! with a minus sign before it is a negative zero decimal. The service +//! refuses it as a literal and takes it as a parameter, which it stores as 0 +//! (measured 2026-11-05). The parser reports the literal, since it knows +//! the signs (see [`is_zero_decimal`]). + +use crate::types::AttributeValue; + +use super::ast::{Operand, Statement, WhereExpr}; +use super::lexer::{Token, TokenKind}; +use super::parse_error; +use super::parser::ParseError; + +/// The item number rule messages, in the service's words. +pub const OVERFLOW_MESSAGE: &str = + "Number overflow. Attempting to store a number with magnitude larger than supported range"; +pub const UNDERFLOW_MESSAGE: &str = + "Number underflow. Attempting to store a number with magnitude smaller than supported range"; +pub const TOO_MANY_DIGITS_MESSAGE: &str = + "Attempting to store more than 38 significant digits in a Number"; +/// A parameter whose `N` text cannot be read as a number. The text is +/// appended after `": "` unless it is empty. +pub const UNCONVERTIBLE_PARAMETER_MESSAGE: &str = + "The parameter cannot be converted to a numeric value"; +/// A number literal that is a negative zero decimal. The position inside the +/// value and the span of the number are appended (see +/// [`negative_zero_refusal`]). +pub const NEGATIVE_ZERO_MESSAGE: &str = "Negative zero decimal is not supported"; + +const MAX_SIGNIFICANT_DIGITS: usize = 38; +/// Largest precision minus scale a number may have (`9.99...E+125`). +const MAX_MAGNITUDE: i64 = 126; +/// Smallest precision minus scale a non-zero number may have (`1E-130`). +const MIN_MAGNITUDE: i64 = -129; +/// At most this many exponent digits remain once leading zeros are skipped. +const MAX_EXPONENT_DIGITS: usize = 10; + +/// Why a number text was refused. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum NumberProblem { + /// The text cannot be read as a number at all. + Unconvertible, + /// More than 38 significant digits. + TooManyDigits, + /// Magnitude above `9.99...E+125`. + Overflow, + /// Magnitude below `1E-130`, including the 32-bit wrap of a huge exponent. + Underflow, +} + +impl NumberProblem { + /// The bare message for the problem. A literal that cannot be converted is + /// reported as a parse failure instead, and a parameter's text is appended + /// to the unconvertible message by [`parameter_message`]. + #[must_use] + pub fn rule_message(self) -> &'static str { + match self { + NumberProblem::Unconvertible => UNCONVERTIBLE_PARAMETER_MESSAGE, + NumberProblem::TooManyDigits => TOO_MANY_DIGITS_MESSAGE, + NumberProblem::Overflow => OVERFLOW_MESSAGE, + NumberProblem::Underflow => UNDERFLOW_MESSAGE, + } + } +} + +/// A number as the decimal reader holds it: the digits as written (sign and +/// point removed) and a 32-bit scale, so the value is `digits * 10^-scale`. +#[derive(Debug, Clone, PartialEq, Eq)] +struct Decimal { + digits: String, + scale: i32, +} + +/// Read `text` with the decimal reader's rules: an optional sign, digits with at +/// most one point (at least one digit), then an optional `e`/`E`, optional +/// sign and exponent digits running to the end of the text. The exponent and +/// the resulting scale must each fit a 32-bit integer. `None` when the text +/// cannot be converted. +fn read_decimal(text: &str) -> Option { + let bytes = text.as_bytes(); + let mut i = usize::from(matches!(bytes.first(), Some(b'+' | b'-'))); + let mut digits = String::new(); + let mut fraction_digits: i64 = 0; + let mut seen_point = false; + let mut exponent: i64 = 0; + while i < bytes.len() { + match bytes[i] { + c @ b'0'..=b'9' => { + digits.push(char::from(c)); + if seen_point { + fraction_digits += 1; + } + } + b'.' if !seen_point => seen_point = true, + b'e' | b'E' => { + exponent = read_exponent(&bytes[i + 1..])?; + break; + } + _ => return None, + } + i += 1; + } + if digits.is_empty() { + return None; + } + let scale = i32::try_from(fraction_digits - exponent).ok()?; + Some(Decimal { digits, scale }) +} + +/// The exponent digits after `e`: an optional sign, then digits to the end of +/// the text. Leading zeros are skipped while more than ten digits remain, and +/// at most ten may be left; the value must fit a 32-bit integer. +fn read_exponent(rest: &[u8]) -> Option { + let (negative, mut digits) = match rest.first() { + Some(b'-') => (true, &rest[1..]), + Some(b'+') => (false, &rest[1..]), + _ => (false, rest), + }; + if digits.is_empty() { + return None; + } + while digits.len() > MAX_EXPONENT_DIGITS && digits[0] == b'0' { + digits = &digits[1..]; + } + if digits.len() > MAX_EXPONENT_DIGITS { + return None; + } + let mut value: i64 = 0; + for &c in digits { + if !c.is_ascii_digit() { + return None; + } + value = value * 10 + i64::from(c - b'0'); + } + let value = if negative { -value } else { value }; + i32::try_from(value).ok()?; + Some(value) +} + +/// Check a number text against the conversion and the item number rules. +/// +/// # Errors +/// +/// Returns the first rule the text breaks: it cannot be converted, then more +/// than 38 significant digits, then the magnitude range. +pub fn check_number(text: &str) -> Result<(), NumberProblem> { + let decimal = read_decimal(text).ok_or(NumberProblem::Unconvertible)?; + let significant = decimal.digits.trim_start_matches('0'); + if significant.is_empty() { + // Zero, whatever its scale. + return Ok(()); + } + if significant.trim_end_matches('0').len() > MAX_SIGNIFICANT_DIGITS { + return Err(NumberProblem::TooManyDigits); + } + // Precision (digits from the first non-zero one) minus scale, which the + // service computes in 32-bit arithmetic. Past `i32::MAX` it wraps to a + // large negative value, which the floor check reports as an underflow. + let precision = i64::try_from(significant.len()).unwrap_or(i64::MAX); + let magnitude = precision.saturating_sub(i64::from(decimal.scale)); + if magnitude > i64::from(i32::MAX) { + return Err(NumberProblem::Underflow); + } + if magnitude > MAX_MAGNITUDE { + return Err(NumberProblem::Overflow); + } + if magnitude < MIN_MAGNITUDE { + return Err(NumberProblem::Underflow); + } + Ok(()) +} + +/// Whether `text`, a number with an optional sign, is a zero written with a +/// decimal point or an exponent: `0.0`, `.0`, `0.`, `00.0`, `0e0`, `0E-3`. +/// With a minus sign among the signs before it, the service refuses such a +/// literal, even when two signs cancel (measured 2026-11-05: `-0.0`, `- -0.0`, +/// `+-0.0`, `-(0.0)` and `-0E-3` are refused). A zero written as an integer +/// (`-0`) and `+0.0` are taken as 0. +#[must_use] +pub(crate) fn is_zero_decimal(text: &str) -> bool { + let unsigned = text.strip_prefix(['+', '-']).unwrap_or(text); + let (mantissa, exponent) = match unsigned.find(['e', 'E']) { + Some(at) => (&unsigned[..at], true), + None => (unsigned, false), + }; + let mut zeros = 0usize; + let mut point = false; + for c in mantissa.chars() { + match c { + '0' => zeros += 1, + '.' => point = true, + _ => return false, + } + } + zeros > 0 && (point || exponent) +} + +/// The refusal of a negative zero decimal at `path` inside its value +/// (`root`, `root[0]`, `root.y`; a bag member is `root`), at `at`, the +/// `line:column:length` span of the number after its signs (measured +/// 2026-11-05). +pub(crate) fn negative_zero_refusal(path: &str, at: &str) -> String { + format!("{NEGATIVE_ZERO_MESSAGE} under key {path} at {at}") +} + +/// The message for a parameter's `N` text that fails [`check_number`]. +#[must_use] +pub fn parameter_message(text: &str, problem: NumberProblem) -> String { + match problem { + NumberProblem::Unconvertible if text.is_empty() => { + UNCONVERTIBLE_PARAMETER_MESSAGE.to_owned() + } + NumberProblem::Unconvertible => format!("{UNCONVERTIBLE_PARAMETER_MESSAGE}: {text}"), + other => other.rule_message().to_owned(), + } +} + +/// Refuse a statement whose text holds a number literal that cannot be +/// converted. The service reports it as a parse failure with nothing after the +/// prefix, ahead of any other check on the statement (measured 2026-11-02). +/// +/// # Errors +/// +/// Returns the parse error for the first such literal. +pub(crate) fn check_number_tokens(tokens: &[Token]) -> Result<(), ParseError> { + for token in tokens { + if let TokenKind::Number(text) = &token.kind + && read_decimal(text).is_none() + { + return Err(unconvertible_literal()); + } + } + Ok(()) +} + +fn unconvertible_literal() -> ParseError { + ParseError { + message: parse_error(""), + } +} + +/// Check every number literal in a parsed statement against the item number +/// rules. The message names the position inside the literal value: `root` for +/// the value itself, `.name` for a map member and `[i]` for a list element +/// (measured 2026-11-02 for a top-level value, a map member and a list +/// element). Literals are visited in statement order; members of one map +/// literal are visited in name order. +/// +/// # Errors +/// +/// Returns the message for the first literal that breaks a rule. +pub fn check_literal_numbers(stmt: &Statement) -> Result<(), ParseError> { + match stmt { + Statement::Select(select) | Statement::Exists(select) => match &select.where_clause { + Some(where_clause) => where_literals(where_clause), + None => Ok(()), + }, + Statement::Insert(insert) => { + for (_, value) in &insert.value { + operand_literals(value)?; + } + Ok(()) + } + Statement::Update(update) => { + for action in &update.sets { + operand_literals(&action.value)?; + } + // The bag of a set_add or set_delete is a value like any other + // (measured 2026-11-02). + for action in &update.set_members { + operand_literals(&action.members)?; + } + match &update.where_clause { + Some(where_clause) => where_literals(where_clause), + None => Ok(()), + } + } + Statement::Delete(delete) => where_literals(&delete.where_clause), + } +} + +/// The refusal [`check_literal_numbers`] gives a number literal `text` at +/// `path` inside its value, or `None` when the number passes. The parser uses +/// it to report such a number ahead of the operator count. +pub(crate) fn literal_number_refusal(text: &str, path: &str) -> Option { + literal_number(text, path).err().map(|e| e.message) +} + +/// The refusal [`check_literal_numbers`] gives the first number literal in +/// `operand` that breaks a rule, or `None` when they all pass. The parser +/// uses it to report a SET value's number ahead of a sign in the clause. +pub(crate) fn operand_number_refusal(operand: &Operand) -> Option { + operand_literals(operand).err().map(|e| e.message) +} + +fn where_literals(expr: &WhereExpr) -> Result<(), ParseError> { + match expr { + WhereExpr::Compare(left, _, right) + | WhereExpr::BeginsWith(left, right) + | WhereExpr::Contains(left, right) + | WhereExpr::AttributeType(left, right) => { + operand_literals(left)?; + operand_literals(right) + } + WhereExpr::Between(value, low, high) => { + operand_literals(value)?; + operand_literals(low)?; + operand_literals(high) + } + WhereExpr::In(value, items) => { + operand_literals(value)?; + items.iter().try_for_each(operand_literals) + } + WhereExpr::Function(_, args) => args.iter().try_for_each(operand_literals), + WhereExpr::IsMissing(..) => Ok(()), + WhereExpr::IsType(operand, ..) => operand_literals(operand), + WhereExpr::And(left, right) | WhereExpr::Or(left, right) => { + where_literals(left)?; + where_literals(right) + } + WhereExpr::Not(inner) | WhereExpr::ComparedWithBoolean(inner, _) => where_literals(inner), + } +} + +fn operand_literals(operand: &Operand) -> Result<(), ParseError> { + match operand { + Operand::Literal(value) => value_numbers(value, &mut "root".to_owned()), + Operand::Arith(left, _, right) => { + operand_literals(left)?; + operand_literals(right) + } + Operand::Function(_, args) => args.iter().try_for_each(operand_literals), + Operand::Parameter(_) | Operand::Path(_) => Ok(()), + } +} + +fn value_numbers(value: &AttributeValue, path: &mut String) -> Result<(), ParseError> { + match value { + AttributeValue::N(text) => literal_number(text, path), + AttributeValue::NS(members) => members.iter().try_for_each(|n| literal_number(n, path)), + AttributeValue::L(items) => { + for (index, item) in items.iter().enumerate() { + let len = path.len(); + path.push_str(&format!("[{index}]")); + value_numbers(item, path)?; + path.truncate(len); + } + Ok(()) + } + AttributeValue::M(members) => { + for (name, member) in members { + let len = path.len(); + path.push('.'); + path.push_str(name); + value_numbers(member, path)?; + path.truncate(len); + } + Ok(()) + } + _ => Ok(()), + } +} + +fn literal_number(text: &str, path: &str) -> Result<(), ParseError> { + match check_number(text) { + Ok(()) => Ok(()), + Err(NumberProblem::Unconvertible) => Err(unconvertible_literal()), + Err(problem) => Err(ParseError { + message: format!("{} under {path}", problem.rule_message()), + }), + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/core/src/partiql/number/tests.rs b/crates/core/src/partiql/number/tests.rs new file mode 100644 index 000000000..02f0fd50c --- /dev/null +++ b/crates/core/src/partiql/number/tests.rs @@ -0,0 +1,346 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Unit tests for the number rules of a statement's literals and parameters. + +use super::*; +use crate::partiql::parse; + +fn problem(text: &str) -> Option { + check_number(text).err() +} + +#[test] +fn an_exponent_outside_32_bits_cannot_be_converted() { + for text in [ + "1e2147483648", + "1e99999999999", + "-1e99999999999", + "1e-99999999999", + "1E+2147483648", + ] { + assert_eq!(problem(text), Some(NumberProblem::Unconvertible), "{text}"); + } +} + +#[test] +fn a_scale_outside_32_bits_cannot_be_converted() { + // The scale is the digits after the point less the exponent. + assert_eq!(problem("1e-2147483648"), Some(NumberProblem::Unconvertible)); + assert_eq!( + problem("1.5e-2147483647"), + Some(NumberProblem::Unconvertible) + ); +} + +#[test] +fn the_largest_exponent_wraps_to_an_underflow() { + assert_eq!(problem("1e2147483647"), Some(NumberProblem::Underflow)); + assert_eq!(problem("-1e2147483647"), Some(NumberProblem::Underflow)); + // 1e2147483646 has precision minus scale 2147483647: no wrap. + assert_eq!(problem("1e2147483646"), Some(NumberProblem::Overflow)); +} + +#[test] +fn the_magnitude_range_matches_the_item_rules() { + assert_eq!(problem("1e126"), Some(NumberProblem::Overflow)); + assert_eq!(problem("1e-131"), Some(NumberProblem::Underflow)); + assert_eq!( + problem("9.9999999999999999999999999999999999999E+125"), + None + ); + assert_eq!(problem("1E-130"), None); + assert_eq!(problem("0.1E-129"), None); + assert_eq!(problem("0.01E-129"), Some(NumberProblem::Underflow)); +} + +#[test] +fn significant_digits_ignore_leading_and_trailing_zeros() { + let digits39 = "123456789012345678901234567890123456789"; + assert_eq!(problem(digits39), Some(NumberProblem::TooManyDigits)); + assert_eq!(problem(&digits39[..38]), None); + assert_eq!(problem(&format!("000{}", &digits39[..38])), None); + assert_eq!(problem(&format!("1{}", "0".repeat(60))), None); + assert_eq!(problem(&format!("0.{}", &digits39[..38])), None); +} + +#[test] +fn zero_is_accepted_at_any_scale() { + for text in ["0", "-0", "0.000", "0e-500", "000e2147483647"] { + assert_eq!(problem(text), None, "{text}"); + } +} + +#[test] +fn forms_the_decimal_reader_accepts() { + // Leading exponent zeros are skipped while more than ten digits remain. + for text in [ + "+1", + ".5", + "5.", + "-.5e1", + "1.e5", + "1e+5", + "1E05", + "1e00000000005", + "1e0000000000005", + ] { + assert_eq!(problem(text), None, "{text}"); + } +} + +#[test] +fn forms_the_decimal_reader_refuses() { + for text in [ + "", + "abc", + " 1", + "1 ", + ".", + "+", + "-", + "1e", + "1e+", + "1e-", + "1.2.3", + "1e5e5", + "1e5.5", + "0x10", + "+-1", + "Infinity", + "NaN", + "1_000", + "e5", + "1e12345678901", + ] { + assert_eq!( + problem(text), + Some(NumberProblem::Unconvertible), + "{text:?}" + ); + } +} + +#[test] +fn parameter_messages_echo_the_text_unless_empty() { + assert_eq!( + parameter_message("1e99999999999", NumberProblem::Unconvertible), + "The parameter cannot be converted to a numeric value: 1e99999999999" + ); + assert_eq!( + parameter_message("", NumberProblem::Unconvertible), + "The parameter cannot be converted to a numeric value" + ); + assert_eq!( + parameter_message("1e2147483647", NumberProblem::Underflow), + UNDERFLOW_MESSAGE + ); + assert_eq!( + parameter_message("1e126", NumberProblem::Overflow), + OVERFLOW_MESSAGE + ); +} + +fn err(sql: &str) -> String { + match parse(sql) { + Err(e) => e.message, + Ok(stmt) => check_literal_numbers(&stmt).unwrap_err().message, + } +} + +#[test] +fn an_unconvertible_literal_is_a_parse_error_with_nothing_after_the_colon() { + for sql in [ + "SELECT * FROM \"t\" WHERE pk = 1e2147483648", + "SELECT * FROM \"t\" WHERE pk = 1e99999999999", + "SELECT * FROM \"t\" WHERE pk = 1.5e-2147483647", + "INSERT INTO \"t\" VALUE {'pk': 1e99999999999}", + ] { + assert_eq!( + err(sql), + "Statement wasn't well formed, can't be processed: ", + "{sql}" + ); + } +} + +#[test] +fn a_literal_that_breaks_a_rule_names_its_position() { + assert_eq!( + err("SELECT * FROM \"t\" WHERE pk = 1e126"), + format!("{OVERFLOW_MESSAGE} under root") + ); + assert_eq!( + err("SELECT * FROM \"t\" WHERE pk = 1e-131"), + format!("{UNDERFLOW_MESSAGE} under root") + ); + assert_eq!( + err("SELECT * FROM \"t\" WHERE pk = 123456789012345678901234567890123456789"), + format!("{TOO_MANY_DIGITS_MESSAGE} under root") + ); + assert_eq!( + err("INSERT INTO \"t\" VALUE {'pk': 1, 'm': {'x': 1e126}}"), + format!("{OVERFLOW_MESSAGE} under root.x") + ); + assert_eq!( + err("INSERT INTO \"t\" VALUE {'pk': 1, 'l': [1, 1e126]}"), + format!("{OVERFLOW_MESSAGE} under root[1]") + ); + assert_eq!( + err("UPDATE \"t\" SET a = 1e-131 WHERE pk = 1"), + format!("{UNDERFLOW_MESSAGE} under root") + ); + assert_eq!( + err("INSERT INTO \"t\" VALUE {'pk': 1, 'm': {'x': [{'y': 1e126}]}}"), + format!("{OVERFLOW_MESSAGE} under root.x[0].y") + ); +} + +#[test] +fn a_literal_in_a_set_add_or_set_delete_bag_names_root() { + for function in ["set_add", "set_delete"] { + for (literal, message) in [ + ("1e126", OVERFLOW_MESSAGE), + ("1e-131", UNDERFLOW_MESSAGE), + ( + "123456789012345678901234567890123456789", + TOO_MANY_DIGITS_MESSAGE, + ), + ] { + let sql = + format!("UPDATE \"t\" SET ns = {function}(ns, <<1, {literal}>>) WHERE pk = 1"); + assert_eq!(err(&sql), format!("{message} under root"), "{sql}"); + } + let sql = format!("UPDATE \"t\" SET ns = {function}(ns, <<1e99999999999>>) WHERE pk = 1"); + assert_eq!( + err(&sql), + "Statement wasn't well formed, can't be processed: ", + "{sql}" + ); + } + let stmt = parse("UPDATE \"t\" SET ns = set_add(ns, <<1, 9.9E125>>) WHERE pk = 1").unwrap(); + assert!(check_literal_numbers(&stmt).is_ok()); +} + +#[test] +fn literals_in_range_pass() { + let stmt = parse( + "UPDATE \"t\" SET a = 9.9E125, b = [1e-130, {'c': 5}] WHERE pk = 1 AND n BETWEEN 1 AND 2", + ) + .unwrap(); + assert!(check_literal_numbers(&stmt).is_ok()); +} + +#[test] +fn a_zero_with_a_point_or_an_exponent_is_a_zero_decimal() { + for text in [ + "0.0", ".0", "0.", "00.0", "0.00", "0e0", "0E-3", "0.0e0", "-0.0", "+0e5", + ] { + assert!(is_zero_decimal(text), "{text}"); + } + for text in ["0", "00", "-0", "0.5", "1.0", "1e0", "10e-1", ".", ""] { + assert!(!is_zero_decimal(text), "{text}"); + } +} + +/// The refusal of a negative zero decimal at `path`, the number `number` +/// being its first occurrence in `sql` from byte `from` on. +fn negative_zero(sql: &str, number: &str, path: &str, from: usize) -> String { + let column = sql[from..].find(number).unwrap() + from + 1; + negative_zero_refusal(path, &format!("1:{column}:{}", number.len())) +} + +#[test] +fn a_negative_zero_decimal_is_refused_at_the_number_after_its_signs() { + let insert = |value: &str| format!("INSERT INTO \"t\" VALUE {{'pk':'a','x':{value}}}"); + for (value, number, path) in [ + ("-0.0", "0.0", "root"), + ("- 0.0", "0.0", "root"), + ("- -0.0", "0.0", "root"), + ("- - -0.0", "0.0", "root"), + ("+-0.0", "0.0", "root"), + ("-+0.0", "0.0", "root"), + ("-00.0", "00.0", "root"), + ("-0.00", "0.00", "root"), + ("-.0", ".0", "root"), + ("-0.", "0.", "root"), + ("-0E-3", "0E-3", "root"), + ("-0.0e0", "0.0e0", "root"), + ("[-0.0]", "0.0", "root[0]"), + ("{'y':-0.0}", "0.0", "root.y"), + ("<<-0.0, 1>>", "0.0", "root"), + ] { + let sql = insert(value); + let from = sql.find("'x'").unwrap(); + assert_eq!(err(&sql), negative_zero(&sql, number, path, from), "{sql}"); + } + // A key attribute too: the parse refuses it ahead of the key type check. + let sql = "INSERT INTO \"t\" VALUE {'pk':-0.0,'sk':'b'}"; + assert_eq!(err(sql), negative_zero(sql, "0.0", "root", 0)); + let update = |value: &str| format!("UPDATE \"t\" SET x = {value} WHERE pk = 'a'"); + for (value, number, path) in [ + ("-0e0", "0e0", "root"), + ("-(0.0)", "0.0", "root"), + ("(-0.0)", "0.0", "root"), + ("-(-0.0)", "0.0", "root"), + ("+(-0.0)", "0.0", "root"), + ("-(+0.0)", "0.0", "root"), + ("{'y': -0.0}", "0.0", "root.y"), + ("[1, -0.0]", "0.0", "root[1]"), + ("set_add(x, <<-0.0>>)", "0.0", "root"), + ] { + let sql = update(value); + assert_eq!(err(&sql), negative_zero(&sql, number, path, 0), "{sql}"); + } + let select = |condition: &str| format!("SELECT * FROM \"t\" WHERE pk = 'a' AND {condition}"); + for (condition, number, path) in [ + ("x = -0.0", "0.0", "root"), + ("x = -0e0", "0e0", "root"), + ("x = - -0.0", "0.0", "root"), + ("x = -+0.0", "0.0", "root"), + ("x IN [-0.0]", "0.0", "root"), + ("x = [-0.0]", "0.0", "root[0]"), + ] { + let sql = select(condition); + assert_eq!(err(&sql), negative_zero(&sql, number, path, 0), "{sql}"); + } +} + +#[test] +fn a_zero_with_no_minus_sign_or_written_as_an_integer_is_taken() { + for sql in [ + "INSERT INTO \"t\" VALUE {'pk':'a','x':-0,'y':+0.0}", + "SELECT * FROM \"t\" WHERE pk = 'a' AND x = -0", + ] { + let stmt = parse(sql).unwrap(); + assert!(check_literal_numbers(&stmt).is_ok(), "{sql}"); + } +} + +#[test] +fn a_negative_zero_decimal_comes_in_statement_order_with_the_number_rules() { + let overflow = |path: &str| format!("{OVERFLOW_MESSAGE} under {path}"); + let insert = |value: &str| format!("INSERT INTO \"t\" VALUE {{'pk':'a',{value}}}"); + // In a value, the number written first is reported. + assert_eq!(err(&insert("'x':1e126,'y':-0.0")), overflow("root")); + let sql = insert("'x':-0.0,'y':1e126"); + assert_eq!(err(&sql), negative_zero(&sql, "0.0", "root", 0)); + // The members of a map in the order they are written, not by name. + let sql = insert("'x':{'b':-0.0,'a':1e126}"); + assert_eq!(err(&sql), negative_zero(&sql, "0.0", "root.b", 0)); + assert_eq!(err(&insert("'x':{'y':1e126},'z':-0.0")), overflow("root.y")); + // Of two zeros, the first. + let sql = insert("'x':-0.0,'y':-0.0"); + assert_eq!(err(&sql), negative_zero(&sql, "0.0", "root", 0)); + let update = + |sets: &str, condition: &str| format!("UPDATE \"t\" SET {sets} WHERE pk = 'a'{condition}"); + let sql = update("x = -0.0, y = 1e126", ""); + assert_eq!(err(&sql), negative_zero(&sql, "0.0", "root", 0)); + assert_eq!(err(&update("x = 1e126, y = -0.0", "")), overflow("root")); + // The condition's numbers come first. + assert_eq!(err(&update("x = -0.0", " AND a = 1e126")), overflow("root")); + let select = |condition: &str| format!("SELECT * FROM \"t\" WHERE pk = 'a' AND {condition}"); + assert_eq!(err(&select("a = 1e126 AND b = -0.0")), overflow("root")); + let sql = select("a = -0.0 AND b = 1e126"); + assert_eq!(err(&sql), negative_zero(&sql, "0.0", "root", 0)); +} diff --git a/crates/core/src/partiql/operand_types.rs b/crates/core/src/partiql/operand_types.rs new file mode 100644 index 000000000..01659450a --- /dev/null +++ b/crates/core/src/partiql/operand_types.rs @@ -0,0 +1,190 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The operand-type check on ordering comparisons. +//! +//! An ordering operator (`<`, `<=`, `>`, `>=`, `BETWEEN`) refuses a value +//! operand whose type has no ordering: anything but `S`, `N` and `B`. The +//! service applies the rule to the statement itself, so it answers before the +//! table is resolved, and the message is the bare sentence with no expression +//! prefix (measured 2026-11-02). A document path is not checked: its type is +//! only known per item. + +use crate::types::AttributeValue; + +use super::ast::{CompareOp, Operand, Statement, WhereExpr}; + +/// Check every ordering comparison in the statement's `WHERE` clause. A `?` +/// operand is checked against its bound parameter; a missing parameter is left +/// to the binding step, which reports the count mismatch. +/// +/// # Errors +/// +/// Returns the service's operand-type message for the first refused operand, +/// left operand before right. +pub fn check_ordering_operands(stmt: &Statement, params: &[AttributeValue]) -> Result<(), String> { + let where_clause = match stmt { + Statement::Select(select) | Statement::Exists(select) => select.where_clause.as_ref(), + Statement::Update(update) => update.where_clause.as_ref(), + Statement::Delete(delete) => Some(&delete.where_clause), + Statement::Insert(_) => None, + }; + match where_clause { + Some(expr) => check_where(expr, params), + None => Ok(()), + } +} + +fn check_where(expr: &WhereExpr, params: &[AttributeValue]) -> Result<(), String> { + match expr { + WhereExpr::Compare(left, op, right) => { + if let Some(symbol) = ordering_symbol(*op) { + check_operand(left, symbol, params)?; + check_operand(right, symbol, params)?; + } + Ok(()) + } + WhereExpr::Between(value, low, high) => { + check_operand(value, "BETWEEN", params)?; + check_operand(low, "BETWEEN", params)?; + check_operand(high, "BETWEEN", params) + } + WhereExpr::And(left, right) | WhereExpr::Or(left, right) => { + check_where(left, params)?; + check_where(right, params) + } + WhereExpr::Not(inner) | WhereExpr::ComparedWithBoolean(inner, _) => { + check_where(inner, params) + } + WhereExpr::In(..) + | WhereExpr::BeginsWith(..) + | WhereExpr::Contains(..) + | WhereExpr::AttributeType(..) + | WhereExpr::Function(..) + | WhereExpr::IsMissing(..) + | WhereExpr::IsType(..) => Ok(()), + } +} + +/// The operator as the message names it, or `None` for `=` and `<>`, which +/// accept every type. +fn ordering_symbol(op: CompareOp) -> Option<&'static str> { + match op { + CompareOp::Lt => Some("<"), + CompareOp::Le => Some("<="), + CompareOp::Gt => Some(">"), + CompareOp::Ge => Some(">="), + CompareOp::Eq | CompareOp::Ne => None, + } +} + +fn check_operand(operand: &Operand, symbol: &str, params: &[AttributeValue]) -> Result<(), String> { + let value = match operand { + Operand::Literal(value) => value, + Operand::Parameter(index) => match params.get(*index) { + Some(value) => value, + None => return Ok(()), + }, + Operand::Path(_) | Operand::Arith(..) | Operand::Function(..) => return Ok(()), + }; + let type_code = match value { + AttributeValue::S(_) | AttributeValue::N(_) | AttributeValue::B(_) => return Ok(()), + AttributeValue::Bool(_) => "BOOL", + AttributeValue::Null => "NULL", + AttributeValue::L(_) => "L", + AttributeValue::M(_) => "M", + AttributeValue::SS(_) => "SS", + AttributeValue::NS(_) => "NS", + AttributeValue::BS(_) => "BS", + }; + Err(format!( + "Incorrect operand type for operator or function; operator or function: {symbol}, \ + operand type: {type_code}" + )) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::partiql::parse; + + fn check(sql: &str, params: &[AttributeValue]) -> Result<(), String> { + check_ordering_operands(&parse(sql).unwrap(), params) + } + + #[test] + fn each_ordering_operator_is_named_as_written() { + for op in ["<", "<=", ">", ">="] { + let sql = format!("SELECT pk FROM \"t\" WHERE pk = ? AND val {op} ?"); + let params = [ + AttributeValue::S("k".to_owned()), + AttributeValue::Bool(true), + ]; + assert_eq!( + check(&sql, ¶ms), + Err(format!( + "Incorrect operand type for operator or function; operator or function: \ + {op}, operand type: BOOL" + )) + ); + } + } + + #[test] + fn between_names_itself() { + let params = [AttributeValue::Bool(true), AttributeValue::Bool(false)]; + let err = check("SELECT * FROM \"t\" WHERE val BETWEEN ? AND ?", ¶ms).unwrap_err(); + assert!( + err.ends_with("operator or function: BETWEEN, operand type: BOOL"), + "{err}" + ); + } + + #[test] + fn every_unordered_type_is_refused_and_ordered_types_pass() { + let unordered = [ + (AttributeValue::Null, "NULL"), + (AttributeValue::L(Vec::new()), "L"), + (AttributeValue::M(std::collections::BTreeMap::new()), "M"), + (AttributeValue::SS(["a".to_owned()].into()), "SS"), + (AttributeValue::NS(["1".to_owned()].into()), "NS"), + (AttributeValue::BS([vec![1u8]].into()), "BS"), + ]; + for (value, code) in unordered { + let err = check("SELECT * FROM \"t\" WHERE val < ?", &[value]).unwrap_err(); + assert!(err.ends_with(&format!("operand type: {code}")), "{err}"); + } + for value in [ + AttributeValue::S("a".to_owned()), + AttributeValue::N("1".to_owned()), + AttributeValue::B(vec![1]), + ] { + assert_eq!(check("SELECT * FROM \"t\" WHERE val < ?", &[value]), Ok(())); + } + } + + #[test] + fn equality_paths_and_missing_parameters_are_left_alone() { + let flag = [AttributeValue::Bool(true)]; + assert_eq!(check("SELECT * FROM \"t\" WHERE val = ?", &flag), Ok(())); + assert_eq!(check("SELECT * FROM \"t\" WHERE val <> ?", &flag), Ok(())); + assert_eq!(check("SELECT * FROM \"t\" WHERE a < b", &[]), Ok(())); + assert_eq!(check("SELECT * FROM \"t\" WHERE a < ?", &[]), Ok(())); + } + + #[test] + fn literals_writes_and_nested_conditions_are_checked() { + assert!(check("SELECT * FROM \"t\" WHERE pk = 'a' AND val > true", &[]).is_err()); + let flag = [AttributeValue::Bool(true)]; + assert!(check("UPDATE \"t\" SET a = 1 WHERE pk = 'a' AND val < ?", &flag).is_err()); + assert!(check("DELETE FROM \"t\" WHERE pk = 'a' AND NOT (val >= ?)", &flag).is_err()); + assert!(check("SELECT * FROM \"t\" WHERE pk = 'a' OR val <= ?", &flag).is_err()); + } + + #[test] + fn the_left_operand_is_reported_first() { + let params = [AttributeValue::Null, AttributeValue::Bool(true)]; + let err = check("SELECT * FROM \"t\" WHERE ? < ?", ¶ms).unwrap_err(); + assert!(err.ends_with("operand type: NULL"), "{err}"); + } +} diff --git a/crates/core/src/partiql/params.rs b/crates/core/src/partiql/params.rs new file mode 100644 index 000000000..03dce7cad --- /dev/null +++ b/crates/core/src/partiql/params.rs @@ -0,0 +1,207 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! PartiQL parameter checks: nesting depth and number conversion. +//! +//! The service converts each statement's `Parameters` before it looks at the +//! statement, and reports the first problem it reaches walking the values in +//! document order: a map or list nested past the item limit, or an `N` (or +//! `NS` member) that cannot be converted or breaks a number rule (measured +//! 2026-11-02). So `[deep, {"N": "abc"}]` is a nesting error and +//! `[{"N": "abc"}, deep]` a conversion error. +//! +//! The walk here runs over the parsed request body: parameters and list +//! elements in request order, the members of a map in the order the parsed +//! JSON object holds them. That is the request's order when the build +//! includes the MongoDB storage crate, whose JSON dependency makes every JSON +//! map keep it (any build of the whole workspace), and name order otherwise. +//! In name order, when two members of one map both have a problem, the one +//! reported can differ from the service's, which walks the request's order +//! (measured 2026-11-02). + +use serde_json::Value; + +use super::number::{check_number, parameter_message}; +use crate::validation::MAX_ITEM_NESTING_DEPTH; + +/// The service's nesting-depth message, the same for parameters as for items. +pub const NESTING_MESSAGE: &str = "Nesting Levels have exceeded supported limits: \ + Attributes in the item have nested levels beyond supported limit"; + +/// Check a statement's parameters (wire attribute values) in order. +/// +/// # Errors +/// +/// Returns the service's message for the first problem: the nesting-depth +/// message, or the parameter number message for an `N` or `NS` member. +pub fn check_parameter_values(parameters: &[Value]) -> Result<(), String> { + parameters.iter().try_for_each(|p| check_value(p, 0)) +} + +/// Walk one wire attribute value. `depth` counts the maps and lists around +/// it. A parameter may hold 32 of them around its leaf, one more than an item +/// attribute; 33 is refused (measured 2026-11-02). The walk stops there, so +/// it recurses at most that deep. +fn check_value(value: &Value, depth: usize) -> Result<(), String> { + let Value::Object(members) = value else { + return Ok(()); + }; + for (tag, payload) in members { + match (tag.as_str(), payload) { + ("M", Value::Object(entries)) => { + let next = enter(depth)?; + for member in entries.values() { + check_value(member, next)?; + } + } + ("L", Value::Array(items)) => { + let next = enter(depth)?; + for item in items { + check_value(item, next)?; + } + } + ("N", Value::String(text)) => check_text(text)?, + ("NS", Value::Array(items)) => { + for item in items { + if let Value::String(text) = item { + check_text(text)?; + } + } + } + _ => {} + } + } + Ok(()) +} + +fn enter(depth: usize) -> Result { + let next = depth + 1; + if next > MAX_ITEM_NESTING_DEPTH { + return Err(NESTING_MESSAGE.to_owned()); + } + Ok(next) +} + +fn check_text(text: &str) -> Result<(), String> { + check_number(text).map_err(|problem| parameter_message(text, problem)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::partiql::number::UNDERFLOW_MESSAGE; + + fn doc(json: &str) -> Vec { + match serde_json::from_str::(json).unwrap() { + Value::Array(items) => items, + other => panic!("expected an array, got {other:?}"), + } + } + + fn deep(levels: usize, leaf: &str) -> String { + let mut v = leaf.to_owned(); + for _ in 0..levels { + v = format!("{{\"M\":{{\"n\":{v}}}}}"); + } + v + } + + const ABC: &str = "The parameter cannot be converted to a numeric value: abc"; + + #[test] + fn parameters_are_walked_in_request_order_and_map_members_as_the_map_holds_them() { + let d = deep(40, "{\"S\":\"leaf\"}"); + let bad = "{\"N\":\"abc\"}"; + assert_eq!( + check_parameter_values(&doc(&format!("[{d},{bad}]"))), + Err(NESTING_MESSAGE.to_owned()) + ); + assert_eq!( + check_parameter_values(&doc(&format!("[{bad},{d}]"))), + Err(ABC.to_owned()) + ); + // Within one map the member the parsed map holds first is reached + // first: the one whose name sorts first, or the one the request wrote + // first in a build whose JSON maps keep the request's order. + for members in [ + format!("\"a\":{d},\"b\":{bad}"), + format!("\"b\":{bad},\"a\":{d}"), + ] { + let params = doc(&format!("[{{\"M\":{{{members}}}}}]")); + let first = params[0]["M"].as_object().unwrap().keys().next().unwrap(); + let want = if first == "a" { NESTING_MESSAGE } else { ABC }; + assert_eq!( + check_parameter_values(¶ms), + Err(want.to_owned()), + "{members}" + ); + } + assert_eq!( + check_parameter_values(&doc(&format!("[{}]", deep(40, bad)))), + Err(NESTING_MESSAGE.to_owned()) + ); + } + + #[test] + fn a_parameter_holds_one_level_more_than_an_item_attribute() { + let ok = deep(MAX_ITEM_NESTING_DEPTH, "{\"S\":\"x\"}"); + assert_eq!(check_parameter_values(&doc(&format!("[{ok}]"))), Ok(())); + let over = deep(MAX_ITEM_NESTING_DEPTH + 1, "{\"S\":\"x\"}"); + assert_eq!( + check_parameter_values(&doc(&format!("[{over}]"))), + Err(NESTING_MESSAGE.to_owned()) + ); + let mut list = "{\"S\":\"x\"}".to_owned(); + for _ in 0..MAX_ITEM_NESTING_DEPTH { + list = format!("{{\"L\":[{list}]}}"); + } + assert_eq!(check_parameter_values(&doc(&format!("[{list}]"))), Ok(())); + let list = format!("{{\"L\":[{list}]}}"); + assert_eq!( + check_parameter_values(&doc(&format!("[{list}]"))), + Err(NESTING_MESSAGE.to_owned()) + ); + } + + #[test] + fn numbers_are_checked_in_every_position() { + for (json, message) in [ + ( + "[{\"N\":\"1e99999999999\"}]", + "The parameter cannot be converted to a numeric value: 1e99999999999", + ), + ( + "[{\"N\":\"\"}]", + "The parameter cannot be converted to a numeric value", + ), + ("[{\"N\":\"1e2147483647\"}]", UNDERFLOW_MESSAGE), + ( + "[{\"L\":[{\"N\":\"1\"},{\"N\":\"1e99999999999\"}]}]", + "The parameter cannot be converted to a numeric value: 1e99999999999", + ), + ( + "[{\"M\":{\"x\":{\"N\":\"1e99999999999\"}}}]", + "The parameter cannot be converted to a numeric value: 1e99999999999", + ), + ( + "[{\"NS\":[\"1\",\"1e99999999999\"]}]", + "The parameter cannot be converted to a numeric value: 1e99999999999", + ), + ] { + assert_eq!( + check_parameter_values(&doc(json)), + Err(message.to_owned()), + "{json}" + ); + } + } + + #[test] + fn other_types_and_good_numbers_pass() { + let params = doc( + "[{\"S\":\"1e99999999999\"},{\"N\":\"12.5\"},{\"NS\":[\"1\",\"2\"]},\ + {\"SS\":[\"abc\"]},{\"BOOL\":true},{\"NULL\":true},{\"B\":\"AQI=\"}]", + ); + assert_eq!(check_parameter_values(¶ms), Ok(())); + } +} diff --git a/crates/core/src/partiql/parser.rs b/crates/core/src/partiql/parser.rs new file mode 100644 index 000000000..a1bf6c309 --- /dev/null +++ b/crates/core/src/partiql/parser.rs @@ -0,0 +1,436 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! PartiQL parser: tokens -> [`ast::Statement`]. +//! +//! Recursive descent. Error messages reproduce the service's +//! `ValidationException` strings verbatim, as measured; a parse failure +//! carries the full message the engine will surface. + +use super::ast::Statement; +use super::lexer::{Token, TokenKind, WordClass, lex, word_class}; +use super::{PARSE_ERROR_PREFIX, parse_error}; + +mod condition; +mod forms; +mod limits; +mod literals; +mod operands; +mod predicates; +mod statements; +mod terms; +mod walk; + +pub use limits::{MAX_STATEMENT_LENGTH, statement_length_refusal}; +use limits::{Nest, Nesting}; +use walk::Findings; + +/// A parse failure. `message` is the full `ValidationException` string the +/// service returns, ready to hand to `DynamoDbError::ValidationException`. +#[derive(Debug, Clone, PartialEq)] +pub struct ParseError { + pub message: String, +} + +impl ParseError { + fn new(message: impl Into) -> Self { + Self { + message: message.into(), + } + } + fn well_formed(reason: &str) -> Self { + Self { + message: parse_error(reason), + } + } + + /// Whether this is a syntax failure (the "wasn't well formed" envelope) + /// rather than a refusal of a statement that parsed. + fn is_syntax(&self) -> bool { + self.message.starts_with(PARSE_ERROR_PREFIX) + } +} + +type PResult = Result; + +/// The refusal of a table alias (measured 2026-11-02 on SELECT, UPDATE and +/// DELETE, with and without `AS`). +const ALIASING: &str = "Aliasing is not supported"; + +const MISSING_RHS: &str = "Missing right-hand side expression of infix operator"; + +/// The refusal of a bag member that is neither a number nor a string. +const BAG_MEMBER_TYPES: &str = + "Unsupported data type in Bag. DynamoDB only supports either numbers or strings in bags"; + +/// How the service names a literal value itself in a refusal of a part of it. +const LITERAL_ROOT: &str = "root"; + +/// The refusal of a statement with more `?` placeholders than the request +/// gives parameters. +pub const PARAMETER_COUNT_MISMATCH: &str = + "Number of parameters in request and statement don't match."; + +/// The refusal of a statement whose `IN` has no list after it, or an empty +/// one. +const IN_OPERANDS: &str = "IN operator must have a left hand argument of type Variable Reference \ + and right hand argument of type Seq with at least one member"; + +/// The refusal of an IS test whose left side is not a name, a literal or a +/// path. +const IS_LEFT_SIDE: &str = + "The left hand side argument to IS must be a Variable Reference, Literal, or Path"; + +/// The refusal of a condition check outside an ExecuteTransaction: a bare +/// expression statement, and `EXISTS(SELECT ...)` in ExecuteStatement. +pub const EXISTS_OUTSIDE_TRANSACTION: &str = + "EXISTS can only be used in ExecuteTransaction write requests."; + +/// The refusal of an operand that is the whole WHERE condition (measured +/// 2026-11-02 with a name, a path, a number, a string, a list, `true` and a +/// parameter). +const NOT_A_CONDITION: &str = "WHERE clause must contain an expression that resolves to a boolean"; + +/// The refusal of an operand that is an argument of AND or OR (measured +/// 2026-11-02: `a AND pk = 'a'`). +const AND_OR_ARGUMENTS: &str = "The arguments to AND/OR must be Expressions"; + +/// The refusal of an operand that is the argument of a NOT (measured +/// 2026-11-05: `NOT a`, `NOT (a)` and `(NOT a)`). +const NOT_ARGUMENT: &str = "The argument to NOT must be a Type or Expression"; + +/// The name of the predicate that stands for an operand that is not a +/// condition. It never runs: the operand is refused (see +/// [`Parser::term_predicate`]). No call can have this name. +const BARE_OPERAND: &str = ""; + +/// The name of the predicate that stands for an operand that is all a group +/// holds, until the group is read as that operand (see `Parser::predicate`). +/// It never leaves the parser. No call can have this name. +const GROUPED_OPERAND: &str = ""; + +/// The syntax error of a `size` call with no comparison after it, where the +/// service's answer is not measured. +const NO_COMPARISON: &str = "Expected a comparison operator"; + +/// Parse a single PartiQL statement. +/// +/// # Errors +/// +/// Returns a [`ParseError`] whose `message` is the service's +/// `ValidationException` text for the failure. +pub fn parse(input: &str) -> PResult { + limits::parse_on_a_fitting_stack(|inline| parse_attempt(input, None, false, inline)) +} + +/// [`parse`] for a request that gives `parameters` values for the statement's +/// `?` placeholders. A statement past the operator limit with more +/// placeholders than that is refused for the parameter count rather than the +/// operator count (measured 2026-11-02). +/// +/// # Errors +/// +/// As [`parse`]. +pub fn parse_for_request(input: &str, parameters: usize) -> PResult { + limits::parse_on_a_fitting_stack(|inline| parse_attempt(input, Some(parameters), false, inline)) +} + +/// [`parse_for_request`] for an ExecuteStatement request, which refuses an +/// `EXISTS(SELECT ...)` statement ahead of the table name rule and of the +/// walk of its condition (measured 2026-11-02 with a table name of two +/// characters and a `+` in the condition). +/// +/// # Errors +/// +/// As [`parse`]. +pub fn parse_for_execute_statement(input: &str, parameters: usize) -> PResult { + limits::parse_on_a_fitting_stack(|inline| parse_attempt(input, Some(parameters), true, inline)) +} + +/// [`parse`] with the full caps on the calling thread, for the tests that +/// measure the stack a deep statement needs. +#[cfg(test)] +pub(crate) fn parse_with_full_caps(input: &str) -> PResult { + parse_attempt(input, None, false, false).expect("only an inline parse runs out of budget") +} + +/// One parse of `input`, for a request with `parameters` values when that is +/// known, refusing an EXISTS statement when `refuse_exists`. `inline` keeps +/// the parse within the depth budget of the caller's thread; `None` means it +/// ran out (see [`limits`]). +fn parse_attempt( + input: &str, + parameters: Option, + refuse_exists: bool, + inline: bool, +) -> Option> { + // A lexing failure carries nothing after the prefix (see the lexer). + let Ok(tokens) = lex(input) else { + return Some(Err(ParseError::well_formed(""))); + }; + let mut p = Parser { + tokens, + pos: 0, + next_param: 0, + parameters, + refuse_exists, + found: Findings::default(), + select_list_call: None, + table: None, + in_where: false, + in_values: false, + after_where: false, + after_not: false, + bare_left: false, + nesting: Nesting::new(inline), + operators: 0, + operator_refusals: 0, + }; + + // A number literal that cannot be converted fails the whole statement + // before parsing starts. + if let Err(e) = super::number::check_number_tokens(&p.tokens) { + return Some(Err(e)); + } + + let result = p.statement().and_then(|stmt| p.finish(stmt)); + if p.nesting.out_of_budget { + return None; + } + // The service reads the whole statement before it checks what the + // statement names, so a syntax error anywhere wins over an alias, an + // empty table name or a refused FROM path, and those win over the checks + // made on a statement that parsed, the LIMIT clause check included + // (measured 2026-11-02: `... t WHERE` is "Unexpected term", + // `DELETE FROM "t" t WHERE pk = 'x'` is the alias refusal). See [`walk`] + // for the order around the operator count. + Some(p.resolve(result)) +} + +// Each flag records one independent fact about the request or the parse so far. +#[allow(clippy::struct_excessive_bools)] +struct Parser { + tokens: Vec, + pos: usize, + next_param: usize, + /// How many parameters the request gives, when the statement is parsed + /// for a request. + parameters: Option, + /// Whether an `EXISTS(SELECT ...)` statement is refused (ExecuteStatement). + refuse_exists: bool, + /// The refusals found in a statement that is still being read (an alias, + /// a refused `FROM` path). They are reported only once the statement + /// parses; see [`walk`]. + found: Findings, + /// The refusal of a call in the select list of a SELECT statement, once + /// the select list has been read (see [`walk`]). + select_list_call: Option, + /// The table the statement names, once it has been read. + table: Option, + /// Set while a WHERE condition is parsed, where only the condition + /// functions are recognized. A refusal that stops the parse there leaves + /// it set. + in_where: bool, + /// Set while the SET and REMOVE clauses of an UPDATE or the value of an + /// INSERT are parsed. + in_values: bool, + /// Set once the WHERE condition has been read. + after_where: bool, + /// Whether the predicate being read follows a NOT. + after_not: bool, + /// Set when an operand that is not a condition was read as the left + /// argument of an AND or an OR, until that operator is read. + bare_left: bool, + /// How deep the parse has recursed (see [`limits`]). + nesting: Nesting, + /// The operators of the WHERE condition read so far. + operators: usize, + /// The operators of the WHERE condition refused so far (see + /// [`Parser::refuse_condition_operator`]). + operator_refusals: usize, +} + +impl Parser { + fn peek(&self) -> &Token { + &self.tokens[self.pos] + } + + fn bump(&mut self) -> Token { + let t = self.tokens[self.pos].clone(); + if self.pos + 1 < self.tokens.len() { + self.pos += 1; + } + t + } + + fn take_param_index(&mut self) -> usize { + let i = self.next_param; + self.next_param += 1; + i + } + + /// A case-insensitive keyword match on the current token, consuming it on hit. + fn eat_keyword(&mut self, kw: &str) -> bool { + if let TokenKind::Ident(s) = &self.peek().kind + && s.eq_ignore_ascii_case(kw) + { + self.bump(); + return true; + } + false + } + + fn is_keyword(&self, kw: &str) -> bool { + matches!(&self.peek().kind, TokenKind::Ident(s) if s.eq_ignore_ascii_case(kw)) + } + + fn eat(&mut self, kind: &TokenKind) -> bool { + if &self.peek().kind == kind { + self.bump(); + true + } else { + false + } + } + + /// Span string `line:column:length` for the current token, for `... at ` messages. + fn span(&self) -> String { + let t = self.peek(); + format!("{}:{}:{}", t.line, t.column, t.length) + } + + /// What may follow a complete statement: nothing, or a semicolon. + fn finish(&mut self, stmt: Statement) -> PResult { + self.eat_final_semicolon(); + match &self.peek().kind { + TokenKind::Eof => Ok(stmt), + // A trailing unsupported clause keyword (e.g. LIMIT, which is a + // request parameter rather than grammar) is refused by name, in + // its place among the refusals of a statement that parses (see + // [`walk`]). The position is the token after it (measured + // 2026-11-02: `LIMIT 2` names the `2`, length 1). LIMIT at the + // end of the statement is not measured and names itself. + TokenKind::Ident(s) if s.eq_ignore_ascii_case("LIMIT") => { + let t = self + .tokens + .get(self.pos + 1) + .filter(|t| !matches!(t.kind, TokenKind::Eof)) + .unwrap_or_else(|| self.peek()); + let refusal = format!("Unsupported clause: LIMIT at {}:{}:1", t.line, t.column); + self.refuse_clause(refusal); + Ok(stmt) + } + _ => Err(self.trailing_error()), + } + } + + /// The error for a token left over after a complete statement or bare + /// expression. + fn trailing_error(&self) -> ParseError { + match self.peek().kind { + TokenKind::Semicolon => ParseError::well_formed( + "Unexpected token after semicolon. (Only one query is allowed.)", + ), + _ => ParseError::well_formed("Unexpected token after expression"), + } + } + + /// The service's error for a token that cannot start a term, by what the + /// token is (measured 2026-11-02): an operator is "Unexpected operator", a + /// reserved keyword "Unexpected keyword", and anything else, the end of + /// the statement included, "Unexpected term". + fn term_error(&self) -> ParseError { + let reason = match &self.peek().kind { + TokenKind::Eq + | TokenKind::Ne + | TokenKind::Lt + | TokenKind::Le + | TokenKind::Gt + | TokenKind::Ge + | TokenKind::Plus + | TokenKind::Minus => "Unexpected operator", + TokenKind::Ident(word) => match word_class(word) { + WordClass::Operator => "Unexpected operator", + WordClass::Keyword => "Unexpected keyword", + _ => "Unexpected term", + }, + _ => "Unexpected term", + }; + ParseError::well_formed(reason) + } + + /// The service refuses an infix operator with nothing after it before it + /// reads a right side (measured 2026-11-02 on a condition ending in `AND` + /// and in `OR`). Call with the operator consumed. + fn require_rhs(&self) -> PResult<()> { + if matches!(self.peek().kind, TokenKind::Eof) { + return Err(ParseError::well_formed(MISSING_RHS)); + } + Ok(()) + } + + /// Whether the current token is an unquoted word usable as a name. + fn at_name(&self) -> bool { + matches!(&self.peek().kind, TokenKind::Ident(s) if word_class(s) == WordClass::Name) + } + + /// An optional alias after a table. The service reads an alias and then + /// refuses it, so the refusal is deferred until the statement parses. + fn optional_alias(&mut self) -> PResult<()> { + if self.read_alias()? { + self.defer(ALIASING); + } + Ok(()) + } + + /// Read an optional alias after a table or a projected path: `AS` and a + /// name, or a bare name. Returns whether there was one. `AS` with no name + /// after it is a syntax error (measured 2026-11-02 in a projection and in + /// FROM). + fn read_alias(&mut self) -> PResult { + let is_name = |p: &Self| p.at_name() || matches!(p.peek().kind, TokenKind::QuotedIdent(_)); + if self.eat_keyword("AS") { + if !is_name(self) { + return Err(ParseError::well_formed("Expected identifier for AS-alias")); + } + } else if !is_name(self) { + return Ok(false); + } + self.bump(); + Ok(true) + } + + /// An optional alias after a path in the select list. It is refused after + /// the table name rule (measured 2026-11-02: `SELECT a AS b FROM "tt"` is + /// refused for the name of the table). + fn projection_alias(&mut self) -> PResult<()> { + if self.read_alias()? { + self.defer_after_name_rule(ALIASING); + } + Ok(()) + } + + /// The aliases a FROM source may carry: `AS` or a bare name, then `AT` + /// and `BY` names, each refused like the first (measured 2026-11-02 with + /// `AT x` and `BY x` after the table). An alias on the table comes ahead + /// of the table name rule (measured 2026-11-02 with and without `AS`). + fn source_aliases(&mut self) -> PResult<()> { + self.optional_alias()?; + for word in ["AT", "BY"] { + if self.is_keyword(word) + && matches!( + self.tokens.get(self.pos + 1).map(|t| &t.kind), + Some(TokenKind::Ident(s)) if word_class(s) == WordClass::Name + ) + { + self.bump(); + self.bump(); + self.defer(ALIASING); + } + } + Ok(()) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/core/src/partiql/parser/condition.rs b/crates/core/src/partiql/parser/condition.rs new file mode 100644 index 000000000..fe24fe84a --- /dev/null +++ b/crates/core/src/partiql/parser/condition.rs @@ -0,0 +1,422 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The WHERE condition: OR, AND and NOT over predicates, comparisons, type names and calls. + +use super::limits::{MAX_OPERATORS, Nest}; +use super::{ + AND_OR_ARGUMENTS, BARE_OPERAND, GROUPED_OPERAND, NO_COMPARISON, NOT_A_CONDITION, NOT_ARGUMENT, + PResult, ParseError, Parser, forms, +}; +use crate::partiql::ast::{CompareOp, Operand, TypeName, WhereExpr}; +use crate::partiql::lexer::{Token, TokenKind}; + +impl Parser { + // --- where expression --- + + pub(super) fn where_expr(&mut self) -> PResult { + let outer = std::mem::replace(&mut self.in_where, true); + // A refusal that stops the parse leaves the flag set, so the + // findings are reported in the order of the part it stopped in. + let expr = self.or_expr()?; + self.in_where = outer; + if !outer { + self.after_where = true; + } + Ok(expr) + } + + fn or_expr(&mut self) -> PResult { + let mut left = self.and_expr()?; + while self.eat_keyword("OR") { + self.count_operator(); + let bare_left = std::mem::take(&mut self.bare_left); + self.require_rhs()?; + let right = self.and_expr()?; + self.end_and_or(bare_left); + if self.within_operator_limit() { + left = WhereExpr::Or(Box::new(left), Box::new(right)); + } + } + Ok(left) + } + + /// AND, any number of times, between predicates. An operand that is not + /// a condition as the left argument of an AND or an OR is refused once + /// the right argument has been read, after what the walk met in both + /// (measured 2026-11-02: `a AND pk = 'a'` is refused for it, `a + 1 AND + /// pk = 'a'` for the `+`). Only a flag is kept for it, so the frames each + /// group nests stay small (see [`limits`]). + fn and_expr(&mut self) -> PResult { + let mut left = self.not_expr()?; + while self.eat_keyword("AND") { + self.count_operator(); + let bare_left = std::mem::take(&mut self.bare_left); + self.require_rhs()?; + let right = self.not_expr()?; + self.end_and_or(bare_left); + if self.within_operator_limit() { + left = WhereExpr::And(Box::new(left), Box::new(right)); + } + } + Ok(left) + } + + /// The right argument of an AND or an OR has been read; `bare_left` is + /// whether the left one is an operand that is not a condition. + fn end_and_or(&mut self, bare_left: bool) { + if bare_left { + self.walk_after_operands(AND_OR_ARGUMENTS); + } + } + + /// `NOT`, any number of times, before a predicate. The run is read in a + /// loop, so its length costs no stack. The predicate is one part of the + /// walk of the condition (see [`walk`]). + fn not_expr(&mut self) -> PResult { + let mut nots = 0; + while self.eat_keyword("NOT") { + self.count_operator(); + nots += 1; + } + self.after_not = nots > 0; + let before = self.walk_part(); + let predicate = self.predicate(); + self.end_walk_part(before); + let mut expr = predicate?; + for _ in 0..nots.min(MAX_OPERATORS + 1) { + expr = WhereExpr::Not(Box::new(expr)); + } + Ok(expr) + } + + /// A predicate: a condition in parentheses, and what may follow it, or a + /// predicate of a term. Parentheses nest through here, so the rest is + /// read in a function of its own, never inlined, and this frame stays + /// small (see [`limits`]). + /// + /// A group that holds one operand and nothing else is that operand, and + /// the predicate goes on from it (measured 2026-11-05: `(pk) = 'a'` + /// reads by key, `((pk)) = 'a'`, `(a) BETWEEN 0 AND 2`, `(a) IN [1]`, + /// `(m) IS NOT MISSING` and `NOT (a) = 2` run, and `(a)` alone is + /// refused as `a` alone is). + fn predicate(&mut self) -> PResult { + if !matches!(self.peek().kind, TokenKind::LParen) { + return self.term_predicate(); + } + let open = self.pos; + let after_not = self.after_not; + let refused_operators = self.operator_refusals; + self.bump(); + let inner = self.nested(Nest::Group, Self::where_expr)?; + // (measured 2026-11-02: `(a = 1 b)`, `(a = 1` and `((a = 1)`) + if !self.eat(&TokenKind::RParen) { + return Err(ParseError::well_formed("Expected RIGHT_PAREN")); + } + match inner { + WhereExpr::Function(name, mut operand) if name == GROUPED_OPERAND => { + self.after_not = after_not; + let left = operand.pop().expect("a grouped operand holds one operand"); + let left = self.steps_after_group(left, open)?; + self.after_operand(left, open, refused_operators) + } + inner => self.after_predicate(inner), + } + } + + /// A predicate that is not in parentheses: a condition function, or an + /// operand and what follows it. + #[inline(never)] + fn term_predicate(&mut self) -> PResult { + // Function-style predicates: begins_with / contains / attribute_type. + if let TokenKind::Ident(name) = &self.peek().kind { + let lname = name.to_ascii_lowercase(); + if matches!( + lname.as_str(), + "begins_with" | "contains" | "attribute_type" + ) && matches!( + self.tokens.get(self.pos + 1).map(|t| &t.kind), + Some(TokenKind::LParen) + ) { + self.count_operator(); + self.bump(); + let args = self.call_args()?; + if args.len() != 2 { + return Err(ParseError::new(format!("{lname} requires two arguments"))); + } + let mut it = args.into_iter(); + let a = it.next().unwrap(); + let b = it.next().unwrap(); + return self.after_predicate(match lname.as_str() { + "begins_with" => WhereExpr::BeginsWith(a, b), + "contains" => WhereExpr::Contains(a, b), + _ => WhereExpr::AttributeType(a, b), + }); + } + if matches!( + lname.as_str(), + "attribute_exists" | "attribute_not_exists" | "size" + ) && matches!( + self.tokens.get(self.pos + 1).map(|t| &t.kind), + Some(TokenKind::LParen) + ) { + self.count_operator(); + self.bump(); + let args = self.call_args()?; + // size(...) can appear as the left of a comparison; handle that below + // by treating the function as an operand. Here with no comparison it is + // a bare boolean function (attribute_exists / attribute_not_exists). + if lname == "size" { + let comparison = self.finish_comparison(Operand::Function(lname, args))?; + return self.after_predicate(comparison); + } + forms::check_presence_operands(&lname, &args)?; + return self.after_predicate(WhereExpr::Function(lname, args)); + } + } + let left_start = self.pos; + let exec = self.is_keyword("EXEC"); + let refused_operators = self.operator_refusals; + let left = self.operand()?; + if exec { + // `EXEC` reads to the end of its arguments and its refusal is + // deferred; this condition stands in for it and never runs. + return Ok(WhereExpr::Function("exec".to_owned(), vec![left])); + } + self.after_operand(left, left_start, refused_operators) + } + + /// The rest of a predicate that starts with the operand `left`, read + /// from token `start` on, the operand itself or a group around it. The + /// condition operators read so far numbered `refused_operators` before + /// the operand was read. + #[inline(never)] + fn after_operand( + &mut self, + left: Operand, + start: usize, + refused_operators: usize, + ) -> PResult { + // A refusal of the operand names its first token inside any group + // around it (measured 2026-11-05: `(a) IS INT4` names the `a`). + let left_span = self.inner_span(start); + let left_start = start; + if self.eat_keyword("IS") { + // An IS after the test is refused for its left side (measured + // 2026-11-02 with `a IS NULL IS NULL`). A comparison with a + // boolean after it is not measured. + let test = self.is_test(left, &left_span)?; + self.trailing_is()?; + return Ok(test); + } + if self.is_keyword("NOT") { + return self.negated_predicate(left); + } + if self.eat_keyword("BETWEEN") { + let between = self.between(left)?; + return self.after_predicate(between); + } + if self.eat_keyword("LIKE") { + let like = self.like(left)?; + return self.after_predicate(like); + } + if self.eat_keyword("IN") { + let in_list = self.in_list(left)?; + return self.after_predicate(in_list); + } + if !self.at_comparison() { + // The operand is all the group around it holds: the group is + // the operand, and the predicate goes on after the group (see + // [`Parser::predicate`]). + if matches!(self.peek().kind, TokenKind::RParen) + && self.nesting.in_group() + && start + .checked_sub(1) + .is_some_and(|i| self.tokens[i].kind == TokenKind::LParen) + { + return Ok(WhereExpr::Function(GROUPED_OPERAND.to_owned(), vec![left])); + } + // A call to a function a condition does not take, and MISSING, + // are refused on their own once the statement is read (measured + // 2026-11-02: `... AND EXISTS(ss)` is "Unrecognized function"), + // and so is an operand holding an operator a condition does not + // take (measured 2026-11-02: `a + 1` is refused for the `+`). + // Each stands for a predicate that never runs. + if let Operand::Function(name, args) = left { + return Ok(WhereExpr::Function(name, args)); + } + let missing = matches!( + &self.tokens[left_start].kind, + TokenKind::Ident(w) if w.eq_ignore_ascii_case("missing") + ); + if missing { + return Ok(WhereExpr::Function("missing".to_owned(), vec![left])); + } + if self.operator_refusals > refused_operators { + return Ok(WhereExpr::Function("operator".to_owned(), vec![left])); + } + // Any other operand is refused, as the whole condition, as an + // argument of AND or OR or as the argument of a NOT (measured + // 2026-11-02 with names, paths, numbers, strings, lists, `true` + // and a parameter as the condition, and `a AND pk = 'a'`; + // measured 2026-11-05 in a group, `(a AND b = 2)` and `(b = 2 AND + // a)`, and after a NOT, `NOT a`, `(NOT a)` and `NOT (a) AND pk = + // 'a'`), after the walk (see [`walk`]). Any other token after the + // operand ends it, and the part of the statement around the + // operand reads that token, so a syntax error there comes first + // (measured 2026-11-05: `a b` is "Unexpected token after + // expression", `a LIMIT 1` is refused for the LIMIT clause and + // `(a b)` is "Expected RIGHT_PAREN"). The check is made here, at + // the end of the recursion, so the frames each group nests stay + // small (see [`limits`]). + self.refuse_bare_operand(left_start); + return Ok(WhereExpr::Function(BARE_OPERAND.to_owned(), vec![left])); + } + let comparison = self.finish_comparison(left)?; + self.after_predicate(comparison) + } + + /// Refuse an operand that is not a condition, starting at token + /// `start`, in its place in the walk: after everything the walk meets + /// in the AND or OR it is an argument of, or in the whole condition. + /// As the right argument the operand is the last thing that operator + /// meets, so its refusal is kept now, after what the operand met and + /// behind what the left argument met. As the left argument it is kept + /// once the right argument has been read (see [`Parser::and_expr`]). An + /// operand after OR and before AND is the left argument of the AND. + /// + /// After a NOT the operand is the NOT's argument, and its refusal is + /// kept now (measured 2026-11-05: `NOT (a) AND pk = 'a'` is refused for + /// the NOT's argument). A group around the operand counts as the + /// operand: `start` is then the token of its `(`. + fn refuse_bare_operand(&mut self, start: usize) { + if self.after_not { + self.walk_after_operands(NOT_ARGUMENT); + return; + } + let before = start.checked_sub(1).and_then(|i| self.tokens.get(i)); + let (after_and, after_or) = (is_word(before, "AND"), is_word(before, "OR")); + let (before_and, before_or) = (self.is_keyword("AND"), self.is_keyword("OR")); + if after_and || (after_or && !before_and) { + self.walk_after_operands(AND_OR_ARGUMENTS); + } else if before_and || before_or { + self.bare_left = true; + } else { + self.walk_after_operands(NOT_A_CONDITION); + } + } + + /// Whether the current token is a comparison operator. + fn at_comparison(&self) -> bool { + matches!( + self.peek().kind, + TokenKind::Eq + | TokenKind::Ne + | TokenKind::Lt + | TokenKind::Le + | TokenKind::Gt + | TokenKind::Ge + ) + } + + fn finish_comparison(&mut self, left: Operand) -> PResult { + let op = match &self.peek().kind { + TokenKind::Eq => CompareOp::Eq, + TokenKind::Ne => CompareOp::Ne, + TokenKind::Lt => CompareOp::Lt, + TokenKind::Le => CompareOp::Le, + TokenKind::Gt => CompareOp::Gt, + TokenKind::Ge => CompareOp::Ge, + _ => return Err(ParseError::well_formed(NO_COMPARISON)), + }; + self.count_operator(); + self.bump(); + self.require_rhs()?; + let right = self.operand()?; + Ok(WhereExpr::Compare(left, op, right)) + } + + /// The type name after `IS` / `IS NOT`: one of the service's type keywords + /// (`DOUBLE PRECISION` and `CHARACTER VARYING` are two words), else + /// "Expected type name" (measured 2026-11-02). + pub(super) fn type_name(&mut self) -> PResult { + let word = match &self.peek().kind { + TokenKind::Ident(w) => w.to_ascii_lowercase(), + _ => return Err(ParseError::well_formed("Expected type name")), + }; + let second = match self.tokens.get(self.pos + 1).map(|t| &t.kind) { + Some(TokenKind::Ident(w)) => w.to_ascii_lowercase(), + _ => String::new(), + }; + let (ty, words) = match (word.as_str(), second.as_str()) { + ("double", "precision") => (TypeName::DoublePrecision, 2), + ("character", "varying") => (TypeName::CharacterVarying, 2), + (w, _) => match TypeName::from_keyword(w) { + Some(ty) => (ty, 1), + None => return Err(ParseError::well_formed("Expected type name")), + }, + }; + for _ in 0..words { + self.bump(); + } + // A length or precision in parentheses, and the time zone of a time, + // are read and do not change the type (measured 2026-11-02: + // `IS VARCHAR(10)` and `IS TIME WITH TIME ZONE` parse). + if self.eat(&TokenKind::LParen) { + loop { + if !matches!(self.peek().kind, TokenKind::Number(_)) { + return Err(ParseError::well_formed("Expected RIGHT_PAREN")); + } + self.bump(); + if !self.eat(&TokenKind::Comma) { + break; + } + } + if !self.eat(&TokenKind::RParen) { + return Err(ParseError::well_formed("Expected RIGHT_PAREN")); + } + } + if matches!(ty, TypeName::Time | TypeName::Timestamp) + && self.eat_keyword("WITH") + && !(self.eat_keyword("TIME") && self.eat_keyword("ZONE")) + { + return Err(ParseError::well_formed("Expected type name")); + } + Ok(ty) + } + + pub(super) fn call_args(&mut self) -> PResult> { + self.arguments(Self::operand) + } + + /// The arguments of a call, with the current token on its `(`, each read + /// with `argument`. + pub(super) fn arguments( + &mut self, + argument: fn(&mut Self) -> PResult, + ) -> PResult> { + self.eat(&TokenKind::LParen); + let mut args = Vec::new(); + if !matches!(self.peek().kind, TokenKind::RParen) { + loop { + args.push(argument(self)?); + if !self.eat(&TokenKind::Comma) { + break; + } + } + } + // (measured 2026-11-02: a call left open at the end of the statement) + if !self.eat(&TokenKind::RParen) { + return Err(ParseError::well_formed("Expected RIGHT_PAREN")); + } + Ok(args) + } +} + +/// Whether `token` is the unquoted word `word`, in any case. +fn is_word(token: Option<&Token>, word: &str) -> bool { + matches!( + token.map(|t| &t.kind), + Some(TokenKind::Ident(s)) if s.eq_ignore_ascii_case(word) + ) +} diff --git a/crates/core/src/partiql/parser/forms.rs b/crates/core/src/partiql/parser/forms.rs new file mode 100644 index 000000000..591b74a68 --- /dev/null +++ b/crates/core/src/partiql/parser/forms.rs @@ -0,0 +1,330 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Statement forms and functions beyond the base productions in the parent +//! module: the FROM-first UPDATE, SET and REMOVE clauses in any number and +//! order, `set_add` and `set_delete` in a SET, `EXISTS(SELECT ...)`, the +//! operand checks on the condition functions, and a condition function +//! compared with a boolean. +//! +//! Every message is the service's (measured 2026-11-02). + +use super::{PResult, ParseError, Parser}; +use crate::partiql::ast::{ + BooleanComparison, Operand, Path, SelectStatement, SetAction, SetMembersAction, SetMembersOp, + UpdateStatement, WhereExpr, +}; +use crate::partiql::lexer::TokenKind; +use crate::types::AttributeValue; + +/// The functions a WHERE condition recognizes. +const CONDITION_FUNCTIONS: [&str; 6] = [ + "attribute_exists", + "attribute_not_exists", + "attribute_type", + "begins_with", + "contains", + "size", +]; + +/// The assignments of one or more SET and REMOVE clauses. +#[derive(Default)] +pub(super) struct DmlClauses { + pub sets: Vec, + pub removes: Vec, + pub set_members: Vec, +} + +impl Parser { + /// The FROM-first UPDATE, after `FROM`: + /// `"table" [WHERE ] (SET ... | REMOVE ...)+ [RETURNING ...]`. + /// + /// The service runs it as the UPDATE it spells out. WHERE is optional in + /// this grammar, so a statement without one parses to its end, where a + /// leftover token (a WHERE written after SET, say) is reported first, and + /// execution then refuses it with the full-key message (measured + /// 2026-11-02). + pub(super) fn update_led_by_from(&mut self) -> PResult { + let table = self.table_name()?; + // The target path and an alias get the refusals of an UPDATE's + // (measured 2026-11-02: an index, a short index name, three + // components, an empty component, `t` and `AS t`). + self.dml_target_rest(&table, true)?; + self.optional_alias()?; + let where_clause = if self.eat_keyword("WHERE") { + Some(self.where_expr()?) + } else { + None + }; + let clauses = self.dml_clauses()?; + let returning = self.returning(false)?; + Ok(UpdateStatement { + table, + sets: clauses.sets, + removes: clauses.removes, + set_members: clauses.set_members, + where_clause, + returning, + }) + } + + /// One or more SET and REMOVE clauses, in any order: an UPDATE may carry + /// two SET clauses, and SET and REMOVE may follow each other (measured + /// 2026-11-02). A SET takes assignments separated by commas, and a REMOVE + /// takes one path: a comma after it is a token left over after the + /// statement (measured 2026-11-02: `SET a = ..., b = ...` runs and + /// `REMOVE m.a, m` is "Unexpected token after expression"). + pub(super) fn dml_clauses(&mut self) -> PResult { + let mut clauses = DmlClauses::default(); + let mut saw_clause = false; + // A refusal that stops the parse leaves the flag set (see `walk`). + self.in_values = true; + loop { + if self.eat_keyword("SET") { + loop { + self.set_assignment(&mut clauses)?; + if !self.eat(&TokenKind::Comma) { + break; + } + } + } else if self.eat_keyword("REMOVE") { + clauses.removes.push(self.simple_path()?); + } else { + break; + } + saw_clause = true; + } + if !saw_clause { + return Err(ParseError::well_formed("Expected data manipulation")); + } + self.in_values = false; + Ok(clauses) + } + + /// One ` = ` of a SET clause. + fn set_assignment(&mut self, clauses: &mut DmlClauses) -> PResult<()> { + let path = self.simple_path()?; + if !self.eat(&TokenKind::Eq) { + return Err(ParseError::well_formed("Expected = in SET clause")); + } + if let Some(op) = self.set_function_ahead() { + let action = self.set_members_call(path, op)?; + clauses.set_members.push(action); + } else { + let value = self.operand()?; + self.note_set_number(&value); + // An IS test as the value parses and is refused (measured + // 2026-11-02 with `SET a = b IS NULL`). + if self.eat_keyword("IS") { + self.eat_keyword("NOT"); + self.type_name()?; + self.defer("IS operator is not supported in update expressions"); + } + clauses.sets.push(SetAction { path, value }); + } + Ok(()) + } + + /// Whether the current token is an identifier opening a call. + fn at_call(&self) -> bool { + matches!(self.peek().kind, TokenKind::Ident(_)) + && matches!( + self.tokens.get(self.pos + 1).map(|t| &t.kind), + Some(TokenKind::LParen) + ) + } + + /// `set_add(` or `set_delete(` at the current token, in any case. + fn set_function_ahead(&self) -> Option { + if !self.at_call() { + return None; + } + match &self.peek().kind { + TokenKind::Ident(name) if name.eq_ignore_ascii_case("set_add") => { + Some(SetMembersOp::Add) + } + TokenKind::Ident(name) if name.eq_ignore_ascii_case("set_delete") => { + Some(SetMembersOp::Delete) + } + _ => None, + } + } + + /// `set_add(, )` or `set_delete(, )` as the value + /// of `SET = ...`, with the current token on the function name. + /// + /// The service's checks, in order: exactly two arguments, the first one + /// the assignment target, the second a set; positions name the function, + /// except the set check, which names the first argument. An empty set + /// literal is refused on its own (all measured 2026-11-02). + fn set_members_call(&mut self, target: Path, op: SetMembersOp) -> PResult { + let function = op.function_name(); + let at = self.span(); + self.bump(); + self.bump(); + let mut args: Vec<(Operand, String)> = Vec::new(); + if !matches!(self.peek().kind, TokenKind::RParen) { + loop { + let arg_at = self.span(); + args.push((self.operand()?, arg_at)); + if !self.eat(&TokenKind::Comma) { + break; + } + } + } + if !self.eat(&TokenKind::RParen) { + return Err(ParseError::well_formed("Expected ) to close function call")); + } + let [(first, first_argument_at), (members, _)]: [(Operand, String); 2] = + args.try_into().map_err(|_| { + ParseError::new(format!( + "{function} must have exactly two arguments at {at}" + )) + })?; + if !matches!(&first, Operand::Path(path) if *path == target) { + return Err(ParseError::new(format!( + "The first argument to {function} must equal the assignment value at {at}" + ))); + } + match &members { + Operand::Literal(AttributeValue::SS(set)) if set.is_empty() => { + return Err(ParseError::new("Empty bags are not supported")); + } + Operand::Literal( + AttributeValue::SS(_) | AttributeValue::NS(_) | AttributeValue::BS(_), + ) + | Operand::Parameter(_) => {} + _ => { + return Err(ParseError::new(op.not_a_set_message(&first_argument_at))); + } + } + Ok(SetMembersAction { + path: target, + op, + members, + first_argument_at, + }) + } + + /// Whether the statement is `EXISTS(SELECT ...)`. + pub(super) fn at_exists_statement(&self) -> bool { + self.is_keyword("EXISTS") + && self.at_call() + && matches!( + self.tokens.get(self.pos + 2).map(|t| &t.kind), + Some(TokenKind::Ident(s)) if s.eq_ignore_ascii_case("SELECT") + ) + } + + /// `EXISTS(SELECT ...)`, with the current token on `EXISTS`. + pub(super) fn exists_statement(&mut self) -> PResult { + self.bump(); + self.bump(); + self.bump(); + let select = self.select(false)?; + if !self.eat(&TokenKind::RParen) { + return Err(ParseError::well_formed("Expected ) to close function call")); + } + Ok(select) + } + + /// A predicate compared with a boolean literal, as in + /// `attribute_exists(a) = true`, any number of times, as in `a = 1 = true + /// = true` (measured 2026-11-02): as a condition, `= true` and `<> false` + /// keep it and `= false` and `<> true` negate it. The comparison is kept + /// as written (see [`WhereExpr::ComparedWithBoolean`]), since a key + /// condition compared with a boolean is read by key (see `plan`). A + /// comparison compared with a number with `=` holds for no item + /// (measured 2026-11-02: `a = 1 = 1` runs and selects nothing). Past the + /// operator limit the statement is refused, and the condition is not + /// built on. + pub(super) fn compared_with_boolean(&mut self, condition: WhereExpr) -> WhereExpr { + let mut condition = condition; + loop { + let equal = match self.peek().kind { + TokenKind::Eq => true, + TokenKind::Ne => false, + _ => return condition, + }; + let literal = match self.tokens.get(self.pos + 1).map(|t| &t.kind) { + Some(TokenKind::Ident(s)) if s.eq_ignore_ascii_case("true") => true, + Some(TokenKind::Ident(s)) if s.eq_ignore_ascii_case("false") => false, + Some(TokenKind::Number(n)) + if equal + && matches!( + condition.without_kept_comparisons(), + WhereExpr::Compare(..) + ) => + { + let n = n.clone(); + self.count_operator(); + self.bump(); + self.bump(); + self.walk_number(&n, super::LITERAL_ROOT); + let never = WhereExpr::Not(Box::new(condition.clone())); + condition = WhereExpr::And(Box::new(condition), Box::new(never)); + continue; + } + _ => return condition, + }; + self.count_operator(); + self.bump(); + self.bump(); + if self.within_operator_limit() { + condition = WhereExpr::ComparedWithBoolean( + Box::new(condition), + BooleanComparison { equal, literal }, + ); + } + } + } + + /// A function call as a top-level INSERT value: once the call parses, it is + /// refused naming the function and its position (measured 2026-11-02 for + /// `set_add`). `None` when the current token does not open a call. + pub(super) fn function_as_item_value(&mut self) -> Option { + if !self.at_call() { + return None; + } + let at = self.span(); + let TokenKind::Ident(name) = self.bump().kind else { + return None; + }; + if let Err(e) = self.call_args() { + return Some(e); + } + Some(ParseError::new(format!( + "Unsupported data type: {} under key root at {at}", + name.to_ascii_lowercase() + ))) + } +} + +/// The operand checks on `attribute_exists` and `attribute_not_exists`: exactly +/// one operand, and that operand a document path rather than a literal or a +/// parameter (measured 2026-11-02). +pub(super) fn check_presence_operands(name: &str, args: &[Operand]) -> PResult<()> { + if args.len() != 1 { + return Err(ParseError::new(format!( + "Incorrect number of operands for operator or function; operator or function: {name}, number of operands: {}", + args.len() + ))); + } + if !matches!(args[0], Operand::Path(_)) { + return Err(ParseError::new(format!( + "Operator or function requires a document path; operator or function: {name}" + ))); + } + Ok(()) +} + +/// A call inside a WHERE condition to anything but a condition function, such +/// as `EXISTS(a)` or `set_add(...)`, is refused naming the function in lower +/// case and its position (measured 2026-11-02). `None` for a condition +/// function. +pub(super) fn where_function_refusal(name: &str, at: &str) -> Option { + (!CONDITION_FUNCTIONS.contains(&name)).then(|| format!("Unrecognized function: {name} at {at}")) +} + +#[cfg(test)] +mod tests; diff --git a/crates/core/src/partiql/parser/forms/tests.rs b/crates/core/src/partiql/parser/forms/tests.rs new file mode 100644 index 000000000..09603a3c1 --- /dev/null +++ b/crates/core/src/partiql/parser/forms/tests.rs @@ -0,0 +1,392 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Unit tests for the statement forms and functions beyond the base productions. + +use crate::partiql::ast::{Operand, SetMembersOp, Statement, WhereExpr}; +use crate::partiql::parse; +use crate::types::AttributeValue; + +fn err(sql: &str) -> String { + parse(sql).unwrap_err().message +} + +fn update(sql: &str) -> crate::partiql::ast::UpdateStatement { + match parse(sql).unwrap() { + Statement::Update(u) => u, + other => panic!("expected an update, got {other:?}"), + } +} + +// `UPDATE "t" SET ` is 15 characters, so a value expression starts at 16. +const SET_AT: &str = "UPDATE \"t\" SET "; + +#[test] +fn set_add_becomes_a_set_members_action() { + let u = update("UPDATE \"t\" SET ss = set_add(ss, <<'a', 'b'>>) WHERE pk = 'p'"); + assert!(u.sets.is_empty()); + assert_eq!(u.set_members.len(), 1); + let action = &u.set_members[0]; + assert_eq!(action.op, SetMembersOp::Add); + assert_eq!(action.path.root, "ss"); + assert!(matches!( + &action.members, + Operand::Literal(AttributeValue::SS(s)) if s.len() == 2 + )); + assert_eq!(action.first_argument_at, "1:29:2"); +} + +#[test] +fn set_functions_are_case_insensitive_and_take_a_parameter() { + let u = update("UPDATE \"t\" SET ns = SET_DELETE(ns, ?) WHERE pk = ?"); + assert_eq!(u.set_members[0].op, SetMembersOp::Delete); + assert!(matches!(u.set_members[0].members, Operand::Parameter(0))); + assert_eq!( + u.where_clause, + Some(WhereExpr::Compare( + Operand::Path(crate::partiql::ast::Path { + root: "pk".to_owned(), + steps: Vec::new() + }), + crate::partiql::ast::CompareOp::Eq, + Operand::Parameter(1) + )) + ); +} + +#[test] +fn two_set_clauses_and_a_remove_parse_in_any_order() { + let u = update( + "UPDATE \"t\" SET a = 1 REMOVE b SET ss = set_add(ss, <<'x'>>) SET c = 2 WHERE pk = 'p'", + ); + assert_eq!(u.sets.len(), 2); + assert_eq!(u.removes.len(), 1); + assert_eq!(u.set_members.len(), 1); +} + +#[test] +fn a_remove_takes_one_path_and_a_set_a_list() { + let leftover = + "Statement wasn't well formed, can't be processed: Unexpected token after expression"; + for sql in [ + "UPDATE \"t\" REMOVE m.a, m WHERE pk = 'p'", + "UPDATE \"t\" SET a = 1, b = 2 REMOVE c, d WHERE pk = 'p'", + "UPDATE \"t\" REMOVE c, d SET a = 1 WHERE pk = 'p'", + "UPDATE \"t\" REMOVE c, WHERE pk = 'p'", + "UPDATE \"t\" REMOVE c, d", + "FROM \"t\" WHERE pk = 'p' REMOVE c, d", + // The comma is reported ahead of a refusal deferred to the end. + "UPDATE \"t\" REMOVE \"\", d WHERE pk = 'p'", + ] { + assert_eq!(err(sql), leftover, "{sql}"); + } + let u = update("UPDATE \"t\" SET a = 1, b = 2 REMOVE c REMOVE d WHERE pk = 'p'"); + assert_eq!(u.sets.len(), 2); + assert_eq!(u.removes.len(), 2); +} + +#[test] +fn set_add_assigned_to_another_attribute_names_the_function() { + let sql = format!("{SET_AT}t = set_add(ss, <<'x'>>) WHERE pk = 'p'"); + assert_eq!( + err(&sql), + "The first argument to SET_ADD must equal the assignment value at 1:20:7" + ); +} + +#[test] +fn set_add_whose_first_argument_is_not_a_path_names_the_function() { + let sql = format!("{SET_AT}ss = set_add(<<'a'>>, <<'b'>>) WHERE pk = 'p'"); + assert_eq!( + err(&sql), + "The first argument to SET_ADD must equal the assignment value at 1:21:7" + ); +} + +#[test] +fn set_add_of_a_value_that_is_not_a_set_names_the_first_argument() { + for value in ["'x'", "['x']", "1", "{'a': 1}", "true", "null", "other"] { + let sql = format!("{SET_AT}ss = set_add(ss, {value}) WHERE pk = 'p'"); + assert_eq!( + err(&sql), + "The second argument to SET_ADD must be a value with type SET at 1:29:2", + "{value}" + ); + } + let sql = format!("{SET_AT}ss = set_delete(ss, 'a') WHERE pk = 'p'"); + assert_eq!( + err(&sql), + "The second argument to SET_DELETE must be a value with type SET at 1:32:2" + ); +} + +#[test] +fn set_add_needs_exactly_two_arguments() { + for args in ["ss", "ss, <<'x'>>, <<'y'>>", ""] { + let sql = format!("{SET_AT}ss = set_add({args}) WHERE pk = 'p'"); + assert_eq!( + err(&sql), + "SET_ADD must have exactly two arguments at 1:21:7", + "{args}" + ); + } +} + +#[test] +fn set_delete_assigned_to_another_attribute_names_the_function() { + let sql = format!("{SET_AT}t = set_delete(ss, <<'a'>>) WHERE pk = 'p'"); + assert_eq!( + err(&sql), + "The first argument to SET_DELETE must equal the assignment value at 1:20:10" + ); +} + +#[test] +fn set_add_of_an_empty_set_literal() { + let sql = format!("{SET_AT}ss = set_add(ss, <<>>) WHERE pk = 'p'"); + assert_eq!(err(&sql), "Empty bags are not supported"); +} + +#[test] +fn set_add_of_a_set_with_a_repeated_member_reports_the_duplicate() { + let sql = format!("{SET_AT}ss = set_add(ss, <<'x', 'x'>>) WHERE pk = 'p'"); + assert_eq!( + err(&sql), + "One or more parameter values were invalid: Input collection [x, x] contains duplicates." + ); +} + +#[test] +fn set_add_in_a_where_is_an_unrecognized_function() { + let sql = "SELECT * FROM \"t\" WHERE pk = 'p' AND set_add(ss, <<'x'>>) = ss"; + assert_eq!(err(sql), "Unrecognized function: set_add at 1:38:7"); +} + +#[test] +fn exists_in_a_where_is_an_unrecognized_function() { + let sql = "SELECT * FROM \"t\" WHERE pk = 'p' AND EXISTS(ss)"; + assert_eq!(err(sql), "Unrecognized function: exists at 1:38:6"); + let sql = "SELECT * FROM \"t\" WHERE pk = 'p' AND a = foo(b)"; + assert_eq!(err(sql), "Unrecognized function: foo at 1:42:3"); +} + +#[test] +fn condition_functions_stay_recognized_in_a_where() { + assert!(parse("SELECT * FROM \"t\" WHERE pk = 'p' AND size(s) = 3").is_ok()); + assert!(parse("SELECT * FROM \"t\" WHERE pk = 'p' AND 3 = size(s)").is_ok()); + assert!(parse("UPDATE \"t\" SET l = list_append(l, [1]) WHERE pk = 'p'").is_ok()); +} + +#[test] +fn set_add_as_an_insert_value_is_an_unsupported_data_type() { + let sql = "INSERT INTO \"t\" VALUE {'pk': 'p', 'ss': set_add(<<'a'>>, <<'b'>>)}"; + assert_eq!( + err(sql), + "Unsupported data type: set_add under key root at 1:41:7" + ); +} + +#[test] +fn attribute_exists_needs_a_document_path() { + for operand in ["'ss'", "?", "1"] { + let sql = format!("SELECT * FROM \"t\" WHERE pk = 'p' AND attribute_exists({operand})"); + assert_eq!( + err(&sql), + "Operator or function requires a document path; operator or function: attribute_exists", + "{operand}" + ); + } + assert_eq!( + err("UPDATE \"t\" SET a = 1 WHERE pk = 'p' AND attribute_not_exists('a')"), + "Operator or function requires a document path; operator or function: attribute_not_exists" + ); +} + +#[test] +fn attribute_exists_needs_exactly_one_operand() { + assert_eq!( + err("SELECT * FROM \"t\" WHERE pk = 'p' AND attribute_exists()"), + "Incorrect number of operands for operator or function; operator or function: attribute_exists, number of operands: 0" + ); + assert_eq!( + err("SELECT * FROM \"t\" WHERE pk = 'p' AND attribute_exists(ss, ns)"), + "Incorrect number of operands for operator or function; operator or function: attribute_exists, number of operands: 2" + ); +} + +#[test] +fn a_condition_function_compared_with_a_boolean() { + let condition = |sql: &str| match parse(sql).unwrap() { + Statement::Select(s) => match s.where_clause.unwrap() { + WhereExpr::And(_, right) => *right, + other => panic!("expected AND, got {other:?}"), + }, + other => panic!("expected a select, got {other:?}"), + }; + let base = "SELECT * FROM \"t\" WHERE pk = 'p' AND "; + // As a condition, `= true` and `<> false` keep the function's test and + // `= false` and `<> true` negate it. + for (tail, negated) in [ + ("attribute_exists(ss) = true", false), + ("attribute_exists(ss) <> FALSE", false), + ("attribute_exists(ss) = false", true), + ("begins_with(s, 'a') <> true", true), + ] { + let condition = condition(&format!("{base}{tail}")); + assert!( + matches!(condition, WhereExpr::ComparedWithBoolean(..)), + "{tail}" + ); + let (negates, inner) = condition.without_nots(); + assert_eq!(negates, negated, "{tail}"); + assert!( + matches!(inner, WhereExpr::Function(..) | WhereExpr::BeginsWith(..)), + "{tail}" + ); + } +} + +#[test] +fn a_comparison_compared_with_booleans_and_with_a_number() { + use crate::partiql::ast::{CompareOp, Path}; + let condition = |sql: &str| match parse(sql).unwrap() { + Statement::Select(s) => s.where_clause.unwrap(), + other => panic!("expected a select, got {other:?}"), + }; + let compare = WhereExpr::Compare( + Operand::Path(Path { + root: "a".to_owned(), + steps: Vec::new(), + }), + CompareOp::Eq, + Operand::Literal(AttributeValue::N("1".to_owned())), + ); + let base = "SELECT * FROM \"t\" WHERE "; + let negation = |sql: &str| { + let condition = condition(sql); + let (negated, inner) = condition.without_nots(); + (negated, inner.clone()) + }; + assert_eq!( + negation(&format!("{base}a = 1 = true = true")), + (false, compare.clone()) + ); + assert_eq!( + negation(&format!("{base}a = 1 = true <> true")), + (true, compare.clone()) + ); + // With a number it holds for no item. + assert_eq!( + condition(&format!("{base}a = 1 = 1")), + WhereExpr::And( + Box::new(compare.clone()), + Box::new(WhereExpr::Not(Box::new(compare.clone()))) + ) + ); + // A comparison with true that keeps the condition still takes a number. + match condition(&format!("{base}a = 1 = true = 1")) { + WhereExpr::And(kept, never) => { + assert_eq!(kept.without_nots(), (false, &compare)); + assert_eq!(never.without_nots(), (true, &compare)); + } + other => panic!("expected AND, got {other:?}"), + } + assert_eq!( + err(&format!("{base}a = 1 = 1e126")), + "Number overflow. Attempting to store a number with magnitude larger than \ + supported range under root" + ); + // `<>` with a number, and a number after any other predicate, are not + // measured. + let leftover = + "Statement wasn't well formed, can't be processed: Unexpected token after expression"; + for tail in ["a = 1 <> 1", "a IN [1] = 1"] { + assert_eq!(err(&format!("{base}{tail}")), leftover, "{tail}"); + } +} + +#[test] +fn from_first_update_parses_where_then_clauses() { + let u = update("from \"t\" where pk = 'p' AND a = 1 set c = 2 remove b returning all new *"); + assert_eq!(u.table, "t"); + assert!(u.where_clause.is_some()); + assert_eq!(u.sets.len(), 1); + assert_eq!(u.removes.len(), 1); + assert_eq!(u.returning, Some(crate::partiql::ast::Returning::AllNew)); +} + +#[test] +fn from_first_update_without_where_parses_for_execution_to_refuse() { + assert!(update("FROM \"t\" SET c = 1").where_clause.is_none()); + assert!(update("FROM \"t\" REMOVE a").where_clause.is_none()); +} + +#[test] +fn from_first_update_with_where_after_set_is_a_leftover_token() { + let leftover = + "Statement wasn't well formed, can't be processed: Unexpected token after expression"; + assert_eq!(err("FROM \"t\" SET c = 1 WHERE pk = 'p'"), leftover); + assert_eq!( + err("FROM \"t\" SET c = 1 WHERE pk = 'p' SET d = 2"), + leftover + ); +} + +#[test] +fn from_first_update_paths_and_aliases_get_the_update_refusals() { + for (sql, expected) in [ + ( + "FROM \"t\".\"idx\" WHERE pk = 'p' SET c = 1", + "This operation is not supported on an index", + ), + ( + "FROM \"t\".\"idx\" WHERE pk = 'p' REMOVE c", + "This operation is not supported on an index", + ), + ( + "FROM \"t\".\"i\" WHERE pk = 'p' SET c = 1", + crate::partiql::from_path::NAME_RULE, + ), + ( + "FROM \"t\".\"a\".\"b\" WHERE pk = 'p' SET c = 1", + "A path may contain at most 2 components in the FROM clause", + ), + ( + "FROM \"t\".\"\" WHERE pk = 'p' SET c = 1", + "Path component cannot be an empty string", + ), + ( + "FROM \"t\" t WHERE pk = 'p' SET c = 1", + "Aliasing is not supported", + ), + ( + "FROM \"t\" AS t WHERE pk = 'p' SET c = 1", + "Aliasing is not supported", + ), + ] { + assert_eq!(err(sql), expected, "{sql}"); + } +} + +#[test] +fn from_first_update_without_a_clause_is_not_data_manipulation() { + assert_eq!( + err("FROM \"t\" WHERE pk = 'p'"), + "Statement wasn't well formed, can't be processed: Expected data manipulation" + ); +} + +#[test] +fn exists_statement_parses_its_select() { + match parse("EXISTS(SELECT * FROM \"t\" WHERE pk = 'p' AND a = 1)").unwrap() { + Statement::Exists(select) => { + assert_eq!(select.from.table, "t"); + assert!(select.where_clause.is_some()); + } + other => panic!("expected EXISTS, got {other:?}"), + } + assert!(matches!( + parse("exists (select * from \"t\" where pk = 'p' and a = 1)").unwrap(), + Statement::Exists(_) + )); +} diff --git a/crates/core/src/partiql/parser/limits.rs b/crates/core/src/partiql/parser/limits.rs new file mode 100644 index 000000000..7842c7fd3 --- /dev/null +++ b/crates/core/src/partiql/parser/limits.rs @@ -0,0 +1,234 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The limits on a statement's size and shape: its length, the operators in +//! its WHERE condition, the nesting of a literal value, and the nesting depth +//! the parser recurses to. +//! +//! The parser recurses once per parenthesized group, literal container and +//! nested operand (a call's arguments, a reserved word that reads an +//! expression). Runs of `NOT` and of signs are read in a loop, so they cost no +//! depth. A statement is first parsed on the calling thread with a small depth +//! budget; one that needs more is parsed again on a thread of its own with a +//! large stack and the full caps, so the depth an accepted statement may reach +//! does not depend on the caller's stack. Past a cap the statement is refused +//! with a syntax error instead of overflowing a stack. +//! +//! Stack use per level on a debug build (measured 2026-11-02 by bisecting +//! the stack of a thread that parses one statement): 8.6 KB per +//! parenthesized group, 6.8 KB per nested operand and 2.5 to 2.8 KB per +//! literal container. The inline budget keeps a parse within 310 KB of the +//! caller's stack, and the worst parse the caps allow used 12.2 MB on its own +//! thread. On a release build the same parses used 2.3 KB, 1.8 KB and 0.5 to +//! 0.7 KB per level, 86 KB of the caller's stack and 3.1 MB at the caps. + +use super::{PResult, ParseError, Parser, Statement}; + +/// The longest statement the service accepts, in characters: 8192 run and +/// 8193 are refused before the statement is read (measured 2026-11-02 with +/// ASCII text). +pub const MAX_STATEMENT_LENGTH: usize = 8192; + +/// The most operators a WHERE condition may hold (measured 2026-11-02: 299 +/// `NOT`s before `a = 1` run and 300 are refused). +pub const MAX_OPERATORS: usize = 300; + +/// The most members an IN list may hold (measured 2026-11-02: 101 and 1000 +/// members are refused, naming the count). +pub const MAX_IN_OPERANDS: usize = 100; + +/// The containers (maps and lists) a literal value may sit inside. A value +/// inside this many is refused, naming its path (measured 2026-11-02 for a +/// list and a map literal 32, 33, 34, 40, 100 and 1000 levels deep, in an +/// INSERT value, a SET value and a WHERE literal). +pub const MAX_LITERAL_NESTING: usize = 32; + +/// Parenthesized groups a condition may nest. The service runs 1000 levels +/// and fails at 2000 (measured 2026-11-02). +pub const MAX_GROUP_DEPTH: usize = 1000; + +/// Literal containers (maps, lists and bags) a value may nest. The service +/// refuses a literal 1000 levels deep with the nesting message and fails at +/// 4000 (measured 2026-11-02). +pub const MAX_LITERAL_DEPTH: usize = 1000; + +/// Operands nested inside operands: a call inside a call's arguments, or a +/// reserved word that reads an expression after it. The service's limit is +/// not measured. +pub const MAX_OPERAND_DEPTH: usize = 100; + +/// The depth (all kinds together) a parse may reach on the calling thread +/// before it moves to a thread of its own. +const INLINE_DEPTH: usize = 32; + +/// The stack of the thread a deep statement is parsed on: about five times +/// what the worst parse the caps allow (1000 groups around 100 nested +/// operands around a 1000-level literal) used on a debug build. +pub const LARGE_STACK_BYTES: usize = 64 << 20; + +/// The reason given for a statement nested past a cap. The service fails +/// such a statement with an internal error instead (measured 2026-11-02 with +/// 2000 groups and 4000 nested lists). +pub const TOO_DEEP: &str = "expression nesting depth exceeded"; + +/// The refusal of a statement longer than [`MAX_STATEMENT_LENGTH`], echoing +/// the statement (measured 2026-11-02: 8192 characters run, 8193 are +/// refused). `None` when the statement is not too long. +#[must_use] +pub fn statement_length_refusal(statement: &str) -> Option { + (statement.chars().count() > MAX_STATEMENT_LENGTH).then(|| { + format!( + "1 validation error detected: Value '{statement}' at 'statement' failed to \ + satisfy constraint: Member must have length less than or equal to \ + {MAX_STATEMENT_LENGTH}" + ) + }) +} + +/// The refusal of a WHERE condition with more than [`MAX_OPERATORS`] +/// operators. The service stops counting at the first one past the limit. +#[must_use] +pub fn too_many_operators() -> String { + format!( + "The expression contains too many operators; operator count: {}", + MAX_OPERATORS + 1 + ) +} + +/// The refusal of an IN list with `members` members, more than +/// [`MAX_IN_OPERANDS`]. +#[must_use] +pub fn too_many_in_operands(members: usize) -> String { + format!("The IN operator is provided with too many operands; number of operands: {members}") +} + +/// The refusal of a literal value at `path` that sits inside +/// [`MAX_LITERAL_NESTING`] containers. +#[must_use] +pub fn literal_nesting_refusal(path: &str) -> String { + format!("Nesting Levels have exceeded supported limits under {path}") +} + +/// A production that recurses. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum Nest { + /// A parenthesized condition. + Group, + /// A map, list or bag literal. + Literal, + /// An operand read inside another one. + Operand, +} + +/// How deep the parser has recursed, by kind, and whether the inline budget +/// still applies. +#[derive(Debug, Default)] +pub(super) struct Nesting { + group: usize, + literal: usize, + operand: usize, + /// Open map and list literals, which the literal nesting rule counts. + pub containers: usize, + /// Whether this parse runs on the caller's thread, under [`INLINE_DEPTH`]. + inline: bool, + /// Set when the inline budget ran out: the parse is abandoned and run + /// again on a thread of its own. + pub out_of_budget: bool, +} + +impl Nesting { + pub(super) fn new(inline: bool) -> Self { + Self { + inline, + ..Self::default() + } + } + + fn enter(&mut self, kind: Nest) -> PResult<()> { + let (depth, cap) = match kind { + Nest::Group => (&mut self.group, MAX_GROUP_DEPTH), + Nest::Literal => (&mut self.literal, MAX_LITERAL_DEPTH), + Nest::Operand => (&mut self.operand, MAX_OPERAND_DEPTH), + }; + if *depth >= cap { + return Err(ParseError::well_formed(TOO_DEEP)); + } + *depth += 1; + if self.inline && self.group + self.literal + self.operand > INLINE_DEPTH { + self.out_of_budget = true; + return Err(ParseError::well_formed(TOO_DEEP)); + } + Ok(()) + } + + /// Whether the parse is inside a parenthesized condition. + pub(super) fn in_group(&self) -> bool { + self.group > 0 + } + + fn leave(&mut self, kind: Nest) { + let depth = match kind { + Nest::Group => &mut self.group, + Nest::Literal => &mut self.literal, + Nest::Operand => &mut self.operand, + }; + *depth -= 1; + } +} + +impl Parser { + /// Run a production that recurses, one level deeper. + pub(super) fn nested( + &mut self, + kind: Nest, + production: impl FnOnce(&mut Self) -> PResult, + ) -> PResult { + self.nesting.enter(kind)?; + let result = production(self); + self.nesting.leave(kind); + result + } + + /// Count one operator of the WHERE condition: AND, OR, NOT, a comparison, + /// BETWEEN, IN, IS, or a function call. + pub(super) fn count_operator(&mut self) { + if self.in_where { + self.operators += 1; + } + } + + /// Whether the condition is still within [`MAX_OPERATORS`]. Past it the + /// statement is refused, so the parser reads the rest of the condition + /// without building it, which keeps the tree it drops shallow. + pub(super) fn within_operator_limit(&self) -> bool { + self.operators <= MAX_OPERATORS + } +} + +/// Parse with `attempt`, on the calling thread when the statement fits the +/// inline budget and on a thread with [`LARGE_STACK_BYTES`] of stack when it +/// does not. `attempt` takes whether the parse is inline and returns `None` +/// when the inline budget ran out. +pub(super) fn parse_on_a_fitting_stack( + attempt: impl Fn(bool) -> Option> + Sync, +) -> PResult { + if let Some(result) = attempt(true) { + return result; + } + std::thread::scope(|scope| { + let spawned = std::thread::Builder::new() + .name("partiql-parse".to_owned()) + .stack_size(LARGE_STACK_BYTES) + .spawn_scoped(scope, || attempt(false)); + match spawned { + Ok(handle) => match handle.join() { + Ok(result) => result.unwrap_or_else(|| Err(ParseError::well_formed(TOO_DEEP))), + Err(panic) => std::panic::resume_unwind(panic), + }, + Err(_) => Err(ParseError::well_formed(TOO_DEEP)), + } + }) +} + +#[cfg(test)] +mod tests; diff --git a/crates/core/src/partiql/parser/limits/tests.rs b/crates/core/src/partiql/parser/limits/tests.rs new file mode 100644 index 000000000..7c0f40592 --- /dev/null +++ b/crates/core/src/partiql/parser/limits/tests.rs @@ -0,0 +1,300 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Unit tests for the limits on a statement's size and shape. + +use super::*; +use crate::partiql::ast::{Operand, WhereExpr}; +use crate::partiql::parse; +use crate::partiql::parser::parse_with_full_caps; +use crate::types::AttributeValue; + +const WF: &str = "Statement wasn't well formed, can't be processed: "; +const SELECT: &str = "SELECT * FROM \"t\" WHERE "; + +fn err(sql: &str) -> String { + parse(sql).unwrap_err().message +} + +fn too_deep() -> String { + format!("{WF}{TOO_DEEP}") +} + +fn groups(n: usize) -> String { + format!("{SELECT}{}a = 1{}", "(".repeat(n), ")".repeat(n)) +} + +fn nots(n: usize) -> String { + format!("{SELECT}{}a = 1", "NOT ".repeat(n)) +} + +fn chain(n: usize, joiner: &str) -> String { + let terms: Vec = (0..n).map(|i| format!("a{i} = 1")).collect(); + format!("{SELECT}{}", terms.join(joiner)) +} + +fn list(n: usize) -> String { + format!( + "INSERT INTO \"t\" VALUE {{'pk': 'x', 'l': {}1{}}}", + "[".repeat(n), + "]".repeat(n) + ) +} + +fn map(n: usize) -> String { + format!( + "INSERT INTO \"t\" VALUE {{'pk': 'x', 'm': {}1{}}}", + "{'a': ".repeat(n), + "}".repeat(n) + ) +} + +fn calls(n: usize) -> String { + format!( + "UPDATE \"t\" SET a = {}x{} WHERE pk = 'p'", + "f(".repeat(n), + ")".repeat(n) + ) +} + +fn nesting(step: &str) -> String { + literal_nesting_refusal(&format!("root{}", step.repeat(MAX_LITERAL_NESTING))) +} + +#[test] +fn a_statement_of_8192_characters_is_not_too_long() { + let head = format!("{SELECT}pk = 'a'"); + let at_limit = format!("{head}{}", " ".repeat(MAX_STATEMENT_LENGTH - head.len())); + assert_eq!(statement_length_refusal(&at_limit), None); + let over = format!("{at_limit} "); + assert_eq!( + statement_length_refusal(&over), + Some(format!( + "1 validation error detected: Value '{over}' at 'statement' failed to \ + satisfy constraint: Member must have length less than or equal to 8192" + )) + ); +} + +#[test] +fn a_condition_may_hold_300_operators() { + // Each NOT and the comparison count. + assert!(parse(¬s(MAX_OPERATORS - 1)).is_ok()); + assert_eq!(err(¬s(MAX_OPERATORS)), too_many_operators()); + assert_eq!( + too_many_operators(), + "The expression contains too many operators; operator count: 301" + ); + // Each AND or OR and each comparison count: 150 terms are 299. + for joiner in [" AND ", " OR "] { + assert!(parse(&chain(150, joiner)).is_ok(), "{joiner}"); + assert_eq!(err(&chain(151, joiner)), too_many_operators(), "{joiner}"); + assert_eq!(err(&chain(299, joiner)), too_many_operators(), "{joiner}"); + } + // A function counts like an operator. + let functions: Vec<&str> = vec!["attribute_exists(a)"; 151]; + assert_eq!( + err(&format!("{SELECT}{}", functions.join(" AND "))), + too_many_operators() + ); +} + +#[test] +fn a_run_of_nots_keeps_its_parity() { + let negations = |sql: &str| { + let Ok(crate::partiql::Statement::Select(s)) = parse(sql) else { + panic!("{sql}"); + }; + let mut depth = 0; + let mut expr = s.where_clause.unwrap(); + while let WhereExpr::Not(inner) = expr { + depth += 1; + expr = *inner; + } + assert!(matches!(expr, WhereExpr::Compare(..)), "{sql}"); + depth + }; + assert_eq!(negations(¬s(299)), 299); + assert_eq!(negations(¬s(2)), 2); +} + +#[test] +fn a_long_condition_is_refused_without_overflowing() { + // 8,189 and about 7,700 characters. + assert_eq!(err(¬s(2040)), too_many_operators()); + assert_eq!(err(&chain(600, " AND ")), too_many_operators()); + assert_eq!(err(&chain(600, " OR ")), too_many_operators()); +} + +#[test] +fn a_condition_may_nest_1000_groups() { + assert!(parse(&groups(MAX_GROUP_DEPTH)).is_ok()); + assert_eq!(err(&groups(MAX_GROUP_DEPTH + 1)), too_deep()); + // 8,029 characters. + assert_eq!(err(&groups(4000)), too_deep()); + // A syntax error before the cap is reached wins. + assert_eq!( + err(&format!("{SELECT}{}a ={}", "(".repeat(10), ")".repeat(10))), + format!("{WF}Unexpected term") + ); +} + +#[test] +fn signs_before_a_name_are_refused_naming_the_operator() { + let refusal = "Unsupported operator in Condition Expression. Operator: +"; + for n in [1, 4000, 8150] { + assert_eq!( + err(&format!("{SELECT}a = {}a", "+".repeat(n))), + refusal, + "{n}" + ); + } + // The sign next to the operand is named. + assert_eq!( + err(&format!("{SELECT}a = + - + -a")), + "Unsupported operator in Condition Expression. Operator: -" + ); + assert_eq!( + err(&format!( + "UPDATE \"t\" SET a = {}b WHERE pk = 'p'", + "+".repeat(4000) + )), + "Incorrect number of operands for operator or function; operator or function: +, \ + number of operands: 1" + ); +} + +#[test] +fn signs_before_a_number_fold_into_it_however_many() { + let literal = |sql: &str| match parse(sql) { + Ok(crate::partiql::Statement::Select(s)) => match s.where_clause { + Some(WhereExpr::Compare(_, _, Operand::Literal(AttributeValue::N(n)))) => n, + other => panic!("{other:?}"), + }, + other => panic!("{other:?}"), + }; + assert_eq!(literal(&format!("{SELECT}a = {}1", "+".repeat(8000))), "1"); + assert_eq!(literal(&format!("{SELECT}a = {}1", "- ".repeat(4000))), "1"); + assert_eq!( + literal(&format!("{SELECT}a = {}1", "- ".repeat(3999))), + "-1" + ); +} + +#[test] +fn two_minus_signs_start_a_comment() { + for n in [2, 10, 100] { + for operand in ["1", "b"] { + assert_eq!( + err(&format!("{SELECT}a = {}{operand}", "-".repeat(n))), + format!("{WF}Missing right-hand side expression of infix operator"), + "{n} {operand}" + ); + } + } +} + +#[test] +fn a_literal_value_inside_32_containers_is_refused_naming_its_path() { + assert!(parse(&list(MAX_LITERAL_NESTING - 1)).is_ok()); + assert!(parse(&map(MAX_LITERAL_NESTING - 1)).is_ok()); + for n in [32, 33, 34, 40, 100, 1000] { + assert_eq!(err(&list(n)), nesting("[0]"), "list {n}"); + assert_eq!(err(&map(n)), nesting(".a"), "map {n}"); + } + for sql in [ + format!("{SELECT}x = {}1{}", "[".repeat(33), "]".repeat(33)), + format!("{SELECT}x = {}1{}", "[".repeat(1000), "]".repeat(1000)), + format!( + "UPDATE \"t\" SET x = {}1{} WHERE pk = 'u'", + "[".repeat(33), + "]".repeat(33) + ), + ] { + assert_eq!(err(&sql), nesting("[0]"), "{sql}"); + } + // The path names where the deep value sits, and a syntax error wins. + assert_eq!( + err(&format!( + "INSERT INTO \"t\" VALUE {{'pk': 'x', 'v': [1, {{'k': {}1{}}}]}}", + "[".repeat(30), + "]".repeat(30) + )), + literal_nesting_refusal(&format!("root[1].k{}", "[0]".repeat(30))) + ); + assert_eq!( + err(&format!( + "INSERT INTO \"t\" VALUE {{'pk':'mix','x':{{'a':[{{'b':{}1{}}}]}}}}}}}}", + "[".repeat(30), + "]".repeat(30) + )), + format!("{WF}Unexpected token after expression") + ); +} + +#[test] +fn a_literal_may_nest_1000_containers() { + assert_eq!(err(&list(MAX_LITERAL_DEPTH + 1)), too_deep()); + assert_eq!(err(&map(MAX_LITERAL_DEPTH + 1)), too_deep()); + // 8,041 characters. + assert_eq!(err(&list(4000)), too_deep()); + let bags = format!("{SELECT}a = {}1{}", "<<".repeat(2000), ">>".repeat(2000)); + assert_eq!(err(&bags), too_deep()); +} + +#[test] +fn an_operand_may_nest_100_operands() { + // The innermost `x` is the hundredth operand. + assert!(parse(&calls(MAX_OPERAND_DEPTH - 1)).is_ok()); + assert_eq!(err(&calls(MAX_OPERAND_DEPTH)), too_deep()); + assert_eq!(err(&calls(2700)), too_deep()); + assert_eq!( + err(&format!("{SELECT}a = {}b", "not ".repeat(2000))), + too_deep() + ); +} + +/// The worst parse the caps allow: groups, operands and a literal each +/// at their cap. +fn worst() -> String { + format!( + "{SELECT}{}a = {}{}1{}{}{}", + "(".repeat(MAX_GROUP_DEPTH), + "size(".repeat(MAX_OPERAND_DEPTH - 1), + "{'a': ".repeat(MAX_LITERAL_DEPTH), + "}".repeat(MAX_LITERAL_DEPTH), + ")".repeat(MAX_OPERAND_DEPTH - 1), + ")".repeat(MAX_GROUP_DEPTH) + ) +} + +#[test] +fn the_worst_parse_fits_half_the_large_stack() { + let statement = worst(); + let answer = std::thread::Builder::new() + .stack_size(LARGE_STACK_BYTES / 2) + .spawn(move || parse_with_full_caps(&statement).map(|_| ())) + .unwrap() + .join() + .unwrap(); + assert_eq!(answer.unwrap_err().message, nesting(".a")); +} + +#[test] +fn a_deep_statement_takes_little_of_the_callers_stack() { + // The inline budget keeps the caller's share to about 310 KB on a + // debug build; the rest of the parse runs on its own thread. + let answers = std::thread::Builder::new() + .stack_size(1 << 20) + .spawn(|| { + [ + parse(&groups(MAX_GROUP_DEPTH)).map(|_| ()), + parse(&worst()).map(|_| ()), + ] + }) + .unwrap() + .join() + .unwrap(); + assert_eq!(answers[0], Ok(())); + assert_eq!(answers[1].clone().unwrap_err().message, nesting(".a")); +} diff --git a/crates/core/src/partiql/parser/literals.rs b/crates/core/src/partiql/parser/literals.rs new file mode 100644 index 000000000..e093247ae --- /dev/null +++ b/crates/core/src/partiql/parser/literals.rs @@ -0,0 +1,343 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Literal values: maps, lists and bags, and the item map of an INSERT. + +use std::collections::{BTreeMap, BTreeSet}; + +use super::limits::{MAX_LITERAL_NESTING, Nest, literal_nesting_refusal}; +use super::{BAG_MEMBER_TYPES, LITERAL_ROOT, PResult, ParseError, Parser}; +use crate::partiql::ast::Operand; +use crate::partiql::lexer::{TokenKind, WordClass, word_class}; +use crate::types::AttributeValue; + +impl Parser { + /// A map literal at `path` inside the value being read (`root`, `root.a`, + /// `root[0]`, ...). + pub(super) fn map_literal(&mut self, path: &str) -> PResult> { + self.nested(Nest::Literal, |p| { + p.nesting.containers += 1; + let map = p.map_members(path); + p.nesting.containers -= 1; + map + }) + } + + fn map_members(&mut self, path: &str) -> PResult> { + // A name given twice is refused at the map's opening brace. + let open = self.span(); + self.eat(&TokenKind::LBrace); + let mut map = BTreeMap::new(); + if !matches!(self.peek().kind, TokenKind::RBrace) { + loop { + let key = match &self.peek().kind { + TokenKind::String(s) => s.clone(), + _ => { + return Err(ParseError::new(format!( + "Tuple keys must be of type String. Actual type: Id under root at {}", + self.span() + ))); + } + }; + if map.contains_key(&key) { + self.refuse_duplicate_key(&key, path, &open); + } + self.bump(); + if !self.eat(&TokenKind::Colon) { + return Err(ParseError::well_formed("Expected : in map literal")); + } + let val = self.literal_value(&format!("{path}.{key}"))?; + map.insert(key, val); + if !self.eat(&TokenKind::Comma) { + break; + } + } + } + if !self.eat(&TokenKind::RBrace) { + return Err(ParseError::well_formed("Expected } to close map literal")); + } + Ok(map) + } + + /// The top-level `VALUE { ... }` item map, returned as ordered pairs. A + /// name given twice is refused at its second occurrence (see + /// [`Parser::refuse_duplicate_key`]), so the pairs name each attribute + /// once. + pub(super) fn item_map(&mut self) -> PResult> { + self.eat(&TokenKind::LBrace); + let mut pairs: Vec<(String, Operand)> = Vec::new(); + if !matches!(self.peek().kind, TokenKind::RBrace) { + loop { + let key = match &self.peek().kind { + TokenKind::String(s) => s.clone(), + _ => { + return Err(ParseError::new(format!( + "Tuple keys must be of type String. Actual type: Id under root at {}", + self.span() + ))); + } + }; + if pairs.iter().any(|(name, _)| *name == key) { + let second = self.span(); + self.refuse_duplicate_key(&key, LITERAL_ROOT, &second); + } + self.bump(); + if !self.eat(&TokenKind::Colon) { + return Err(ParseError::well_formed("Expected : in item value")); + } + // A VALUE field may be a literal or a ? parameter. + let val = if matches!(self.peek().kind, TokenKind::Param) { + self.bump(); + Operand::Parameter(self.take_param_index()) + } else if let Some(refusal) = self.function_as_item_value() { + return Err(refusal); + } else if let TokenKind::Ident(name) = &self.peek().kind + && self.at_name_in_literal(name) + { + // A name as the whole value is refused naming itself + // (measured 2026-11-02: `{'pk': pk}` is "Unsupported + // data type: pk under key root"). + let refusal = format!( + "Unsupported data type: {name} under key {LITERAL_ROOT} at {}", + self.span() + ); + self.bump(); + self.defer(&refusal); + Operand::Literal(AttributeValue::Null) + } else { + Operand::Literal(self.literal_value(LITERAL_ROOT)?) + }; + pairs.push((key, val)); + if !self.eat(&TokenKind::Comma) { + break; + } + } + } + if !self.eat(&TokenKind::RBrace) { + return Err(ParseError::well_formed("Expected } to close item value")); + } + Ok(pairs) + } + + /// A list literal at `path` inside the value being read. + pub(super) fn list_literal(&mut self, path: &str) -> PResult> { + self.nested(Nest::Literal, |p| { + p.nesting.containers += 1; + let list = p.list_elements(path); + p.nesting.containers -= 1; + list + }) + } + + fn list_elements(&mut self, path: &str) -> PResult> { + self.eat(&TokenKind::LBracket); + let mut items = Vec::new(); + if !matches!(self.peek().kind, TokenKind::RBracket) { + loop { + let element = format!("{path}[{}]", items.len()); + items.push(self.literal_value(&element)?); + if !self.eat(&TokenKind::Comma) { + break; + } + } + } + if !self.eat(&TokenKind::RBracket) { + return Err(ParseError::well_formed("Expected ] to close list literal")); + } + Ok(items) + } + + /// A bag (set) literal at `path` inside the value being read. A bag is + /// not a container for the literal nesting rule: its members are numbers + /// or strings. + pub(super) fn bag_literal(&mut self, path: &str) -> PResult { + self.nested(Nest::Literal, |p| p.bag_members(path)) + } + + fn bag_members(&mut self, path: &str) -> PResult { + self.eat(&TokenKind::LDoubleAngle); + let mut strings: BTreeSet = BTreeSet::new(); + let mut numbers: BTreeSet = BTreeSet::new(); + let mut saw_string = false; + let mut saw_number = false; + // Members in source order, for the service's duplicate message which + // echoes the whole collection ("Input collection [a, a] ..."). + let mut members: Vec = Vec::new(); + if !matches!(self.peek().kind, TokenKind::RDoubleAngle) { + loop { + // A parameter as a member parses and is refused at its + // position (measured 2026-11-02 with `<>`). + if matches!(self.peek().kind, TokenKind::Param) { + let at = self.span(); + self.bump(); + self.take_param_index(); + self.defer(&format!("{BAG_MEMBER_TYPES} at {at}")); + } else { + match self.literal_value(path)? { + AttributeValue::S(s) => { + saw_string = true; + strings.insert(s.clone()); + members.push(s); + } + AttributeValue::N(n) => { + saw_number = true; + numbers.insert(n.clone()); + members.push(n); + } + _ => return Err(ParseError::new(BAG_MEMBER_TYPES)), + } + } + if !self.eat(&TokenKind::Comma) { + break; + } + } + } + // (measured 2026-11-02: `<<1.2.3>>`, a member followed by a second + // number, in set_add and set_delete) + if !self.eat(&TokenKind::RDoubleAngle) { + return Err(ParseError::well_formed( + "Expected RIGHT_DOUBLE_ANGLE_BRACKET", + )); + } + if saw_string && saw_number { + return Err(ParseError::new(BAG_MEMBER_TYPES)); + } + let distinct = if saw_number { + numbers.len() + } else { + strings.len() + }; + if distinct != members.len() { + return Err(ParseError::new(duplicate_bag_message(&members))); + } + if saw_number { + Ok(AttributeValue::NS(numbers)) + } else { + Ok(AttributeValue::SS(strings)) + } + } + + /// A literal value in a map/list/bag position (no paths), at `path` inside + /// the value being read. A value inside [`MAX_LITERAL_NESTING`] maps and + /// lists is refused once the statement parses, naming its path; the + /// first such value in the statement is named. + pub(super) fn literal_value(&mut self, path: &str) -> PResult { + if self.nesting.containers >= MAX_LITERAL_NESTING { + self.defer(&literal_nesting_refusal(path)); + } + if let Some(number) = self.signed_number() { + self.note_number(&number, path); + return Ok(AttributeValue::N(number.text)); + } + match self.peek().kind.clone() { + TokenKind::LParen => self.literal_in_parentheses(path), + // A parameter inside a literal parses and is refused at its + // position, naming where it sits in the value (measured + // 2026-11-02 in INSERT values, SET values and WHERE literals: + // `{'k':?}` is root.k, `[1,?]` is root[1]). The value stands in + // for it and is never used. + TokenKind::Param => { + let at = self.span(); + self.bump(); + self.take_param_index(); + self.defer(&format!( + "Unsupported data type: Parameter under key {path} at {at}" + )); + Ok(AttributeValue::Null) + } + TokenKind::String(s) => { + self.bump(); + Ok(AttributeValue::S(s)) + } + TokenKind::LBrace => Ok(AttributeValue::M(self.map_literal(path)?)), + TokenKind::LBracket => Ok(AttributeValue::L(self.list_literal(path)?)), + TokenKind::LDoubleAngle => self.bag_literal(path), + TokenKind::Ident(name) if self.at_name_in_literal(&name) => { + self.refuse_name_in_literal(path); + Ok(AttributeValue::Null) + } + TokenKind::Ident(name) => { + let l = name.to_ascii_lowercase(); + self.bump(); + match l.as_str() { + "true" => Ok(AttributeValue::Bool(true)), + "false" => Ok(AttributeValue::Bool(false)), + "null" => Ok(AttributeValue::Null), + _ => Err(ParseError::well_formed("Expected a literal value")), + } + } + _ => Err(ParseError::well_formed("Expected a literal value")), + } + } + + /// A literal value in parentheses at `path`, with the current token on + /// the `(`. The parentheses only group it, and a comma of their own + /// makes a list (measured 2026-11-05: the INSERT value `{'pk': ('k'), 'x': + /// (5), 'l': [(5)], 'p': (1, 2)}` stores k, 5, [5] and [1, 2]). + fn literal_in_parentheses(&mut self, path: &str) -> PResult { + if self.list_in_parentheses_ahead() { + return Ok(AttributeValue::L(self.list_in_parentheses(path)?)); + } + self.bump(); + let value = self.nested(Nest::Group, |p| p.literal_value(path))?; + if !self.eat(&TokenKind::RParen) { + return Err(ParseError::well_formed("Expected RIGHT_PAREN")); + } + Ok(value) + } + + /// Whether the current token, the word `word`, is a name standing alone + /// inside a literal. A name that starts a path or a call is not measured + /// there, and keeps the syntax error. + pub(super) fn at_name_in_literal(&self, word: &str) -> bool { + word_class(word) == WordClass::Name + && !matches!( + self.tokens.get(self.pos + 1).map(|t| &t.kind), + Some(TokenKind::Dot | TokenKind::LBracket | TokenKind::LParen) + ) + } + + /// Refuse `key`, a key of the map at `path` that a member before it + /// already names, at the position `at` (measured 2026-11-03, before the + /// statement is authorized and with nothing written): + /// + /// * at the top level of an INSERT VALUE, the position of the second + /// occurrence, its length the quoted key's: `{'pk': 'd1', 'pk': 'd2', + /// 'sk': 's'}` is "Duplicate keys in tuple. Key: pk under root at + /// 1:47:4", for the partition key, the sort key and another attribute; + /// * in any other map (one nested in a value, a map in a list, a SET + /// value, a WHERE literal), the position of that map's opening `{`, + /// length 1: in `'m': {'y': {'x': 1, 'x': 2}}` the inner `{` with + /// "Key: x under root.y". + /// + /// Names differing in case are different names. Like the other refusals + /// of a literal, it is reported once the statement parses, in its place + /// among them; of two maps that each repeat a name, the one whose second + /// name comes first is reported (not measured). + fn refuse_duplicate_key(&mut self, key: &str, path: &str, at: &str) { + self.defer(&format!( + "Duplicate keys in tuple. Key: {key} under {path} at {at}" + )); + } + + /// Read a name inside a literal, at `path` inside the value: it parses + /// and is refused at its position (measured 2026-11-02: in `a IN ((pk, + /// 1))` the `pk` is root[0]). The value read stands in for it and is + /// never used. Kept out of [`Parser::literal_value`], which nests. + fn refuse_name_in_literal(&mut self, path: &str) { + let at = self.span(); + self.bump(); + self.defer(&format!( + "Unsupported data type: Id under key {path} at {at}" + )); + } +} + +fn duplicate_bag_message(members: &[String]) -> String { + // The service echoes the whole collection in source order, e.g. + // `Input collection [a, a] contains duplicates.` + format!( + "One or more parameter values were invalid: Input collection [{}] contains duplicates.", + members.join(", ") + ) +} diff --git a/crates/core/src/partiql/parser/operands.rs b/crates/core/src/partiql/parser/operands.rs new file mode 100644 index 000000000..0ca78883c --- /dev/null +++ b/crates/core/src/partiql/parser/operands.rs @@ -0,0 +1,498 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Paths and operands: the signs and arithmetic of an operand and the term it starts with. + +use super::limits::Nest; +use super::{LITERAL_ROOT, PResult, ParseError, Parser, forms, terms}; +use crate::partiql::ast::{ArithOp, Operand, Path, PathStep}; +use crate::partiql::lexer::{TokenKind, WordClass, is_function_keyword, word_class}; +use crate::types::AttributeValue; + +/// A number literal with the signs written before it folded in (see +/// [`Parser::signed_number`]). +pub(super) struct SignedNumber { + /// The number's text, with a `-` when the signs negate it. + pub text: String, + /// The `line:column:length` span of the number itself. + pub at: String, + /// Whether the number is a zero written with a decimal point or an + /// exponent and a minus sign is among the signs before it (see + /// `number::is_zero_decimal`). + pub negative_zero: bool, +} + +impl Parser { + // --- paths --- + + /// An empty quoted name in a path is refused once the statement parses + /// (measured 2026-11-02 in a condition, as the path and as a step). + fn refuse_empty_component(&mut self, name: &str) { + if name.is_empty() { + self.defer("Path component cannot be an empty string"); + } + } + + pub(super) fn path(&mut self) -> PResult { + let root = match &self.peek().kind { + TokenKind::Ident(s) + if matches!(word_class(s), WordClass::Name | WordClass::Literal) => + { + let s = s.clone(); + self.bump(); + s + } + TokenKind::QuotedIdent(s) => { + let s = s.clone(); + self.bump(); + self.refuse_empty_component(&s); + s + } + _ => return Err(self.term_error()), + }; + self.path_steps(Path { + root, + steps: Vec::new(), + }) + } + + /// The steps after `path`: `.name` and what brackets hold (see + /// [`Parser::bracket_step`]), as many as follow. + pub(super) fn path_steps(&mut self, mut path: Path) -> PResult { + loop { + if self.eat(&TokenKind::Dot) { + match &self.peek().kind { + TokenKind::QuotedIdent(s) => { + let s = s.clone(); + self.bump(); + self.refuse_empty_component(&s); + path.steps.push(PathStep::Attribute(s)); + } + TokenKind::Ident(s) if word_class(s) == WordClass::Name => { + let s = s.clone(); + self.bump(); + path.steps.push(PathStep::Attribute(s)); + } + TokenKind::Star => { + return Err(ParseError::new(format!( + "Unexpected path component at {}", + self.span() + ))); + } + // A reserved word (measured 2026-11-02 with `inner`), or + // anything else that is not a name, after a dot. + _ => return Err(ParseError::well_formed("Invalid path dot component")), + } + } else if self.eat(&TokenKind::LBracket) { + let step = self.bracket_step()?; + // (measured 2026-11-05: `m['k'`, `m[k`, `m['k' 'j']` and `m[k + // j]`; a bracket left open wins over a refusal of what it + // holds, as `m[-1` and `m[k.j` show) + if !self.eat(&TokenKind::RBracket) { + return Err(ParseError::well_formed("Expected RIGHT_BRACKET")); + } + path.steps.push(step); + } else { + break; + } + } + Ok(path) + } + + /// What a pair of brackets after a path holds, with the current token + /// after the `[`. A number is a list index. A string, a quoted name or + /// a name is a map member, as after a dot (measured 2026-11-05: `m['k']`, + /// `m["k"]` and `m[k]` read as `m.k` in a condition, a SET target, a + /// REMOVE, a SET value and the select list, and `m['my key']` names a + /// member a dot cannot). A reserved word there is a syntax error. Anything + /// else parses and is refused once the statement parses, at its first + /// token (measured 2026-11-05: `m[?]`, `m[1.5]`, `m[true]`, `m[k.j]`, + /// `m['k'.x]` and `m[-1.5]`, at the number); a minus before a whole + /// number is refused as an index out of range. The step returned for a + /// refused content stands for it and is never used. + fn bracket_step(&mut self) -> PResult { + let at = self.span(); + let refuse = |p: &mut Self, at: &str| { + p.defer(&format!("Unexpected path component at {at}")); + Ok(PathStep::Index(0)) + }; + match self.peek().kind.clone() { + TokenKind::Number(n) => { + self.bump(); + match n.parse::() { + Ok(index) => Ok(PathStep::Index(index)), + Err(_) => refuse(self, &at), + } + } + TokenKind::String(name) | TokenKind::QuotedIdent(name) => { + if matches!(self.nth_kind(1), Some(TokenKind::Dot | TokenKind::LBracket)) { + self.bump(); + self.path_steps(Path { + root: name, + steps: Vec::new(), + })?; + return refuse(self, &at); + } + self.bump(); + self.refuse_empty_component(&name); + Ok(PathStep::Attribute(name)) + } + TokenKind::Ident(word) => match word_class(&word) { + WordClass::Name => { + if matches!(self.nth_kind(1), Some(TokenKind::Dot | TokenKind::LBracket)) { + self.path()?; + return refuse(self, &at); + } + self.bump(); + Ok(PathStep::Attribute(word)) + } + WordClass::Literal => { + self.bump(); + refuse(self, &at) + } + WordClass::Operator => Err(ParseError::well_formed("Unexpected operator")), + WordClass::Keyword | WordClass::Special => { + Err(ParseError::well_formed("Unexpected term")) + } + }, + TokenKind::Minus if matches!(self.nth_kind(1), Some(TokenKind::Number(_))) => { + self.bump(); + let at = self.span(); + let TokenKind::Number(n) = self.bump().kind else { + unreachable!("a number follows the minus"); + }; + if n.bytes().all(|b| b.is_ascii_digit()) { + self.defer(&format!( + "List index is not within the allowable range; index: [-{n}] at {at}" + )); + Ok(PathStep::Index(0)) + } else { + refuse(self, &at) + } + } + TokenKind::Param => { + self.bump(); + self.take_param_index(); + refuse(self, &at) + } + _ => Err(ParseError::new(format!( + "Unexpected path component at {at}" + ))), + } + } + + // --- operands --- + + /// A condition takes no arithmetic: the service refuses any operator in + /// it once the statement parses (measured 2026-11-02 with `+`, `-`, `%`, + /// `/` and `||`, between numbers and on a path). The walk meets the + /// operator before the operands the current part has read (see [`walk`]). + pub(super) fn refuse_condition_operator(&mut self, op: &str) { + if self.in_where { + self.operator_refusals += 1; + self.walk_operator(format!( + "Unsupported operator in Condition Expression. Operator: {op}" + )); + } + } + + /// An operand, one part of the walk of a condition (see [`walk`]). + pub(super) fn operand(&mut self) -> PResult { + let before = self.walk_part(); + let operand = self.nested(Nest::Operand, Self::operand_unnested); + self.end_walk_part(before); + operand + } + + fn operand_unnested(&mut self) -> PResult { + let left = self.operand_atom()?; + if matches!(self.peek().kind, TokenKind::Plus | TokenKind::Minus) { + let op = if matches!(self.peek().kind, TokenKind::Plus) { + ArithOp::Add + } else { + ArithOp::Sub + }; + self.refuse_condition_operator(if op == ArithOp::Add { "+" } else { "-" }); + self.bump(); + self.require_rhs()?; + let right = self.operand_atom()?; + return Ok(Operand::Arith(Box::new(left), op, Box::new(right))); + } + // `*`, `/`, `%` and `||` in a condition read like `+` and `-`. + // Elsewhere they are not measured: `/`, `%` and `||` keep the answer + // they had when they did not lex, and `*` is left for the caller. + let op = match self.peek().kind { + TokenKind::Star if self.in_where => "*", + TokenKind::Slash => "/", + TokenKind::Percent => "%", + TokenKind::Concat => "||", + _ => return Ok(left), + }; + if !self.in_where { + return Err(ParseError::well_formed("")); + } + self.refuse_condition_operator(op); + self.bump(); + self.require_rhs()?; + self.operand_atom()?; + Ok(left) + } + + /// A number literal with any number of leading signs (`-7.3`, `+5`, + /// `- .5`, `- -7`), and in any number of parentheses with signs before + /// and inside them (measured 2026-11-05: `-(5)` and `- (5)` are -5, + /// `-(-(5))` is 5, `(-0)` is 0): the service folds the signs into the + /// number. Returns `None`, consuming nothing, when no number follows the + /// signs and parentheses or the parentheses do not close right after it. + /// The run is read in a loop, so its length costs no stack. + pub(super) fn signed_number(&mut self) -> Option { + let start = self.pos; + let mut negative = false; + let mut minus = false; + let mut open = 0usize; + loop { + match self.peek().kind { + TokenKind::Minus => { + negative = !negative; + minus = true; + } + TokenKind::Plus => {} + TokenKind::LParen => open += 1, + TokenKind::Number(_) => break, + _ => { + self.pos = start; + return None; + } + } + self.bump(); + } + let at = self.span(); + let TokenKind::Number(n) = self.bump().kind else { + unreachable!("the loop stops on a number"); + }; + for _ in 0..open { + if !self.eat(&TokenKind::RParen) { + self.pos = start; + return None; + } + } + let negative_zero = minus && crate::partiql::number::is_zero_decimal(&n); + let text = if negative { + match n.strip_prefix('-') { + Some(positive) => positive.to_owned(), + None => format!("-{n}"), + } + } else { + n + }; + Some(SignedNumber { + text, + at, + negative_zero, + }) + } + + fn operand_atom(&mut self) -> PResult { + if let Some(number) = self.signed_number() { + self.note_number(&number, LITERAL_ROOT); + return self + .literal_steps(Operand::Literal(AttributeValue::N(number.text)), &number.at); + } + // A sign before anything but a number is an operator with one + // operand, refused once the statement parses (measured 2026-11-02: + // `-n` in a condition names the operator, as a SET value it counts + // the operands, and 4000 `+` before a name get the answer one does). + // In a run of signs the one next to the operand is named. + let mut sign = None; + while let TokenKind::Plus | TokenKind::Minus = self.peek().kind { + sign = Some(if matches!(self.peek().kind, TokenKind::Plus) { + "+" + } else { + "-" + }); + self.bump(); + } + let at = self.span(); + let Some(op) = sign else { + return self.unsigned_atom(&at); + }; + // The sign is an operator of its own in the walk, met before its + // operand. + let before = self.walk_part(); + let inner = self.unsigned_atom(&at); + if inner.is_ok() { + if self.in_where { + self.refuse_condition_operator(op); + } else { + self.defer_sign(&format!( + "Incorrect number of operands for operator or function; operator or function: {op}, number of operands: 1" + )); + } + } + self.end_walk_part(before); + inner + } + + /// An operand in parentheses, with the current token on the `(`. The + /// parentheses only group what they hold (measured 2026-11-05: `pk = + /// ('a')`, `pk = (?)`, `SET x = (a + 1)`, `SET x = (a) + 1` and + /// `begins_with((sk), 'b')` run as written without them), a path in them + /// takes the steps after them (measured 2026-11-05: `(m).k = 1` and + /// `(m)['k'] = 1` read `m.k`), and a literal followed by steps is + /// refused at the literal. A comma of their own makes a list (measured + /// 2026-11-05: `SET x = ('a', 'b')` stores a list). + fn operand_in_parentheses(&mut self) -> PResult { + if self.list_in_parentheses_ahead() { + let list = self.list_in_parentheses(LITERAL_ROOT)?; + return Ok(Operand::Literal(AttributeValue::L(list))); + } + let open = self.pos; + self.bump(); + let inner = self.nested(Nest::Group, Self::operand)?; + // (measured 2026-11-05: `a = (1` and `pk = ('a'`) + if !self.eat(&TokenKind::RParen) { + return Err(ParseError::well_formed("Expected RIGHT_PAREN")); + } + self.steps_after_group(inner, open) + } + + /// The `line:column:length` span of the first token inside the group + /// that opens at token `open`, past any `(` of groups inside it. + pub(super) fn inner_span(&self, open: usize) -> String { + self.tokens[open..] + .iter() + .find(|t| t.kind != TokenKind::LParen) + .map(|t| format!("{}:{}:{}", t.line, t.column, t.length)) + .unwrap_or_default() + } + + /// The operand a group that opens at token `open` holds, with the path + /// steps written after the group: a path takes them (measured + /// 2026-11-05: `(m).k` and `(m)['k']` read `m.k`), and a literal is + /// refused for them at its first token (measured 2026-11-05: `('x').y` + /// names the `'x'`). + pub(super) fn steps_after_group(&mut self, operand: Operand, open: usize) -> PResult { + match operand { + Operand::Path(path) => Ok(Operand::Path(self.path_steps(path)?)), + literal @ Operand::Literal(_) => { + let at = self.inner_span(open); + self.literal_steps(literal, &at) + } + other => Ok(other), + } + } + + /// An operand that does not start with a sign, at the span `at`. + fn unsigned_atom(&mut self, at: &str) -> PResult { + match self.peek().kind.clone() { + TokenKind::LParen => self.operand_in_parentheses(), + // A backtick literal parses and is refused at its position, by + // its Ion type (measured 2026-11-02 with an integer). + TokenKind::Ion(text) => { + self.bump(); + self.defer(&format!( + "Ion Literals are not supported: {} under key root at {at}", + terms::ion_type_name(&text) + )); + Ok(Operand::Literal(AttributeValue::Null)) + } + TokenKind::Param => { + self.bump(); + let index = self.take_param_index(); + self.walk_parameter(index); + Ok(Operand::Parameter(index)) + } + TokenKind::String(s) => { + self.bump(); + self.literal_steps(Operand::Literal(AttributeValue::S(s)), at) + } + TokenKind::LBrace => Ok(Operand::Literal(AttributeValue::M( + self.map_literal(LITERAL_ROOT)?, + ))), + TokenKind::LBracket => Ok(Operand::Literal(AttributeValue::L( + self.list_literal(LITERAL_ROOT)?, + ))), + TokenKind::LDoubleAngle => Ok(Operand::Literal(self.bag_literal(LITERAL_ROOT)?)), + TokenKind::Ident(name) => { + let lname = name.to_ascii_lowercase(); + let literal = match lname.as_str() { + "true" => Some(AttributeValue::Bool(true)), + "false" => Some(AttributeValue::Bool(false)), + "null" => Some(AttributeValue::Null), + _ => None, + }; + if let Some(value) = literal { + self.bump(); + return self.literal_steps(Operand::Literal(value), at); + } + // (measured 2026-11-02: as a comparison operand and as the + // argument of attribute_exists, ahead of the table lookup). + // The rest is read as a name, so a syntax error after it wins. + if lname == "missing" && self.in_where { + self.defer("MISSING must be compared using IS operator"); + return Ok(Operand::Path(self.path()?)); + } + // function call vs path + let call = matches!( + self.tokens.get(self.pos + 1).map(|t| &t.kind), + Some(TokenKind::LParen) + ); + let class = word_class(&name); + let callable = class == WordClass::Name + || (class == WordClass::Keyword && is_function_keyword(&name)); + if call && callable { + self.count_operator(); + let at = self.span(); + let refusal = if self.in_where { + forms::where_function_refusal(&lname, &at) + } else { + None + }; + self.bump(); + // In a condition an argument may hold a LIKE (see + // [`Parser::argument_with_like`]). + let argument: fn(&mut Self) -> PResult = if self.in_where { + Self::argument_with_like + } else { + Self::operand + }; + let args = self.arguments(argument)?; + // The walk meets a call's arguments before its name + // (measured 2026-11-02: `foo(1e126)` is refused for the + // number, `foo(1 + 1)` for the `+`). + if let Some(refusal) = refusal { + self.defer(&refusal); + } + Ok(Operand::Function(lname, args)) + } else if matches!(class, WordClass::Name | WordClass::Literal) { + // `missing` outside a condition is read as a name. + Ok(Operand::Path(self.path()?)) + } else { + self.reserved_word_term() + } + } + TokenKind::QuotedIdent(_) => Ok(Operand::Path(self.path()?)), + _ => Err(self.term_error()), + } + } + + /// Path steps after a literal: the statement parses, and the path is + /// refused at the literal (measured 2026-11-02: `pk = 1.2.e3` is + /// "Unexpected path component at" the `1.2`, and `1.2.` and `1.2..3` are + /// "Invalid path dot component"). + fn literal_steps(&mut self, literal: Operand, at: &str) -> PResult { + if !matches!(self.peek().kind, TokenKind::Dot) { + return Ok(literal); + } + while self.eat(&TokenKind::Dot) { + match &self.peek().kind { + TokenKind::QuotedIdent(_) => {} + TokenKind::Ident(s) if word_class(s) == WordClass::Name => {} + _ => return Err(ParseError::well_formed("Invalid path dot component")), + } + self.bump(); + } + self.defer(&format!("Unexpected path component at {at}")); + Ok(literal) + } +} diff --git a/crates/core/src/partiql/parser/predicates.rs b/crates/core/src/partiql/parser/predicates.rs new file mode 100644 index 000000000..8c5745801 --- /dev/null +++ b/crates/core/src/partiql/parser/predicates.rs @@ -0,0 +1,412 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The predicates that follow an operand in a condition: `IN`, `BETWEEN`, +//! `LIKE` and their `NOT` forms, and `IS`; and what may follow a predicate. + +use super::{IN_OPERANDS, IS_LEFT_SIDE, LITERAL_ROOT, Nest, PResult, ParseError, Parser}; +use crate::partiql::ast::{CompareOp, Operand, TypeName, WhereExpr}; +use crate::partiql::lexer::{TokenKind, WordClass, is_function_keyword, word_class}; +use crate::types::AttributeValue; + +impl Parser { + /// ` NOT IN ...` and `NOT BETWEEN ...`, with the current token on + /// the NOT: the predicate after the NOT, negated. The NOT counts as an + /// operator of its own (measured 2026-11-02: `a NOT IN [1]` and `a NOT + /// BETWEEN 1 AND 2` count two operators and run wherever a condition + /// does). `NOT LIKE` gets the answers LIKE gets (measured 2026-11-02). + /// + /// Before anything else a NOT ends the expression at the operand, and + /// what encloses it reports the NOT (measured 2026-11-02: `a NOT = 1`, + /// `a NOT IS MISSING` and `a NOT NOT IN [1]` are a token left over after + /// the statement, `(a NOT = 1)` a group without its right parenthesis). + pub(super) fn negated_predicate(&mut self, left: Operand) -> PResult { + let word = match self.tokens.get(self.pos + 1).map(|t| &t.kind) { + Some(TokenKind::Ident(word)) => word.to_ascii_uppercase(), + _ => String::new(), + }; + if !matches!(word.as_str(), "IN" | "BETWEEN" | "LIKE") { + return Err(ParseError::well_formed(if self.nesting.in_group() { + "Expected RIGHT_PAREN" + } else { + "Unexpected token after expression" + })); + } + self.bump(); + self.count_operator(); + self.bump(); + let predicate = match word.as_str() { + "IN" => self.in_list(left)?, + "BETWEEN" => self.between(left)?, + _ => self.like(left)?, + }; + self.after_predicate(WhereExpr::Not(Box::new(predicate))) + } + + /// The rest of ` BETWEEN AND ` after BETWEEN + /// (measured 2026-11-02: nothing after BETWEEN, after its low bound and + /// after its AND each have their own message, with and without NOT). + pub(super) fn between(&mut self, value: Operand) -> PResult { + self.count_operator(); + self.require_rhs()?; + let low = self.operand()?; + if !self.eat_keyword("AND") { + return Err(ParseError::well_formed("Expected AND keyword")); + } + if matches!(self.peek().kind, TokenKind::Eof) { + return Err(ParseError::well_formed("Expected expression after AND")); + } + let high = self.operand()?; + Ok(WhereExpr::Between(value, low, high)) + } + + /// The rest of ` LIKE [ESCAPE ]` after LIKE. + /// The statement parses, and LIKE is refused as an operator a condition + /// does not take (see [`Parser::like_pattern`]). The comparison returned + /// stands for it and never runs. + pub(super) fn like(&mut self, value: Operand) -> PResult { + let pattern = self.like_pattern()?; + Ok(WhereExpr::Compare(value, CompareOp::Eq, pattern)) + } + + /// The pattern of a LIKE and its ESCAPE, after LIKE. LIKE is refused as + /// an operator a condition does not take, with or without NOT and with or + /// without ESCAPE (measured 2026-11-02: `a LIKE 'x'`, `a NOT LIKE 'x'` + /// and `a LIKE 'x' ESCAPE 'y'` name the operator `like`). Nothing after + /// LIKE is refused like any infix operator, and nothing after ESCAPE has + /// a message of its own (measured 2026-11-02). + fn like_pattern(&mut self) -> PResult { + self.count_operator(); + self.require_rhs()?; + self.refuse_condition_operator("like"); + let pattern = self.operand()?; + if self.eat_keyword("ESCAPE") { + if matches!(self.peek().kind, TokenKind::Eof) { + return Err(ParseError::well_formed("Expected expression after ESCAPE")); + } + self.operand()?; + } + Ok(pattern) + } + + /// An argument of a call read as an operand of a condition: an operand, + /// or an operand and a LIKE, which is refused there like anywhere in the + /// condition (measured 2026-11-02: `foo(a LIKE 'x')` names the operator + /// `like`). The argument is one part of the walk, so the LIKE comes + /// after what the arguments before it met. The operand stands for the + /// LIKE and never runs. NOT LIKE there, and LIKE in the arguments of a + /// condition function that starts a predicate, are not measured, and keep + /// the syntax error of a call left open. + pub(super) fn argument_with_like(&mut self) -> PResult { + let before = self.walk_part(); + let argument = self.operand().and_then(|argument| { + if self.eat_keyword("LIKE") { + self.like_pattern()?; + } + Ok(argument) + }); + self.end_walk_part(before); + argument + } + + /// The rest of ` IS [NOT] ` after IS, with `left_span` + /// the position of the operand. The left side must be a name, a literal + /// or a path (measured 2026-11-02 with a parameter), and a type with no + /// filter is refused at the left side's position (measured 2026-11-02: + /// `a IS INT4` names the `a`). The walk meets the IS before its left + /// side (measured 2026-11-02: `? IS NULL` with no parameter is refused + /// for its left side), and the refusal of `INT4` after the left side + /// (measured 2026-11-02: `1e126 IS INT4` is refused for the number). The + /// other types with no filter are not measured against their left side, + /// and keep the place of an operator, before it. + pub(super) fn is_test(&mut self, left: Operand, left_span: &str) -> PResult { + self.count_operator(); + let negated = self.eat_keyword("NOT"); + if negated { + self.count_operator(); + } + let ty = self.type_name()?; + if matches!( + left, + Operand::Parameter(_) | Operand::Function(..) | Operand::Arith(..) + ) { + self.walk_operator(IS_LEFT_SIDE.to_owned()); + // It stands for the test and never runs. + return Ok(WhereExpr::Function("is".to_owned(), vec![left])); + } + if ty == TypeName::Missing { + return match left { + Operand::Path(path) => Ok(WhereExpr::IsMissing(path, !negated)), + // (measured 2026-11-02: `1 IS MISSING` and `'x' IS NOT + // MISSING`). Other literals are not measured. + Operand::Literal(AttributeValue::N(_) | AttributeValue::S(_)) => { + Err(ParseError::new( + "Operator or function requires a document path; operator or function: \ + attribute_not_exists", + )) + } + _ => Err(ParseError::new("IS MISSING requires an attribute path")), + }; + } + if ty != TypeName::Null && ty.attribute_type().is_none() { + let refusal = format!("{} at {left_span}", ty.unsupported_filter_message()); + if ty == TypeName::Integer4 { + self.walk_after_operands(&refusal); + } else { + self.walk_operator(refusal); + } + } + Ok(WhereExpr::IsType(left, ty, negated)) + } + + /// The rest of ` IN ` after IN: the members in brackets, + /// in parentheses or in a bag (measured 2026-11-02: `a IN [1, 2]`, `a IN + /// (1, 2)` and `a IN <<1, 5>>` run, with and without NOT). A bag there is + /// not checked as a set literal (measured 2026-11-02: `a IN <<1, 1>>` and + /// `a IN <<'x', 1>>` run). Anything else after IN, an empty list and an + /// empty bag are refused naming the operands IN takes (measured + /// 2026-11-02 with `IN 1`, `IN []` and `IN <<>>`); in parentheses a member + /// must follow (measured 2026-11-02: `IN ()` and `IN (1,)` are + /// "Unexpected term"). + pub(super) fn in_list(&mut self, value: Operand) -> PResult { + self.count_operator(); + self.require_rhs()?; + let items = match self.peek().kind { + TokenKind::LBracket => { + self.bump(); + let items = self.in_members(&TokenKind::RBracket)?; + if !self.eat(&TokenKind::RBracket) { + return Err(ParseError::new("Expected ] to close IN list")); + } + items + } + TokenKind::LParen => { + self.bump(); + let mut items = vec![self.in_member()?]; + while self.eat(&TokenKind::Comma) { + items.push(self.in_member()?); + } + if !self.eat(&TokenKind::RParen) { + return Err(ParseError::well_formed("Expected RIGHT_PAREN")); + } + items + } + TokenKind::LDoubleAngle => self.nested(Nest::Literal, Self::in_bag)?, + _ => return Err(ParseError::new(IN_OPERANDS)), + }; + if items.is_empty() { + return Err(ParseError::new(IN_OPERANDS)); + } + self.count_in_operands(items.len()); + Ok(WhereExpr::In(value, items)) + } + + /// The members of an IN list up to the token that closes it, which is + /// left unread. + fn in_members(&mut self, close: &TokenKind) -> PResult> { + let mut items = Vec::new(); + if &self.peek().kind != close { + loop { + items.push(self.operand()?); + if !self.eat(&TokenKind::Comma) { + break; + } + } + } + Ok(items) + } + + /// One member of an IN list in parentheses. A member in parentheses of + /// its own is a list when a comma separates what they hold, and what they + /// hold otherwise (measured 2026-11-02: in `a IN ((1, 2))` the member is + /// the list `[1, 2]`, and in `a IN ((5))` the number 5). + fn in_member(&mut self) -> PResult { + if !matches!(self.peek().kind, TokenKind::LParen) { + return self.operand(); + } + if self.list_in_parentheses_ahead() { + let list = self.list_in_parentheses(LITERAL_ROOT)?; + return Ok(Operand::Literal(AttributeValue::L(list))); + } + self.bump(); + let member = self.nested(Nest::Group, Self::in_member)?; + if !self.eat(&TokenKind::RParen) { + return Err(ParseError::well_formed("Expected RIGHT_PAREN")); + } + Ok(member) + } + + /// Whether the parentheses opening at the current token hold a comma of + /// their own, outside any bracket, brace or parenthesis inside them. + pub(super) fn list_in_parentheses_ahead(&self) -> bool { + let mut depth = 0usize; + for token in &self.tokens[self.pos..] { + match token.kind { + TokenKind::LParen + | TokenKind::LBracket + | TokenKind::LBrace + | TokenKind::LDoubleAngle => depth += 1, + TokenKind::RParen + | TokenKind::RBracket + | TokenKind::RBrace + | TokenKind::RDoubleAngle => { + depth -= 1; + if depth == 0 { + return false; + } + } + TokenKind::Comma if depth == 1 => return true, + TokenKind::Eof => return false, + _ => {} + } + } + false + } + + /// A list literal written in parentheses, at `path` inside the value + /// being read: its elements are read as those of a list in brackets. + pub(super) fn list_in_parentheses(&mut self, path: &str) -> PResult> { + self.nested(Nest::Literal, |p| { + p.nesting.containers += 1; + let list = p.elements_in_parentheses(path); + p.nesting.containers -= 1; + list + }) + } + + fn elements_in_parentheses(&mut self, path: &str) -> PResult> { + self.bump(); + let mut items = Vec::new(); + loop { + let element = format!("{path}[{}]", items.len()); + if self.term_in_parentheses(&element)? { + // It stands for the term and is never used. + items.push(AttributeValue::Null); + } else { + items.push(self.literal_value(&element)?); + } + if !self.eat(&TokenKind::Comma) { + break; + } + } + if !self.eat(&TokenKind::RParen) { + return Err(ParseError::well_formed("Expected RIGHT_PAREN")); + } + Ok(items) + } + + /// A quoted name, a path or a call as an element of a list in + /// parentheses, at `path` inside the list: it parses and is refused at + /// its position, named by what it is (measured 2026-11-02: in `a IN + /// (("pk", 1))`, `a IN ((m.b, 1))` and `a IN ((foo(b), 1))` the first + /// member is refused as an Id, a Path and a Call under root[0], at the + /// first token). Returns false, having read nothing, for any other + /// element; a name alone is refused by [`Parser::literal_value`]. In a + /// list in brackets these are not measured. + fn term_in_parentheses(&mut self, path: &str) -> PResult { + let next = self.tokens.get(self.pos + 1).map(|t| &t.kind); + let name = match &self.peek().kind { + TokenKind::Ident(w) => word_class(w) == WordClass::Name, + TokenKind::QuotedIdent(_) => true, + _ => false, + }; + let callable = matches!( + &self.peek().kind, + TokenKind::Ident(w) if word_class(w) == WordClass::Name || is_function_keyword(w) + ); + let kind = match next { + Some(TokenKind::Dot | TokenKind::LBracket) if name => "Path", + Some(TokenKind::LParen) if callable => "Call", + Some(TokenKind::LParen) => return Ok(false), + _ if matches!(self.peek().kind, TokenKind::QuotedIdent(_)) => "Id", + _ => return Ok(false), + }; + self.defer(&format!( + "Unsupported data type: {kind} under key {path} at {}", + self.span() + )); + match kind { + "Path" => { + self.path()?; + } + "Call" => { + // What the arguments meet comes after the refusal. + self.bump(); + let before = self.walk_part(); + let args = self.call_args(); + self.end_walk_part(before); + args?; + } + _ => { + self.bump(); + } + } + Ok(true) + } + + /// The members of a bag after IN, with the current token on its `<<`: + /// the operands of the IN in the order they are written, each a value, a + /// parameter or a name (measured 2026-11-02: `a IN <>` and `a IN + /// <>` run). A member is named as the value itself in a refusal + /// (measured 2026-11-02: `a IN <<1e126>>` is refused under root). + fn in_bag(&mut self) -> PResult> { + self.bump(); + let mut members = Vec::new(); + if !matches!(self.peek().kind, TokenKind::RDoubleAngle) { + loop { + members.push(if matches!(self.peek().kind, TokenKind::Param) { + self.bump(); + let index = self.take_param_index(); + self.walk_parameter(index); + Operand::Parameter(index) + } else if let TokenKind::Ident(w) = &self.peek().kind + && self.at_name_in_literal(w) + { + Operand::Path(self.path()?) + } else { + Operand::Literal(self.literal_value(LITERAL_ROOT)?) + }); + if !self.eat(&TokenKind::Comma) { + break; + } + } + } + if !self.eat(&TokenKind::RDoubleAngle) { + return Err(ParseError::well_formed( + "Expected RIGHT_DOUBLE_ANGLE_BRACKET", + )); + } + Ok(members) + } + + /// What may follow a predicate: a comparison with a boolean (measured + /// 2026-11-02: `a NOT IN [1] = true`, `(a = 1) = false` and `pk = 'a' = + /// true` run and match what the predicate does or does not), or an IS + /// test (see [`Parser::trailing_is`]). + pub(super) fn after_predicate(&mut self, predicate: WhereExpr) -> PResult { + if self.trailing_is()? { + return Ok(predicate); + } + Ok(self.compared_with_boolean(predicate)) + } + + /// An IS test after a predicate, which parses and is refused for its + /// left side (measured 2026-11-02 after IN, NOT IN, BETWEEN, `=` and an + /// IS test). The walk meets the IS before the predicate. Returns whether + /// there was one. + pub(super) fn trailing_is(&mut self) -> PResult { + if !self.eat_keyword("IS") { + return Ok(false); + } + self.count_operator(); + if self.eat_keyword("NOT") { + self.count_operator(); + } + self.type_name()?; + self.walk_operator(IS_LEFT_SIDE.to_owned()); + Ok(true) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/core/src/partiql/parser/predicates/tests.rs b/crates/core/src/partiql/parser/predicates/tests.rs new file mode 100644 index 000000000..ea7840149 --- /dev/null +++ b/crates/core/src/partiql/parser/predicates/tests.rs @@ -0,0 +1,418 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Unit tests for the predicates that follow an operand in a condition. + +use crate::partiql::ast::{CompareOp, Operand, Path, Statement, WhereExpr}; +use crate::partiql::parse; +use crate::partiql::parser::{IN_OPERANDS, limits}; +use crate::types::AttributeValue; + +const WF: &str = "Statement wasn't well formed, can't be processed: "; + +fn err(sql: &str) -> String { + parse(sql).unwrap_err().message +} + +fn wf(reason: &str) -> String { + format!("{WF}{reason}") +} + +fn where_of(sql: &str) -> WhereExpr { + match parse(sql).unwrap() { + Statement::Select(s) => s.where_clause.unwrap(), + other => panic!("expected select, got {other:?}"), + } +} + +fn path(name: &str) -> Operand { + Operand::Path(Path { + root: name.to_owned(), + steps: Vec::new(), + }) +} + +fn number(n: &str) -> Operand { + Operand::Literal(AttributeValue::N(n.to_owned())) +} + +#[test] +fn not_in_and_not_between_negate_the_predicate_after_the_not() { + let not_in = WhereExpr::Not(Box::new(WhereExpr::In( + path("a"), + vec![number("1"), number("2")], + ))); + assert_eq!( + where_of("SELECT * FROM \"t\" WHERE a NOT IN [1, 2]"), + not_in + ); + assert_eq!( + where_of("SELECT * FROM \"t\" WHERE a not in (1, 2)"), + not_in + ); + assert_eq!( + where_of("SELECT * FROM \"t\" WHERE a NOT BETWEEN 1 AND 2"), + WhereExpr::Not(Box::new(WhereExpr::Between( + path("a"), + number("1"), + number("2") + ))) + ); + // A NOT before the operand applies to the whole predicate. + assert_eq!( + where_of("SELECT * FROM \"t\" WHERE NOT a NOT IN [1, 2]"), + WhereExpr::Not(Box::new(not_in)) + ); +} + +#[test] +fn in_takes_its_members_in_parentheses() { + assert_eq!( + where_of("SELECT * FROM \"t\" WHERE a IN (1, 2)"), + WhereExpr::In(path("a"), vec![number("1"), number("2")]) + ); + assert_eq!( + where_of("SELECT * FROM \"t\" WHERE a IN (?, ?)"), + WhereExpr::In( + path("a"), + vec![Operand::Parameter(0), Operand::Parameter(1)] + ) + ); +} + +#[test] +fn the_not_forms_parse_wherever_a_condition_does() { + for sql in [ + "SELECT * FROM \"t\" WHERE pk = 'a' AND a NOT IN [1]", + "SELECT * FROM \"t\" WHERE a NOT IN [?, ?] OR a NOT BETWEEN ? AND ?", + "SELECT * FROM \"t\".\"gsi\" WHERE g = 1 AND a NOT BETWEEN 1 AND 2", + "UPDATE \"t\" SET q = 1 WHERE pk = 'a' AND a NOT IN (1, 2)", + "UPDATE \"t\" SET q = 1 WHERE pk = 'a' AND a NOT BETWEEN 5 AND 6 RETURNING ALL NEW *", + "DELETE FROM \"t\" WHERE pk = 'a' AND a NOT IN [1]", + "EXISTS(SELECT * FROM \"t\" WHERE pk = 'a' AND a NOT IN [1])", + "FROM \"t\" WHERE pk = 'a' AND a NOT BETWEEN 1 AND 2 SET q = 1", + ] { + assert!(parse(sql).is_ok(), "{sql}"); + } +} + +#[test] +fn a_not_form_counts_two_operators() { + let nots = + |n: usize, term: &str| format!("SELECT * FROM \"t\" WHERE {}{term}", "NOT ".repeat(n)); + for term in [ + "a NOT IN [1]", + "a NOT IN (1, 2, 3)", + "a NOT BETWEEN 1 AND 2", + ] { + assert!(parse(¬s(298, term)).is_ok(), "{term}"); + assert_eq!( + err(¬s(299, term)), + limits::too_many_operators(), + "{term}" + ); + } + // IN in parentheses counts one, however many members it has. + assert!(parse(¬s(299, "a IN (1, 2, 3)")).is_ok()); +} + +#[test] +fn malformed_not_forms_get_the_messages_of_their_predicates() { + for (condition, expected) in [ + ("a NOT IN 1", IN_OPERANDS.to_owned()), + ("a NOT IN []", IN_OPERANDS.to_owned()), + ("a IN ()", wf("Unexpected term")), + ("a NOT IN ()", wf("Unexpected term")), + ("a IN (1,)", wf("Unexpected term")), + ("(a NOT = 1)", wf("Expected RIGHT_PAREN")), + ("a NOT", wf("Unexpected token after expression")), + ( + "a NOT AND pk = 'a'", + wf("Unexpected token after expression"), + ), + ("a NOT NOT IN [1]", wf("Unexpected token after expression")), + ("a NOT BETWEEN 1", wf("Expected AND keyword")), + ("a NOT = 1", wf("Unexpected token after expression")), + ("a NOT IS MISSING", wf("Unexpected token after expression")), + ("a IS NOT IN [1]", wf("Expected type name")), + ( + "a NOT IN", + wf("Missing right-hand side expression of infix operator"), + ), + ( + "a NOT BETWEEN", + wf("Missing right-hand side expression of infix operator"), + ), + ("a NOT BETWEEN 1 AND", wf("Expected expression after AND")), + ("a IN (1, 2", wf("Expected RIGHT_PAREN")), + ] { + assert_eq!( + err(&format!("SELECT * FROM \"t\" WHERE {condition}")), + expected, + "{condition}" + ); + } +} + +fn list(items: &[&str]) -> Operand { + Operand::Literal(AttributeValue::L( + items + .iter() + .map(|n| AttributeValue::N((*n).to_owned())) + .collect(), + )) +} + +#[test] +fn a_member_in_parentheses_of_its_own_is_a_list_or_what_they_hold() { + for (condition, members) in [ + ("a IN ((1, 2))", vec![list(&["1", "2"])]), + ("a IN ((5))", vec![number("5")]), + ("a IN ([5])", vec![list(&["5"])]), + ("a IN ([1], 2)", vec![list(&["1"]), number("2")]), + ("a IN (1)", vec![number("1")]), + ("a IN (pk, 1)", vec![path("pk"), number("1")]), + ] { + assert_eq!( + where_of(&format!("SELECT * FROM \"t\" WHERE {condition}")), + WhereExpr::In(path("a"), members), + "{condition}" + ); + } +} + +fn string(s: &str) -> Operand { + Operand::Literal(AttributeValue::S(s.to_owned())) +} + +#[test] +fn in_takes_its_members_in_a_bag() { + assert_eq!( + where_of("SELECT * FROM \"t\" WHERE a IN <<1, 5>>"), + WhereExpr::In(path("a"), vec![number("1"), number("5")]) + ); + // The bag is not checked as a set literal. + assert_eq!( + where_of("SELECT * FROM \"t\" WHERE a IN <<1, 1>>"), + WhereExpr::In(path("a"), vec![number("1"), number("1")]) + ); + assert_eq!( + where_of("SELECT * FROM \"t\" WHERE a IN <<'x', 1>>"), + WhereExpr::In(path("a"), vec![string("x"), number("1")]) + ); + assert_eq!(err("SELECT * FROM \"t\" WHERE a IN <<>>"), IN_OPERANDS); +} + +#[test] +fn a_name_inside_a_list_in_parentheses_is_refused_at_its_position() { + let sql = "SELECT * FROM \"t\" WHERE a IN ((pk, 1))"; + let column = sql.find("pk").unwrap() + 1; + assert_eq!( + err(sql), + format!("Unsupported data type: Id under key root[0] at 1:{column}:2") + ); +} + +#[test] +fn a_predicate_may_be_compared_with_a_boolean() { + let not_in = WhereExpr::Not(Box::new(WhereExpr::In(path("a"), vec![number("1")]))); + for condition in ["a NOT IN [1] = true", "(a NOT IN [1]) = true"] { + assert_eq!( + where_of(&format!("SELECT * FROM \"t\" WHERE {condition}")).without_kept_comparisons(), + ¬_in, + "{condition}" + ); + } + let pk = WhereExpr::Compare(path("pk"), CompareOp::Eq, string("a")); + let compared = |equal, literal| { + WhereExpr::ComparedWithBoolean( + Box::new(pk.clone()), + crate::partiql::ast::BooleanComparison { equal, literal }, + ) + }; + assert_eq!( + where_of("SELECT * FROM \"t\" WHERE a IN [1] AND pk = 'a' = true"), + WhereExpr::And( + Box::new(WhereExpr::In(path("a"), vec![number("1")])), + Box::new(compared(true, true)) + ) + ); + assert_eq!( + where_of("SELECT * FROM \"t\" WHERE (pk = 'a') = false"), + compared(true, false) + ); + assert_eq!( + where_of("SELECT * FROM \"t\" WHERE pk = 'a' <> true"), + compared(false, true) + ); + assert_eq!( + where_of("SELECT * FROM \"t\" WHERE pk = 'a' <> FALSE"), + compared(false, false) + ); +} + +#[test] +fn an_is_test_after_a_predicate_is_refused_for_its_left_side() { + for condition in [ + "a IN (1, 2) IS MISSING", + "a NOT IN (1, 2) IS MISSING", + "a IN [1] IS NOT MISSING", + "a BETWEEN 1 AND 2 IS MISSING", + "a = 1 IS MISSING", + ] { + assert_eq!( + err(&format!("SELECT * FROM \"t\" WHERE {condition}")), + super::super::IS_LEFT_SIDE, + "{condition}" + ); + } +} + +#[test] +fn an_in_list_of_more_than_100_members_is_refused() { + let select = |n: usize| { + let members: Vec = (0..n).map(|i| i.to_string()).collect(); + format!("SELECT * FROM \"t\" WHERE a IN [{}]", members.join(", ")) + }; + for n in [101, 1000] { + assert_eq!(err(&select(n)), limits::too_many_in_operands(n), "{n}"); + } + assert!(parse(&select(100)).is_ok()); +} + +#[test] +fn like_is_refused_as_an_operator_a_condition_does_not_take() { + let like = "Unsupported operator in Condition Expression. Operator: like"; + for condition in ["a LIKE 'x'", "a NOT LIKE 'x'", "a LIKE 'x' ESCAPE 'y'"] { + assert_eq!( + err(&format!("SELECT * FROM \"t\" WHERE {condition}")), + like, + "{condition}" + ); + } + let missing_rhs = wf("Missing right-hand side expression of infix operator"); + assert_eq!(err("SELECT * FROM \"t\" WHERE a NOT LIKE"), missing_rhs); + // A syntax error after it wins. + assert_eq!(err("SELECT * FROM \"t\" WHERE a LIKE 'x' AND"), missing_rhs); +} + +#[test] +fn like_inside_the_argument_of_a_call_is_refused_as_like() { + let like = "Unsupported operator in Condition Expression. Operator: like"; + for condition in [ + "foo(a LIKE 'x')", + "foo(b, a LIKE 'x' ESCAPE 'y')", + "b = foo(a LIKE 'x')", + ] { + assert_eq!( + err(&format!("SELECT * FROM \"t\" WHERE {condition}")), + like, + "{condition}" + ); + } + // An argument before it is met first. + assert_eq!( + err("SELECT * FROM \"t\" WHERE foo(1e126, a LIKE 'x')"), + "Number overflow. Attempting to store a number with magnitude larger than \ + supported range under root" + ); + // NOT LIKE there, a condition function that starts a predicate and + // LIKE in a SET value are not measured. + for condition in ["foo(a NOT LIKE 'x')", "size(a LIKE 'x') = 1"] { + assert_eq!( + err(&format!("SELECT * FROM \"t\" WHERE {condition}")), + wf("Expected RIGHT_PAREN"), + "{condition}" + ); + } + assert_eq!( + err("UPDATE \"t\" SET q = foo(a LIKE 'x') WHERE pk = 'a'"), + wf("Expected RIGHT_PAREN") + ); +} + +#[test] +fn escape_with_nothing_after_it() { + assert_eq!( + err("SELECT * FROM \"t\" WHERE a LIKE 'x' ESCAPE"), + wf("Expected expression after ESCAPE") + ); +} + +#[test] +fn is_missing_on_a_number_or_a_string_needs_a_document_path() { + let path = "Operator or function requires a document path; operator or function: \ + attribute_not_exists"; + for condition in ["1 IS MISSING", "'x' IS NOT MISSING"] { + assert_eq!( + err(&format!("SELECT * FROM \"t\" WHERE {condition}")), + path, + "{condition}" + ); + } + // Other literals are not measured. + assert_eq!( + err("SELECT * FROM \"t\" WHERE [1] IS MISSING"), + "IS MISSING requires an attribute path" + ); +} + +#[test] +fn an_is_after_an_is_test_is_refused_for_its_left_side() { + for condition in ["a IS NULL IS NULL", "a IS MISSING IS NOT NULL"] { + assert_eq!( + err(&format!("SELECT * FROM \"t\" WHERE {condition}")), + super::super::IS_LEFT_SIDE, + "{condition}" + ); + } + // A comparison with a boolean after it is not measured. + assert_eq!( + err("SELECT * FROM \"t\" WHERE a IS NULL = true"), + wf("Unexpected token after expression") + ); +} + +#[test] +fn a_name_in_a_bag_after_in_is_an_operand() { + assert_eq!( + where_of("SELECT * FROM \"t\" WHERE a IN <>"), + WhereExpr::In(path("a"), vec![path("pk"), number("1")]) + ); +} + +#[test] +fn a_quoted_name_a_path_or_a_call_in_a_list_in_parentheses_is_refused_at_its_position() { + for (members, token, kind, length) in [ + ("((\"pk\", 1))", "\"pk\"", "Id", 4), + ("((m.b, 1))", "m.b", "Path", 1), + ("((foo(b), 1))", "foo", "Call", 3), + ("((1, foo(1 + 1)))", "foo", "Call", 3), + ] { + let sql = format!("SELECT * FROM \"t\" WHERE a IN {members}"); + let column = sql.find(token).unwrap() + 1; + let index = usize::from(members.starts_with("((1")); + assert_eq!( + err(&sql), + format!("Unsupported data type: {kind} under key root[{index}] at 1:{column}:{length}"), + "{members}" + ); + } + // In a list in brackets they are not measured. + assert_eq!( + err("SELECT * FROM \"t\" WHERE a = [\"pk\"]"), + wf("Expected a literal value") + ); +} + +#[test] +fn positions_after_a_not_form_are_unchanged() { + assert_eq!( + err("SELECT * FROM \"t\" WHERE a NOT IN [1] AND foo(b)"), + "Unrecognized function: foo at 1:42:3" + ); + assert_eq!( + err("SELECT * FROM \"t\" WHERE a NOT BETWEEN 1 AND 2 AND\nfoo(b) = 1"), + "Unrecognized function: foo at 2:1:3" + ); +} diff --git a/crates/core/src/partiql/parser/statements.rs b/crates/core/src/partiql/parser/statements.rs new file mode 100644 index 000000000..abcf6b96a --- /dev/null +++ b/crates/core/src/partiql/parser/statements.rs @@ -0,0 +1,499 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The statements: SELECT, INSERT, UPDATE and DELETE, their targets and their clauses. + +use super::{PResult, ParseError, Parser}; +use crate::partiql::ast::{ + DeleteStatement, FromSource, InsertStatement, OrderBy, OrderTerm, Path, Projection, Returning, + SelectStatement, Statement, UpdateStatement, +}; +use crate::partiql::from_path; +use crate::partiql::lexer::{TokenKind, WordClass, is_function_keyword, word_class}; + +impl Parser { + // --- statements --- + + pub(super) fn statement(&mut self) -> PResult { + if self.is_keyword("SELECT") { + self.bump(); + Ok(Statement::Select(self.select(true)?)) + } else if self.is_keyword("INSERT") { + self.bump(); + Ok(Statement::Insert(self.insert()?)) + } else if self.is_keyword("UPDATE") { + self.bump(); + Ok(Statement::Update(self.update()?)) + } else if self.is_keyword("DELETE") { + self.bump(); + Ok(Statement::Delete(self.delete()?)) + } else if self.is_keyword("FROM") { + self.bump(); + Ok(Statement::Update(self.update_led_by_from()?)) + } else if self.at_exists_statement() { + Ok(Statement::Exists(self.exists_statement()?)) + } else { + // A statement that starts with none of the statement keywords is + // read as a bare expression and refused by what it is. + self.bare_statement() + } + } + + /// A SELECT after its keyword. `statement` is true for a SELECT statement, + /// false for the query of an EXISTS. + pub(super) fn select(&mut self, statement: bool) -> PResult { + let mut projection_error = None; + let projection = self.projection(&mut projection_error)?; + if statement { + self.select_list_call.clone_from(&projection_error); + } + if !self.eat_keyword("FROM") { + return Err(ParseError::well_formed("Expected FROM after SELECT list")); + } + let from = self.parse_from_source()?; + let where_clause = if self.eat_keyword("WHERE") { + Some(self.where_expr()?) + } else { + None + }; + let order_by = if self.eat_keyword("ORDER") { + if !self.eat_keyword("BY") { + return Err(ParseError::well_formed("Expected BY after ORDER")); + } + // A list of names, each with a direction or none (measured + // 2026-11-03: `ORDER BY pk DESC, sk`, `pk, sk ASC` and `sk DESC, + // pk` run). A comma that ends the statement is "Unexpected term" + // (measured 2026-11-03). A comma before anything else but a name + // stays a token left over after the statement; the service + // answers `ORDER BY pk, 1` with an internal error. + let mut terms = vec![self.order_term()?]; + while matches!(self.peek().kind, TokenKind::Comma) { + if matches!(self.nth_kind(1), Some(TokenKind::Eof)) { + self.bump(); + return Err(self.term_error()); + } + if !self.name_follows() { + break; + } + self.bump(); + terms.push(self.order_term()?); + } + Some(OrderBy { terms }) + } else { + None + }; + Ok(SelectStatement { + projection, + from, + where_clause, + order_by, + projection_error, + }) + } + + /// One path of an ORDER BY list and its direction, ascending unless it + /// says `DESC`. + fn order_term(&mut self) -> PResult { + let at = self.span(); + let path = self.path()?; + let descending = self.eat_keyword("DESC"); + if !descending { + self.eat_keyword("ASC"); + } + Ok(OrderTerm { + path, + descending, + at, + }) + } + + fn projection(&mut self, deferred: &mut Option) -> PResult { + // `SELECT ALL` and `SELECT DISTINCT` read a select list after the + // quantifier, and `SELECT VALUE` an expression (measured 2026-11-02: + // `SELECT DISTINCT FROM "t"` names FROM, `SELECT VALUE FROM "t"` is the + // error of a FROM-led form). Past those errors the forms are not + // measured, and the quantifier keeps its term error. + if self.is_keyword("ALL") || self.is_keyword("DISTINCT") { + let quantifier = self.term_error(); + self.bump(); + return Err(self.select_list_keyword().unwrap_or(quantifier)); + } + if self.is_keyword("VALUE") { + let value = self.term_error(); + self.bump(); + self.operand()?; + return Err(value); + } + if self.eat(&TokenKind::Star) { + return Ok(Projection::All); + } + let mut paths = vec![self.projection_path(deferred)?]; + self.projection_alias()?; + while self.eat(&TokenKind::Comma) { + paths.push(self.projection_path(deferred)?); + self.projection_alias()?; + } + Ok(Projection::Paths(paths)) + } + + /// A path in the SELECT list. An identifier immediately followed by `(` + /// (e.g. `COUNT(*)`) and a literal are unsupported path components + /// reported at their span (measured 2026-11-02 with COUNT, null, true, + /// false and missing). Neither is a parse failure: the item is skipped and + /// its refusal goes in `deferred` (see [`SelectStatement::projection_error`]). + /// A reserved word gets the answer it gets where any term starts, and a + /// path may have any number of steps (measured 2026-11-02: `SELECT m.a.b` + /// runs). + fn projection_path(&mut self, deferred: &mut Option) -> PResult { + if let Some(e) = self.select_list_keyword() { + return Err(e); + } + let (line, column) = { + let t = self.peek(); + (t.line, t.column) + }; + let root_len = match &self.peek().kind { + TokenKind::Ident(s) | TokenKind::QuotedIdent(s) => s.len(), + _ => 0, + }; + // `COUNT(*)` and the like: a function call is not a projectable path + // component. Checked before the path is read because a keyword such as + // COUNT is not a name. An argument list that is never closed is + // reported at once. + let callee = match &self.peek().kind { + TokenKind::Ident(s) if word_class(s) == WordClass::Name || is_function_keyword(s) => { + Some(s.clone()) + } + TokenKind::QuotedIdent(s) => Some(s.clone()), + _ => None, + }; + if let Some(root) = callee + && matches!( + self.tokens.get(self.pos + 1).map(|t| &t.kind), + Some(TokenKind::LParen) + ) + { + let message = format!("Unexpected path component at {line}:{column}:{root_len}"); + self.bump(); + if !self.skip_parenthesized() { + return Err(ParseError::new(message)); + } + deferred.get_or_insert(message); + return Ok(Path { + root, + steps: Vec::new(), + }); + } + let literal = match &self.peek().kind { + TokenKind::Ident(s) => word_class(s) == WordClass::Literal, + TokenKind::Number(_) | TokenKind::String(_) => true, + _ => false, + }; + if literal { + deferred.get_or_insert(format!("Unexpected path component at {}", self.span())); + self.bump(); + return Ok(Path { + root: String::new(), + steps: Vec::new(), + }); + } + if matches!(&self.peek().kind, TokenKind::Ident(s) if word_class(s) != WordClass::Name) { + // Only a form whose refusal waits for the end of the statement + // reads to its end here. + self.operand()?; + return Ok(Path { + root: String::new(), + steps: Vec::new(), + }); + } + self.path() + } + + /// Consume a parenthesized group starting at the current `(`, nested groups + /// included. Returns false, having consumed the rest of the statement, + /// when the group is never closed. + fn skip_parenthesized(&mut self) -> bool { + let mut depth = 0usize; + loop { + match self.peek().kind { + TokenKind::LParen => depth += 1, + TokenKind::RParen => depth -= 1, + TokenKind::Eof => return false, + _ => {} + } + self.bump(); + if depth == 0 { + return true; + } + } + } + + fn parse_from_source(&mut self) -> PResult { + let table = match &self.peek().kind { + // A quoted or an unquoted name (measured 2026-11-02: `FROM t` runs). + TokenKind::QuotedIdent(s) => { + let s = s.clone(); + self.bump(); + s + } + TokenKind::Ident(s) if word_class(s) == WordClass::Name => { + let s = s.clone(); + self.bump(); + s + } + // A reserved word gets the answer it gets where any term starts + // (measured 2026-11-02: `FROM where` is "Unexpected keyword", `FROM + // and` "Unexpected operator", `FROM set` "Expected identifier for + // simple path"); a literal word is not a table. + TokenKind::Ident(s) if word_class(s) != WordClass::Literal => { + self.operand()?; + return Err(ParseError::new("Unexpected from source")); + } + // Where no term can start (the end of the statement, an + // operator), the service reports the term error (measured + // 2026-11-02: `SELECT * FROM` is "Unexpected term"). + TokenKind::Eof + | TokenKind::Semicolon + | TokenKind::Comma + | TokenKind::Dot + | TokenKind::Star + | TokenKind::Colon + | TokenKind::RParen + | TokenKind::RBracket + | TokenKind::RBrace + | TokenKind::RDoubleAngle + | TokenKind::Eq + | TokenKind::Ne + | TokenKind::Lt + | TokenKind::Le + | TokenKind::Gt + | TokenKind::Ge => return Err(self.term_error()), + // Any other term (a bare name, a literal) is not a table. + _ => return Err(ParseError::new("Unexpected from source")), + }; + self.table = Some(table.clone()); + let mut rest = self.path_steps_after_table()?; + let index = if rest.is_empty() { + None + } else { + Some(rest.remove(0)) + }; + let mut components = vec![table.as_str()]; + components.extend(index.as_deref()); + components.extend(rest.iter().map(String::as_str)); + let refusal = from_path::refusal(&components); + // An alias is refused ahead of the path (measured 2026-11-02: `FROM + // "t"."" x` is the alias refusal). + self.source_aliases()?; + if let Some(refusal) = refusal { + self.defer(&refusal); + } + Ok(FromSource { table, index }) + } + + /// The `.component` steps after the table name of a FROM path. A step is + /// a name or a quoted name; anything else after the dot, a reserved word + /// or the end of the path included, is "Invalid path dot component" + /// (measured 2026-11-02 with `"t".index` and `"t". WHERE`). + fn path_steps_after_table(&mut self) -> PResult> { + let mut rest = Vec::new(); + while self.eat(&TokenKind::Dot) { + match &self.peek().kind { + TokenKind::QuotedIdent(s) => { + rest.push(s.clone()); + self.bump(); + } + TokenKind::Ident(s) if word_class(s) == WordClass::Name => { + rest.push(s.clone()); + self.bump(); + } + _ => return Err(ParseError::well_formed("Invalid path dot component")), + } + } + Ok(rest) + } + + /// The rest of an `UPDATE` (`update`) or `DELETE` target after its table + /// name. A path naming an index parses and is refused once the statement + /// has parsed: an UPDATE's after the table name rule (measured + /// 2026-11-02: `UPDATE "tt"."idx"` is refused for the name of the table). + /// A DELETE's is not measured against the rule and keeps its place ahead + /// of it. + pub(super) fn dml_target_rest(&mut self, table: &str, update: bool) -> PResult<()> { + let rest = self.path_steps_after_table()?; + let mut components = vec![table]; + components.extend(rest.iter().map(String::as_str)); + if let Some(refusal) = from_path::refusal(&components) { + self.defer(&refusal); + } else if components.len() == 2 { + if update { + self.defer_after_name_rule(from_path::ON_INDEX); + } else { + self.defer(from_path::ON_INDEX); + } + } + Ok(()) + } + + fn insert(&mut self) -> PResult { + if !self.eat_keyword("INTO") { + return Err(ParseError::well_formed("Unexpected keyword")); + } + let table_span = self.span(); + let table = self.table_name()?; + if !self.path_steps_after_table()?.is_empty() { + self.defer(&from_path::insert_refusal(&table_span)); + } + if self.is_keyword("VALUES") { + return Err(ParseError::well_formed( + "Expected LEFT_PAREN for row value constructor", + )); + } + if !self.eat_keyword("VALUE") { + return Err(self.insert_without_value()); + } + if !matches!(self.peek().kind, TokenKind::LBrace) { + let span = self.span(); + return Err(ParseError::new(format!( + "Unsupported format in item expression. Please use struct format {{...}} to express item values for insert statement. at {span}" + ))); + } + self.in_values = true; + let value = self.item_map()?; + self.in_values = false; + Ok(InsertStatement { table, value }) + } + + /// `INSERT INTO
` followed by something other than VALUE: the + /// service reads an expression there, the items to insert, and refuses a + /// statement that ends after it (measured 2026-11-02 with a map, a name + /// and a string). Tokens after the expression are a leftover token + /// (`INSERT INTO "t" t VALUE {...}`), and no expression at all is the term + /// error ("Unexpected term" at the end of the statement). + fn insert_without_value(&mut self) -> ParseError { + if let Err(e) = self.operand() { + return e; + } + self.eat_final_semicolon(); + if !matches!(self.peek().kind, TokenKind::Eof) { + return self.trailing_error(); + } + ParseError::new( + "Unsupported operation: Inserting multiple items in a single statement is not supported, use \"INSERT INTO tableName VALUE item\" instead", + ) + } + + fn update(&mut self) -> PResult { + let table = self.table_name()?; + self.dml_target_rest(&table, true)?; + self.optional_alias()?; + let clauses = self.dml_clauses()?; + // An UPDATE without WHERE parses; execution refuses it with the + // full-key message, and a transaction cancels at that member with a + // ValidationError (measured 2026-11-02 in ExecuteTransaction). + let where_clause = if self.eat_keyword("WHERE") { + Some(self.where_expr()?) + } else { + None + }; + let returning = self.returning(false)?; + Ok(UpdateStatement { + table, + sets: clauses.sets, + removes: clauses.removes, + set_members: clauses.set_members, + where_clause, + returning, + }) + } + + fn delete(&mut self) -> PResult { + if !self.eat_keyword("FROM") { + return Err(ParseError::well_formed("Expected FROM after DELETE")); + } + let table = self.table_name()?; + self.dml_target_rest(&table, false)?; + self.optional_alias()?; + if !self.eat_keyword("WHERE") { + return Err(ParseError::new( + "Where clause does not contain a mandatory equality on all key attributes", + )); + } + let where_clause = self.where_expr()?; + let returning = self.returning(true)?; + Ok(DeleteStatement { + table, + where_clause, + returning, + }) + } + + /// The RETURNING clause of an UPDATE, or of a DELETE when `delete`. + pub(super) fn returning(&mut self, delete: bool) -> PResult> { + if !self.eat_keyword("RETURNING") { + return Ok(None); + } + let old_new; + let modified = self.eat_keyword("MODIFIED"); + if self.eat_keyword("ALL") { + old_new = match self.old_or_new() { + // A syntax error (measured 2026-11-02 with `RETURNING ALL` at + // the end of a DELETE). An UPDATE's is not measured. + Err(_) if delete => { + return Err(ParseError::well_formed( + "Expected ( MODIFIED | ALL ) ( NEW | OLD ) in each returning element.", + )); + } + old_new => old_new?, + }; + self.eat(&TokenKind::Star); + Ok(Some(match (modified, old_new) { + (false, false) => Returning::AllOld, + (false, true) => Returning::AllNew, + (true, false) => Returning::ModifiedOld, + (true, true) => Returning::ModifiedNew, + })) + } else if modified { + old_new = self.old_or_new()?; + self.eat(&TokenKind::Star); + Ok(Some(if old_new { + Returning::ModifiedNew + } else { + Returning::ModifiedOld + })) + } else { + Err(ParseError::new("Invalid returning clause")) + } + } + + /// Returns true for NEW, false for OLD. + fn old_or_new(&mut self) -> PResult { + if self.eat_keyword("OLD") { + Ok(false) + } else if self.eat_keyword("NEW") { + Ok(true) + } else { + Err(ParseError::new("Invalid returning clause")) + } + } + + /// The table an INSERT, UPDATE or DELETE names, quoted or not (measured + /// 2026-11-02: unquoted names run). Where no name starts (the end of the + /// statement, a reserved word, a literal) the service reports "Expected + /// identifier for simple path" (measured 2026-11-02 on `DELETE FROM` with + /// nothing, a semicolon, WHERE and other reserved words after it). + pub(super) fn table_name(&mut self) -> PResult { + let table = match &self.peek().kind { + TokenKind::QuotedIdent(s) => s.clone(), + TokenKind::Ident(s) if word_class(s) == WordClass::Name => s.clone(), + _ => { + return Err(ParseError::well_formed( + "Expected identifier for simple path", + )); + } + }; + self.bump(); + self.table = Some(table.clone()); + Ok(table) + } +} diff --git a/crates/core/src/partiql/parser/terms.rs b/crates/core/src/partiql/parser/terms.rs new file mode 100644 index 000000000..24ad702bb --- /dev/null +++ b/crates/core/src/partiql/parser/terms.rs @@ -0,0 +1,429 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The service's answers for a reserved word where a term starts, for the +//! first word of a select-list item, for a SET or REMOVE target that is not a +//! name, and for a statement that is a bare expression. +//! +//! The service reads a statement with a grammar wider than what DynamoDB +//! runs, so a reserved word where a term starts often begins a form of its own +//! (`CASE `, `DATE '...'`, `FROM ... SET`, `EXEC `), +//! and the answer is the error that form meets at the next token: `WHERE case +//! = 1` is "Unexpected operator" for the `=`, `attribute_exists(case)` is +//! "Unexpected term" for the `)`. Every answer here was measured on +//! 2026-11-02 for 274 words in four positions: a condition (`WHERE = 1`), +//! the argument of `attribute_exists()`, a SET target and a projection +//! (`SELECT FROM ...`). Where such a form reads to its end, the answer is +//! not measured and the word keeps the term error of its class. + +use super::{PResult, ParseError, Parser}; +use crate::partiql::ast::{Operand, Path, Statement}; +use crate::partiql::lexer::{TokenKind, WordClass, word_class}; +use crate::partiql::parse_error; +use crate::types::AttributeValue; + +/// The keywords that end a select list rather than start an item. In an item's +/// place they get a message of their own; every other reserved word gets the +/// term error. +const SELECT_LIST_KEYWORDS: [&str; 7] = [ + "delete", "from", "remove", "select", "set", "update", "where", +]; + +/// The calls that name the missing parenthesis in their message. +const PAREN_NAMED: [&str; 5] = [ + "can_cast", + "can_lossless_cast", + "cast", + "coalesce", + "nullif", +]; + +/// The calls whose missing parenthesis is "Expected LEFT_PAREN". +const PAREN_UNNAMED: [&str; 5] = ["date_add", "date_diff", "extract", "substring", "trim"]; + +const SIMPLE_PATH: &str = "Expected identifier for simple path"; + +/// How a bare expression statement is refused: by the kind of expression it is. +enum BareForm { + /// An operator expression (`x = 1`, `NOT x`, `x IS NOT INT4`) or a call + /// (`size(x)`): the service takes it for a condition check, which only a + /// transaction runs. + Operator, + /// `x IS `. + IsType, + /// A single term, named as the service names it (`Id`, `Lit`, ...). + Term(&'static str), +} + +impl Parser { + pub(super) fn nth_kind(&self, n: usize) -> Option<&TokenKind> { + self.tokens.get(self.pos + n).map(|t| &t.kind) + } + + /// Whether the token after the current one is a name: an unquoted word + /// that is not reserved, or a quoted identifier. + pub(super) fn name_follows(&self) -> bool { + match self.nth_kind(1) { + Some(TokenKind::Ident(w)) => word_class(w) == WordClass::Name, + Some(TokenKind::QuotedIdent(_)) => true, + _ => false, + } + } + + /// The answer for a reserved word where a term starts, with the current + /// token on the word. `Ok` only for `EXEC`, whose refusal is deferred to + /// the end of the statement (see [`Parser::exec_term`]); the operand it + /// returns stands in for the form and is never run. + pub(super) fn reserved_word_term(&mut self) -> PResult { + let TokenKind::Ident(word) = &self.peek().kind else { + return Err(self.term_error()); + }; + let word = word.to_ascii_lowercase(); + let fallback = self.term_error(); + let next = self.nth_kind(1).cloned(); + match word.as_str() { + // A prefix and two forms that take an expression first. + "not" | "case" | "pivot" => { + self.bump(); + self.operand()?; + Err(fallback) + } + // A query: its select list starts at the next token. + "select" => { + self.bump(); + if let Some(e) = self.select_list_keyword() { + return Err(e); + } + self.operand()?; + Err(fallback) + } + // `FROM [WHERE ...]` followed by a data manipulation. + "from" => { + self.bump(); + self.operand()?; + if self.eat_keyword("WHERE") { + self.where_expr()?; + } + if self.is_keyword("SET") || self.is_keyword("REMOVE") { + return Err(fallback); + } + Err(ParseError::well_formed("Expected data manipulation")) + } + "exec" => self.exec_term(), + "values" if next != Some(TokenKind::LParen) => Err(ParseError::well_formed( + "Expected LEFT_PAREN for row value constructor", + )), + // These are calls when `(` follows; the call is read elsewhere. + w if PAREN_NAMED.contains(&w) => Err(ParseError::well_formed(&format!( + "Missing left parenthesis after {w}" + ))), + w if PAREN_UNNAMED.contains(&w) => Err(ParseError::well_formed("Expected LEFT_PAREN")), + "date" => match next { + Some(TokenKind::String(_)) => Err(fallback), + _ => Err(ParseError::well_formed( + "Expected date string followed by the keyword DATE, found SYMBOL", + )), + }, + // A keyword after TIME must be WITH (`TIME WITH TIME ZONE '...'`); + // anything else must be the time string. + "time" => match next { + Some(TokenKind::String(_)) => Err(fallback), + Some(TokenKind::Ident(w)) if w.eq_ignore_ascii_case("with") => Err(fallback), + Some(TokenKind::Ident(w)) if word_class(&w) == WordClass::Keyword => { + Err(ParseError::well_formed("Expected WITH keyword")) + } + _ => Err(ParseError::well_formed( + "Expected time string followed by the keyword TIME OR TIME WITH TIME ZONE, found SYMBOL", + )), + }, + "create" | "drop" => match next { + Some(TokenKind::Ident(w)) + if w.eq_ignore_ascii_case("table") || w.eq_ignore_ascii_case("index") => + { + Err(fallback) + } + _ => Err(ParseError::well_formed(&format!( + "Unexpected token following {}", + word.to_ascii_uppercase() + ))), + }, + "delete" => match next { + Some(TokenKind::Ident(w)) if w.eq_ignore_ascii_case("from") => Err(fallback), + _ => Err(ParseError::well_formed("Expected FROM after DELETE")), + }, + "set" | "remove" | "update" if !self.name_follows() => { + Err(ParseError::well_formed(SIMPLE_PATH)) + } + _ => Err(fallback), + } + } + + /// `EXEC [, ...]` where a term starts. Any token is read as + /// the procedure's name, then the arguments while terms follow. The refusal + /// names the token read as the name and waits for the end of the + /// statement, so a syntax error after the form wins (measured 2026-11-02: + /// `WHERE exec = 1` is "Keyword = only expected at the top level in the + /// query", `attribute_exists(exec)` is "Expected RIGHT_PAREN" and + /// `SELECT exec FROM "t"` is "Expected FROM after SELECT list"). + fn exec_term(&mut self) -> PResult { + self.bump(); + if matches!(self.peek().kind, TokenKind::Eof) { + return Err(self.term_error()); + } + let name = token_text(&self.bump().kind); + if self.at_term_start() { + loop { + self.operand()?; + if !self.eat(&TokenKind::Comma) { + break; + } + } + } + self.defer(&parse_error(&format!( + "Keyword {name} only expected at the top level in the query" + ))); + Ok(Operand::Literal(AttributeValue::Null)) + } + + /// Whether the current token can start a term. + fn at_term_start(&self) -> bool { + match &self.peek().kind { + TokenKind::Ident(w) => word_class(w) == WordClass::Name, + TokenKind::QuotedIdent(_) + | TokenKind::String(_) + | TokenKind::Number(_) + | TokenKind::Ion(_) + | TokenKind::Param + | TokenKind::LParen + | TokenKind::LBracket + | TokenKind::LBrace + | TokenKind::LDoubleAngle + | TokenKind::Plus + | TokenKind::Minus => true, + _ => false, + } + } + + /// The refusal of a keyword that ends a select list where an item should + /// start (measured 2026-11-02 with FROM, SELECT, DELETE, WHERE, SET, UPDATE + /// and REMOVE, and with FROM after `SELECT ALL` and `SELECT DISTINCT`). + pub(super) fn select_list_keyword(&self) -> Option { + let TokenKind::Ident(word) = &self.peek().kind else { + return None; + }; + let word = word.to_ascii_lowercase(); + SELECT_LIST_KEYWORDS.contains(&word.as_str()).then(|| { + ParseError::well_formed(&format!( + "Expected identifier or an expression but found unexpected keyword '{word}' in a select list." + )) + }) + } + + /// A SET or REMOVE target, or the target of an UPDATE or DELETE: a path + /// whose first step is a name. Anything else there, a reserved word or a + /// literal included, is "Expected identifier for simple path" (measured + /// 2026-11-02 for every reserved word as a SET target and as a DELETE + /// table). + pub(super) fn simple_path(&mut self) -> PResult { + let is_name = match &self.peek().kind { + TokenKind::Ident(w) => word_class(w) == WordClass::Name, + TokenKind::QuotedIdent(_) => true, + _ => false, + }; + if !is_name { + return Err(ParseError::well_formed(SIMPLE_PATH)); + } + self.path() + } + + /// A statement that is an expression rather than a query or a write. It + /// parses, then is refused by what it is (measured 2026-11-02): `x` is + /// "Unsupported operation: Id", `1` is "Unsupported operation: Lit", + /// `x IS ` is "Unsupported operation: IsType", and an operator + /// expression (`x = 1`, `NOT x`, `x IS NOT INT4`) or a call (`size(x)`) is + /// taken for a condition check, which only a transaction runs. A trailing + /// semicolon is accepted. + pub(super) fn bare_statement(&mut self) -> PResult { + let form = self.bare_expression()?; + self.eat_final_semicolon(); + if !matches!(self.peek().kind, TokenKind::Eof) { + return Err(self.trailing_error()); + } + // The checks of a statement that would run do not apply to one that + // is refused by its form. + self.forget_findings(); + Err(ParseError::new(match form { + BareForm::Operator => super::EXISTS_OUTSIDE_TRANSACTION.to_owned(), + BareForm::IsType => "Unsupported operation: IsType".to_owned(), + BareForm::Term(name) => format!("Unsupported operation: {name}"), + })) + } + + fn bare_expression(&mut self) -> PResult { + let mut form = self.bare_unit()?; + while self.eat_keyword("AND") || self.eat_keyword("OR") { + self.require_rhs()?; + self.bare_unit()?; + form = BareForm::Operator; + } + Ok(form) + } + + fn bare_unit(&mut self) -> PResult { + // A run of NOTs is read in a loop, so its length costs no stack. + let mut negated = false; + while self.eat_keyword("NOT") { + negated = true; + } + let form = self.bare_term()?; + Ok(if negated { BareForm::Operator } else { form }) + } + + fn bare_term(&mut self) -> PResult { + let operand = self.operand()?; + // `*` outside a condition is left to the caller (see `operand`). + if self.eat(&TokenKind::Star) { + self.require_rhs()?; + self.operand()?; + return Ok(BareForm::Operator); + } + if self.eat_keyword("IS") { + let negated = self.eat_keyword("NOT"); + self.type_name()?; + return Ok(if negated { + BareForm::Operator + } else { + BareForm::IsType + }); + } + let comparison = matches!( + self.peek().kind, + TokenKind::Eq + | TokenKind::Ne + | TokenKind::Lt + | TokenKind::Le + | TokenKind::Gt + | TokenKind::Ge + ); + if comparison || self.is_keyword("LIKE") || self.is_keyword("IN") { + self.bump(); + self.require_rhs()?; + self.operand()?; + return Ok(BareForm::Operator); + } + if self.eat_keyword("BETWEEN") { + self.require_rhs()?; + self.operand()?; + if !self.eat_keyword("AND") { + return Err(ParseError::well_formed("Expected AND keyword")); + } + self.operand()?; + return Ok(BareForm::Operator); + } + Ok(match operand { + Operand::Path(path) if path.steps.is_empty() => BareForm::Term("Id"), + Operand::Path(_) => BareForm::Term("Path"), + Operand::Literal(AttributeValue::M(_)) => BareForm::Term("Struct"), + Operand::Literal(AttributeValue::L(_)) => BareForm::Term("List"), + Operand::Literal(AttributeValue::SS(_) | AttributeValue::NS(_)) => { + BareForm::Term("Bag") + } + Operand::Literal(_) => BareForm::Term("Lit"), + Operand::Parameter(_) => BareForm::Term("Parameter"), + // A call is taken for a condition check like an operator + // expression (measured 2026-11-02 with `size(x)`). + Operand::Function(..) | Operand::Arith(..) => BareForm::Operator, + }) + } + + /// Consume a semicolon that ends the statement: the service accepts one + /// (measured 2026-11-02 after a SELECT, an UPDATE and a bare `IS` test). + pub(super) fn eat_final_semicolon(&mut self) { + if matches!(self.peek().kind, TokenKind::Semicolon) + && matches!(self.nth_kind(1), Some(TokenKind::Eof)) + { + self.bump(); + } + } +} + +/// The text of a token as a message names it. +fn token_text(kind: &TokenKind) -> String { + match kind { + TokenKind::Ident(s) + | TokenKind::QuotedIdent(s) + | TokenKind::String(s) + | TokenKind::Number(s) + | TokenKind::Ion(s) => s.clone(), + TokenKind::Param => "?".to_owned(), + TokenKind::Comma => ",".to_owned(), + TokenKind::Dot => ".".to_owned(), + TokenKind::Star => "*".to_owned(), + TokenKind::LParen => "(".to_owned(), + TokenKind::RParen => ")".to_owned(), + TokenKind::LBracket => "[".to_owned(), + TokenKind::RBracket => "]".to_owned(), + TokenKind::LDoubleAngle => "<<".to_owned(), + TokenKind::RDoubleAngle => ">>".to_owned(), + TokenKind::LBrace => "{".to_owned(), + TokenKind::RBrace => "}".to_owned(), + TokenKind::Colon => ":".to_owned(), + TokenKind::Eq => "=".to_owned(), + TokenKind::Ne => "<>".to_owned(), + TokenKind::Lt => "<".to_owned(), + TokenKind::Le => "<=".to_owned(), + TokenKind::Gt => ">".to_owned(), + TokenKind::Ge => ">=".to_owned(), + TokenKind::Plus => "+".to_owned(), + TokenKind::Minus => "-".to_owned(), + TokenKind::Percent => "%".to_owned(), + TokenKind::Slash => "/".to_owned(), + TokenKind::Concat => "||".to_owned(), + TokenKind::Semicolon => ";".to_owned(), + TokenKind::Eof => String::new(), + } +} + +/// The Ion type name of a backtick literal's text, as the service names it +/// when it refuses the literal (measured 2026-11-02 for an integer, INT). +pub(super) fn ion_type_name(text: &str) -> &'static str { + let t = text.trim(); + let numeric = |s: &str| { + let s = s.strip_prefix('-').unwrap_or(s); + !s.is_empty() && s.chars().all(|c| c.is_ascii_digit() || c == '_') + }; + if numeric(t) { + "INT" + } else if t.starts_with("{{") { + "BLOB" + } else if t.starts_with('{') { + "STRUCT" + } else if t.starts_with('[') { + "LIST" + } else if t.starts_with('(') { + "SEXP" + } else if t.starts_with('"') { + "STRING" + } else if t == "true" || t == "false" { + "BOOL" + } else if t.starts_with("null") { + "NULL" + } else if t.contains(['e', 'E']) + && t.chars() + .next() + .is_some_and(|c| c.is_ascii_digit() || c == '-') + { + "FLOAT" + } else if t + .chars() + .next() + .is_some_and(|c| c.is_ascii_digit() || c == '-') + { + "DECIMAL" + } else { + "SYMBOL" + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/core/src/partiql/parser/terms/tests.rs b/crates/core/src/partiql/parser/terms/tests.rs new file mode 100644 index 000000000..af5fb233a --- /dev/null +++ b/crates/core/src/partiql/parser/terms/tests.rs @@ -0,0 +1,252 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Unit tests for the answers for a reserved word where a term or a name starts. + +use crate::partiql::{Statement, parse}; + +const WF: &str = "Statement wasn't well formed, can't be processed: "; + +fn wf(reason: &str) -> String { + format!("{WF}{reason}") +} + +/// The parse failure, or the projection refusal a SELECT carries, or +/// "parsed". +fn answer(sql: &str) -> String { + match parse(sql) { + Err(e) => e.message, + Ok(Statement::Select(s)) => s.projection_error.unwrap_or_else(|| "parsed".to_owned()), + Ok(_) => "parsed".to_owned(), + } +} + +/// A word's answers in the four measured positions: a condition, the +/// argument of attribute_exists, a SET target and a projection. +fn positions(word: &str) -> [String; 4] { + [ + answer(&format!("SELECT * FROM \"tbl\" WHERE {word} = 1")), + answer(&format!( + "SELECT * FROM \"tbl\" WHERE attribute_exists({word})" + )), + answer(&format!("UPDATE \"tbl\" SET {word} = 1 WHERE pk = 'a'")), + answer(&format!("SELECT {word} FROM \"tbl\"")), + ] +} + +fn in_select_list(word: &str) -> String { + wf(&format!( + "Expected identifier or an expression but found unexpected keyword '{word}' in a select list." + )) +} + +#[test] +fn a_reserved_word_in_the_four_positions() { + let keyword = wf("Unexpected keyword"); + let operator = wf("Unexpected operator"); + let term = wf("Unexpected term"); + let simple = wf("Expected identifier for simple path"); + let manipulation = wf("Expected data manipulation"); + let date = wf("Expected date string followed by the keyword DATE, found SYMBOL"); + let time = wf( + "Expected time string followed by the keyword TIME OR TIME WITH TIME ZONE, found SYMBOL", + ); + let cases: Vec<(&str, [String; 4])> = vec![ + ("order", [keyword.clone(), keyword.clone(), simple.clone(), keyword.clone()]), + ("INNER", [keyword.clone(), keyword.clone(), simple.clone(), keyword.clone()]), + ("count", [keyword.clone(), keyword.clone(), simple.clone(), keyword.clone()]), + ("insert", [keyword.clone(), keyword.clone(), simple.clone(), keyword.clone()]), + ("and", [operator.clone(), operator.clone(), simple.clone(), operator.clone()]), + ("between", [operator.clone(), operator.clone(), simple.clone(), operator.clone()]), + ("as", [term.clone(), term.clone(), simple.clone(), term.clone()]), + ("desc", [term.clone(), term.clone(), simple.clone(), term.clone()]), + ("not", [operator.clone(), term.clone(), simple.clone(), manipulation.clone()]), + ("case", [operator.clone(), term.clone(), simple.clone(), manipulation.clone()]), + ("pivot", [operator.clone(), term.clone(), simple.clone(), manipulation.clone()]), + ("select", [operator.clone(), term.clone(), simple.clone(), in_select_list("select")]), + ("from", [operator.clone(), term.clone(), simple.clone(), in_select_list("from")]), + ("all", [keyword.clone(), keyword.clone(), simple.clone(), in_select_list("from")]), + ("distinct", [keyword.clone(), keyword.clone(), simple.clone(), in_select_list("from")]), + ("value", [keyword.clone(), keyword.clone(), simple.clone(), manipulation.clone()]), + ("where", [keyword.clone(), keyword.clone(), simple.clone(), in_select_list("where")]), + ( + "delete", + [ + wf("Expected FROM after DELETE"), + wf("Expected FROM after DELETE"), + simple.clone(), + in_select_list("delete"), + ], + ), + ("set", [simple.clone(), simple.clone(), simple.clone(), in_select_list("set")]), + ("update", [simple.clone(), simple.clone(), simple.clone(), in_select_list("update")]), + ("remove", [simple.clone(), simple.clone(), simple.clone(), in_select_list("remove")]), + ( + "exec", + [ + wf("Keyword = only expected at the top level in the query"), + wf("Expected RIGHT_PAREN"), + simple.clone(), + wf("Expected FROM after SELECT list"), + ], + ), + ("date", [date.clone(), date.clone(), simple.clone(), date.clone()]), + ("time", [time.clone(), time.clone(), simple.clone(), wf("Expected WITH keyword")]), + ( + "cast", + [ + wf("Missing left parenthesis after cast"), + wf("Missing left parenthesis after cast"), + simple.clone(), + wf("Missing left parenthesis after cast"), + ], + ), + ( + "nullif", + [ + wf("Missing left parenthesis after nullif"), + wf("Missing left parenthesis after nullif"), + simple.clone(), + wf("Missing left parenthesis after nullif"), + ], + ), + ( + "trim", + [ + wf("Expected LEFT_PAREN"), + wf("Expected LEFT_PAREN"), + simple.clone(), + wf("Expected LEFT_PAREN"), + ], + ), + ( + "values", + [ + wf("Expected LEFT_PAREN for row value constructor"), + wf("Expected LEFT_PAREN for row value constructor"), + simple.clone(), + wf("Expected LEFT_PAREN for row value constructor"), + ], + ), + ( + "create", + [ + wf("Unexpected token following CREATE"), + wf("Unexpected token following CREATE"), + simple.clone(), + wf("Unexpected token following CREATE"), + ], + ), + ( + "drop", + [ + wf("Unexpected token following DROP"), + wf("Unexpected token following DROP"), + simple.clone(), + wf("Unexpected token following DROP"), + ], + ), + ( + "missing", + [ + "MISSING must be compared using IS operator".to_owned(), + "MISSING must be compared using IS operator".to_owned(), + simple.clone(), + "Unexpected path component at 1:8:7".to_owned(), + ], + ), + ( + "null", + [ + "parsed".to_owned(), + "Operator or function requires a document path; operator or function: attribute_exists" + .to_owned(), + simple.clone(), + "Unexpected path component at 1:8:4".to_owned(), + ], + ), + ( + "false", + [ + "parsed".to_owned(), + "Operator or function requires a document path; operator or function: attribute_exists" + .to_owned(), + simple.clone(), + "Unexpected path component at 1:8:5".to_owned(), + ], + ), + ]; + for (word, expected) in cases { + assert_eq!(positions(word), expected, "{word}"); + } +} + +#[test] +fn a_word_the_service_does_not_reserve_is_a_name_everywhere() { + for word in ["year", "partition", "leading", "upsert", "timezone_hour"] { + assert_eq!( + positions(word), + ["parsed", "parsed", "parsed", "parsed"], + "{word}" + ); + } +} + +#[test] +fn a_reserved_word_where_a_from_source_starts() { + for (word, expected) in [ + ("where", wf("Unexpected keyword")), + ("count", wf("Unexpected keyword")), + ("as", wf("Unexpected term")), + ("and", wf("Unexpected operator")), + ("set", wf("Expected identifier for simple path")), + ("select", in_select_list("where")), + ("null", "Unexpected from source".to_owned()), + ] { + assert_eq!( + answer(&format!("SELECT * FROM {word} WHERE pk = 'p'")), + expected, + "{word}" + ); + } + for word in [ + "where", "value", "select", "inner", "as", "and", "null", "set", + ] { + assert_eq!( + answer(&format!("DELETE FROM {word} WHERE pk = 'p'")), + wf("Expected identifier for simple path"), + "{word}" + ); + } +} + +#[test] +fn the_forms_that_take_an_expression_report_the_next_token() { + // FROM-led form with a WHERE and no data manipulation. + assert_eq!( + answer("SELECT case FROM \"tbl\" WHERE pk = 'p'"), + wf("Expected data manipulation") + ); + // EXEC reads to the end of its arguments; the next clause is checked. + assert_eq!( + answer("SELECT exec FROM \"tbl\" WHERE pk = 'p'"), + wf("Expected FROM after SELECT list") + ); + assert_eq!( + answer("SELECT * FROM \"tbl\" WHERE pk = 'p' AND exec = 1"), + wf("Keyword = only expected at the top level in the query") + ); +} + +#[test] +fn ion_type_names() { + use super::ion_type_name; + assert_eq!(ion_type_name("1"), "INT"); + assert_eq!(ion_type_name("-12"), "INT"); + assert_eq!(ion_type_name("1.5"), "DECIMAL"); + assert_eq!(ion_type_name("1e0"), "FLOAT"); + assert_eq!(ion_type_name("\"a\""), "STRING"); + assert_eq!(ion_type_name("{a: 1}"), "STRUCT"); + assert_eq!(ion_type_name("[1]"), "LIST"); + assert_eq!(ion_type_name("abc"), "SYMBOL"); +} diff --git a/crates/core/src/partiql/parser/tests.rs b/crates/core/src/partiql/parser/tests.rs new file mode 100644 index 000000000..ab76c024b --- /dev/null +++ b/crates/core/src/partiql/parser/tests.rs @@ -0,0 +1,29 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Shared helpers and module wiring for the parser's unit tests. + +use super::*; +use crate::partiql::ast::{Operand, Returning, TypeName, WhereExpr}; +use crate::types::AttributeValue; + +fn err(sql: &str) -> String { + parse(sql).unwrap_err().message +} + +const WF: &str = "Statement wasn't well formed, can't be processed: "; + +fn wf(reason: &str) -> String { + format!("{WF}{reason}") +} + +fn where_of(sql: &str) -> WhereExpr { + match parse(sql).unwrap() { + Statement::Select(s) => s.where_clause.unwrap(), + other => panic!("expected select, got {other:?}"), + } +} + +mod conditions; +mod statements; +mod values; diff --git a/crates/core/src/partiql/parser/tests/conditions.rs b/crates/core/src/partiql/parser/tests/conditions.rs new file mode 100644 index 000000000..32f826930 --- /dev/null +++ b/crates/core/src/partiql/parser/tests/conditions.rs @@ -0,0 +1,392 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Unit tests for the WHERE condition: its predicates, operators and groups. + +use super::*; + +#[test] +fn empty_in_list_is_rejected() { + let m = err("SELECT * FROM \"t\" WHERE pk='a' AND sk IN []"); + assert!( + m.starts_with("IN operator must have a left hand argument"), + "{m}" + ); +} + +#[test] +fn where_with_nothing_after_it_is_an_unexpected_term() { + for sql in [ + "SELECT * FROM \"t\" WHERE", + "SELECT * FROM \"t\" WHERE;", + "SELECT * FROM \"t\" WHERE ", + "SELECT pk FROM \"t\" WHERE", + "UPDATE \"t\" SET a = 1 WHERE", + "DELETE FROM \"t\" WHERE", + "SELECT * FROM \"t\" WHERE NOT", + "SELECT * FROM \"t\" WHERE (", + "SELECT * FROM \"t\".\"gsi\" WHERE", + ] { + assert_eq!(err(sql), wf("Unexpected term"), "{sql}"); + } +} + +#[test] +fn an_infix_operator_with_nothing_after_it_is_missing_its_right_side() { + for sql in [ + "SELECT * FROM \"t\" WHERE pk = 'p1' AND", + "SELECT * FROM \"t\" WHERE pk = 'p1' OR", + "SELECT * FROM \"t\" WHERE pk =", + "SELECT * FROM \"t\" WHERE pk = 'p' AND n <", + "SELECT * FROM \"t\" WHERE pk = 'p' AND n = 1 +", + "SELECT * FROM \"t\" WHERE pk = 'p' AND n IN", + "SELECT * FROM \"t\" WHERE pk = 'p' AND n BETWEEN", + ] { + assert_eq!( + err(sql), + wf("Missing right-hand side expression of infix operator"), + "{sql}" + ); + } + // A semicolon after the operator is a term that cannot start. + assert_eq!( + err("SELECT * FROM \"t\" WHERE pk = 'p1' AND;"), + wf("Unexpected term") + ); +} + +#[test] +fn a_term_that_cannot_start_is_named_by_what_it_is() { + assert_eq!( + err("SELECT * FROM \"t\" WHERE ORDER BY pk"), + wf("Unexpected keyword") + ); + assert_eq!( + err("SELECT * FROM \"t\" WHERE IS NULL"), + wf("Unexpected operator") + ); + assert_eq!( + err("SELECT * FROM \"t\" WHERE pk = LIKE"), + wf("Unexpected operator") + ); + assert_eq!( + err("SELECT * FROM \"t\" WHERE pk = 'a' AND value = 1"), + wf("Unexpected keyword") + ); + assert_eq!( + err("SELECT * FROM \"t\" WHERE = 1"), + wf("Unexpected operator") + ); + assert_eq!(err("SELECT * FROM \"t\" WHERE , 1"), wf("Unexpected term")); + assert_eq!( + err("SELECT * FROM \"t\" WHERE asc = 1"), + wf("Unexpected term") + ); +} + +#[test] +fn a_keyword_function_name_is_still_a_call() { + assert!(parse("SELECT * FROM \"t\" WHERE pk = 'a' AND size(s) = 3").is_ok()); +} + +#[test] +fn is_with_a_name_that_is_not_a_type() { + assert_eq!( + err("SELECT * FROM \"t\" WHERE pk = 'a' AND x IS FOO"), + wf("Expected type name") + ); +} + +#[test] +fn is_null_and_type_tests_parse_as_filters() { + match where_of("SELECT * FROM \"t\" WHERE pk = 'a' AND nul IS NULL") { + WhereExpr::And(_, b) => { + assert!(matches!(*b, WhereExpr::IsType(_, TypeName::Null, false))); + } + other => panic!("{other:?}"), + } + assert!(matches!( + where_of("SELECT * FROM \"t\" WHERE s IS NOT NULL"), + WhereExpr::IsType(_, TypeName::Null, true) + )); + assert!(matches!( + where_of("SELECT * FROM \"t\" WHERE flag IS BOOL"), + WhereExpr::IsType(_, TypeName::Boolean, false) + )); + assert!(matches!( + where_of("SELECT * FROM \"t\" WHERE s IS string"), + WhereExpr::IsType(_, TypeName::String, false) + )); + assert!(matches!( + where_of("SELECT * FROM \"t\" WHERE s IS MISSING"), + WhereExpr::IsMissing(_, true) + )); +} + +#[test] +fn an_is_filter_on_a_type_with_no_filter_is_refused_at_its_left_side() { + assert_eq!( + err("SELECT * FROM \"t\" WHERE pk = 'a' AND x IS INT4"), + "Unsupported data type for IS operator: Integer4Type. Supported data types are \ + [MISSING, NULL, BOOLEAN, BLOB, STRING, NUMERIC, STRUCT, TUPLE, LIST] at 1:38:1" + ); + // A length after the type is read. + assert_eq!( + err("SELECT * FROM \"t\" WHERE pk = 'a' AND sv IS VARCHAR(10)"), + "Unsupported data type for IS operator: CharacterVaryingType. Supported data \ + types are [MISSING, NULL, BOOLEAN, BLOB, STRING, NUMERIC, STRUCT, TUPLE, LIST] \ + at 1:38:2" + ); +} + +#[test] +fn a_between_that_breaks_off() { + assert_eq!( + err("SELECT * FROM \"t\" WHERE n BETWEEN 1 AND"), + wf("Expected expression after AND") + ); + assert_eq!( + err("SELECT * FROM \"t\" WHERE n BETWEEN 1"), + wf("Expected AND keyword") + ); + assert_eq!(err("SELECT * FROM \"t\" WHERE n BETWEEN"), wf(MISSING_RHS)); +} + +#[test] +fn like_with_nothing_after_it_is_missing_its_right_side() { + assert_eq!(err("SELECT * FROM \"t\" WHERE s LIKE"), wf(MISSING_RHS)); +} + +#[test] +fn a_condition_takes_no_arithmetic() { + for (condition, op) in [ + ("n = -n", "-"), + ("n = +n", "+"), + ("pk = 1-2", "-"), + ("pk = 1+2", "+"), + ("n % 2 = 1", "%"), + ("n / 2 = 1", "/"), + ("s || 'x' = 'y'", "||"), + ] { + assert_eq!( + err(&format!("SELECT * FROM \"t\" WHERE {condition}")), + format!("Unsupported operator in Condition Expression. Operator: {op}"), + "{condition}" + ); + } + // A sign before a number is part of the number. + assert!(parse("SELECT * FROM \"t\" WHERE n = -5").is_ok()); +} + +#[test] +fn an_operand_with_an_operator_as_the_condition_is_refused_for_the_operator() { + assert_eq!( + err("SELECT * FROM \"t\" WHERE a + 1"), + "Unsupported operator in Condition Expression. Operator: +" + ); + // A syntax error after it wins. + assert_eq!(err("SELECT * FROM \"t\" WHERE a + 1 AND"), wf(MISSING_RHS)); + // So it does for an operand AND or OR takes. + assert_eq!( + err("SELECT * FROM \"t\" WHERE a + 1 AND pk = 'a'"), + "Unsupported operator in Condition Expression. Operator: +" + ); +} + +#[test] +fn an_operand_that_is_the_whole_condition_is_refused() { + for condition in ["a", "1", "'x'", "?", "[1]", "m.b", "true", "a;"] { + assert_eq!( + err(&format!("SELECT * FROM \"t\" WHERE {condition}")), + NOT_A_CONDITION, + "{condition}" + ); + } + // After what the walk meets in it. + assert_eq!( + parse_for_request("SELECT * FROM \"tbl\" WHERE ?", 0) + .unwrap_err() + .message, + PARAMETER_COUNT_MISMATCH + ); + // A syntax error after it wins. + assert_eq!( + err("SELECT * FROM \"t\" WHERE a AND ("), + wf("Unexpected term") + ); +} + +#[test] +fn an_operand_that_and_or_or_takes_is_refused() { + for condition in [ + "a AND pk = 'a'", + "pk = 'a' AND a", + "a OR pk = 'a'", + "pk = 'a' AND a OR b = 1", + ] { + assert_eq!( + err(&format!("SELECT * FROM \"t\" WHERE {condition}")), + AND_OR_ARGUMENTS, + "{condition}" + ); + } + // After what the walk meets in the arguments of its operator, and + // ahead of what it meets after them. + let overflow = "Number overflow. Attempting to store a number with magnitude larger \ + than supported range under root"; + for condition in [ + "a AND b = 1e126", + "c = 1e126 AND a", + "a OR b = 1 AND c = 1 AND d = 1e126", + "b = 1 OR a AND c = 1e126", + ] { + assert_eq!( + err(&format!("SELECT * FROM \"t\" WHERE {condition}")), + overflow, + "{condition}" + ); + } + for condition in ["pk = 'a' AND a OR c = 1e126", "b = 1 OR a OR c = 1e126"] { + assert_eq!( + err(&format!("SELECT * FROM \"t\" WHERE {condition}")), + AND_OR_ARGUMENTS, + "{condition}" + ); + } +} + +#[test] +fn an_operand_in_a_group_or_after_a_not_is_refused_for_its_place() { + for (condition, want) in [ + ("(a)", NOT_A_CONDITION), + ("((a))", NOT_A_CONDITION), + ("pk = 'a' AND (a)", AND_OR_ARGUMENTS), + ("(a) OR pk = 'a'", AND_OR_ARGUMENTS), + ("(a AND b = 1)", AND_OR_ARGUMENTS), + ("(b = 1 AND a)", AND_OR_ARGUMENTS), + ("((a) AND b = 1)", AND_OR_ARGUMENTS), + ("NOT a", NOT_ARGUMENT), + ("NOT (a)", NOT_ARGUMENT), + ("NOT ((a))", NOT_ARGUMENT), + ("(NOT a)", NOT_ARGUMENT), + ("NOT (a) AND pk = 'a'", NOT_ARGUMENT), + // The NOT's refusal is met before the AND's, which comes after both + // of its arguments. + ("a AND NOT b", NOT_ARGUMENT), + ] { + assert_eq!( + err(&format!("SELECT * FROM \"t\" WHERE {condition}")), + want, + "{condition}" + ); + } + // Any other token after the operand ends it, and the part of the + // statement around the operand reads that token: a syntax error there + // comes first. + // The LIMIT refusal names the clause's value, the last character. + let sql = "SELECT * FROM \"t\" WHERE a LIMIT 1"; + assert_eq!( + err(sql), + format!("Unsupported clause: LIMIT at 1:{}:1", sql.len()) + ); + for (condition, want) in [ + ("a b", "Unexpected token after expression"), + ("pk = 'a' AND a b", "Unexpected token after expression"), + ("(a b)", "Expected RIGHT_PAREN"), + ("pk = 'a' AND (a b)", "Expected RIGHT_PAREN"), + ] { + assert_eq!( + err(&format!("SELECT * FROM \"t\" WHERE {condition}")), + wf(want), + "{condition}" + ); + } +} + +#[test] +fn an_operand_alone_in_parentheses_is_that_operand() { + for condition in [ + "(pk) = 'a'", + "((pk)) = (('a'))", + "pk = ('a')", + "pk = (?)", + "(a) BETWEEN 0 AND 2", + "(a) IN [1]", + "(a) NOT IN [2]", + "(m) IS NOT MISSING", + "NOT (a) = 2", + "(a) = 1 = true", + "begins_with((sk), 'b')", + "attribute_exists((a))", + "(m).k = 1", + "(m)['k'] = 1", + "a = -(-1)", + ] { + let sql = format!("SELECT * FROM \"t\" WHERE {condition}"); + assert!(parse(&sql).is_ok(), "{condition}: {:?}", parse(&sql)); + } + assert_eq!( + where_of("SELECT * FROM \"t\" WHERE ((pk)) = ('a')"), + where_of("SELECT * FROM \"t\" WHERE pk = 'a'") + ); + assert_eq!( + where_of("SELECT * FROM \"t\" WHERE (m)['k'] = -(1)"), + where_of("SELECT * FROM \"t\" WHERE m.k = -1") + ); + assert_eq!( + err("SELECT * FROM \"t\" WHERE (a) IS INT4"), + "Unsupported data type for IS operator: Integer4Type. Supported data types are \ + [MISSING, NULL, BOOLEAN, BLOB, STRING, NUMERIC, STRUCT, TUPLE, LIST] at 1:26:1" + ); + for (condition, want) in [ + ("a = (1", wf("Expected RIGHT_PAREN")), + ("a = (1))", wf("Unexpected token after expression")), + ("a = ()", wf("Unexpected term")), + ( + "(a + 1) = 2", + "Unsupported operator in Condition Expression. Operator: +".to_owned(), + ), + ] { + assert_eq!( + err(&format!("SELECT * FROM \"t\" WHERE {condition}")), + want, + "{condition}" + ); + } +} + +#[test] +fn a_group_left_open_is_missing_its_right_parenthesis() { + for condition in ["(a = 1 b)", "(a = 1", "((a = 1)"] { + assert_eq!( + err(&format!("SELECT * FROM \"t\" WHERE {condition}")), + wf("Expected RIGHT_PAREN"), + "{condition}" + ); + } +} + +#[test] +fn a_call_left_open_is_missing_its_right_parenthesis() { + assert_eq!( + err("SELECT * FROM \"t\" WHERE attribute_exists(a"), + wf("Expected RIGHT_PAREN") + ); +} + +#[test] +fn an_is_test_on_a_parameter_is_refused() { + assert_eq!( + err("SELECT * FROM \"t\" WHERE pk = 'a' AND ? IS NULL"), + "The left hand side argument to IS must be a Variable Reference, Literal, or Path" + ); + assert!(parse("SELECT * FROM \"t\" WHERE pk = 'a' AND 'a' IS STRING").is_ok()); +} + +#[test] +fn an_is_test_as_a_set_value_is_refused() { + assert_eq!( + err("UPDATE \"t\" SET a = b IS NULL WHERE pk = 'a'"), + "IS operator is not supported in update expressions" + ); +} diff --git a/crates/core/src/partiql/parser/tests/statements.rs b/crates/core/src/partiql/parser/tests/statements.rs new file mode 100644 index 000000000..5597460b8 --- /dev/null +++ b/crates/core/src/partiql/parser/tests/statements.rs @@ -0,0 +1,466 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Unit tests for the statements, their targets and their clauses. + +use super::*; + +#[test] +fn select_missing_from() { + assert_eq!( + err("SELECT * WHERE pk='a'"), + "Statement wasn't well formed, can't be processed: Expected FROM after SELECT list" + ); +} + +#[test] +fn non_dml_leading_token() { + assert_eq!( + err("FOO * FROM \"t\""), + "Statement wasn't well formed, can't be processed: Expected data manipulation" + ); +} + +#[test] +fn an_unquoted_table_name_is_a_table() { + let Statement::Select(s) = parse("SELECT * FROM t WHERE pk='a'").unwrap() else { + panic!("expected select"); + }; + assert_eq!(s.from.table, "t"); + for sql in [ + "UPDATE t SET a = 1 WHERE pk = 'a'", + "DELETE FROM t WHERE pk = 'a'", + "INSERT INTO t VALUE {'pk': 'a'}", + ] { + assert!(parse(sql).is_ok(), "{sql}"); + } + // A literal is not a table. + assert_eq!( + err("SELECT * FROM null WHERE pk='a'"), + "Unexpected from source" + ); + assert_eq!( + err("SELECT * FROM 't' WHERE pk='a'"), + "Unexpected from source" + ); +} + +#[test] +fn delete_missing_from() { + assert_eq!( + err("DELETE \"t\" WHERE pk='a' AND sk='b'"), + "Statement wasn't well formed, can't be processed: Expected FROM after DELETE" + ); +} + +#[test] +fn update_without_set_or_remove() { + assert_eq!( + err("UPDATE \"t\" WHERE pk='a' AND sk='b'"), + "Statement wasn't well formed, can't be processed: Expected data manipulation" + ); +} + +#[test] +fn insert_values_plural_is_rejected() { + assert_eq!( + err("INSERT INTO \"t\" VALUES {'pk':'a'}"), + "Statement wasn't well formed, can't be processed: Expected LEFT_PAREN for row value constructor" + ); +} + +#[test] +fn insert_double_quoted_key_is_rejected() { + let m = err("INSERT INTO \"t\" VALUE {\"pk\":'a'}"); + assert!( + m.starts_with("Tuple keys must be of type String. Actual type: Id under root at "), + "{m}" + ); +} + +#[test] +fn two_statements_separated_by_semicolon() { + assert_eq!( + err("SELECT * FROM \"t\" WHERE pk='a';SELECT * FROM \"t\" WHERE pk='b'"), + "Statement wasn't well formed, can't be processed: Unexpected token after semicolon. (Only one query is allowed.)" + ); +} + +#[test] +fn aliasing_is_rejected() { + assert_eq!( + err("SELECT name AS n FROM \"t\" WHERE pk='a'"), + "Aliasing is not supported" + ); +} + +#[test] +fn limit_clause_is_an_unsupported_clause() { + // The column is that of the token after LIMIT, with length 1. + let sql = "SELECT * FROM \"t\" WHERE pk='a' LIMIT 2"; + let column = sql.find('2').unwrap() + 1; + assert_eq!( + err(sql), + format!("Unsupported clause: LIMIT at 1:{column}:1") + ); +} + +#[test] +fn a_projected_path_may_have_any_number_of_steps() { + assert!(parse("SELECT a.b.c FROM \"t\" WHERE pk='a'").is_ok()); + // A reserved word after a dot is still refused. + assert_eq!( + err("SELECT a.inner.c FROM \"t\" WHERE pk='a'"), + "Statement wasn't well formed, can't be processed: Invalid path dot component" + ); +} + +#[test] +fn single_dot_projection_is_allowed() { + assert!(parse("SELECT a.b FROM \"t\" WHERE pk='a'").is_ok()); +} + +#[test] +fn count_star_is_an_unexpected_path_component() { + // COUNT starts at column 8 (after "SELECT ") and is 5 chars. The + // statement parses and carries the refusal for the executor. + let Statement::Select(s) = parse("SELECT COUNT(*) FROM \"t\" WHERE pk='a'").unwrap() else { + panic!("expected select"); + }; + assert_eq!( + s.projection_error.as_deref(), + Some("Unexpected path component at 1:8:5") + ); + assert_eq!(s.from.table, "t"); + assert!(s.where_clause.is_some()); +} + +#[test] +fn a_projected_call_keeps_the_first_refusal_and_parses_on() { + let Statement::Select(s) = parse("SELECT a, size(b(c)), COUNT(*) FROM \"t\"").unwrap() else { + panic!("expected select"); + }; + assert_eq!( + s.projection_error.as_deref(), + Some("Unexpected path component at 1:11:4") + ); +} + +#[test] +fn a_syntax_error_after_a_projected_call_wins() { + let m = err("SELECT COUNT(*) FROM \"t\" WHERE"); + assert!( + m.starts_with("Statement wasn't well formed, can't be processed: "), + "{m}" + ); + assert_eq!( + err("SELECT COUNT(* FROM \"t\""), + "Unexpected path component at 1:8:5" + ); +} + +#[test] +fn select_happy_path_parses() { + let s = + parse("SELECT a, b.c, d[0] FROM \"t\".\"gsi1\" WHERE pk='a' AND sk > 'b' ORDER BY sk DESC") + .unwrap(); + match s { + Statement::Select(sel) => { + assert_eq!(sel.from.table, "t"); + assert_eq!(sel.from.index.as_deref(), Some("gsi1")); + let order = sel.order_by.unwrap(); + assert_eq!(order.terms.len(), 1); + assert!(order.terms[0].descending); + } + _ => panic!("expected select"), + } +} + +#[test] +fn order_by_takes_a_list_of_names_each_with_a_direction() { + let order = |sql: &str| match parse(sql).unwrap() { + Statement::Select(s) => s + .order_by + .unwrap() + .terms + .iter() + .map(|term| (term.path.as_simple().unwrap().to_owned(), term.descending)) + .collect::>(), + other => panic!("expected select, got {other:?}"), + }; + let base = "SELECT * FROM \"t\" WHERE pk IN ['a', 'b'] ORDER BY "; + let term = |name: &str, descending| (name.to_owned(), descending); + for (tail, terms) in [ + ("pk", vec![term("pk", false)]), + ("pk DESC", vec![term("pk", true)]), + ("pk ASC", vec![term("pk", false)]), + ("pk, sk", vec![term("pk", false), term("sk", false)]), + ("pk DESC, sk", vec![term("pk", true), term("sk", false)]), + ("pk ASC, sk", vec![term("pk", false), term("sk", false)]), + ("pk, sk DESC", vec![term("pk", false), term("sk", true)]), + ("sk DESC, pk", vec![term("sk", true), term("pk", false)]), + ("pk DESC, sk DESC", vec![term("pk", true), term("sk", true)]), + ( + "pk, \"sk\", n", + vec![term("pk", false), term("sk", false), term("n", false)], + ), + ] { + assert_eq!(order(&format!("{base}{tail}")), terms, "{tail}"); + } + // A comma that ends the statement is an unexpected term (measured + // 2026-11-03). A comma before anything else but a name stays a token + // left over; the service answers it with an internal error. + assert_eq!(err(&format!("{base}pk,")), wf("Unexpected term")); + assert_eq!( + err(&format!("{base}pk, 1")), + wf("Unexpected token after expression") + ); +} + +#[test] +fn insert_rich_literal_parses() { + let s = parse( + "INSERT INTO \"t\" VALUE {'pk':'p','n':3.5,'b':true,'z':null,'m':{'x':1},'l':[1,'two'],'ss':<<'a','b'>>}", + ) + .unwrap(); + match s { + Statement::Insert(i) => assert_eq!(i.value.len(), 7), + _ => panic!("expected insert"), + } +} + +#[test] +fn update_with_params_assigns_positional_indices() { + let s = parse("UPDATE \"t\" SET a = ? WHERE pk = ? AND sk = ?").unwrap(); + match s { + Statement::Update(u) => { + assert!(matches!(u.sets[0].value, Operand::Parameter(0))); + } + _ => panic!("expected update"), + } +} + +#[test] +fn a_bare_is_type_test_is_an_unsupported_operation() { + for sql in [ + "x IS INT4", + "x IS INT8", + "x IS DOUBLE PRECISION", + "x IS CHARACTER VARYING", + "x IS NULL", + ] { + assert_eq!(err(sql), "Unsupported operation: IsType", "{sql}"); + } + assert_eq!(err("x IS FOO"), wf("Expected type name")); + assert_eq!(err("x IS INT4 y"), wf("Unexpected token after expression")); +} + +#[test] +fn a_bare_expression_is_refused_by_what_it_is() { + const CHECK: &str = "EXISTS can only be used in ExecuteTransaction write requests."; + for (sql, expected) in [ + ("x", "Unsupported operation: Id"), + ("1", "Unsupported operation: Lit"), + ("x = 1", CHECK), + ("NOT x", CHECK), + ("x IS NOT INT4", CHECK), + ("x IS VARCHAR(10)", "Unsupported operation: IsType"), + ("x IS TIME WITH TIME ZONE", "Unsupported operation: IsType"), + ("'a' IS STRING", "Unsupported operation: IsType"), + ("? IS NULL", "Unsupported operation: IsType"), + ("x IS INT4;", "Unsupported operation: IsType"), + ] { + assert_eq!(err(sql), expected, "{sql}"); + } + assert_eq!(err("x IS ES_ANY"), wf("Expected type name")); + // `*` then a FROM-led form that has no data manipulation. + assert_eq!(err("FOO * FROM \"t\""), wf("Expected data manipulation")); + // A call is a condition check like an operator expression. + for sql in ["size(x)", "attribute_exists(x)", "foo(x)"] { + assert_eq!(err(sql), CHECK, "{sql}"); + } +} + +#[test] +fn a_statement_may_end_with_a_semicolon() { + assert!(parse("SELECT * FROM \"t\" WHERE pk = 'p';").is_ok()); + assert!(parse("UPDATE \"t\" SET a = 1 WHERE pk = 'p';").is_ok()); + assert_eq!( + err("SELECT * FROM \"t\" WHERE pk = 'p'; x"), + wf("Unexpected token after semicolon. (Only one query is allowed.)") + ); +} + +#[test] +fn an_insert_without_value_reads_the_items_to_insert() { + const MANY: &str = "Unsupported operation: Inserting multiple items in a single \ + statement is not supported, use \"INSERT INTO tableName VALUE \ + item\" instead"; + for sql in [ + "INSERT INTO \"t\" {'pk': 'x'}", + "INSERT INTO \"t\" t", + "INSERT INTO \"t\" 'x'", + ] { + assert_eq!(err(sql), MANY, "{sql}"); + } + assert_eq!(err("INSERT INTO \"t\""), wf("Unexpected term")); + assert_eq!( + err("INSERT INTO \"t\" t VALUE {'pk': 'x'}"), + wf("Unexpected token after expression") + ); +} + +#[test] +fn as_with_no_name_after_it_is_a_syntax_error() { + for sql in [ + "SELECT pk AS FROM \"t\" WHERE pk = 'p'", + "SELECT * FROM \"t\" AS WHERE pk = 'p'", + ] { + assert_eq!(err(sql), wf("Expected identifier for AS-alias"), "{sql}"); + } +} + +#[test] +fn at_and_by_after_a_table_are_aliases() { + for sql in [ + "SELECT * FROM \"t\" AT x WHERE pk = 'p'", + "SELECT * FROM \"t\" BY x WHERE pk = 'p'", + ] { + assert_eq!(err(sql), "Aliasing is not supported", "{sql}"); + } +} + +#[test] +fn an_alias_is_refused_ahead_of_an_empty_index_name() { + assert_eq!( + err("SELECT * FROM \"t\".\"\" x WHERE pk = 'p'"), + "Aliasing is not supported" + ); +} + +#[test] +fn a_from_path_step_that_is_not_a_name() { + for sql in [ + "SELECT * FROM \"t\".index WHERE pk = 'p'", + "SELECT * FROM \"t\". WHERE pk = 'p'", + ] { + assert_eq!(err(sql), wf("Invalid path dot component"), "{sql}"); + } +} + +#[test] +fn an_update_without_where_parses_for_execution_to_refuse() { + let Statement::Update(u) = parse("UPDATE \"t\" SET a = 1").unwrap() else { + panic!("expected update"); + }; + assert!(u.where_clause.is_none()); + assert_eq!(err("UPDATE \"t\" t SET a = 1"), "Aliasing is not supported"); + assert_eq!( + err("UPDATE \"t\" SET a = 1 FOO"), + wf("Unexpected token after expression") + ); +} + +#[test] +fn a_dml_statement_with_no_table() { + for sql in [ + "DELETE FROM", + "DELETE FROM ", + "DELETE FROM;", + "DELETE FROM WHERE pk = 'a'", + "UPDATE SET a = 1 WHERE pk = 'a'", + "INSERT INTO VALUE {'pk': 'a'}", + ] { + assert_eq!(err(sql), wf("Expected identifier for simple path"), "{sql}"); + } +} + +#[test] +fn select_from_with_no_table() { + assert_eq!(err("SELECT * FROM"), wf("Unexpected term")); + assert_eq!( + err("SELECT * FROM WHERE pk = 'a'"), + wf("Unexpected keyword") + ); +} + +#[test] +fn a_table_alias_is_refused() { + for sql in [ + "SELECT * FROM \"t\" t WHERE pk = 'x'", + "SELECT * FROM \"t\" AS t WHERE pk = 'x'", + "UPDATE \"t\" t SET a = 3 WHERE pk = 'x'", + "DELETE FROM \"t\" t WHERE pk = 'x'", + "SELECT a b FROM \"t\" WHERE pk = 'x'", + ] { + assert_eq!(err(sql), "Aliasing is not supported", "{sql}"); + } + // An INSERT target takes no alias: the name is a leftover token. + assert_eq!( + err("INSERT INTO \"t\" t VALUE {'pk': 'x'}"), + wf("Unexpected token after expression") + ); +} + +#[test] +fn a_syntax_error_is_reported_ahead_of_an_alias() { + assert_eq!(err("SELECT * FROM \"t\" t WHERE"), wf("Unexpected term")); +} + +#[test] +fn a_lexing_failure_has_nothing_after_the_prefix() { + assert_eq!(err("SELECT * FROM \"t\" WHERE pk = 1 # note"), WF); + assert_eq!(err("SELECT * FROM \"t\" WHERE pk = 1 /* note"), WF); +} + +#[test] +fn comments_read_as_whitespace() { + for sql in [ + "SELECT * FROM \"t\" WHERE pk = 'a' -- a note", + "SELECT * FROM \"t\" -- a note\nWHERE pk = 'a'", + "-- a note\nSELECT * FROM \"t\" WHERE pk = 'a'", + "SELECT * FROM \"t\" /* a note */ WHERE pk = 'a'", + "SELECT * FROM \"t\" WHERE pk = 'a' /* a note */", + ] { + let w = where_of(sql); + assert!(matches!(w, WhereExpr::Compare(..)), "{sql}"); + } +} + +#[test] +fn an_empty_from_component_is_refused_after_the_parse() { + assert_eq!( + err("SELECT * FROM \"\" WHERE pk = 'x'"), + "Path component cannot be an empty string" + ); + assert_eq!( + err("SELECT * FROM \"t\".\"\" WHERE pk = 'x'"), + "Path component cannot be an empty string" + ); + assert_eq!(err("SELECT * FROM \"\" WHERE"), wf("Unexpected term")); +} + +#[test] +fn delete_returning_all_old_parses() { + let s = parse("DELETE FROM \"t\" WHERE pk='a' AND sk='b' RETURNING ALL OLD *").unwrap(); + match s { + Statement::Delete(d) => assert_eq!(d.returning, Some(super::Returning::AllOld)), + _ => panic!("expected delete"), + } +} + +#[test] +fn a_delete_returning_all_with_nothing_after_it_is_a_syntax_error() { + let element = wf("Expected ( MODIFIED | ALL ) ( NEW | OLD ) in each returning element."); + for sql in [ + "DELETE FROM \"t\" WHERE pk = 'a' RETURNING ALL", + "DELETE FROM \"t\" WHERE pk = 'a' AND c = 1e126 RETURNING ALL", + ] { + assert_eq!(err(sql), element, "{sql}"); + } + // An UPDATE's is not measured. + assert_eq!( + err("UPDATE \"t\" SET a = 1 WHERE pk = 'a' RETURNING ALL"), + "Invalid returning clause" + ); +} diff --git a/crates/core/src/partiql/parser/tests/values.rs b/crates/core/src/partiql/parser/tests/values.rs new file mode 100644 index 000000000..b97c470e5 --- /dev/null +++ b/crates/core/src/partiql/parser/tests/values.rs @@ -0,0 +1,448 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Unit tests for literal values, parameters, numbers and paths. + +use super::*; +use crate::partiql::ast::PathStep; +use crate::partiql::number::OVERFLOW_MESSAGE; + +#[test] +fn duplicate_bag_members_are_named() { + assert_eq!( + err("INSERT INTO \"t\" VALUE {'pk':'a','s':<<'a','a'>>}"), + "One or more parameter values were invalid: Input collection [a, a] contains duplicates." + ); +} + +/// The refusal of the first `?` inside a literal, at its column. +fn parameter_in_literal(sql: &str, path: &str) -> String { + let column = sql.find('?').unwrap() + 1; + format!("Unsupported data type: Parameter under key {path} at 1:{column}:1") +} + +#[test] +fn a_parameter_inside_a_literal_is_refused_at_its_position() { + for (sql, path) in [ + ("INSERT INTO \"t\" VALUE {'pk':'m1','w':{'k':?}}", "root.k"), + ("INSERT INTO \"t\" VALUE {'pk':'m2','w':[?]}", "root[0]"), + ("INSERT INTO \"t\" VALUE {'pk':'m3','w':[1,?]}", "root[1]"), + ( + "INSERT INTO \"t\" VALUE {'pk':'m4','w':{'a':{'b':?}}}", + "root.a.b", + ), + ( + "INSERT INTO \"t\" VALUE {'pk':'m7','w':{'k':?, 'j':?}}", + "root.k", + ), + ( + "INSERT INTO \"t\" VALUE {'pk':'m8','w':{'a':[1, {'b':?}]}}", + "root.a[1].b", + ), + ("UPDATE \"t\" SET w = {'k':?} WHERE pk = 'p'", "root.k"), + ("UPDATE \"t\" SET w = [?] WHERE pk = 'p'", "root[0]"), + ( + "SELECT * FROM \"t\" WHERE pk = 'p' AND w = {'k':?}", + "root.k", + ), + ("SELECT * FROM \"t\" WHERE pk = 'p' AND w = [?]", "root[0]"), + ] { + assert_eq!(err(sql), parameter_in_literal(sql, path), "{sql}"); + } + // A parameter as the whole value is a parameter, and a syntax error + // after the literal wins. + assert!(parse("INSERT INTO \"t\" VALUE {'pk':'m6','w':?}").is_ok()); + assert_eq!( + err("SELECT * FROM \"t\" WHERE pk = 'p' AND w = {'k':?} AND"), + wf(MISSING_RHS) + ); +} + +#[test] +fn a_value_that_names_an_attribute_twice_is_refused_at_the_second_name() { + // Measured 2026-11-03 with table names of these lengths: the line, + // column and length of the second occurrence, the quotes counted. + for (sql, key, at) in [ + ( + "INSERT INTO \"pqa-abcdef-c\" VALUE {'pk': 'd1', 'pk': 'd2', 'sk': 's'}", + "pk", + "1:47:4", + ), + ( + "INSERT INTO \"pqa-abcdef-c\" VALUE {'pk': 'a', 'pk': 'b', 'sk': 'dup'}", + "pk", + "1:46:4", + ), + ( + "INSERT INTO \"pqa-abcdef-c\" VALUE {'pk': 'd3', 'sk': 's', 'v': '1', 'v': '2'}", + "v", + "1:68:3", + ), + ( + "INSERT INTO \"pqa-abcdef-c\" VALUE {'pk': 'd4', 'sk': 's1', 'sk': 's2'}", + "sk", + "1:59:4", + ), + ( + "INSERT INTO \"rv-0123456789\" VALUE {'pk': 'd1', 'pk': 'd2', 'sk': 's'}", + "pk", + "1:48:4", + ), + ( + "INSERT INTO \"rv-0123456789\" VALUE {'pk': 'd3', 'sk': 's', 'v': '1', 'v': '2'}", + "v", + "1:69:3", + ), + ] { + assert_eq!( + err(sql), + format!("Duplicate keys in tuple. Key: {key} under root at {at}"), + "{sql}" + ); + } +} + +#[test] +fn any_other_map_that_names_a_key_twice_is_refused_at_its_opening_brace() { + // Measured 2026-11-03 with a table name of this length: the position of + // the `{` of the map that names the key twice, length 1, and the path + // of that map inside the value. + for (sql, path, at) in [ + ( + "INSERT INTO \"rv-0123456789\" VALUE {'pk': 'n1', 'sk': 's', 'm': {'x': 1, 'x': 2}}", + "root", + "1:64:1", + ), + ( + "INSERT INTO \"rv-0123456789\" VALUE {'pk': 'n2', 'sk': 's', 'l': [{'x': 1, 'x': 2}]}", + "root[0]", + "1:65:1", + ), + ( + "INSERT INTO \"rv-0123456789\" VALUE {'pk': 'n4', 'sk': 's', 'm': {'y': {'x': 1, 'x': \ + 2}}}", + "root.y", + "1:70:1", + ), + ( + "UPDATE \"rv-0123456789\" SET m = {'x': 1, 'x': 2} WHERE pk = 'a' AND sk = 'x'", + "root", + "1:32:1", + ), + ( + "SELECT * FROM \"rv-0123456789\" WHERE pk = 'a' AND m = {'x': 1, 'x': 2}", + "root", + "1:54:1", + ), + ] { + assert_eq!( + err(sql), + format!("Duplicate keys in tuple. Key: x under {path} at {at}"), + "{sql}" + ); + } + // The same rule elsewhere in a value, and for a name given three times. + for (sql, path) in [ + ( + "INSERT INTO \"t\" VALUE {'pk':'a','m':{'n':{'x':1,'x':2}}}", + "root.n", + ), + ( + "INSERT INTO \"t\" VALUE {'pk':'a','l':[1,{'x':1,'x':2}]}", + "root[1]", + ), + ( + "INSERT INTO \"t\" VALUE {'pk':'a','m':{'x':1,'x':2,'x':3}}", + "root", + ), + ( + "UPDATE \"t\" SET m = [{'x':1,'x':2}] WHERE pk = 'a'", + "root[0]", + ), + ] { + let column = sql.find("{'x'").unwrap() + 1; + assert_eq!( + err(sql), + format!("Duplicate keys in tuple. Key: x under {path} at 1:{column}:1"), + "{sql}" + ); + } +} + +#[test] +fn a_repeated_key_is_one_refusal_among_the_others_of_a_value() { + // The first refusal in the value is reported, and a syntax error after + // the literal wins. + let sql = "INSERT INTO \"t\" VALUE {'pk':'a','w':{'k':?},'pk':'b'}"; + assert_eq!(err(sql), parameter_in_literal(sql, "root.k")); + let sql = "INSERT INTO \"t\" VALUE {'pk':'a','pk':'b','w':{'k':?}}"; + let column = sql.rfind("'pk'").unwrap() + 1; + assert_eq!( + err(sql), + format!("Duplicate keys in tuple. Key: pk under root at 1:{column}:4") + ); + assert_eq!( + err("SELECT * FROM \"t\" WHERE pk = 'p' AND w = {'k':1,'k':2} AND"), + wf(MISSING_RHS) + ); + // Measured 2026-11-03. + assert_eq!( + err("INSERT INTO \"t\" VALUE {'pk': 'n7', 'pk': 'n8', 'sk': 's'} WHERE"), + wf("Unexpected token after expression") + ); + // Names are compared as written: a name in another case is another + // attribute (measured 2026-11-03 for 'v' and 'V'). + assert!(parse("INSERT INTO \"t\" VALUE {'pk':'a','PK':'b'}").is_ok()); + assert!(parse("INSERT INTO \"t\" VALUE {'pk': 'n3', 'sk': 's', 'v': 1, 'V': 2}").is_ok()); + assert!(parse("UPDATE \"t\" SET m = {'x':1,'X':2} WHERE pk = 'a'").is_ok()); +} + +#[test] +fn a_parameter_in_a_bag_is_refused_at_its_position() { + let sql = "INSERT INTO \"t\" VALUE {'pk':'m5','w':<>}"; + let column = sql.find('?').unwrap() + 1; + assert_eq!( + err(sql), + format!( + "Unsupported data type in Bag. DynamoDB only supports either numbers or strings \ + in bags at 1:{column}:1" + ) + ); +} + +#[test] +fn a_sign_before_a_set_value_that_is_not_a_number() { + assert_eq!( + err("UPDATE \"t\" SET a = -b WHERE pk = 'p'"), + "Incorrect number of operands for operator or function; operator or function: -, \ + number of operands: 1" + ); + assert!(parse("UPDATE \"t\" SET a = b - 1 WHERE pk = 'p'").is_ok()); +} + +#[test] +fn a_backtick_literal_is_refused_by_its_ion_type() { + assert_eq!( + err("SELECT * FROM \"t\" WHERE s = `1`"), + "Ion Literals are not supported: INT under key root at 1:29:3" + ); +} + +#[test] +fn at_and_dollar_words_are_names() { + let Statement::Select(s) = parse("SELECT * FROM \"t\" WHERE @n = 1 AND $x = 2").unwrap() else { + panic!("expected select"); + }; + let Some(WhereExpr::And(a, b)) = s.where_clause else { + panic!("expected AND"); + }; + assert!(matches!(*a, WhereExpr::Compare(Operand::Path(ref p), _, _) if p.root == "n")); + assert!(matches!(*b, WhereExpr::Compare(Operand::Path(ref p), _, _) if p.root == "$x")); +} + +#[test] +fn number_text_in_a_condition() { + let where_ = |n: &str| err(&format!("SELECT * FROM \"t\" WHERE pk = {n}")); + for n in ["1.2.3", ".5.5", "1e5.5", "1e5e5", "1_000", "1a", "1.2a"] { + assert_eq!(where_(n), wf("Unexpected token after expression"), "{n}"); + } + for n in ["1..2", "5..", "1.5e", "1e+", "5.e", "1ee5", "1e.5"] { + assert_eq!(where_(n), wf(""), "{n}"); + } + for n in ["1.2.", "1.2..3"] { + assert_eq!(where_(n), wf("Invalid path dot component"), "{n}"); + } + // The path is refused at the number it starts from. + assert_eq!(where_("1.2.e3"), "Unexpected path component at 1:30:3"); + for n in ["..5", ".e5"] { + assert_eq!(where_(n), wf("Unexpected term"), "{n}"); + } + for n in ["1.e3", "1.", ".5", "00.5", "1e05", "1E5"] { + assert!( + parse(&format!("SELECT * FROM \"t\" WHERE pk = {n}")).is_ok(), + "{n}" + ); + } +} + +#[test] +fn an_empty_name_in_a_condition_is_refused_after_the_parse() { + for condition in ["\"\" = 1", "m.\"\" = 1"] { + assert_eq!( + err(&format!( + "SELECT * FROM \"t\" WHERE pk = 'p' AND {condition}" + )), + "Path component cannot be an empty string", + "{condition}" + ); + } + assert_eq!( + err("SELECT * FROM \"t\" WHERE pk = 'p' AND \"\" = 1 AND"), + wf(MISSING_RHS) + ); +} + +#[test] +fn a_bag_whose_member_runs_on_is_missing_its_closing_brackets() { + for function in ["set_add", "set_delete"] { + assert_eq!( + err(&format!( + "UPDATE \"t\" SET ns = {function}(ns, <<1.2.3>>) WHERE pk = 'p'" + )), + wf("Expected RIGHT_DOUBLE_ANGLE_BRACKET"), + "{function}" + ); + } +} + +#[test] +fn a_reserved_word_after_a_dot_is_an_invalid_path_dot_component() { + assert_eq!( + err("SELECT * FROM \"t\" WHERE pk = 'a' AND attribute_exists(m.inner)"), + wf("Invalid path dot component") + ); + assert_eq!( + err("UPDATE \"t\" SET m.inner = 1 WHERE pk = 'a'"), + wf("Invalid path dot component") + ); + assert_eq!( + err("SELECT * FROM \"t\" WHERE pk = 'a' AND m. = 1"), + wf("Invalid path dot component") + ); + // A quoted name after a dot is a name. + assert!(parse("SELECT * FROM \"t\" WHERE pk = 'a' AND m.\"inner\" = 1").is_ok()); +} + +#[test] +fn a_number_with_no_leading_digit() { + let literal = |sql: &str| match where_of(sql) { + WhereExpr::Compare(_, _, Operand::Literal(AttributeValue::N(n))) => n, + other => panic!("{other:?}"), + }; + assert_eq!(literal("SELECT * FROM \"t\" WHERE pk = .73e1"), ".73e1"); + assert_eq!(literal("SELECT * FROM \"t\" WHERE pk = -.73e1"), "-.73e1"); + assert_eq!(literal("SELECT * FROM \"t\" WHERE pk = - 7.3"), "-7.3"); + assert_eq!(literal("SELECT * FROM \"t\" WHERE pk = - -7"), "7"); + match parse("INSERT INTO \"t\" VALUE {'pk': .7325e1, 'n': -5}").unwrap() { + Statement::Insert(i) => { + assert_eq!( + i.value[0].1, + Operand::Literal(AttributeValue::N(".7325e1".to_owned())) + ); + assert_eq!( + i.value[1].1, + Operand::Literal(AttributeValue::N("-5".to_owned())) + ); + } + other => panic!("{other:?}"), + } +} + +#[test] +fn a_name_as_an_insert_value_is_refused_naming_itself() { + let sql = "INSERT INTO \"t\" VALUE {'pk': pk}"; + let column = sql.find("pk}").unwrap() + 1; + assert_eq!( + err(sql), + format!("Unsupported data type: pk under key root at 1:{column}:2") + ); + // Inside a value it is an Id. + let sql = "INSERT INTO \"t\" VALUE {'pk': 'a', 'x': [pk]}"; + let column = sql.find("pk]").unwrap() + 1; + assert_eq!( + err(sql), + format!("Unsupported data type: Id under key root[0] at 1:{column}:2") + ); +} + +#[test] +fn a_string_or_a_name_in_brackets_is_a_map_member() { + let member = |name: &str| PathStep::Attribute(name.to_owned()); + for (condition, steps) in [ + ("m['k'] = 1", vec![member("k")]), + ("m[\"k\"] = 1", vec![member("k")]), + ("m[k] = 1", vec![member("k")]), + ("m['my key'] = 1", vec![member("my key")]), + ("m['k'][0] = 1", vec![member("k"), PathStep::Index(0)]), + ("m[k].j = 1", vec![member("k"), member("j")]), + ("m[2] = 1", vec![PathStep::Index(2)]), + ] { + let sql = format!("SELECT * FROM \"t\" WHERE {condition}"); + let WhereExpr::Compare(Operand::Path(path), ..) = where_of(&sql) else { + panic!("not a comparison of a path: {condition}"); + }; + assert_eq!(path.root, "m", "{condition}"); + assert_eq!(path.steps, steps, "{condition}"); + } + let Ok(Statement::Update(update)) = + parse("UPDATE \"t\" SET m['k'] = 5 REMOVE m[k] WHERE pk = 'p'") + else { + panic!("not an update"); + }; + assert_eq!(update.sets[0].path.steps, vec![member("k")]); + assert_eq!(update.removes[0].steps, vec![member("k")]); +} + +#[test] +fn other_contents_of_brackets_are_refused() { + let condition = |c: &str| err(&format!("SELECT * FROM \"t\" WHERE pk = 'p' AND {c}")); + // What the brackets hold starts at column 40. + for (brackets, want) in [ + ( + "m[''] = 1", + "Path component cannot be an empty string".to_owned(), + ), + ( + "m[\"\"] = 1", + "Path component cannot be an empty string".to_owned(), + ), + ("m[?] = 1", "Unexpected path component at 1:40:1".to_owned()), + ( + "m[1.5] = 1", + "Unexpected path component at 1:40:3".to_owned(), + ), + ( + "m[true] = 1", + "Unexpected path component at 1:40:4".to_owned(), + ), + ( + "m[k.j] = 1", + "Unexpected path component at 1:40:1".to_owned(), + ), + ( + "m[k[0]] = 1", + "Unexpected path component at 1:40:1".to_owned(), + ), + ( + "m['k'.x] = 1", + "Unexpected path component at 1:40:3".to_owned(), + ), + ( + "m[-1] = 1", + "List index is not within the allowable range; index: [-1] at 1:41:1".to_owned(), + ), + ( + "m[-1.5] = 1", + "Unexpected path component at 1:41:3".to_owned(), + ), + ("m[select] = 1", wf("Unexpected term")), + ("m[desc] = 1", wf("Unexpected term")), + ("m[and] = 1", wf("Unexpected operator")), + ("m['k' 'j'] = 1", wf("Expected RIGHT_BRACKET")), + ("m['k'", wf("Expected RIGHT_BRACKET")), + ("m[0", wf("Expected RIGHT_BRACKET")), + ("m[-1", wf("Expected RIGHT_BRACKET")), + ("m[k.j", wf("Expected RIGHT_BRACKET")), + ] { + assert_eq!(condition(brackets), want, "{brackets}"); + } + // In statement order with the other problems the walk meets. + assert_eq!( + condition("a = 1e126 AND m[k.j] = 1"), + format!("{OVERFLOW_MESSAGE} under root") + ); + assert_eq!( + condition("m[k.j] = 1e126"), + "Unexpected path component at 1:40:1" + ); +} diff --git a/crates/core/src/partiql/parser/walk.rs b/crates/core/src/partiql/parser/walk.rs new file mode 100644 index 000000000..3c2f60e20 --- /dev/null +++ b/crates/core/src/partiql/parser/walk.rs @@ -0,0 +1,365 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The refusals of a statement that parses, and the order they are reported +//! in. +//! +//! The parser reads on after it finds a refusal, so that a syntax error +//! anywhere in the statement still wins, and reports the refusal once the +//! statement has been read. The service's order is (measured 2026-11-02): +//! +//! 1. a syntax error anywhere in the statement; +//! 2. a refusal of the statement's target, found before the WHERE condition +//! (an alias on the table, a FROM path); +//! 3. in ExecuteStatement, an `EXISTS(SELECT ...)` statement; +//! 4. for a request, a table name outside the name rule; +//! 5. an alias in the select list, and an UPDATE whose target is an index; +//! 6. a LIMIT clause; +//! 7. the first problem the walk of the WHERE condition meets (below); +//! 8. the IN list limit and the operator count of the condition, whichever +//! the condition meets first; +//! 9. the refusals in an UPDATE's SET and REMOVE clauses and in an INSERT's +//! value, wherever they are written, then those after the condition. +//! +//! Measured: an alias on the table before a FROM path refusal and before the +//! name rule, and an empty FROM path component and a path of three +//! components before the name rule; EXISTS in ExecuteStatement before the +//! name rule and `+`; the name rule before an alias in the select list, an +//! UPDATE of an index, a LIMIT clause, `+`, a number that breaks an item +//! number rule with 301 operators, 301 operators and an IN list of 101 +//! members; LIMIT before a number, a missing parameter, `+`, 301 operators +//! and an IN list of 101 members in the condition; the walk before the IN +//! limit and the count; the count before a SET value, a SET parameter, a +//! unary plus, an IS test and a parameter inside a list in a SET clause, in +//! both UPDATE forms; the walk of the condition before a SET clause written +//! before it. Not measured: an alias on the table against EXISTS, an +//! INSERT's value against the name rule, and a DELETE of an index against +//! the name rule (it keeps its place in 2). +//! +//! In the SET clause, a sign before a value that is not a number comes after +//! the number literals of the clause that break an item number rule, in +//! either order (measured 2026-11-02 with a unary plus), and after a +//! negative zero decimal (measured 2026-11-05). Those numbers are otherwise +//! left to the checks made once the statement parses; a negative zero +//! decimal in the SET, REMOVE and VALUE parts is reported here, in its +//! place among them (see [`Parser::note_number`]). +//! +//! The walk goes over the condition as a tree, while the parser reads it in +//! the order it is written. The walk meets an operator before its operands +//! (`b = 1e126 + 1` is refused for the `+`, `? IS NULL` with no parameter +//! for the left side of the IS), the operands in the order they are written, +//! and the arguments of a call before its name (`foo(1e126)` is refused for +//! the number, `foo([?])` for the parameter). It meets the refusals found +//! while the condition is read (an operator a condition does not take, an +//! unknown function, MISSING as a value, a literal nested too deep, a +//! parameter inside a literal), a number literal that breaks an item number +//! rule or is a negative zero decimal, also inside a list or map literal, +//! and, for a request, a `?` with no parameter left in it. The order is the same within the operator limit and +//! past it. Some checks are met after the operands they apply to (measured +//! 2026-11-02): the refusal of `IS INT4` after its left side (`1e126 IS INT4` +//! is refused for the number), an operand that is not a condition, as an +//! argument of AND or OR, after both arguments, and an operand that is the +//! whole condition after the rest of the walk. +//! +//! A SELECT whose select list holds a call (`COUNT(*)`) is refused for the +//! call ahead of every number literal and operand type in its condition +//! (measured 2026-11-02), and the engine reports the call once it has +//! resolved the target. So the parser leaves the condition of such a SELECT +//! to the engine. The call comes ahead of an IN with no list after it too +//! (measured 2026-11-02 with `a IN 1`), which stops the parse: the parser +//! reports the call itself then. That a refusal found while reading the +//! condition, the IN limit and the operator count come after the call is +//! not measured. +//! +//! A refusal the parse stops at comes after those found before it that come +//! ahead of its part of the statement in this order. + +use super::limits::{MAX_IN_OPERANDS, too_many_in_operands, too_many_operators}; +use super::operands::SignedNumber; +use super::{ + EXISTS_OUTSIDE_TRANSACTION, IN_OPERANDS, PARAMETER_COUNT_MISMATCH, PResult, ParseError, Parser, + Statement, +}; +use crate::partiql::ast::Operand; +use crate::partiql::from_path; + +/// The refusals found while a statement is read, by the part of the +/// statement they were found in. +#[derive(Debug, Default)] +pub(super) struct Findings { + /// The first refusal found before the WHERE condition, outside the SET, + /// REMOVE and VALUE parts, that comes ahead of the table name rule. + before_where: Option, + /// The first refusal found before the WHERE condition that comes after + /// the table name rule: an alias in the select list, an UPDATE of an + /// index. + after_name_rule: Option, + /// The refusal of the LIMIT clause a statement ends with. + clause: Option, + /// The first problem the walk of the WHERE condition has met in the part + /// of the condition being read (see [`Parser::walk_part`]). + walk: Option, + /// The refusal of the first IN list with too many members, when the + /// condition meets it before its 301st operator. + in_operands: Option, + /// The first refusal in an UPDATE's SET and REMOVE clauses or an INSERT's + /// value. + values: Option, + /// Whether [`Findings::values`] is the refusal of a sign before a SET + /// value. + values_is_sign: bool, + /// The first number literal in a SET value that breaks an item number + /// rule, or the first negative zero decimal in the clause when it comes + /// ahead of those numbers (see [`Parser::note_number`]). + set_number: Option, + /// Whether a number literal read in the SET, REMOVE and VALUE parts + /// breaks an item number rule. The engine reports that number once the + /// statement parses. + value_rule_number: bool, + /// The first refusal found after the WHERE condition. + after_where: Option, +} + +/// The parts of a statement, in the order their refusals are reported. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +enum Part { + BeforeWhere, + Where, + Values, + AfterWhere, +} + +fn keep_first(slot: &mut Option, message: &str) { + if slot.is_none() { + *slot = Some(message.to_owned()); + } +} + +impl Parser { + /// The part of the statement being read. + fn part(&self) -> Part { + if self.in_where { + Part::Where + } else if self.in_values { + Part::Values + } else if self.after_where { + Part::AfterWhere + } else { + Part::BeforeWhere + } + } + + /// Keep a refusal found while the statement is still being read. + pub(super) fn defer(&mut self, message: &str) { + let slot = match self.part() { + Part::BeforeWhere => &mut self.found.before_where, + Part::Where => &mut self.found.walk, + Part::Values => &mut self.found.values, + Part::AfterWhere => &mut self.found.after_where, + }; + keep_first(slot, message); + } + + /// Keep a refusal found before the WHERE condition that comes after the + /// table name rule. + pub(super) fn defer_after_name_rule(&mut self, message: &str) { + keep_first(&mut self.found.after_name_rule, message); + } + + /// Keep the refusal of a sign before a value that is not a number. In a + /// SET clause it comes after the clause's number literals that break an + /// item number rule (see [`Parser::note_set_number`]). + pub(super) fn defer_sign(&mut self, message: &str) { + if self.part() == Part::Values && self.found.values.is_none() { + self.found.values = Some(message.to_owned()); + self.found.values_is_sign = true; + } else { + self.defer(message); + } + } + + /// Note the value of a SET assignment: the first of its number literals + /// that breaks an item number rule, if any. + pub(super) fn note_set_number(&mut self, value: &Operand) { + if self.found.set_number.is_none() { + self.found.set_number = crate::partiql::number::operand_number_refusal(value); + } + } + + /// Keep the refusal of a clause the statement ends with. + pub(super) fn refuse_clause(&mut self, message: String) { + if self.found.clause.is_none() { + self.found.clause = Some(message); + } + } + + /// Start a part of the condition, an operand or a predicate: what the + /// walk met before it is set aside, and handed back to + /// [`Parser::end_walk_part`], which puts it ahead of what the part meets. + /// Within the part, an operator read after its first operand can then + /// be put ahead of what that operand met (see [`Parser::walk_operator`]). + pub(super) fn walk_part(&mut self) -> Option { + self.found.walk.take() + } + + /// End a part of the condition started with [`Parser::walk_part`]. + pub(super) fn end_walk_part(&mut self, before: Option) { + if before.is_some() { + self.found.walk = before; + } + } + + /// The refusal of an operator of the condition, read after the operands + /// the current part has read so far: the walk meets an operator before + /// its operands. + pub(super) fn walk_operator(&mut self, refusal: String) { + if self.in_where { + self.found.walk = Some(refusal); + } + } + + /// The refusal of a check of the condition that the walk meets after the + /// operands it applies to. + pub(super) fn walk_after_operands(&mut self, refusal: &str) { + if self.in_where { + keep_first(&mut self.found.walk, refusal); + } + } + + /// Note a number literal read with its signs at `path` inside its value + /// (`root`, `root[0]`, `root.k`). A negative zero decimal is refused at + /// the number (see `number::is_zero_decimal`), before the statement's + /// target is resolved (measured 2026-11-05: on a table that does not + /// exist, and ahead of the key type check of an INSERT). In the + /// condition the walk meets it as it meets a number that breaks an item + /// number rule (see [`Parser::walk_number`]). In the SET, REMOVE and + /// VALUE parts the two come in the order they are written (measured + /// 2026-11-05: `{'x':1e126,'y':-0.0}` is refused for the overflow, + /// `{'x':-0.0,'y':1e126}` and `{'x':{'b':-0.0,'a':1e126}}` for the + /// zero), so a number that breaks a rule first leaves the clause to the + /// engine. A sign before a value that is not a number comes after a + /// negative zero, as after those numbers (measured 2026-11-05 with + /// `SET x = +b, y = -0.0`). + pub(super) fn note_number(&mut self, number: &SignedNumber, path: &str) { + let zero = number + .negative_zero + .then(|| crate::partiql::number::negative_zero_refusal(path, &number.at)); + match self.part() { + Part::Where => match zero { + Some(refusal) => keep_first(&mut self.found.walk, &refusal), + None => self.walk_number(&number.text, path), + }, + Part::Values if !self.found.value_rule_number => match zero { + Some(refusal) if self.found.values_is_sign => { + self.found.set_number.get_or_insert(refusal); + } + Some(refusal) => keep_first(&mut self.found.values, &refusal), + None => { + self.found.value_rule_number = + crate::partiql::number::literal_number_refusal(&number.text, path) + .is_some(); + } + }, + _ => {} + } + } + + /// Note a number literal read in the WHERE condition at `path` inside its + /// value (`root`, `root[0]`, `root.k`), for the walk. + pub(super) fn walk_number(&mut self, text: &str, path: &str) { + if self.in_where + && self.found.walk.is_none() + && let Some(refusal) = crate::partiql::number::literal_number_refusal(text, path) + { + self.found.walk = Some(refusal); + } + } + + /// Note the `?` with index `index` read in the WHERE condition, for the + /// walk: the request may give no parameter for it. + pub(super) fn walk_parameter(&mut self, index: usize) { + if self.in_where && self.parameters.is_some_and(|given| index >= given) { + keep_first(&mut self.found.walk, PARAMETER_COUNT_MISMATCH); + } + } + + /// Note an IN list of `members` members, read with the operators counted + /// so far. + pub(super) fn count_in_operands(&mut self, members: usize) { + if members > MAX_IN_OPERANDS && self.within_operator_limit() { + let refusal = too_many_in_operands(members); + keep_first(&mut self.found.in_operands, &refusal); + } + } + + /// Forget the refusals found so far, for a statement refused by its form. + pub(super) fn forget_findings(&mut self) { + self.found = Findings::default(); + } + + /// For a request, the refusal of a table name outside the name rule. + fn table_name_refusal(&self) -> Option { + let table = self + .table + .as_deref() + .filter(|_| self.parameters.is_some())?; + from_path::breaks_name_rule(table).then(|| from_path::NAME_RULE.to_owned()) + } + + /// The answer for a statement once it has been read, in the order the + /// module documents. `result` is what the parse returned. + pub(super) fn resolve(&mut self, result: PResult) -> PResult { + let found = std::mem::take(&mut self.found); + let call = self.select_list_call.take(); + // The parts read in full: all of them, or those before the part a + // refusal stopped the parse in. + let reached = match &result { + Err(e) if e.is_syntax() => return result, + Err(_) => self.part(), + Ok(_) => Part::AfterWhere, + }; + let stopped_at_in = matches!(&result, Err(e) if e.message == IN_OPERANDS); + let condition = match &result { + Ok(Statement::Select(s)) => s.projection_error.is_none(), + Err(_) if stopped_at_in => call.is_none(), + _ => true, + }; + let exists = (self.refuse_exists && matches!(&result, Ok(Statement::Exists(_)))) + .then(|| EXISTS_OUTSIDE_TRANSACTION.to_owned()); + let mut refusal = found + .before_where + .or(exists) + .or_else(|| self.table_name_refusal()) + .or(found.after_name_rule) + .or(found.clause); + if reached >= Part::Where && condition { + refusal = refusal.or(found.walk); + } + if reached > Part::Where && condition { + refusal = refusal + .or(found.in_operands) + .or_else(|| (!self.within_operator_limit()).then(too_many_operators)); + } + if reached >= Part::Values { + let values = match found.values { + Some(sign) if found.values_is_sign => Some(found.set_number.unwrap_or(sign)), + values => values, + }; + refusal = refusal.or(values); + } + if reached >= Part::AfterWhere { + refusal = refusal.or(found.after_where); + } + // A statement that parses leaves its call to the engine. + if stopped_at_in { + refusal = refusal.or(call); + } + match (result, refusal) { + (_, Some(refusal)) => Err(ParseError::new(refusal)), + (result, None) => result, + } + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/core/src/partiql/parser/walk/tests.rs b/crates/core/src/partiql/parser/walk/tests.rs new file mode 100644 index 000000000..06bd2f2d0 --- /dev/null +++ b/crates/core/src/partiql/parser/walk/tests.rs @@ -0,0 +1,260 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Unit tests for the order in which the refusals of a statement are reported. + +use crate::partiql::ast::Statement; +use crate::partiql::from_path::NAME_RULE; +use crate::partiql::number::{OVERFLOW_MESSAGE, TOO_MANY_DIGITS_MESSAGE, UNDERFLOW_MESSAGE}; +use crate::partiql::parse; +use crate::partiql::parser::limits::{ + literal_nesting_refusal, too_many_in_operands, too_many_operators, +}; +use crate::partiql::parser::{ + IN_OPERANDS, IS_LEFT_SIDE, PARAMETER_COUNT_MISMATCH, parse_for_request, +}; + +const WF: &str = "Statement wasn't well formed, can't be processed: "; +const SELECT: &str = "SELECT * FROM \"tbl\" WHERE "; +const PLUS: &str = "Unsupported operator in Condition Expression. Operator: +"; +const MISSING_VALUE: &str = "MISSING must be compared using IS operator"; + +fn err(sql: &str) -> String { + parse(sql).unwrap_err().message +} + +fn refusal(sql: &str, given: usize) -> String { + parse_for_request(sql, given).unwrap_err().message +} + +fn many() -> String { + "NOT ".repeat(300) + "a = 1" +} + +fn overflow() -> String { + format!("{OVERFLOW_MESSAGE} under root") +} + +fn deep() -> String { + format!("{}1{}", "[".repeat(33), "]".repeat(33)) +} + +fn nesting() -> String { + literal_nesting_refusal(&format!("root{}", "[0]".repeat(32))) +} + +/// The refusal of the call to `foo` in `sql`. +fn unknown_function(sql: &str) -> String { + let column = sql.find("foo(").unwrap() + 1; + format!("Unrecognized function: foo at 1:{column}:3") +} + +/// The refusal of the first `?` in `sql`, a parameter inside a list. +fn parameter_in_a_list(sql: &str) -> String { + let column = sql.find('?').unwrap() + 1; + format!("Unsupported data type: Parameter under key root[0] at 1:{column}:1") +} + +/// Assert the answer for `condition` within the operator limit and past +/// it, for a request with no parameters. +fn assert_walk(condition: &str, expected: impl Fn(&str) -> String) { + let many = many(); + for sql in [ + format!("{SELECT}{condition}"), + format!("{SELECT}{condition} AND {many}"), + ] { + assert_eq!(refusal(&sql, 0), expected(&sql), "{sql}"); + } +} + +#[test] +fn a_number_breaking_a_rule_in_the_condition_comes_before_the_operator_count() { + let many = many(); + let digits39 = "123456789012345678901234567890123456789"; + for (sql, expected) in [ + (format!("{SELECT}{many} AND b = 1e126"), overflow()), + ( + format!("{SELECT}{many} AND b = 1e-131"), + format!("{UNDERFLOW_MESSAGE} under root"), + ), + ( + format!("{SELECT}{many} AND b = {digits39}"), + format!("{TOO_MANY_DIGITS_MESSAGE} under root"), + ), + (format!("{SELECT}b = 1e126 AND {many}"), overflow()), + // The first in the statement, named by its place in the value. + ( + format!("{SELECT}b = [1, 1e-131] AND c = 1e126 AND {many}"), + format!("{UNDERFLOW_MESSAGE} under root[1]"), + ), + ( + format!("{SELECT}b = {{'k': 1e-131}} AND {many}"), + format!("{UNDERFLOW_MESSAGE} under root.k"), + ), + ( + format!("{SELECT}c = 1e126 AND d = {digits39} AND {many}"), + overflow(), + ), + ] { + assert_eq!(err(&sql), expected, "{sql}"); + } + // A number within the rules leaves the operator count. + assert_eq!( + err(&format!("{SELECT}b = 9.9E125 AND {many}")), + too_many_operators() + ); +} + +#[test] +fn the_walk_reports_the_first_problem_in_the_condition() { + let like = "Unsupported operator in Condition Expression. Operator: like"; + let deep = deep(); + for (condition, expected) in [ + ("b = 1e126 AND c = ?".to_owned(), overflow()), + ( + "c = ? AND b = 1e126".to_owned(), + PARAMETER_COUNT_MISMATCH.to_owned(), + ), + ("b = 1 + 1 AND c = 1e126".to_owned(), PLUS.to_owned()), + ("c = 1e126 AND b = 1 + 1".to_owned(), overflow()), + ("c = 1e126 AND foo(b)".to_owned(), overflow()), + ("c = 1e126 AND b = MISSING".to_owned(), overflow()), + ( + "b = MISSING AND c = 1e126".to_owned(), + MISSING_VALUE.to_owned(), + ), + ("c = 1e126 AND b = 'x' || 'y'".to_owned(), overflow()), + ("c = 1e126 AND a LIKE 'x'".to_owned(), overflow()), + ("a LIKE 'x' AND c = 1e126".to_owned(), like.to_owned()), + ("c = 1e126 AND a IS INT4".to_owned(), overflow()), + (format!("c = 1e126 AND b = {deep}"), overflow()), + (format!("b = {deep} AND c = 1e126"), nesting()), + ] { + assert_walk(&condition, |_| expected.clone()); + } + assert_walk("foo(b) AND c = 1e126", unknown_function); +} + +#[test] +fn the_walk_meets_an_operator_before_its_operands() { + for condition in ["b = 1e126 + 1", "b = 1 + 1e126"] { + assert_walk(condition, |_| PLUS.to_owned()); + } + assert_walk("? IS NULL AND c = 1e126", |_| IS_LEFT_SIDE.to_owned()); +} + +#[test] +fn the_walk_meets_the_arguments_of_a_call_before_its_name() { + let deep = deep(); + assert_walk("foo(1e126)", |_| overflow()); + assert_walk(&format!("foo({deep})"), |_| nesting()); + assert_walk("foo(1 + 1)", |_| PLUS.to_owned()); + assert_walk("foo([?])", parameter_in_a_list); + assert_walk("foo(b)", unknown_function); +} + +#[test] +fn a_parameter_the_request_does_not_give_is_met_by_the_walk() { + let many = many(); + for sql in [ + format!("{SELECT}{many} AND b = ?"), + format!("{SELECT}b = ? AND {many}"), + ] { + assert_eq!(refusal(&sql, 0), PARAMETER_COUNT_MISMATCH, "{sql}"); + // Enough parameters, or more, leave the operator count. + assert_eq!(refusal(&sql, 1), too_many_operators(), "{sql}"); + assert_eq!(refusal(&sql, 2), too_many_operators(), "{sql}"); + // Without the request the count is not known. + assert_eq!(err(&sql), too_many_operators(), "{sql}"); + } + let two = format!("{SELECT}b = ? AND {many} AND c = ?"); + assert_eq!(refusal(&two, 1), PARAMETER_COUNT_MISMATCH); + // The placeholders are numbered in statement order, the SET clause's + // first. + let update = format!("UPDATE \"tbl\" SET q = ? WHERE pk = 'a' AND c = ? AND {many}"); + assert_eq!(refusal(&update, 1), PARAMETER_COUNT_MISMATCH); + assert_eq!(refusal(&update, 2), too_many_operators()); + // Within the operator limit too, in its place in the walk. + assert_eq!( + refusal(&format!("{SELECT}c = ? AND b = 1 + 1"), 0), + PARAMETER_COUNT_MISMATCH + ); + assert_eq!(refusal(&format!("{SELECT}b = 1 + 1 AND c = ?"), 0), PLUS); + assert!(parse_for_request(&format!("{SELECT}b = ?"), 1).is_ok()); + // Without the request it is left to the execution. + assert!(parse(&format!("{SELECT}b = ?")).is_ok()); +} + +#[test] +fn the_in_list_limit_comes_after_the_walk() { + let members: Vec = (0..101).map(|i| i.to_string()).collect(); + let list = format!("a IN [{}]", members.join(", ")); + for (condition, expected) in [ + (format!("{list} AND c = 1e126"), overflow()), + (format!("{list} AND b = 1 + 1"), PLUS.to_owned()), + ( + format!("{list} AND c = ?"), + PARAMETER_COUNT_MISMATCH.to_owned(), + ), + (list.clone(), too_many_in_operands(101)), + ] { + let sql = format!("{SELECT}{condition}"); + assert_eq!(refusal(&sql, 0), expected, "{condition}"); + } +} + +#[test] +fn an_unknown_function_or_missing_waits_for_a_syntax_error() { + for condition in ["foo(b) AND (", "b = MISSING AND ("] { + assert_eq!( + err(&format!("{SELECT}{condition}")), + format!("{WF}Unexpected term"), + "{condition}" + ); + } + // Within the operator limit each keeps its place in the walk. + assert_eq!( + err(&format!("{SELECT}b = MISSING AND c = 1 + 1")), + MISSING_VALUE + ); + assert_eq!(err(&format!("{SELECT}c = 1 + 1 AND b = MISSING")), PLUS); + assert_eq!( + err(&format!("{SELECT}attribute_exists(missing)")), + MISSING_VALUE + ); +} + +#[test] +fn the_refusal_of_is_int4_comes_after_its_left_side() { + let int4 = |column: usize| { + format!( + "Unsupported data type for IS operator: Integer4Type. Supported data types are \ + [MISSING, NULL, BOOLEAN, BLOB, STRING, NUMERIC, STRUCT, TUPLE, LIST] at 1:{column}:1" + ) + }; + assert_walk("1e126 IS INT4", |_| overflow()); + assert_walk("a IS INT4 AND c = 1e126", |sql| { + int4(sql.find("a IS").unwrap() + 1) + }); + // The other types keep the place of an operator. + assert_walk("1e126 IS INT8", |sql| { + let column = sql.find("1e126").unwrap() + 1; + format!( + "Unsupported data type for IS operator: Integer8Type. Supported data types are \ + [MISSING, NULL, BOOLEAN, BLOB, STRING, NUMERIC, STRUCT, TUPLE, LIST] at 1:{column}:5" + ) + }); +} + +#[test] +fn a_refusal_that_stops_the_parse_comes_after_those_ahead_of_its_part() { + assert_eq!(err(&format!("{SELECT}b = 1 + 1 AND a IN 1")), PLUS); + assert_eq!(err(&format!("{SELECT}c = 1e126 AND a IN 1")), overflow()); + // A SET clause written before the condition comes after it. + assert_eq!( + err("UPDATE \"tbl\" SET q = +b WHERE pk = 'a' AND a IN 1"), + IN_OPERANDS + ); +} + +mod statement; diff --git a/crates/core/src/partiql/parser/walk/tests/statement.rs b/crates/core/src/partiql/parser/walk/tests/statement.rs new file mode 100644 index 000000000..46f4d5571 --- /dev/null +++ b/crates/core/src/partiql/parser/walk/tests/statement.rs @@ -0,0 +1,297 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Unit tests for the order of the refusals of a statement around the walk of its condition. + +use super::*; + +#[test] +fn a_set_clause_comes_after_the_operator_count() { + let many = many(); + for value in [ + "1e126", + "[1, 1e126]", + "{'k': 1e126}", + "?", + "+b", + "b IS NULL", + "[?]", + ] { + let sql = format!("UPDATE \"tbl\" SET q = {value} WHERE pk = 'a' AND {many}"); + assert_eq!(refusal(&sql, 0), too_many_operators(), "{value}"); + } + // Also when the clause is written after the condition. + let sql = format!("FROM \"tbl\" WHERE pk = 'a' AND {many} SET q = +b"); + assert_eq!(refusal(&sql, 0), too_many_operators()); + // And after the walk within the operator limit. + assert_eq!( + refusal("UPDATE \"tbl\" SET q = +b WHERE pk = 'a' AND c = 1 + 1", 0), + PLUS + ); + // The SET value is still checked when the condition is within it. + assert_eq!( + refusal("UPDATE \"tbl\" SET q = +b WHERE pk = 'a'", 0), + "Incorrect number of operands for operator or function; operator or function: +, \ + number of operands: 1" + ); + assert!(parse("UPDATE \"tbl\" SET q = 1e126 WHERE pk = 'a'").is_ok()); +} + +#[test] +fn the_table_name_rule_comes_first_for_a_request() { + let many = many(); + let members: Vec = (0..101).map(|i| i.to_string()).collect(); + for condition in [ + "b = 1 + 1".to_owned(), + many.clone(), + format!("c = 1e126 AND {many}"), + format!("a IN [{}]", members.join(", ")), + "c = ?".to_owned(), + "a IN 1".to_owned(), + ] { + let sql = format!("SELECT * FROM \"tt\" WHERE {condition}"); + assert_eq!(refusal(&sql, 0), NAME_RULE, "{condition}"); + } + // An empty FROM path component and a syntax error come before it. + assert_eq!( + refusal("SELECT * FROM \"\" WHERE b = 1 + 1", 0), + "Path component cannot be an empty string" + ); + assert_eq!( + refusal("SELECT * FROM \"tt\" WHERE b = 1 +", 0), + format!("{WF}Missing right-hand side expression of infix operator") + ); + // Without a request the rule is left to the checks made once the + // statement parses. + assert!(parse("SELECT * FROM \"tt\" WHERE pk = 'a'").is_ok()); + assert_eq!(err("SELECT * FROM \"tt\" WHERE b = 1 + 1"), PLUS); +} + +#[test] +fn an_alias_comes_before_the_walk() { + let sql = format!("SELECT * FROM \"tbl\" AS x WHERE c = 1e126 AND {}", many()); + assert_eq!(err(&sql), "Aliasing is not supported"); +} + +#[test] +fn the_refusals_found_while_reading_still_come_before_the_operator_count() { + let many = "NOT ".repeat(300) + "a = 1"; + for (tail, expected) in [ + ( + "a = 1 AND b = 1 + 1 AND c = 1e126".to_owned(), + PLUS.to_owned(), + ), + ( + "a = 1 AND foo(b) AND c = 1e126".to_owned(), + format!( + "Unrecognized function: foo at 1:{}:3", + SELECT.len() + many.len() + " AND a = 1 AND ".len() + 1 + ), + ), + ("a = 1 AND b = MISSING".to_owned(), MISSING_VALUE.to_owned()), + (format!("a = 1 AND b = {}", deep()), nesting()), + ( + "a = 1 AND".to_owned(), + format!("{WF}Missing right-hand side expression of infix operator"), + ), + // The argument count of a call is checked after the operator count. + ("a = 1 AND size(b, c) = 1".to_owned(), too_many_operators()), + ] { + let sql = format!("{SELECT}{many} AND {tail}"); + assert_eq!(err(&sql), expected, "{tail}"); + } +} + +#[test] +fn a_select_with_a_call_in_its_select_list_leaves_its_condition_to_the_engine() { + for condition in ["pk = 1e126".to_owned(), "b = 1 + 1".to_owned(), many()] { + let sql = format!("SELECT COUNT(*) FROM \"tbl\" WHERE {condition}"); + let Ok(Statement::Select(select)) = parse_for_request(&sql, 0) else { + panic!("{sql}"); + }; + assert_eq!( + select.projection_error.as_deref(), + Some("Unexpected path component at 1:8:5"), + "{condition}" + ); + } + // What comes before the condition still comes first. + assert_eq!( + refusal("SELECT COUNT(*) FROM \"tbl\" AS x WHERE b = 1 + 1", 0), + "Aliasing is not supported" + ); + assert_eq!( + refusal("SELECT COUNT(*) FROM \"tt\" WHERE b = 1 + 1", 0), + NAME_RULE + ); +} + +#[test] +fn the_table_name_rule_comes_before_a_select_list_alias_and_an_update_of_an_index() { + for sql in [ + "SELECT a AS b FROM \"tt\" WHERE pk = 'a'", + "UPDATE \"tt\".\"idx\" SET q = 1 WHERE pk = 'a'", + "FROM \"tt\".\"idx\" WHERE pk = 'a' SET q = 1", + ] { + assert_eq!(refusal(sql, 0), NAME_RULE, "{sql}"); + } + // Both still come before the walk, and an alias on the table before + // the rule. + assert_eq!( + refusal("SELECT a AS b FROM \"tbl\" WHERE b = 1 + 1", 0), + "Aliasing is not supported" + ); + assert_eq!( + refusal("UPDATE \"tbl\".\"idx\" SET q = 1 WHERE b = 1 + 1", 0), + "This operation is not supported on an index" + ); + assert_eq!( + refusal("UPDATE \"tt\" AS x SET q = 1 WHERE pk = 'a'", 0), + "Aliasing is not supported" + ); + // A DELETE of an index is not measured against the rule. + assert_eq!( + refusal("DELETE FROM \"tt\".\"idx\" WHERE pk = 'a'", 0), + "This operation is not supported on an index" + ); +} + +#[test] +fn execute_statement_refuses_exists_before_the_table_name_rule() { + use crate::partiql::parser::{EXISTS_OUTSIDE_TRANSACTION, parse_for_execute_statement}; + for sql in [ + "EXISTS(SELECT * FROM \"tt\" WHERE pk = 'a' AND b = 1 + 1)", + "EXISTS(SELECT * FROM \"tbl\" WHERE pk = 'a' AND b = 1e126)", + ] { + assert_eq!( + parse_for_execute_statement(sql, 0).unwrap_err().message, + EXISTS_OUTSIDE_TRANSACTION, + "{sql}" + ); + // Elsewhere the statement keeps the order of its checks. + assert_ne!(refusal(sql, 0), EXISTS_OUTSIDE_TRANSACTION, "{sql}"); + } + assert_eq!( + refusal( + "EXISTS(SELECT * FROM \"tt\" WHERE pk = 'a' AND b = 1 + 1)", + 0 + ), + NAME_RULE + ); + // An alias on the table and a syntax error come first. + assert_eq!( + parse_for_execute_statement("EXISTS(SELECT * FROM \"tbl\" x WHERE pk = 'a')", 0) + .unwrap_err() + .message, + "Aliasing is not supported" + ); +} + +#[test] +fn a_limit_clause_comes_after_the_table_name_rule_and_before_the_walk() { + let members: Vec = (0..101).map(|i| i.to_string()).collect(); + for condition in [ + "c = 1e126".to_owned(), + "c = ?".to_owned(), + "b = 1 + 1".to_owned(), + many(), + format!("a IN [{}]", members.join(", ")), + ] { + let sql = format!("{SELECT}{condition} LIMIT 2"); + // The token after LIMIT is named. + let column = sql.find("LIMIT 2").unwrap() + "LIMIT ".len() + 1; + assert_eq!( + refusal(&sql, 0), + format!("Unsupported clause: LIMIT at 1:{column}:1"), + "{condition}" + ); + } + assert_eq!( + refusal("SELECT * FROM \"tt\" WHERE pk = 'a' LIMIT 2", 0), + NAME_RULE + ); +} + +#[test] +fn a_select_list_call_comes_before_an_in_with_no_list() { + let call = "Unexpected path component at 1:8:5"; + for sql in [ + "SELECT COUNT(*) FROM \"tbl\" WHERE a IN 1", + "SELECT COUNT(*) FROM \"tbl\" WHERE c = 1e126 AND a IN 1", + ] { + assert_eq!(refusal(sql, 0), call, "{sql}"); + } + // Without a call, and in an EXISTS, the IN keeps its refusal. + assert_eq!(refusal(&format!("{SELECT}a IN 1"), 0), IN_OPERANDS); + assert_eq!( + refusal("EXISTS(SELECT COUNT(*) FROM \"tbl\" WHERE a IN 1)", 0), + IN_OPERANDS + ); + // What comes before the condition still comes first. + assert_eq!( + refusal("SELECT COUNT(*) FROM \"tt\" WHERE a IN 1", 0), + NAME_RULE + ); +} + +#[test] +fn a_sign_in_a_set_clause_comes_after_its_numbers() { + const SIGN: &str = "Incorrect number of operands for operator or function; operator or \ + function: +, number of operands: 1"; + for sets in [ + "q = 1e126, r = +b", + "r = +b, q = 1e126", + "r = +b, q = [1e126]", + ] { + let sql = format!("UPDATE \"tbl\" SET {sets} WHERE pk = 'a'"); + let expected = if sets.contains('[') { + format!("{OVERFLOW_MESSAGE} under root[0]") + } else { + overflow() + }; + assert_eq!(refusal(&sql, 0), expected, "{sets}"); + } + assert_eq!( + refusal("UPDATE \"tbl\" SET r = +b, q = 1 WHERE pk = 'a'", 0), + SIGN + ); + // The other refusals of the clause keep their order. + assert_eq!( + refusal("UPDATE \"tbl\" SET r = +b, s = b IS NULL WHERE pk = 'a'", 0), + SIGN + ); + assert_eq!( + refusal( + "UPDATE \"tbl\" SET s = b IS NULL, q = 1e126, r = +b WHERE pk = 'a'", + 0 + ), + "IS operator is not supported in update expressions" + ); + // A number alone is left to the checks made once the statement + // parses. + assert!(parse("UPDATE \"tbl\" SET q = 1e126 WHERE pk = 'a'").is_ok()); +} + +#[test] +fn a_negative_zero_decimal_keeps_its_place_among_the_refusals_of_a_value() { + let zero = |sql: &str, path: &str| { + let column = sql.find("-0.0").unwrap() + 2; + format!("Negative zero decimal is not supported under key {path} at 1:{column}:3") + }; + // With a parameter inside a literal, the one written first. + let sql = "INSERT INTO \"tbl\" VALUE {'pk':'a','x':[-0.0, ?]}"; + assert_eq!(refusal(sql, 1), zero(sql, "root[0]")); + let sql = "INSERT INTO \"tbl\" VALUE {'pk':'a','x':[?, -0.0]}"; + assert_eq!(refusal(sql, 1), parameter_in_a_list(sql)); + // Ahead of a sign before a SET value that is not a number. + let sql = "UPDATE \"tbl\" SET x = +b, y = -0.0 WHERE pk = 'a'"; + assert_eq!(refusal(sql, 0), zero(sql, "root")); + // After what the walk of the condition meets. + assert_eq!( + refusal( + "UPDATE \"tbl\" SET x = -0.0 WHERE pk = 'a' AND a = 1 + 1", + 0 + ), + PLUS + ); +} diff --git a/crates/core/src/partiql/plan.rs b/crates/core/src/partiql/plan.rs new file mode 100644 index 000000000..abbe71cb2 --- /dev/null +++ b/crates/core/src/partiql/plan.rs @@ -0,0 +1,565 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! How a WHERE clause reads its target (a table or an index). +//! +//! The service reads a WHERE clause as an OR of disjuncts. It splits at every +//! OR with a condition on a key attribute on either side, and at every IN on a +//! key attribute (one disjunct per value), and AND distributes over the split. +//! It then takes the disjuncts in order (measured 2026-11-02): +//! +//! * A disjunct reads by key when it holds one distinct equality of the +//! partition key with a literal or parameter and no other predicate on the +//! partition key, and on the sort key either nothing or one distinct key +//! condition (`=`, `<`, `<=`, `>`, `>=`, `BETWEEN`, `begins_with` with +//! values) and nothing else. A value on the left is the mirrored +//! comparison: `1 < sk` is `sk > 1`. +//! * Any other predicate on a key attribute (`<>`, `NOT` over one condition, +//! a function of it, a second value, an IN with a path in its list) makes +//! the disjunct a scan, and the first disjunct that is a scan makes the +//! whole statement one: nothing after it is examined. +//! * A key attribute compared with another path (`sk = a`), a path into a key +//! attribute (`sk.x`), and a NOT over an AND or an OR that names a key +//! attribute, are not predicates on the key: they stay in the disjunct's +//! filter, which a keyed read refuses. +//! * A key condition (an equality of the partition key with a value, a key +//! condition on the sort key, or an AND of them) compared with a boolean +//! literal is read by that key condition, and the literal alone decides: +//! compared with true (`= true` or `<> true`) the disjunct returns every +//! item it reads, compared with false (`= false` or `<> false`) none +//! (measured 2026-11-03: `pk = 'a' <> true` returns partition a, `pk = 'a' +//! AND sk > 'x' <> true` the items after x, `(pk = 'a' AND sk = 'x') <> +//! true` a/x, `pk = 'a' <> false` and `pk = 'a' AND sk = 'x' = false` +//! nothing, and `pk = 'a' = false OR pk = 'b'` partition b). Any other +//! condition compared with a boolean is the condition, or its negation for +//! `= false` and `<> true` (measured 2026-11-03: `pk IN ['a'] <> true` and +//! `pk = 'a' AND n = 1 = false` read as NOTs). +//! +//! [`WhereAnalysis`] holds that reading. The SELECT executor, the index rules, +//! the point reads of `BatchExecuteStatement` and `ExecuteTransaction` and the +//! overlap refusal all work from it; [`key_equality`] gives a write's WHERE +//! the same reading of a single condition. + +use std::cmp::Ordering; +use std::collections::HashMap; + +use super::ast::{BooleanComparison, CompareOp, Operand, WhereExpr}; +use crate::types::AttributeValue; + +mod ranges; +mod roles; +mod split; + +use ranges::{Bound, Interval, interval}; +use roles::{Role, is_key_condition, role}; +use split::{Split, key_read, split}; + +/// The key attribute names of the target (table or index). +#[derive(Debug, Clone, Copy)] +pub struct KeyNames<'a> { + pub partition: &'a str, + pub sort: Option<&'a str>, +} + +impl KeyNames<'_> { + fn is_key(&self, name: &str) -> bool { + name == self.partition || Some(name) == self.sort + } +} + +/// A key condition on the sort key, written with the key on the left. +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum SortCondition<'a> { + /// `sk value` for `=`, `<`, `<=`, `>` and `>=`. + Compare(CompareOp, &'a Operand), + /// `sk BETWEEN low AND high`. + Between(&'a Operand, &'a Operand), + /// `begins_with(sk, prefix)`. + BeginsWith(&'a Operand), +} + +impl<'a> SortCondition<'a> { + /// The values the condition compares the sort key with, in statement order. + #[must_use] + pub fn values(&self) -> Vec<&'a Operand> { + match *self { + Self::Compare(_, value) | Self::BeginsWith(value) => vec![value], + Self::Between(low, high) => vec![low, high], + } + } +} + +/// A disjunct the service reads by key. +#[derive(Debug, Clone, PartialEq)] +pub struct KeyRead<'a> { + /// The value the partition key equals. + pub partition: &'a Operand, + /// The key condition on the sort key, when the disjunct has one. + pub sort: Option>, + /// The disjunct's other conditions, in statement order. + pub filter: Vec<&'a WhereExpr>, + /// The comparisons with a boolean its key conditions are under, in + /// statement order (see the module documentation). + pub comparisons: Vec, +} + +impl KeyRead<'_> { + /// Whether the disjunct returns no item of those it reads: one of its + /// key conditions is compared with false. + #[must_use] + pub fn returns_nothing(&self) -> bool { + self.comparisons + .iter() + .any(|comparison| !comparison.literal) + } +} + +/// The disjuncts of a WHERE clause, read as the service reads them. +#[derive(Debug, Clone)] +pub struct WhereAnalysis<'a> { + /// The WHERE clause read. + condition: &'a WhereExpr, + /// The disjuncts read by key, in statement order, up to the first one + /// that is a scan. + reads: Vec>, + /// Whether every disjunct reads by key. + keyed: bool, + /// How many disjuncts the WHERE splits into, at most `u64::MAX`. + disjuncts: u64, +} + +impl<'a> WhereAnalysis<'a> { + /// Read `where_clause` against the target's key names. `value_of` + /// resolves a literal or a bound parameter to its value and returns + /// `None` for anything else; two equalities on one key are the same + /// condition when their values are equal. + /// + /// Past [`MAX_BRANCHES`] disjuncts only the first [`MAX_BRANCHES`] are + /// read; the statement reads by key when every disjunct does, those past + /// the first [`MAX_BRANCHES`] too, and its disjuncts are counted to the + /// last. + #[must_use] + pub fn new( + where_clause: &'a WhereExpr, + keys: KeyNames<'_>, + value_of: &dyn Fn(&Operand) -> Option, + ) -> Self { + let Split { + branches, + complete, + tally, + } = split(where_clause, keys, value_of); + let mut reads = Vec::with_capacity(branches.len()); + for branch in &branches { + match key_read(&branch.conjuncts, value_of) { + Some(read) => reads.push(read), + None => { + return Self { + condition: where_clause, + reads, + keyed: false, + disjuncts: tally.count, + }; + } + } + } + Self { + condition: where_clause, + keyed: (complete || tally.reads_by_key()) && !reads.is_empty(), + reads, + disjuncts: tally.count, + } + } + + /// How many disjuncts the WHERE splits into, those past the first + /// [`MAX_BRANCHES`] too (at most `u64::MAX`): one for each value of an + /// IN on a key attribute and for each side of an OR that splits, + /// multiplied across an AND. + #[must_use] + pub fn disjunct_count(&self) -> u64 { + self.disjuncts + } + + /// The refusal of a WHERE of a table read that reads a path into a key + /// attribute: in the filter of a disjunct read by key, naming the first + /// such disjunct (see [`scalar_key_refusal`]), else in any condition of + /// the WHERE, naming the first in statement order. Measured 2026-11-02: + /// `sk.x = 1` alone and `x = 1 OR (pk = 'a' AND sk.x = 1)` are refused, + /// as `pk = 'a' AND sk.x = 1` is. A NOT over an AND or an OR is one + /// condition here, as it is to the key reading: a path into a key under + /// it gets the filter refusal of a disjunct read by key instead. + #[must_use] + pub fn path_into_key_refusal(&self, keys: KeyNames<'_>) -> Option { + self.reads + .iter() + .find_map(|read| scalar_key_refusal(&read.filter, keys)) + .or_else(|| first_path_into_key(self.condition, keys).map(scalar_message)) + } + + /// The disjuncts read by key before the first one that is a scan: every + /// disjunct when the statement reads by key. The service checks each of + /// these in order. + #[must_use] + pub fn reads(&self) -> &[KeyRead<'a>] { + &self.reads + } + + /// Whether every disjunct reads by key, so the statement is not a scan. + #[must_use] + pub fn is_keyed(&self) -> bool { + self.keyed + } + + /// The one key read, when the statement reads by key with one disjunct. + #[must_use] + pub fn single(&self) -> Option<&KeyRead<'a>> { + match self.reads.as_slice() { + [read] if self.keyed => Some(read), + _ => None, + } + } + + /// The filter conditions every disjunct shares, in statement order, when + /// the statement reads by key: the conditions beside the key conditions + /// at the top of the WHERE. A condition inside one OR branch is not among + /// them (measured 2026-11-02 through an index: an unprojected attribute + /// beside an IN on the index key is refused, inside one OR branch it is + /// not). + #[must_use] + pub fn shared_filter(&self) -> Option> { + if !self.keyed { + return None; + } + let mut seen: HashMap<*const WhereExpr, usize> = HashMap::new(); + for read in &self.reads { + let mut once: Vec<*const WhereExpr> = Vec::new(); + for term in &read.filter { + let id: *const WhereExpr = *term; + if !once.contains(&id) { + once.push(id); + *seen.entry(id).or_default() += 1; + } + } + } + let first = self.reads.first()?; + Some( + first + .filter + .iter() + .copied() + .filter(|term| { + let id: *const WhereExpr = *term; + seen.get(&id) == Some(&self.reads.len()) + }) + .collect(), + ) + } + + /// Refuse a statement whose disjuncts read overlapping parts of one + /// partition. + /// + /// When every disjunct reads by key, two on the same partition whose + /// sort-key ranges overlap are refused; a disjunct with no sort-key + /// condition covers the whole partition. A statement that is a scan + /// refuses nothing. Measured 2026-11-02: a disjunct with a non-key filter + /// beside a sort-key equality disjunct on the same partition is refused, + /// and a disjunct whose sort-key condition is negated or written with + /// `<>` makes the statement run. + /// + /// # Errors + /// + /// Returns [`OVERLAP_MESSAGE`] for overlapping disjuncts. + pub fn check_overlap( + &self, + value_of: &dyn Fn(&Operand) -> Option, + ) -> Result<(), &'static str> { + if !self.keyed { + return Ok(()); + } + let mut reads: Vec<(Bound, Interval)> = Vec::new(); + for read in &self.reads { + let Some(partition) = value_of(read.partition) + .as_ref() + .and_then(Bound::from_value) + else { + return Ok(()); + }; + let interval = match &read.sort { + Some(condition) => match interval(condition, value_of) { + Some(interval) => interval, + None => return Ok(()), + }, + None => Interval::UNBOUNDED, + }; + reads.push((partition, interval)); + } + let mut partitions: Vec<&Bound> = Vec::new(); + for (partition, _) in &reads { + if !partitions.iter().any(|p| p.same(partition)) { + partitions.push(partition); + } + } + for partition in partitions { + let mut intervals: Vec<&Interval> = reads + .iter() + .filter(|(p, _)| p.same(partition)) + .map(|(_, i)| i) + .collect(); + if intervals.len() < 2 { + continue; + } + // Every bound must be comparable with every other for the order + // below to mean anything; a key value of the wrong type is + // reported before this check. + let comparable = intervals.iter().all(|a| { + intervals + .iter() + .all(|b| Interval::cmp_lower(a, b).is_some() && a.overlaps(b).is_some()) + }); + if !comparable { + return Ok(()); + } + intervals.sort_by(|a, b| Interval::cmp_lower(a, b).unwrap_or(Ordering::Equal)); + if intervals + .windows(2) + .any(|pair| pair[0].overlaps(pair[1]) == Some(true)) + { + return Err(OVERLAP_MESSAGE); + } + } + Ok(()) + } +} + +/// The service's refusal of a `SELECT` whose OR branches read overlapping +/// sort-key ranges of one partition (measured 2026-11-02). +pub const OVERLAP_MESSAGE: &str = + "Overlapping conditions with range keys are not supported in where clause"; + +/// The refusal of a keyed read whose filter compares a key attribute with +/// another path, naming the key of the first such condition (measured +/// 2026-11-02: `pk = 'a' AND sk = a`, `a = sk`, `sk = pk` and `pk = a` name +/// the key compared). +#[must_use] +pub fn filter_key_refusal(filter: &[&WhereExpr], keys: KeyNames<'_>) -> Option { + filter.iter().find_map(|term| match role(term, keys) { + Role::KeyAndPath(key) => Some(format!( + "Filter Expression can only contain non-primary key attributes: Primary key \ + attribute: {key}" + )), + _ => None, + }) +} + +/// The refusal of a keyed read whose filter reads a path into a key +/// attribute, naming the key of the first such condition (measured +/// 2026-11-02: `sk.x = 1` and `sk[0] = 1` beside `pk = 'a'`). +#[must_use] +pub fn scalar_key_refusal(filter: &[&WhereExpr], keys: KeyNames<'_>) -> Option { + filter.iter().find_map(|term| match role(term, keys) { + Role::IntoKey(key) => Some(scalar_message(key)), + _ => None, + }) +} + +/// The refusal of a path into a key attribute, before the key's name. +pub const SCALAR_KEY_REFUSAL: &str = "Key attributes must be scalars; list random access '[]' and map lookup '.' are not \ + allowed: Key "; + +/// The refusal of a path into a key attribute among the conditions a write +/// checks against its item, before the key's name (see +/// [`condition_key_path`]). +pub const CONDITION_KEY_REFUSAL: &str = "Key attributes must be scalars; list random access '[]' and map lookup '.' are \ + not allowed: Key: "; + +/// The key attribute of a condition of a write's WHERE that tests a path +/// into it against values alone: a comparison with a value on either side, +/// BETWEEN with values, IN with a list of values, begins_with, contains or +/// attribute_type with a value as the second argument, an IS test of a +/// type, or one of these compared with a boolean (measured 2026-11-05 on +/// UPDATE and DELETE: `pk.c = 1`, `1 = pk.c`, `sk[0] = 'z'`, `pk.c BETWEEN 1 +/// AND 2`, `pk.c IN [1]`, `begins_with(pk.c, 'x')`, `pk.c IS NULL` and `pk.c +/// = 1 = false` are refused with [`SCALAR_KEY_REFUSAL`]). `None` for any +/// other condition. +#[must_use] +pub fn path_tested_against_values<'a>(expr: &'a WhereExpr, keys: KeyNames<'_>) -> Option<&'a str> { + roles::value_test_of_path_into_key(expr, keys) +} + +/// The key attribute the refusal of a path into a key attribute among a +/// write's other conditions names: the sort key when any of `conditions` +/// reads a path into it, else the partition key when one reads a path into +/// that (measured 2026-11-05: `NOT pk.c = 1 AND NOT sk.d = 1`, either way +/// round, and `pk.c = sk.d` name sk; `pk.c = 1 OR sk = 'b'` names pk). +/// These are the conditions [`path_tested_against_values`] does not take: +/// OR, NOT, a function of the path, IS MISSING, IS NOT, and the path +/// compared with another path. +#[must_use] +pub fn condition_key_path<'a>(conditions: &[&'a WhereExpr], keys: KeyNames<'_>) -> Option<&'a str> { + let mut named = Vec::new(); + for condition in conditions { + roles::keys_with_paths_into_them(condition, keys, &mut named); + } + named + .iter() + .find(|name| keys.sort == Some(**name)) + .or_else(|| named.first()) + .copied() +} + +/// The refusal of a path into the key attribute `key`. +fn scalar_message(key: &str) -> String { + format!("{SCALAR_KEY_REFUSAL}{key}") +} + +/// The key attribute of the first condition of a WHERE, in statement order, +/// that reads a path into a key attribute. The walk goes through AND and OR +/// only, with its own stack, and asks each other condition its role. +fn first_path_into_key<'a>(expr: &'a WhereExpr, keys: KeyNames<'_>) -> Option<&'a str> { + let mut pending = vec![expr]; + while let Some(expr) = pending.pop() { + match expr { + WhereExpr::And(a, b) | WhereExpr::Or(a, b) => { + pending.push(b); + pending.push(a); + } + condition => { + if let Role::IntoKey(key) = role(condition, keys) { + return Some(key); + } + } + } + } + None +} + +/// The order in which two disjuncts read by key are read: by partition +/// value, then by where their sort-key ranges start (a disjunct with no +/// sort-key condition, or one with no lower bound, first; on an equal start +/// an inclusive one first). Measured 2026-11-02: `(pk = 'b' AND sk = 'x') OR +/// (pk = 'a' AND sk > 'a')` reads a/x and a/y before b/x. Values that do not +/// resolve, or that have two types, compare equal. +#[must_use] +pub fn read_order( + a: &KeyRead<'_>, + b: &KeyRead<'_>, + value_of: &dyn Fn(&Operand) -> Option, +) -> Ordering { + read_order_in(a, b, Direction::default(), value_of) +} + +/// The directions of a read under an ORDER BY: of its partitions, and of +/// the items of one partition. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct Direction { + pub partitions_descending: bool, + pub items_descending: bool, +} + +/// [`read_order`] with the partitions, and the sort-key ranges of one +/// partition, in the directions `direction` gives. +#[must_use] +pub fn read_order_in( + a: &KeyRead<'_>, + b: &KeyRead<'_>, + direction: Direction, + value_of: &dyn Fn(&Operand) -> Option, +) -> Ordering { + let directed = |order: Ordering, descending: bool| { + if descending { order.reverse() } else { order } + }; + let bound = |op: &Operand| value_of(op).as_ref().and_then(Bound::from_value); + let partitions = match (bound(a.partition), bound(b.partition)) { + (Some(x), Some(y)) => x.compare(&y).unwrap_or(Ordering::Equal), + _ => Ordering::Equal, + }; + directed(partitions, direction.partitions_descending).then_with(|| { + let range = |read: &KeyRead<'_>| match &read.sort { + None => Some(Interval::UNBOUNDED), + Some(condition) => interval(condition, value_of), + }; + let ranges = match (range(a), range(b)) { + (Some(x), Some(y)) => Interval::cmp_lower(&x, &y).unwrap_or(Ordering::Equal), + _ => Ordering::Equal, + }; + directed(ranges, direction.items_descending) + }) +} + +/// A WHERE of a statement read by key, as a filter: each key condition +/// compared with a boolean (see the module documentation) is the key +/// condition when compared with true, and holds for no item when compared +/// with false. The walk follows the AND and OR nesting, as [`split`] does. +#[must_use] +pub fn keyed_condition(expr: &WhereExpr, keys: KeyNames<'_>) -> WhereExpr { + match expr { + WhereExpr::And(a, b) => WhereExpr::And( + Box::new(keyed_condition(a, keys)), + Box::new(keyed_condition(b, keys)), + ), + WhereExpr::Or(a, b) => WhereExpr::Or( + Box::new(keyed_condition(a, keys)), + Box::new(keyed_condition(b, keys)), + ), + WhereExpr::ComparedWithBoolean(inner, comparison) if is_key_condition(inner, keys) => { + let inner = keyed_condition(inner, keys); + if comparison.literal { + inner + } else { + let never = WhereExpr::Not(Box::new(inner.clone())); + WhereExpr::And(Box::new(inner), Box::new(never)) + } + } + WhereExpr::ComparedWithBoolean(inner, comparison) if !comparison.negates() => { + keyed_condition(inner, keys) + } + other => other.clone(), + } +} + +/// The key attribute a single WHERE condition pins by equality. +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum KeyEquality<'a> { + /// The partition key equals this value. + Partition(&'a Operand), + /// The sort key equals this value. + Sort(&'a Operand), +} + +/// `term` as an equality of a key attribute with a literal or parameter, +/// written either way round (`pk = 'a'` or `'a' = pk`). +#[must_use] +pub fn key_equality<'a>(term: &'a WhereExpr, keys: KeyNames<'_>) -> Option> { + match role(term, keys) { + Role::Partition(value) => Some(KeyEquality::Partition(value)), + Role::Sort(SortCondition::Compare(CompareOp::Eq, value)) => Some(KeyEquality::Sort(value)), + _ => None, + } +} + +/// Whether two value operands name the same key value: equal values (a +/// number compared by its value), or the same operand when either does not +/// resolve. +#[must_use] +pub fn same_value( + a: &Operand, + b: &Operand, + value_of: &dyn Fn(&Operand) -> Option, +) -> bool { + match (value_of(a), value_of(b)) { + (Some(x), Some(y)) => match (Bound::from_value(&x), Bound::from_value(&y)) { + (Some(x), Some(y)) => x.same(&y), + _ => x == y, + }, + _ => a == b, + } +} + +/// The most disjuncts of a WHERE that are listed. AND distributes over OR, +/// so the count multiplies: k ANDed pairs of ORed key equalities make 2^k +/// disjuncts. The ones past these are counted, and whether they read by key +/// is known, without listing them (see [`WhereAnalysis::new`]). +pub const MAX_BRANCHES: usize = 1024; + +#[cfg(test)] +mod tests; diff --git a/crates/core/src/partiql/plan/ranges.rs b/crates/core/src/partiql/plan/ranges.rs new file mode 100644 index 000000000..7873200cc --- /dev/null +++ b/crates/core/src/partiql/plan/ranges.rs @@ -0,0 +1,158 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The sort-key range a key condition reads, and the key values that bound it. + +use std::cmp::Ordering; +use std::str::FromStr; + +use bigdecimal::BigDecimal; + +use super::SortCondition; +use crate::partiql::ast::{CompareOp, Operand}; +use crate::types::AttributeValue; +use crate::validation::number::validate_and_normalize_number; + +/// The sort-key range a key condition reads, or `None` when a value does not +/// resolve to a key value. +pub(super) fn interval( + condition: &SortCondition<'_>, + value_of: &dyn Fn(&Operand) -> Option, +) -> Option { + let bound = |op: &Operand| value_of(op).as_ref().and_then(Bound::from_value); + Some(match *condition { + SortCondition::Compare(op, value) => { + let v = bound(value)?; + match op { + CompareOp::Eq => Interval::new(Some(v.clone()), true, Some(v), true), + CompareOp::Ge => Interval::new(Some(v), true, None, false), + CompareOp::Gt => Interval::new(Some(v), false, None, false), + CompareOp::Le => Interval::new(None, false, Some(v), true), + CompareOp::Lt => Interval::new(None, false, Some(v), false), + CompareOp::Ne => return None, + } + } + SortCondition::Between(low, high) => { + Interval::new(Some(bound(low)?), true, Some(bound(high)?), true) + } + // Every key that starts with the prefix: from the prefix up to the + // prefix with its last byte raised by one. + SortCondition::BeginsWith(prefix) => { + let low = bound(prefix)?; + let high = match &low { + Bound::S(bytes) => next_prefix(bytes).map(Bound::S), + Bound::B(bytes) => next_prefix(bytes).map(Bound::B), + // A number has no prefix. + Bound::N(_) => return None, + }; + Interval::new(Some(low), true, high, false) + } + }) +} + +/// A comparable key value. +#[derive(Debug, Clone)] +pub(super) enum Bound { + S(Vec), + N(BigDecimal), + B(Vec), +} + +impl Bound { + pub(super) fn from_value(value: &AttributeValue) -> Option { + match value { + AttributeValue::S(s) => Some(Self::S(s.as_bytes().to_vec())), + AttributeValue::B(b) => Some(Self::B(b.clone())), + AttributeValue::N(n) => { + let normalized = validate_and_normalize_number(n).ok()?; + BigDecimal::from_str(&normalized).ok().map(Self::N) + } + _ => None, + } + } + + pub(super) fn compare(&self, other: &Self) -> Option { + match (self, other) { + (Self::S(a), Self::S(b)) | (Self::B(a), Self::B(b)) => Some(a.cmp(b)), + (Self::N(a), Self::N(b)) => Some(a.cmp(b)), + _ => None, + } + } + + pub(super) fn same(&self, other: &Self) -> bool { + self.compare(other) == Some(Ordering::Equal) + } +} + +/// The least byte string greater than every string with this prefix: the +/// prefix with its last byte raised by one (carrying), or `None` when there +/// is none. +fn next_prefix(prefix: &[u8]) -> Option> { + let mut next = prefix.to_vec(); + while let Some(last) = next.pop() { + if last < u8::MAX { + next.push(last + 1); + return Some(next); + } + } + None +} + +/// A sort-key range; a missing bound is unbounded. +#[derive(Debug, Clone)] +pub(super) struct Interval { + min: Option, + min_inclusive: bool, + max: Option, + max_inclusive: bool, +} + +impl Interval { + pub(super) const UNBOUNDED: Self = Self { + min: None, + min_inclusive: false, + max: None, + max_inclusive: false, + }; + + fn new( + min: Option, + min_inclusive: bool, + max: Option, + max_inclusive: bool, + ) -> Self { + Self { + min, + min_inclusive, + max, + max_inclusive, + } + } + + /// Order by lower bound, an unbounded one first; on equal bounds an + /// inclusive one first. + pub(super) fn cmp_lower(a: &Self, b: &Self) -> Option { + match (&a.min, &b.min) { + (None, None) => Some(Ordering::Equal), + (None, Some(_)) => Some(Ordering::Less), + (Some(_), None) => Some(Ordering::Greater), + (Some(x), Some(y)) => Some(match x.compare(y)? { + Ordering::Equal => b.min_inclusive.cmp(&a.min_inclusive), + other => other, + }), + } + } + + /// Whether `next`, which sorts at or after `self`, starts before `self` + /// ends. + pub(super) fn overlaps(&self, next: &Self) -> Option { + let (Some(end), Some(start)) = (&self.max, &next.min) else { + return Some(true); + }; + Some(match end.compare(start)? { + Ordering::Greater => true, + Ordering::Equal => self.max_inclusive && next.min_inclusive, + Ordering::Less => false, + }) + } +} diff --git a/crates/core/src/partiql/plan/roles.rs b/crates/core/src/partiql/plan/roles.rs new file mode 100644 index 000000000..fc0aa2cba --- /dev/null +++ b/crates/core/src/partiql/plan/roles.rs @@ -0,0 +1,412 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! What each condition of a WHERE clause is to the key reading. + +use super::{KeyNames, SortCondition}; +use crate::partiql::ast::{BooleanComparison, CompareOp, Operand, Path, WhereExpr}; + +/// What one condition of a disjunct is to the key reading. +#[derive(Debug, Clone, Copy, PartialEq)] +pub(super) enum Role<'a> { + /// An equality of the partition key with a value. + Partition(&'a Operand), + /// The comparison with a boolean that key conditions of the disjunct are + /// under (see `split`). It adds no condition of its own. + Compared(BooleanComparison), + /// A key condition on the sort key. + Sort(SortCondition<'a>), + /// Any other predicate on a key attribute. + OnKey, + /// A key attribute compared with something that is not a value. + KeyAndPath(&'a str), + /// A path into a key attribute. + IntoKey(&'a str), + /// A condition on other attributes. + Filter, +} + +impl Role<'_> { + /// Whether an OR with this condition on either side is split. + pub(super) fn splits(self) -> bool { + !matches!(self, Role::IntoKey(_) | Role::Filter) + } +} + +/// The role of one condition (not an AND). +pub(super) fn role<'a>(expr: &'a WhereExpr, keys: KeyNames<'_>) -> Role<'a> { + match expr { + WhereExpr::Compare(l, op, r) => { + if let Some(key) = bare_key(l, keys) + && is_value(r) + { + return key_comparison(key, *op, r, keys); + } + if let Some(key) = bare_key(r, keys) + && is_value(l) + { + return key_comparison(key, mirror(*op), l, keys); + } + comparison_role(&[l, r], keys) + } + WhereExpr::Between(v, low, high) => match bare_key(v, keys) { + Some(key) if is_value(low) && is_value(high) => { + if Some(key) == keys.sort { + Role::Sort(SortCondition::Between(low, high)) + } else { + Role::OnKey + } + } + _ => comparison_role(&[v, low, high], keys), + }, + WhereExpr::BeginsWith(v, prefix) => match bare_key(v, keys) { + Some(key) if is_value(prefix) => { + if Some(key) == keys.sort { + Role::Sort(SortCondition::BeginsWith(prefix)) + } else { + Role::OnKey + } + } + _ => comparison_role(&[v, prefix], keys), + }, + // An IN on a key attribute with values only is split into one + // equality per value before a role is asked for. With a path in the + // list it is one more predicate on the key (measured 2026-11-02: + // `pk = 'a' AND sk IN [a]` runs). + WhereExpr::In(v, items) => { + if bare_key(v, keys).is_some() { + return Role::OnKey; + } + let operands: Vec<&Operand> = std::iter::once(v).chain(items).collect(); + comparison_role(&operands, keys) + } + // A NOT over an AND or an OR is not read for key conditions: it stays + // in the disjunct's filter (measured 2026-11-02: `pk = 'a' AND NOT + // (sk = 'x' AND a = 1) OR pk = 'a' AND sk = 'y'` is refused as + // overlapping, so its first disjunct reads partition 'a' by key). A + // path into a key attribute under it gets the refusal a key compared + // with a path gets (measured 2026-11-02: `pk = 'a' AND NOT (sk.q = 1 + // AND n = 1)` names sk in the filter refusal). See [`named_key`] for + // the key it names. + WhereExpr::Not(_) => negation_role(expr, keys), + // A comparison with a boolean that keeps the condition is the + // condition, and one that negates it is read as a NOT. A key + // condition compared with a boolean is read by `split` before a role + // is asked for. + WhereExpr::ComparedWithBoolean(inner, comparison) => { + if comparison.negates() { + negation_role(expr, keys) + } else { + role(inner, keys) + } + } + other => other_role(other, keys), + } +} + +/// Whether a condition is a key condition, or an AND of key conditions: an +/// equality of the partition key with a value, or a key condition on the +/// sort key. +pub(super) fn is_key_condition(expr: &WhereExpr, keys: KeyNames<'_>) -> bool { + let mut pending = vec![expr]; + while let Some(expr) = pending.pop() { + match expr { + WhereExpr::And(a, b) => { + pending.push(a); + pending.push(b); + } + other => { + if !matches!(role(other, keys), Role::Partition(_) | Role::Sort(_)) { + return false; + } + } + } + } + true +} + +/// The role of a negated condition: a NOT over an AND or an OR stays in +/// the filter (see [`role`]), and any other is read as what it holds. +fn negation_role<'a>(expr: &'a WhereExpr, keys: KeyNames<'_>) -> Role<'a> { + if matches!( + expr.without_nots().1, + WhereExpr::And(..) | WhereExpr::Or(..) + ) { + match named_key(expr, keys) { + Some(key) => Role::KeyAndPath(key), + None => Role::Filter, + } + } else { + other_role(expr, keys) + } +} + +/// The role of a condition that is no key condition: a predicate on a key +/// attribute, a path into one, or a filter. +fn other_role<'a>(expr: &'a WhereExpr, keys: KeyNames<'_>) -> Role<'a> { + if mentions_key(expr, keys) { + Role::OnKey + } else if let Some(key) = path_into_key(expr, keys) { + Role::IntoKey(key) + } else { + Role::Filter + } +} + +/// A comparison of `key` with a value, the key on the left. +pub(super) fn key_comparison<'a>( + key: &str, + op: CompareOp, + value: &'a Operand, + keys: KeyNames<'_>, +) -> Role<'a> { + if key == keys.partition { + if op == CompareOp::Eq { + Role::Partition(value) + } else { + Role::OnKey + } + } else if op == CompareOp::Ne { + Role::OnKey + } else { + Role::Sort(SortCondition::Compare(op, value)) + } +} + +/// The role of a comparison that is not a key compared with values. +fn comparison_role<'a>(operands: &[&'a Operand], keys: KeyNames<'_>) -> Role<'a> { + if let Some(key) = operands.iter().find_map(|op| bare_key(op, keys)) { + return Role::KeyAndPath(key); + } + if operands.iter().any(|op| operand_key(op, keys).is_some()) { + return Role::OnKey; + } + match operands + .iter() + .find_map(|op| operand_path_into_key(op, keys)) + { + Some(key) => Role::IntoKey(key), + None => Role::Filter, + } +} + +/// The comparison with its two sides swapped: `1 < sk` is `sk > 1`. +fn mirror(op: CompareOp) -> CompareOp { + match op { + CompareOp::Eq => CompareOp::Eq, + CompareOp::Ne => CompareOp::Ne, + CompareOp::Lt => CompareOp::Gt, + CompareOp::Le => CompareOp::Ge, + CompareOp::Gt => CompareOp::Lt, + CompareOp::Ge => CompareOp::Le, + } +} + +/// The key attribute an operand names as a bare path. +pub(super) fn bare_key<'a>(op: &'a Operand, keys: KeyNames<'_>) -> Option<&'a str> { + simple_path(op).filter(|name| keys.is_key(name)) +} + +/// The attribute name when the operand is a bare attribute path, else `None`. +fn simple_path(op: &Operand) -> Option<&str> { + match op { + Operand::Path(Path { root, steps }) if steps.is_empty() => Some(root.as_str()), + _ => None, + } +} + +pub(super) fn is_value(op: &Operand) -> bool { + matches!(op, Operand::Literal(_) | Operand::Parameter(_)) +} + +/// The key attribute of the first path into one in the operand. +fn operand_path_into_key<'a>(op: &'a Operand, keys: KeyNames<'_>) -> Option<&'a str> { + match op { + Operand::Path(path) if !path.steps.is_empty() && keys.is_key(&path.root) => { + Some(path.root.as_str()) + } + Operand::Function(_, args) => args.iter().find_map(|a| operand_path_into_key(a, keys)), + Operand::Arith(l, _, r) => { + operand_path_into_key(l, keys).or_else(|| operand_path_into_key(r, keys)) + } + _ => None, + } +} + +/// What a condition tests: an operand, or the path of an `IS [NOT] MISSING`. +enum Leaf<'a> { + Operand(&'a Operand), + Missing(&'a Path), +} + +/// The leaves of a condition, left to right. An explicit stack keeps a run +/// of NOTs and a long AND or OR chain off the call stack. +fn leaves(expr: &WhereExpr) -> Vec> { + let mut out = Vec::new(); + let mut pending = vec![expr]; + while let Some(expr) = pending.pop() { + match expr { + WhereExpr::Compare(a, _, b) + | WhereExpr::BeginsWith(a, b) + | WhereExpr::Contains(a, b) + | WhereExpr::AttributeType(a, b) => { + out.push(Leaf::Operand(a)); + out.push(Leaf::Operand(b)); + } + WhereExpr::Between(v, low, high) => { + out.extend([v, low, high].map(Leaf::Operand)); + } + WhereExpr::In(v, items) => { + out.push(Leaf::Operand(v)); + out.extend(items.iter().map(Leaf::Operand)); + } + WhereExpr::Function(_, args) => out.extend(args.iter().map(Leaf::Operand)), + WhereExpr::IsMissing(path, _) => out.push(Leaf::Missing(path)), + WhereExpr::IsType(operand, ..) => out.push(Leaf::Operand(operand)), + WhereExpr::And(a, b) | WhereExpr::Or(a, b) => { + pending.push(b); + pending.push(a); + } + WhereExpr::Not(inner) | WhereExpr::ComparedWithBoolean(inner, _) => pending.push(inner), + } + } + out +} + +/// Whether a condition tests a key attribute itself anywhere. +fn mentions_key(expr: &WhereExpr, keys: KeyNames<'_>) -> bool { + first_key(expr, keys).is_some() +} + +/// The first key attribute a condition tests itself, left to right. +fn first_key<'a>(expr: &'a WhereExpr, keys: KeyNames<'_>) -> Option<&'a str> { + leaves(expr).into_iter().find_map(|leaf| match leaf { + Leaf::Operand(op) => operand_key(op, keys), + Leaf::Missing(path) => path.as_simple().filter(|name| keys.is_key(name)), + }) +} + +/// The first key attribute an operand is or holds as a function's argument. +fn operand_key<'a>(op: &'a Operand, keys: KeyNames<'_>) -> Option<&'a str> { + match op { + Operand::Path(_) => bare_key(op, keys), + Operand::Function(_, args) => args.iter().find_map(|a| operand_key(a, keys)), + Operand::Arith(l, _, r) => operand_key(l, keys).or_else(|| operand_key(r, keys)), + Operand::Literal(_) | Operand::Parameter(_) => None, + } +} + +/// The key attribute the filter refusal of a NOT over an AND or an OR names: +/// the sort key when the condition names it, itself or as the root of a +/// path, else the partition key (measured 2026-11-02: `pk = 'a' AND NOT +/// (sk.q = 1 AND pk = b)` and `pk = 'a' AND NOT (pk = b AND sk.q = 1)` name +/// sk). +fn named_key<'a>(expr: &'a WhereExpr, keys: KeyNames<'_>) -> Option<&'a str> { + let mut named = Vec::new(); + for leaf in leaves(expr) { + match leaf { + Leaf::Operand(op) => key_roots(op, keys, &mut named), + Leaf::Missing(path) if keys.is_key(&path.root) => named.push(path.root.as_str()), + Leaf::Missing(_) => {} + } + } + named + .iter() + .find(|name| keys.sort == Some(**name)) + .or_else(|| named.first()) + .copied() +} + +/// Every key attribute an operand names, itself or as the root of a path. +fn key_roots<'a>(op: &'a Operand, keys: KeyNames<'_>, out: &mut Vec<&'a str>) { + match op { + Operand::Path(path) if keys.is_key(&path.root) => out.push(path.root.as_str()), + Operand::Function(_, args) => { + for arg in args { + key_roots(arg, keys, out); + } + } + Operand::Arith(l, _, r) => { + key_roots(l, keys, out); + key_roots(r, keys, out); + } + Operand::Path(_) | Operand::Literal(_) | Operand::Parameter(_) => {} + } +} + +/// The key attribute of a condition that tests a path into one against +/// values alone (see `plan::path_tested_against_values`). +pub(super) fn value_test_of_path_into_key<'a>( + expr: &'a WhereExpr, + keys: KeyNames<'_>, +) -> Option<&'a str> { + let into_key = |op: &'a Operand| match op { + Operand::Path(path) if !path.steps.is_empty() && keys.is_key(&path.root) => { + Some(path.root.as_str()) + } + _ => None, + }; + match expr { + WhereExpr::Compare(l, _, r) => into_key(l) + .filter(|_| is_value(r)) + .or_else(|| into_key(r).filter(|_| is_value(l))), + WhereExpr::Between(v, low, high) => into_key(v).filter(|_| is_value(low) && is_value(high)), + WhereExpr::In(v, items) => into_key(v).filter(|_| items.iter().all(is_value)), + WhereExpr::BeginsWith(a, b) + | WhereExpr::Contains(a, b) + | WhereExpr::AttributeType(a, b) => into_key(a).filter(|_| is_value(b)), + WhereExpr::IsType(operand, _, false) => into_key(operand), + WhereExpr::ComparedWithBoolean(inner, _) => value_test_of_path_into_key(inner, keys), + _ => None, + } +} + +/// Every key attribute a condition reads a path into, in statement order. +pub(super) fn keys_with_paths_into_them<'a>( + expr: &'a WhereExpr, + keys: KeyNames<'_>, + out: &mut Vec<&'a str>, +) { + for leaf in leaves(expr) { + let path = match leaf { + Leaf::Operand(op) => { + paths_into_keys(op, keys, out); + continue; + } + Leaf::Missing(path) => path, + }; + if !path.steps.is_empty() && keys.is_key(&path.root) { + out.push(path.root.as_str()); + } + } +} + +/// Every key attribute an operand reads a path into, itself or as a +/// function's argument or an arithmetic operand. +fn paths_into_keys<'a>(op: &'a Operand, keys: KeyNames<'_>, out: &mut Vec<&'a str>) { + match op { + Operand::Path(path) if !path.steps.is_empty() && keys.is_key(&path.root) => { + out.push(path.root.as_str()); + } + Operand::Function(_, args) => { + for arg in args { + paths_into_keys(arg, keys, out); + } + } + Operand::Arith(l, _, r) => { + paths_into_keys(l, keys, out); + paths_into_keys(r, keys, out); + } + Operand::Path(_) | Operand::Literal(_) | Operand::Parameter(_) => {} + } +} + +/// The key attribute of the first path into one in a condition. +fn path_into_key<'a>(expr: &'a WhereExpr, keys: KeyNames<'_>) -> Option<&'a str> { + leaves(expr).into_iter().find_map(|leaf| match leaf { + Leaf::Operand(op) => operand_path_into_key(op, keys), + Leaf::Missing(path) => { + (!path.steps.is_empty() && keys.is_key(&path.root)).then_some(path.root.as_str()) + } + }) +} diff --git a/crates/core/src/partiql/plan/split.rs b/crates/core/src/partiql/plan/split.rs new file mode 100644 index 000000000..7acb97bca --- /dev/null +++ b/crates/core/src/partiql/plan/split.rs @@ -0,0 +1,349 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The split of a WHERE clause into the disjuncts read separately, and the key read of each. + +use super::roles::{Role, bare_key, is_key_condition, is_value, key_comparison, role}; +use super::{KeyNames, KeyRead, MAX_BRANCHES, SortCondition, same_value}; +use crate::partiql::ast::{CompareOp, Operand, WhereExpr}; +use crate::types::AttributeValue; + +/// One condition of a disjunct, with its role. An IN on a key attribute +/// contributes one value per disjunct and no WHERE term of its own. +#[derive(Debug, Clone, Copy)] +pub(super) struct Conjunct<'a> { + term: Option<&'a WhereExpr>, + role: Role<'a>, +} + +#[derive(Debug, Clone)] +pub(super) struct Branch<'a> { + pub(super) conjuncts: Vec>, + has_key: bool, +} + +/// The first [`MAX_BRANCHES`] disjuncts of a WHERE, in statement order, +/// whether they are all of them, and the tally of all of them. +pub(super) struct Split<'a> { + pub(super) branches: Vec>, + pub(super) complete: bool, + pub(super) tally: Tally<'a>, +} + +impl<'a> Split<'a> { + fn one(term: &'a WhereExpr, role: Role<'a>, has_key: bool) -> Self { + Self { + branches: vec![Branch { + conjuncts: vec![Conjunct { + term: Some(term), + role, + }], + has_key, + }], + complete: true, + tally: Tally::of(role), + } + } +} + +/// What the key reading needs to know of every disjunct of a WHERE, those +/// past the first [`MAX_BRANCHES`] too: how many there are, and whether each +/// one reads by key (see [`key_read`]). It is kept without listing the +/// disjuncts, whose number AND multiplies. +#[derive(Debug, Clone, Copy)] +pub(super) struct Tally<'a> { + /// How many disjuncts there are, at most `u64::MAX`. + pub(super) count: u64, + /// Whether some disjunct is a scan for what it holds: a predicate on a + /// key attribute that is no key condition, two partition key values, or + /// two sort key conditions. + clash: bool, + /// Whether some disjunct holds no equality of the partition key. + unkeyed: bool, + /// The partition key values the disjuncts hold. + partitions: Held<&'a Operand>, + /// The sort key conditions the disjuncts hold. + sorts: Held>, +} + +impl<'a> Tally<'a> { + /// The tally of no disjunct, which an OR with any other leaves as it is. + const NONE: Self = Self { + count: 0, + clash: false, + unkeyed: false, + partitions: Held::Nothing, + sorts: Held::Nothing, + }; + + /// The tally of one disjunct of one condition. + fn of(role: Role<'a>) -> Self { + let mut tally = Self { + count: 1, + clash: false, + unkeyed: true, + partitions: Held::Nothing, + sorts: Held::Nothing, + }; + match role { + Role::Partition(value) => { + tally.unkeyed = false; + tally.partitions = Held::One(value); + } + Role::Sort(condition) => tally.sorts = Held::One(condition), + Role::OnKey => tally.clash = true, + Role::Compared(_) | Role::KeyAndPath(_) | Role::IntoKey(_) | Role::Filter => {} + } + tally + } + + /// The tally of the disjuncts of either side of an OR. + fn or(self, other: Self, value_of: &dyn Fn(&Operand) -> Option) -> Self { + Self { + count: self.count.saturating_add(other.count), + clash: self.clash || other.clash, + unkeyed: self.unkeyed || other.unkeyed, + partitions: self + .partitions + .either(other.partitions, |a, b| same_value(a, b, value_of)), + sorts: self + .sorts + .either(other.sorts, |a, b| same_condition(a, b, value_of)), + } + } + + /// The tally of the disjuncts of an AND: each disjunct of one side with + /// each disjunct of the other. + fn and(self, other: Self, value_of: &dyn Fn(&Operand) -> Option) -> Self { + let same_partition = |a: &&Operand, b: &&Operand| same_value(a, b, value_of); + let same_sort = + |a: &SortCondition<'_>, b: &SortCondition<'_>| same_condition(a, b, value_of); + Self { + count: self.count.saturating_mul(other.count), + clash: self.clash + || other.clash + || self.partitions.clash(other.partitions, same_partition) + || self.sorts.clash(other.sorts, same_sort), + unkeyed: self.unkeyed && other.unkeyed, + partitions: self.partitions.either(other.partitions, same_partition), + sorts: self.sorts.either(other.sorts, same_sort), + } + } + + /// Whether every disjunct reads by key. + pub(super) fn reads_by_key(&self) -> bool { + !self.clash && !self.unkeyed + } +} + +/// The values of one key attribute that a set of disjuncts holds: none, one, +/// or more than one distinct value. +#[derive(Debug, Clone, Copy)] +enum Held { + Nothing, + One(T), + Several, +} + +impl Held { + /// The values the disjuncts of two sets hold together. + fn either(self, other: Self, same: impl Fn(&T, &T) -> bool) -> Self { + match (self, other) { + (Self::Nothing, held) | (held, Self::Nothing) => held, + (Self::One(a), Self::One(b)) if same(&a, &b) => Self::One(a), + _ => Self::Several, + } + } + + /// Whether a disjunct of one set ANDed with a disjunct of the other can + /// hold two distinct values. + fn clash(self, other: Self, same: impl Fn(&T, &T) -> bool) -> bool { + match (self, other) { + (Self::Nothing, _) | (_, Self::Nothing) => false, + (Self::One(a), Self::One(b)) => !same(&a, &b), + _ => true, + } + } +} + +/// Split a WHERE into the disjuncts the service reads separately. The +/// recursion follows the AND and OR nesting, which the operator limit keeps +/// within 300 levels; a run of NOTs is one condition. `value_of` resolves a +/// value as [`key_read`] does, for the tally. +pub(super) fn split<'a>( + expr: &'a WhereExpr, + keys: KeyNames<'_>, + value_of: &dyn Fn(&Operand) -> Option, +) -> Split<'a> { + match expr { + WhereExpr::Or(a, b) => { + let left = split(a, keys, value_of); + let right = split(b, keys, value_of); + let splits = !left.complete + || !right.complete + || left + .branches + .iter() + .chain(&right.branches) + .any(|b| b.has_key); + if !splits { + return Split::one(expr, role(expr, keys), false); + } + let complete = left.complete + && right.complete + && left.branches.len() + right.branches.len() <= MAX_BRANCHES; + let tally = left.tally.or(right.tally, value_of); + let mut branches = left.branches; + if left.complete { + branches.extend(right.branches); + } + branches.truncate(MAX_BRANCHES); + Split { + branches, + complete, + tally, + } + } + WhereExpr::And(a, b) => { + let left = split(a, keys, value_of); + let right = split(b, keys, value_of); + let complete = left.complete + && right.complete + && left.branches.len().saturating_mul(right.branches.len()) <= MAX_BRANCHES; + Split { + branches: product(&left.branches, &right.branches), + complete, + tally: left.tally.and(right.tally, value_of), + } + } + WhereExpr::In(left, items) => match bare_key(left, keys) { + Some(name) if items.iter().all(is_value) => { + let equality = |item| key_comparison(name, CompareOp::Eq, item, keys); + let tally = items.iter().fold(Tally::NONE, |tally, item| { + tally.or(Tally::of(equality(item)), value_of) + }); + Split { + branches: items + .iter() + .take(MAX_BRANCHES) + .map(|item| Branch { + conjuncts: vec![Conjunct { + term: None, + role: equality(item), + }], + has_key: true, + }) + .collect(), + complete: items.len() <= MAX_BRANCHES, + tally, + } + } + _ => { + let role = role(expr, keys); + Split::one(expr, role, role.splits()) + } + }, + // A negated condition is never split (measured 2026-11-02: a branch + // whose sort-key condition is negated makes the statement run). + WhereExpr::Not(_) => Split::one(expr, role(expr, keys), false), + // A key condition compared with a boolean is read by key, and the + // comparison goes with its disjunct, in an OR as anywhere else + // (measured 2026-11-03: `pk = 'a' <> true OR pk = 'b'` reads both + // partitions). Any other condition compared with a boolean is the + // condition when the comparison keeps it, and is read as a NOT when + // it negates it. + WhereExpr::ComparedWithBoolean(inner, comparison) => { + if is_key_condition(inner, keys) { + let mut split = split(inner, keys, value_of); + for branch in &mut split.branches { + branch.conjuncts.push(Conjunct { + term: None, + role: Role::Compared(*comparison), + }); + } + split + } else if comparison.negates() { + Split::one(expr, role(expr, keys), false) + } else { + split(inner, keys, value_of) + } + } + other => { + let role = role(other, keys); + Split::one(other, role, role.splits()) + } + } +} + +/// The disjuncts of an AND: each disjunct of one side with each disjunct of +/// the other, in statement order, the first [`MAX_BRANCHES`] of them. +fn product<'a>(left: &[Branch<'a>], right: &[Branch<'a>]) -> Vec> { + let mut branches = Vec::new(); + 'product: for l in left { + for r in right { + if branches.len() == MAX_BRANCHES { + break 'product; + } + let mut conjuncts = l.conjuncts.clone(); + conjuncts.extend(r.conjuncts.iter().copied()); + branches.push(Branch { + conjuncts, + has_key: l.has_key || r.has_key, + }); + } + } + branches +} + +/// The key read of one disjunct, or `None` when the disjunct is a scan. +pub(super) fn key_read<'a>( + conjuncts: &[Conjunct<'a>], + value_of: &dyn Fn(&Operand) -> Option, +) -> Option> { + let mut partition: Option<&'a Operand> = None; + let mut sort: Option> = None; + let mut filter = Vec::new(); + let mut comparisons = Vec::new(); + for conjunct in conjuncts { + match conjunct.role { + Role::Partition(value) => match partition { + Some(existing) if !same_value(existing, value, value_of) => return None, + Some(_) => {} + None => partition = Some(value), + }, + Role::Compared(comparison) => comparisons.push(comparison), + Role::Sort(condition) => match &sort { + Some(existing) if !same_condition(existing, &condition, value_of) => return None, + Some(_) => {} + None => sort = Some(condition), + }, + Role::OnKey => return None, + Role::KeyAndPath(_) | Role::IntoKey(_) | Role::Filter => { + filter.extend(conjunct.term); + } + } + } + Some(KeyRead { + partition: partition?, + sort, + filter, + comparisons, + }) +} + +fn same_condition( + a: &SortCondition<'_>, + b: &SortCondition<'_>, + value_of: &dyn Fn(&Operand) -> Option, +) -> bool { + match (a, b) { + (SortCondition::Compare(op1, v1), SortCondition::Compare(op2, v2)) => { + op1 == op2 && same_value(v1, v2, value_of) + } + (SortCondition::Between(l1, h1), SortCondition::Between(l2, h2)) => { + same_value(l1, l2, value_of) && same_value(h1, h2, value_of) + } + (SortCondition::BeginsWith(p), SortCondition::BeginsWith(q)) => same_value(p, q, value_of), + _ => false, + } +} diff --git a/crates/core/src/partiql/plan/tests.rs b/crates/core/src/partiql/plan/tests.rs new file mode 100644 index 000000000..07eecb05a --- /dev/null +++ b/crates/core/src/partiql/plan/tests.rs @@ -0,0 +1,516 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Unit tests for the reading of a WHERE clause. + +use super::*; +use crate::partiql::ast::Statement; +use crate::partiql::parse; + +fn where_of(sql: &str) -> WhereExpr { + match parse(sql).unwrap() { + Statement::Select(s) => s.where_clause.unwrap(), + _ => panic!("not a select"), + } +} + +const KEYS: KeyNames<'static> = KeyNames { + partition: "pk", + sort: Some("sk"), +}; + +const HASH_ONLY: KeyNames<'static> = KeyNames { + partition: "pk", + sort: None, +}; + +fn literal(op: &Operand) -> Option { + match op { + Operand::Literal(v) => Some(v.clone()), + _ => None, + } +} + +fn select(condition: &str) -> String { + format!("SELECT * FROM \"t\" WHERE {condition}") +} + +/// The reads of a WHERE as (partition, sort condition kind, filter +/// length) and whether every disjunct reads by key. +fn reading(condition: &str, keys: KeyNames<'_>) -> (Vec, bool) { + let w = where_of(&select(condition)); + let analysis = WhereAnalysis::new(&w, keys, &literal); + let reads = analysis + .reads() + .iter() + .map(|read| { + let value = |op: &Operand| match literal(op) { + Some(AttributeValue::S(s)) => s, + Some(AttributeValue::N(n)) => n, + other => format!("{other:?}"), + }; + let sort = match &read.sort { + None => String::new(), + Some(SortCondition::Compare(op, v)) => format!(" sk {op:?} {}", value(v)), + Some(SortCondition::Between(l, h)) => { + format!(" sk between {} {}", value(l), value(h)) + } + Some(SortCondition::BeginsWith(p)) => format!(" sk begins {}", value(p)), + }; + format!("{}{sort} +{}", value(read.partition), read.filter.len()) + }) + .collect(); + (reads, analysis.is_keyed()) +} + +#[test] +fn a_disjunct_reads_by_key_with_one_partition_equality() { + assert_eq!(reading("pk = 'a'", KEYS), (vec!["a +0".to_owned()], true)); + assert_eq!( + reading("pk = 'a' AND pk = 'a' AND x = 1", KEYS), + (vec!["a +1".to_owned()], true) + ); + assert_eq!( + reading("'a' = pk", HASH_ONLY), + (vec!["a +0".to_owned()], true) + ); + for condition in [ + "pk = 'a' AND pk = 'b'", + "pk = 'a' AND pk = 1", + "pk = 1 AND pk > 'a'", + "pk = 1 AND NOT pk = 'a'", + "pk = 'a' AND pk <> 1", + "pk = 'a' AND size(pk) = 1", + "pk = 'a' AND attribute_exists(pk)", + "pk > 'a'", + "begins_with(pk, 'a')", + "pk BETWEEN 'a' AND 'b'", + "sk = 1", + "x = 1", + "NOT (pk = 1 AND sk = 1)", + "pk = a", + ] { + assert_eq!(reading(condition, KEYS), (vec![], false), "{condition}"); + } +} + +#[test] +fn the_sort_key_holds_one_key_condition_or_nothing() { + for (condition, read) in [ + ("pk = 'a' AND sk = 'x'", "a sk Eq x +0"), + ("pk = 'a' AND sk > 'x'", "a sk Gt x +0"), + ("pk = 'a' AND 'x' < sk", "a sk Gt x +0"), + ("pk = 'a' AND 'x' >= sk", "a sk Le x +0"), + ("pk = 'a' AND sk BETWEEN 'x' AND 'y'", "a sk between x y +0"), + ("pk = 'a' AND begins_with(sk, 'x')", "a sk begins x +0"), + ("pk = 'a' AND sk = 'x' AND 'x' = sk", "a sk Eq x +0"), + ("1 = sk AND 'a' = pk", "a sk Eq 1 +0"), + // A key compared with a path, or a path into a key, is a filter. + ("pk = 'a' AND sk = 1 AND sk = a", "a sk Eq 1 +1"), + ("pk = 'a' AND sk.x = 1", "a +1"), + ] { + assert_eq!( + reading(condition, KEYS), + (vec![read.to_owned()], true), + "{condition}" + ); + } + for condition in [ + "pk = 'a' AND sk = 'x' AND sk = 1", + "pk = 'a' AND sk < 1 AND sk > 0", + "pk = 'a' AND sk > 'a' AND sk = 1", + "pk = 'a' AND begins_with(sk, 'x') AND sk = 1", + "pk = 'a' AND begins_with(sk, 'x') AND begins_with(sk, 'y')", + "pk = 'a' AND sk = 1 AND NOT sk = 'y'", + "pk = 'a' AND sk = 1 AND contains(sk, 'x')", + "pk = 'a' AND sk = 1 AND attribute_exists(sk)", + "pk = 'a' AND sk = 1 AND sk <> 'y'", + "pk = 'a' AND sk = 1 AND sk IS NOT MISSING", + "pk = 'a' AND size(sk) = 1", + "pk = 'a' AND NOT sk IN ['1', '3']", + "pk = 'a' AND NOT sk BETWEEN 'a' AND 'b'", + // An IN with a path in its list is one more predicate on the key. + "pk = 'a' AND sk IN [a]", + "pk = 'a' AND sk IN ['x', a]", + "pk = 'a' AND pk IN [a]", + ] { + assert_eq!(reading(condition, KEYS), (vec![], false), "{condition}"); + } +} + +#[test] +fn a_key_condition_compared_with_a_boolean_is_read_by_key() { + // The read, and whether it returns nothing: the literal alone decides. + let reads = |condition: &str, keys: KeyNames<'_>| { + let w = where_of(&select(condition)); + let analysis = WhereAnalysis::new(&w, keys, &literal); + let nothing: Vec = analysis + .reads() + .iter() + .map(KeyRead::returns_nothing) + .collect(); + (reading(condition, keys), nothing) + }; + let one = |read: &str| (vec![read.to_owned()], true); + for (condition, keys, read, nothing) in [ + ("pk = 'a' = false", HASH_ONLY, one("a +0"), vec![true]), + ("pk = 'a' <> false", HASH_ONLY, one("a +0"), vec![true]), + ("pk = 'a' <> true", HASH_ONLY, one("a +0"), vec![false]), + ("pk = 'a' = true", HASH_ONLY, one("a +0"), vec![false]), + ("'a' = pk = false", KEYS, one("a +0"), vec![true]), + ("(pk = 'a') = false", KEYS, one("a +0"), vec![true]), + ( + "pk = 'a' AND sk = 'x' = false", + KEYS, + one("a sk Eq x +0"), + vec![true], + ), + ( + "pk = 'a' AND sk > 'x' <> true", + KEYS, + one("a sk Gt x +0"), + vec![false], + ), + ( + "(pk = 'a' AND sk = 'x') <> true", + KEYS, + one("a sk Eq x +0"), + vec![false], + ), + ("pk = 'a' <> true AND n = 1", KEYS, one("a +1"), vec![false]), + ("pk = 'a' = false AND n = 1", KEYS, one("a +1"), vec![true]), + ( + "pk = 'a' = false AND sk = 'x'", + KEYS, + one("a sk Eq x +0"), + vec![true], + ), + ( + "pk = 'a' AND pk = 'a' = false", + KEYS, + one("a +0"), + vec![true], + ), + ( + "pk = 'a' = false OR pk = 'b'", + KEYS, + (vec!["a +0".to_owned(), "b +0".to_owned()], true), + vec![true, false], + ), + ( + "pk = 'a' <> true OR pk = 'b'", + KEYS, + (vec!["a +0".to_owned(), "b +0".to_owned()], true), + vec![false, false], + ), + ] { + assert_eq!(reads(condition, keys), (read, nothing), "{condition}"); + } + // A condition that is no key condition keeps its meaning: an IN, a NOT, + // two comparisons with false, a filter, and another partition value. + for condition in [ + "pk IN ['a'] <> true", + "pk IN ['a'] = false", + "NOT pk = 'a'", + "pk = 'a' = false = false", + "pk = 'b' AND pk = 'a' = false", + ] { + assert_eq!(reading(condition, KEYS), (vec![], false), "{condition}"); + } + assert_eq!( + reading("pk = 'a' AND n = 1 <> true", KEYS), + (vec!["a +1".to_owned()], true) + ); +} + +#[test] +fn a_key_condition_compared_with_a_boolean_is_written_by_its_literal() { + let keyed = |condition: &str| keyed_condition(&where_of(&select(condition)), KEYS); + let pk_a = where_of(&select("pk = 'a'")); + // Compared with true: the key condition itself. + assert_eq!(keyed("pk = 'a' <> true"), pk_a); + assert_eq!(keyed("pk = 'a' = true"), pk_a); + // Compared with false: the key condition and its negation, which no item + // passes. + let never = WhereExpr::And( + Box::new(pk_a.clone()), + Box::new(WhereExpr::Not(Box::new(pk_a.clone()))), + ); + assert_eq!(keyed("pk = 'a' <> false"), never); + assert_eq!( + keyed("pk = 'a' = false OR n = 1"), + WhereExpr::Or(Box::new(never), Box::new(where_of(&select("n = 1")))) + ); + // Anything else is left as it is. + let w = where_of(&select("pk IN ['a'] <> true")); + assert_eq!(keyed_condition(&w, KEYS), w); +} + +#[test] +fn a_not_over_an_and_or_an_or_stays_in_the_filter() { + for (condition, read) in [ + ("pk = 'a' AND NOT (sk = 'x' AND a = 1)", "a +1"), + ("pk = 'a' AND NOT (sk = 'x' OR a = 1)", "a +1"), + ("pk = 'a' AND NOT NOT (sk = 'x' AND a = 1)", "a +1"), + ( + "pk = 'a' AND sk = 'y' AND NOT (pk = 'b' AND a = 1)", + "a sk Eq y +1", + ), + ("pk = 'a' AND NOT (a = 1 AND b = 2)", "a +1"), + ] { + assert_eq!( + reading(condition, KEYS), + (vec![read.to_owned()], true), + "{condition}" + ); + } + // A NOT over one condition on a key makes a scan. + for condition in [ + "pk = 'a' AND NOT (sk = 'x')", + "pk = 'a' AND NOT NOT sk = 'x'", + ] { + assert_eq!(reading(condition, KEYS), (vec![], false), "{condition}"); + } +} + +#[test] +fn key_values_are_ordered_by_bytes_or_by_number() { + let s = |v: &str| AttributeValue::S(v.to_owned()); + let n = |v: &str| AttributeValue::N(v.to_owned()); + let compare_key_values = |a: &AttributeValue, b: &AttributeValue| { + Bound::from_value(a)?.compare(&Bound::from_value(b)?) + }; + assert_eq!(compare_key_values(&s("a"), &s("zz")), Some(Ordering::Less)); + assert_eq!( + compare_key_values(&s("b"), &s("a")), + Some(Ordering::Greater) + ); + assert_eq!( + compare_key_values(&n("10"), &n("9")), + Some(Ordering::Greater) + ); + assert_eq!( + compare_key_values(&n("1.0"), &n("1")), + Some(Ordering::Equal) + ); + assert_eq!( + compare_key_values( + &AttributeValue::B(vec![0xff]), + &AttributeValue::B(vec![0x01, 0x02]) + ), + Some(Ordering::Greater) + ); + assert_eq!(compare_key_values(&s("1"), &n("1")), None); +} + +#[test] +fn disjuncts_are_read_in_order_until_the_first_scan() { + for (condition, reads, keyed) in [ + ("a = 1 OR pk = 1", vec![], false), + ("pk = 1 OR a = 1", vec!["1 +0"], false), + ("pk = 'a' OR a = 1 OR pk = 1", vec!["a +0"], false), + ("pk = 'a' OR pk = 1 OR a = 1", vec!["a +0", "1 +0"], false), + ("pk = 'a' AND (pk = 1 OR a = 1)", vec![], false), + ( + "pk = 'a' AND (a = 1 OR sk = 1)", + vec!["a +1", "a sk Eq 1 +0"], + true, + ), + ("pk = 'a' AND (sk = 'x' OR a = 1) AND sk = 1", vec![], false), + ( + "pk IN [1, 'b'] AND sk = 1", + vec!["1 sk Eq 1 +0", "b sk Eq 1 +0"], + true, + ), + ( + "pk = 'a' AND sk IN ['x', 'y']", + vec!["a sk Eq x +0", "a sk Eq y +0"], + true, + ), + ( + "(pk = 1 AND sk = 'x') OR (pk = 'b' AND sk = 1)", + vec!["1 sk Eq x +0", "b sk Eq 1 +0"], + true, + ), + ("pk = 'a' OR sk = 1", vec!["a +0"], false), + ( + "pk = 'a' AND sk = 'x' OR sk = 1", + vec!["a sk Eq x +0"], + false, + ), + // An OR with no key attribute on either side is one condition. + ("pk = 'a' AND (x = 1 OR y = 2)", vec!["a +1"], true), + ] { + let reads: Vec = reads.into_iter().map(str::to_owned).collect(); + assert_eq!(reading(condition, KEYS), (reads, keyed), "{condition}"); + } +} + +#[test] +fn the_shared_filter_is_the_conditions_beside_the_keys() { + let shared = |condition: &str| { + let w = where_of(&select(condition)); + WhereAnalysis::new(&w, KEYS, &literal) + .shared_filter() + .map(|f| f.len()) + }; + assert_eq!(shared("pk IN ['a', 'b'] AND x = 1 AND y = 2"), Some(2)); + assert_eq!(shared("(pk = 'a' AND x = 1) OR pk = 'b'"), Some(0)); + assert_eq!(shared("pk = 'a' AND (x = 1 OR y = 2)"), Some(1)); + assert_eq!(shared("pk = 'a' AND x = 1 OR y = 1"), None); +} + +#[test] +fn a_key_against_a_path_or_into_a_key_is_named_in_the_filter() { + let refusal = |condition: &str, keys: KeyNames<'_>| { + let w = where_of(&select(condition)); + let analysis = WhereAnalysis::new(&w, keys, &literal); + let filter = analysis.shared_filter().unwrap(); + scalar_key_refusal(&filter, keys).or_else(|| filter_key_refusal(&filter, keys)) + }; + let path = |key: &str| { + Some(format!( + "Filter Expression can only contain non-primary key attributes: Primary key \ + attribute: {key}" + )) + }; + let scalar = |key: &str| { + Some(format!( + "Key attributes must be scalars; list random access '[]' and map lookup '.' \ + are not allowed: Key {key}" + )) + }; + for (condition, want) in [ + ("pk = 'a' AND sk = a", path("sk")), + ("pk = 'a' AND a = sk", path("sk")), + ("pk = 'a' AND sk = 'x' AND sk = a", path("sk")), + ("pk = 'a' AND a = 'x' AND sk = a", path("sk")), + ("pk = 'a' AND sk = pk", path("sk")), + ("pk = 'a' AND pk = a", path("pk")), + ("pk = 'a' AND sk.x = 1", scalar("sk")), + ("pk = 'a' AND sk[0] = 1", scalar("sk")), + ("pk = 'a' AND a = 1", None), + ("pk = 'a' AND NOT (sk = 'x' AND a = 1)", path("sk")), + ("pk = 'a' AND NOT (a = 1 OR pk = 'b')", path("pk")), + // Under a NOT over an AND or an OR, a path into a key attribute + // gets the filter refusal. + ("pk = 'a' AND NOT (a = 1 AND sk.x = 1)", path("sk")), + ("pk = 'a' AND NOT (sk.q = 1 AND n = 1)", path("sk")), + ("pk = 'a' AND NOT (n = 1 OR sk[0] = 1)", path("sk")), + ("pk = 'a' AND NOT (pk.q = 1 AND n = 1)", path("pk")), + ("pk = 'a' AND NOT (a = 1 AND b = 2)", None), + // Naming both keys, it names the sort key. + ("pk = 'a' AND NOT (sk.q = 1 AND pk = b)", path("sk")), + ("pk = 'a' AND NOT (pk = b AND sk.q = 1)", path("sk")), + ] { + assert_eq!(refusal(condition, KEYS), want, "{condition}"); + } + assert_eq!(refusal("pk = 'a' AND a = pk", HASH_ONLY), path("pk")); + assert_eq!(refusal("pk = 'a' AND pk.x = 1", HASH_ONLY), scalar("pk")); +} + +#[test] +fn a_path_into_a_key_is_refused_anywhere_in_the_where() { + let refusal = |condition: &str| { + let w = where_of(&select(condition)); + WhereAnalysis::new(&w, KEYS, &literal).path_into_key_refusal(KEYS) + }; + let scalar = |key: &str| Some(scalar_message(key)); + for (condition, want) in [ + // In a scan, and in a disjunct read by key after one. + ("sk.x = 1", scalar("sk")), + ("x = 1 OR (pk = 'a' AND sk.x = 1)", scalar("sk")), + // In the filter of a disjunct read by key. + ("pk = 'a' AND sk.x = 1", scalar("sk")), + ("(pk = 'a' AND sk.x = 1) OR x = 1", scalar("sk")), + ("pk = 'a' AND pk[0] = 1", scalar("pk")), + // The disjuncts read by key come first, then statement order. + ("(pk = 'a' OR sk.x = 1) AND pk.y = 1", scalar("pk")), + ("x = 1 OR sk.x = 1 OR pk.y = 1", scalar("sk")), + // Under a NOT over an AND or an OR it is the filter refusal's. + ("pk = 'a' AND NOT (sk.q = 1 AND n = 1)", None), + ("pk = 'a' AND sk = 'x'", None), + ("x = 1 OR sk = a", None), + ] { + assert_eq!(refusal(condition), want, "{condition}"); + } +} + +#[test] +fn disjuncts_are_read_by_partition_then_by_where_their_range_starts() { + let order = |condition: &str, keys: KeyNames<'_>| { + let w = where_of(&select(condition)); + let analysis = WhereAnalysis::new(&w, keys, &literal); + let mut reads: Vec<(usize, &KeyRead<'_>)> = analysis.reads().iter().enumerate().collect(); + reads.sort_by(|a, b| read_order(a.1, b.1, &literal)); + reads.into_iter().map(|(i, _)| i).collect::>() + }; + assert_eq!(order("pk IN ['c', 'a', 'b']", HASH_ONLY), vec![1, 2, 0]); + assert_eq!(order("pk IN [10, 9, 100]", HASH_ONLY), vec![1, 0, 2]); + assert_eq!( + order("(pk = 'b' AND sk = 'x') OR (pk = 'a' AND sk > 'a')", KEYS), + vec![1, 0] + ); + assert_eq!( + order( + "(pk = 'a' AND sk > 'y') OR (pk = 'a' AND sk = 'x') OR (pk = 'a' AND sk < 'b')", + KEYS + ), + vec![2, 1, 0] + ); + assert_eq!( + order("pk = 'b' OR (pk = 'a' AND sk BETWEEN 'x' AND 'y')", KEYS), + vec![1, 0] + ); + // Values of two types keep statement order. + assert_eq!(order("pk = 'b' OR pk = 1", HASH_ONLY), vec![0, 1]); +} + +#[test] +fn a_write_condition_pins_a_key_written_either_way_round() { + let equality = |condition: &str| { + let w = where_of(&select(condition)); + key_equality(&w, KEYS).map(|e| match e { + KeyEquality::Partition(v) => ("pk", literal(v)), + KeyEquality::Sort(v) => ("sk", literal(v)), + }) + }; + let s = |v: &str| Some(AttributeValue::S(v.to_owned())); + assert_eq!(equality("pk = 'a'"), Some(("pk", s("a")))); + assert_eq!(equality("'a' = pk"), Some(("pk", s("a")))); + assert_eq!(equality("'x' = sk"), Some(("sk", s("x")))); + // A comparison with a boolean that keeps the condition is the condition. + assert_eq!(equality("pk = 'a' = true"), Some(("pk", s("a")))); + assert_eq!(equality("pk = 'a' <> false"), Some(("pk", s("a")))); + for condition in [ + "sk > 'x'", + "pk <> 'a'", + "pk = a", + "x = 'a'", + "pk IN ['a']", + "pk = 'a' = false", + "pk = 'a' <> true", + ] { + assert_eq!(equality(condition), None, "{condition}"); + } +} + +#[test] +fn values_are_the_same_by_value() { + let n = |v: &str| Operand::Literal(AttributeValue::N(v.to_owned())); + let s = |v: &str| Operand::Literal(AttributeValue::S(v.to_owned())); + assert!(same_value(&n("1"), &n("1.0"), &literal)); + assert!(!same_value(&n("1"), &s("1"), &literal)); + assert!(same_value( + &Operand::Parameter(0), + &Operand::Parameter(0), + &literal + )); + assert!(!same_value( + &Operand::Parameter(0), + &Operand::Parameter(1), + &literal + )); +} + +mod overlap; diff --git a/crates/core/src/partiql/plan/tests/overlap.rs b/crates/core/src/partiql/plan/tests/overlap.rs new file mode 100644 index 000000000..98208239e --- /dev/null +++ b/crates/core/src/partiql/plan/tests/overlap.rs @@ -0,0 +1,215 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Unit tests for the overlap refusal, the bound on the disjuncts listed, +//! and the count of all of them. + +use super::*; +use crate::partiql::ast::Path; + +fn overlap(condition: &str) -> Result<(), &'static str> { + let w = where_of(&select(condition)); + WhereAnalysis::new(&w, KEYS, &literal).check_overlap(&literal) +} + +#[test] +fn a_non_key_branch_beside_a_sort_key_branch_on_one_partition_overlaps() { + for cond in ["x = '1'", "x <> '1'"] { + let condition = format!("pk = 'a' AND {cond} OR pk = 'a' AND sk = '2'"); + assert_eq!(overlap(&condition), Err(OVERLAP_MESSAGE), "{condition}"); + } + assert_eq!( + overlap("pk = 'a' AND (sk = a OR a = 'x')"), + Err(OVERLAP_MESSAGE) + ); + assert_eq!( + overlap("pk = 'a' AND NOT (sk = 'x' AND a = 1) OR pk = 'a' AND sk = 'y'"), + Err(OVERLAP_MESSAGE) + ); +} + +#[test] +fn a_negated_or_not_equal_sort_key_branch_makes_a_scan() { + for left in [ + "pk = 'p' AND NOT sk = '1'", + "pk = 'p' AND sk <> '1'", + "pk = 'p' AND NOT begins_with(sk, 'x')", + "pk = 'p' AND NOT sk IN ['1', '3']", + "pk = 'p' AND NOT sk > '5'", + "pk = 'p' AND NOT (sk = '1')", + ] { + let condition = format!("{left} OR pk = 'p' AND sk = '2'"); + assert_eq!(overlap(&condition), Ok(()), "{condition}"); + } + assert_eq!(overlap("pk = 'p' OR pk = 'p' AND NOT sk = '1'"), Ok(())); +} + +#[test] +fn distinct_sort_key_values_and_partitions_do_not_overlap() { + assert_eq!(overlap("pk = 'p' AND sk IN ['s1', 's3']"), Ok(())); + assert_eq!(overlap("pk = 'a' OR pk = 'b'"), Ok(())); + assert_eq!( + overlap("pk = 'a' AND sk < '2' OR pk = 'a' AND sk > '2'"), + Ok(()) + ); + assert_eq!( + overlap("pk = 'a' AND sk <= '2' OR pk = 'a' AND '2' <= sk"), + Err(OVERLAP_MESSAGE) + ); +} + +/// `k` copies of `( OR )` joined with AND: 2^k branches. +fn anded_ors(k: usize, a: &str, b: &str) -> String { + let pair = format!("({a} OR {b})"); + vec![pair; k].join(" AND ") +} + +#[test] +fn forty_anded_ors_return_at_once() { + let started = std::time::Instant::now(); + assert_eq!(overlap(&anded_ors(40, "pk = 'a'", "pk = 'b'")), Ok(())); + // 2^40 disjuncts that read partition 'a', each one by key. + assert_eq!( + overlap(&anded_ors(40, "pk = 'a'", "pk = 'a'")), + Err(OVERLAP_MESSAGE) + ); + let values = |n: usize| { + (0..n) + .map(|i| format!("'v{i}'")) + .collect::>() + .join(", ") + }; + let ins = format!("pk IN [{}] AND sk IN [{}]", values(40), values(40)); + assert_eq!(overlap(&ins), Ok(())); + assert!( + started.elapsed() < std::time::Duration::from_secs(2), + "{:?}", + started.elapsed() + ); +} + +#[test] +fn the_branch_count_is_bounded() { + // 2^10 branches are listed and compared; past MAX_BRANCHES the first + // ones are compared, and the statement still reads by key when every + // disjunct does. + assert_eq!(1 << 10, MAX_BRANCHES); + for k in [10, 11] { + let w = where_of(&select(&anded_ors(k, "pk = 'a'", "pk = 'a'"))); + let analysis = WhereAnalysis::new(&w, KEYS, &literal); + assert_eq!(analysis.check_overlap(&literal), Err(OVERLAP_MESSAGE)); + assert_eq!(analysis.reads().len(), MAX_BRANCHES, "{k}"); + assert_eq!(analysis.disjunct_count(), 1 << k); + } + // An IN list this long is refused by the parser (more than 100 + // members), so the condition is built here. + let ins = |n: usize| pk_in(vec![AttributeValue::S("a".to_owned()); n]); + let overlap_of = |w: &WhereExpr| WhereAnalysis::new(w, KEYS, &literal).check_overlap(&literal); + assert_eq!(overlap_of(&ins(MAX_BRANCHES)), Err(OVERLAP_MESSAGE)); + assert_eq!(overlap_of(&ins(MAX_BRANCHES + 1)), Err(OVERLAP_MESSAGE)); +} + +/// `pk IN []`. +fn pk_in(values: Vec) -> WhereExpr { + WhereExpr::In( + Operand::Path(Path { + root: "pk".to_owned(), + steps: Vec::new(), + }), + values.into_iter().map(Operand::Literal).collect(), + ) +} + +#[test] +fn past_the_branch_count_the_first_disjuncts_are_still_read() { + let w = pk_in( + (0..=MAX_BRANCHES) + .map(|i| AttributeValue::N(i.to_string())) + .collect(), + ); + let analysis = WhereAnalysis::new(&w, KEYS, &literal); + assert_eq!(analysis.reads().len(), MAX_BRANCHES); + assert!(analysis.is_keyed()); + assert_eq!(analysis.disjunct_count(), 1025); +} + +/// Whether a WHERE reads by key, and how many disjuncts it splits into. +fn tally(condition: &str, keys: KeyNames<'_>) -> (bool, u64) { + let w = where_of(&select(condition)); + let analysis = WhereAnalysis::new(&w, keys, &literal); + (analysis.is_keyed(), analysis.disjunct_count()) +} + +/// ` IN [000, 001, ...]` with `n` values. +fn listed(name: &str, prefix: &str, n: usize) -> String { + let values: Vec = (0..n).map(|i| format!("'{prefix}{i:03}'")).collect(); + format!("{name} IN [{}]", values.join(", ")) +} + +#[test] +fn every_disjunct_is_counted_and_read_by_key_past_the_branch_count() { + let products = format!("{} AND {}", listed("pk", "v", 100), listed("sk", "s", 100)); + for (condition, want) in [ + (listed("pk", "v", 51), (true, 51)), + ( + format!("{} AND x = 1 AND y = 2", listed("pk", "v", 50)), + (true, 50), + ), + ( + format!("{} AND {}", listed("pk", "v", 17), listed("sk", "s", 3)), + (true, 51), + ), + ( + format!("{} AND (sk = 's' OR sk = 't')", listed("pk", "v", 26)), + (true, 52), + ), + ( + format!("{} OR {}", listed("pk", "v", 26), listed("pk", "w", 25)), + (true, 51), + ), + (products.clone(), (true, 10_000)), + // A disjunct that is a scan makes the statement one, past the + // listed disjuncts too. + (format!("({products}) OR x = 1"), (false, 10_001)), + (format!("x = 1 OR {}", listed("pk", "v", 51)), (false, 52)), + ( + format!("{} AND {}", listed("pk", "v", 100), listed("pk", "w", 100)), + (false, 10_000), + ), + ( + format!("({products}) AND (sk = 'a' OR pk = 'b')"), + (false, 20_000), + ), + ( + format!("({products}) AND (n = 1 OR pk <> 'b')"), + (false, 20_000), + ), + ( + format!("({products}) OR (pk = 'a' AND pk = 'b')"), + (false, 10_001), + ), + ( + format!("({products}) OR (pk = 'a' AND sk = 'x')"), + (true, 10_001), + ), + // The first 1024 of these read by key, the 1101st holds two sort + // key conditions and no partition key. + ( + format!( + "({} OR sk = 'q') AND {}", + listed("pk", "v", 11), + listed("sk", "s", 100) + ), + (false, 1_200), + ), + ] { + let shown: String = condition.chars().take(50).collect(); + assert_eq!(tally(&condition, KEYS), want, "{shown}"); + } + assert_eq!(tally(&listed("pk", "v", 100), HASH_ONLY), (true, 100)); +} + +#[test] +fn a_branch_without_a_partition_equality_makes_a_scan() { + assert_eq!(overlap("pk = 'a' OR x = 1"), Ok(())); +} diff --git a/crates/core/src/validation/mod.rs b/crates/core/src/validation/mod.rs index a849f2c95..18a3bd00a 100755 --- a/crates/core/src/validation/mod.rs +++ b/crates/core/src/validation/mod.rs @@ -2266,6 +2266,15 @@ pub const UPDATE_LIST_INDEX_TARGET_COST: usize = 1; /// `finished` is the whole finished item; `actions` are the statement's clauses /// with `#name` placeholders still in place, resolved here through `maps`. /// +/// An update translated from a PartiQL `UPDATE` (`maps.finished_item_sizing`) +/// is sized by the finished item alone, as +/// [`validate_transact_update_item_size`] sizes a transacted Update: at a +/// three-byte key, `UPDATE ... SET p = ?` writes a finished item of exactly +/// 409,600 bytes that `UpdateItem` with `SET p = :p` refuses, two `SET` +/// clauses or an `if_not_exists` add nothing, and one byte more is refused +/// with the update wording, in `ExecuteStatement` and `BatchExecuteStatement` +/// alike (measured us-east-1, 2026-11-04). +/// /// # Errors /// /// Returns `DynamoDbError::ValidationException` with the update wording when @@ -2276,6 +2285,9 @@ pub fn validate_update_item_size( maps: &crate::expression::ExpressionMaps, max_bytes: usize, ) -> Result<(), DynamoDbError> { + if maps.finished_item_sizing { + return validate_transact_update_item_size(finished, max_bytes); + } let statement = if maps.legacy_attribute_updates { legacy_update_statement_size(finished, actions, maps)? } else { @@ -4800,6 +4812,21 @@ mod update_size_tests { assert!(validate_update_item_size(&finished, &actions, &maps, 5).is_err()); } + #[test] + fn a_partiql_update_is_sized_by_its_finished_item_alone() { + // The 33-byte statement of the test above is refused at 32 as an + // UpdateItem. Translated from a PartiQL UPDATE, only the finished item + // ({p: 10 bytes}, 11 bytes) counts. + let (actions, mut maps) = build("SET p = :v", &[("v", s("0123456789"))]); + maps.finished_item_sizing = true; + let mut finished = Item::new(); + finished.insert("p".to_owned(), s("0123456789")); + + assert!(validate_update_item_size(&finished, &actions, &maps, 11).is_ok()); + let err = validate_update_item_size(&finished, &actions, &maps, 10).unwrap_err(); + assert_eq!(err.to_string(), ITEM_SIZE_TO_UPDATE_EXCEEDED); + } + #[test] fn legacy_attribute_updates_size_the_finished_value_not_the_statement() { // The legacy form charges 3 plus the name and finished value of each diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml index 9f58bea6e..4cc448eb4 100755 --- a/crates/engine/Cargo.toml +++ b/crates/engine/Cargo.toml @@ -22,3 +22,15 @@ sha2 = "0.10" hex = "0.4" tokio = { workspace = true } futures = { workspace = true } + +[dev-dependencies] +# A storage engine for the tests that need an `OperationContext`. +extenddb-storage-sqlite = { workspace = true } + +[features] +default = ["partiql"] +# PartiQL surface (ExecuteStatement, BatchExecuteStatement, ExecuteTransaction), +# a pure translation over the existing data-plane handlers. Removable: with this +# off, the three operations are not known and the server answers them with +# UnknownOperationException exactly as before. +partiql = ["extenddb-core/partiql"] diff --git a/crates/engine/src/item_authorizer.rs b/crates/engine/src/item_authorizer.rs new file mode 100644 index 000000000..8b87a2a8a --- /dev/null +++ b/crates/engine/src/item_authorizer.rs @@ -0,0 +1,57 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The authorization of item reads and writes that a request names inside +//! its body. +//! +//! The server authorizes most operations before dispatch, from the request +//! fields that name the table they read or write. An operation that names its +//! tables inside its body, as the PartiQL operations do in their statements, +//! asks the [`ItemAuthorizer`] the server puts in its +//! [`OperationContext`](crate::OperationContext) to allow each read or write +//! before it makes it. A context without one refuses every such read and +//! write. + +use std::future::Future; +use std::pin::Pin; + +use extenddb_core::error::DynamoDbError; +use extenddb_core::types::AttributeValue; + +/// One read or write of items, as the authorizer evaluates it: the action, +/// the resource, and the item-level condition keys the request carries for +/// it. A field that is `None` leaves its condition key absent. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct ItemAccess { + /// The action, without the `dynamodb:` prefix (`PartiQLSelect`). + pub action: &'static str, + /// The table read or written, by name. + pub table: String, + /// The index read, for a read of an index: the resource is then the + /// index, not the table. + pub index: Option, + /// The partition key values read or written (`dynamodb:LeadingKeys`). + pub partition_keys: Option>, + /// The top-level attribute names (`dynamodb:Attributes`). + pub attributes: Option>, + /// `dynamodb:Select`. + pub select: Option<&'static str>, + /// `dynamodb:ReturnValues`. + pub return_values: Option<&'static str>, + /// `dynamodb:FullTableScan`. + pub full_table_scan: Option, + /// `dynamodb:EnclosingOperation`. + pub enclosing_operation: Option<&'static str>, +} + +/// The answer an [`ItemAuthorizer`] gives. +pub type ItemAuthorization<'a> = + Pin> + Send + 'a>>; + +/// Allows or refuses the item reads and writes of one request, for the +/// identity that signed it. +pub trait ItemAuthorizer: Send + Sync { + /// `Ok` when `access` is allowed. A refusal is the error the request + /// answers with: `AccessDeniedException` with the IAM message. + fn authorize<'a>(&'a self, access: &'a ItemAccess) -> ItemAuthorization<'a>; +} diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 187f4f59f..88a28ed82 100755 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -24,8 +24,11 @@ mod get_item; mod import_export; mod import_export_io; mod index_helpers; +mod item_authorizer; mod legacy_filter; mod list_tables; +#[cfg(feature = "partiql")] +mod partiql; mod put_item; mod query; mod read_helpers; @@ -43,6 +46,8 @@ mod ttl; mod update_item; mod update_table; mod vector_gate; +#[cfg(feature = "partiql")] +use partiql::PartiqlExecutor; pub use batch_get_item::handle_batch_get_item; pub use batch_write_item::handle_batch_write_item; @@ -55,6 +60,7 @@ pub use describe_limits::handle_describe_limits; pub use describe_table::handle_describe_table; pub use get_item::handle_get_item; pub use import_export::{handle_export_table, handle_import_table}; +pub use item_authorizer::{ItemAccess, ItemAuthorization, ItemAuthorizer}; pub use list_tables::handle_list_tables; pub use put_item::handle_put_item; pub use query::handle_query; @@ -126,8 +132,24 @@ pub fn is_known_operation(operation: &str) -> bool { | "DescribeContinuousBackups" | "UpdateContinuousBackups" | "RestoreTableToPointInTime" + ) || is_known_partiql_operation(operation) +} + +/// PartiQL operations, known only when the `partiql` feature is on. With the +/// feature off this is always false, so the server answers them with +/// `UnknownOperationException`. +#[cfg(feature = "partiql")] +fn is_known_partiql_operation(operation: &str) -> bool { + matches!( + operation, + "ExecuteStatement" | "BatchExecuteStatement" | "ExecuteTransaction" ) } + +#[cfg(not(feature = "partiql"))] +fn is_known_partiql_operation(_operation: &str) -> bool { + false +} use serde::Serialize; /// Serialize an operation output to JSON, logging and sanitizing any serialization failure. @@ -370,6 +392,10 @@ pub struct OperationContext { /// read. When unset (e.g. unit tests), the engine falls back to direct /// storage lookups. pub table_key_info_lookup: Option>, + /// Authorizes the item reads and writes a request names inside its body + /// (the statements of the PartiQL operations), which the server does not + /// authorize before dispatch. `None` refuses every such read and write. + pub item_authorizer: Option>, } impl OperationContext { @@ -529,6 +555,12 @@ pub async fn dispatch( "RestoreTableToPointInTime" => backup::handle_restore_table_to_point_in_time(body, ctx) .await .map(DispatchResult::body_only), + #[cfg(feature = "partiql")] + "ExecuteStatement" => ctx.execute_statement(body).await, + #[cfg(feature = "partiql")] + "BatchExecuteStatement" => ctx.batch_execute_statement(body).await, + #[cfg(feature = "partiql")] + "ExecuteTransaction" => ctx.execute_transaction(body).await, _ => Err(DynamoDbError::UnknownOperationException(String::new())), } } diff --git a/crates/engine/src/partiql/authorize.rs b/crates/engine/src/partiql/authorize.rs new file mode 100644 index 000000000..7354f1cfd --- /dev/null +++ b/crates/engine/src/partiql/authorize.rs @@ -0,0 +1,311 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The authorization of each statement of a PartiQL request. +//! +//! The service authorizes a PartiQL request statement by statement, through +//! the [`ItemAuthorizer`] of the request's context, and needs no action for +//! the operation itself (measured 2026-11-03): +//! +//! * The action is the statement's type: `PartiQLSelect` (also for an +//! `EXISTS` check), `PartiQLInsert`, `PartiQLUpdate`, `PartiQLDelete`. The +//! resource is the table, or for a read of an index the index alone. +//! * A statement is authorized at a point among its checks: some come before +//! it, the others after it (see [`gate`]). +//! * The condition keys are those of [`access`]. An `ExecuteStatement` read +//! by key authorizes each of its disjuncts with its own partition value. A +//! refused disjunct is held back, the way a disjunct that fails a check +//! made when it is read is held back (see +//! `select::read_plan::check_reads`), and a statement whose every disjunct +//! is refused is refused at once. + +use extenddb_core::error::DynamoDbError; +use extenddb_core::partiql::Statement; +use extenddb_core::partiql::ast::{Operand, WhereExpr}; +use extenddb_core::partiql::plan::{KeyNames, KeyRead, WhereAnalysis, read_order}; +use extenddb_core::types::{ + AttributeValue, KeySchemaElement, TableKeyInfo, is_multipart_key_schema, +}; +use extenddb_storage::error::StorageError; + +use super::SingleRequest; +use super::common::{self, bound_value}; +use super::select::MAX_DECOMPOSED_READS; +use crate::create_table::storage_err_to_dynamo; +use crate::{ItemAccess, ItemAuthorizer, OperationContext}; + +pub(super) use access::written_value; +pub(super) use gate::Gate; + +use access::{Target, access}; + +mod access; +mod gate; + +/// The request a statement is a member of. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum Surface { + /// `ExecuteStatement`. + Statement, + /// `BatchExecuteStatement`. + Batch, + /// `ExecuteTransaction`. + Transaction, +} + +impl Surface { + /// The `dynamodb:EnclosingOperation` a statement of this request carries + /// (measured 2026-11-03: absent in `ExecuteStatement`, where + /// "ExecuteStatement" matches nothing). + fn enclosing_operation(self) -> Option<&'static str> { + match self { + Self::Statement => None, + Self::Batch => Some("BatchExecuteStatement"), + Self::Transaction => Some("ExecuteTransaction"), + } + } +} + +/// The decision on each disjunct of an `ExecuteStatement` SELECT read by +/// key, by partition value as the request writes it. Empty for every other +/// statement, which is authorized as a whole. +#[derive(Debug, Default)] +pub(super) struct Refused { + /// Whether the statement's disjuncts were authorized one by one. + by_disjunct: bool, + /// Each partition value authorized, with its refusal when it is refused. + by_partition: Vec<(Option, Option)>, +} + +impl Refused { + /// The refusal of the disjunct `read` of a statement of `req`, when it + /// is refused. A disjunct whose partition value was not authorized, one + /// past [`MAX_DECOMPOSED_READS`], is refused: its statement has more + /// disjuncts than may run, and is refused whatever the decision. + pub(super) fn refusal(&self, read: &KeyRead<'_>, req: &SingleRequest) -> Option { + if !self.by_disjunct { + return None; + } + match self.decision(read, req) { + Some(decision) => decision.clone(), + None => Some(unauthorized()), + } + } + + /// The decision on the partition value of `read`, when one was made. + fn decision(&self, read: &KeyRead<'_>, req: &SingleRequest) -> Option<&Option> { + let value = written_value(read.partition, req); + self.by_partition + .iter() + .find(|(partition, _)| *partition == value) + .map(|(_, decision)| decision) + } +} + +/// Authorize `stmt`, a member of a request of `surface`, whose table has the +/// key info `key_info` (`None` for a missing table). +/// +/// # Errors +/// +/// The refusal of the statement (`AccessDeniedException` with the IAM +/// message), or a failed read of the index the statement names. +pub(super) async fn statement( + ctx: &OperationContext, + stmt: &Statement, + key_info: Option<&TableKeyInfo>, + req: &SingleRequest, + surface: Surface, +) -> Result { + let schema = target_schema(ctx, stmt, key_info).await?; + let target = Target { + table_keys: key_info.map(|info| { + info.key_schema + .iter() + .map(|key| key.attribute_name.as_str()) + .collect() + }), + keys: schema.as_deref().and_then(single_attribute_keys), + }; + let value_of = |op: &Operand| bound_value(op, &req.parameters).cloned(); + let analysis = match (where_clause(stmt), target.keys) { + (Some(where_clause), Some(keys)) => Some(WhereAnalysis::new(where_clause, keys, &value_of)), + _ => None, + }; + let access = access(stmt, &target, analysis.as_ref(), req, surface); + let authorizer = ctx.item_authorizer.as_deref(); + match (stmt, surface, &analysis) { + (Statement::Select(_), Surface::Statement, Some(analysis)) if analysis.is_keyed() => { + each_disjunct(authorizer, &access, analysis, req, &value_of).await + } + _ => { + authorize_access(authorizer, &access).await?; + Ok(Refused::default()) + } + } +} + +/// Ask `authorizer` to allow `access`, refusing it when there is none. +/// +/// # Errors +/// +/// The authorizer's refusal, or `AccessDeniedException` without one. +pub(super) async fn authorize_access( + authorizer: Option<&dyn ItemAuthorizer>, + access: &ItemAccess, +) -> Result<(), DynamoDbError> { + if let Some(authorizer) = authorizer { + authorizer.authorize(access).await + } else { + tracing::error!( + action = access.action, + table = %access.table, + "no item authorizer in the request context; refusing the statement" + ); + Err(unauthorized()) + } +} + +/// The refusal of what no authorizer allowed. +fn unauthorized() -> DynamoDbError { + DynamoDbError::AccessDeniedException("Not authorized to perform this action".to_owned()) +} + +/// Authorize each disjunct of an `ExecuteStatement` SELECT read by key with +/// its own partition value (measured 2026-11-03: under `ForAllValues` +/// `["a"]`, `pk IN ['a', 'b']` and `pk = 'a' OR pk = 'b'` return the items +/// of a with a NextToken, and `pk = 'b'` is refused). A partition value is +/// authorized once, whatever number of disjuncts read it. +/// +/// At most [`MAX_DECOMPOSED_READS`] partition values are authorized, the +/// most disjuncts a statement that runs may have: a statement with more is +/// refused before it is authorized (see `select::read_plan::check_keys`), +/// unless a check that comes ahead of that refusal fails. Then the answer +/// only tells AccessDenied from that check's refusal, and is AccessDenied +/// when the values authorized are all refused. +/// +/// # Errors +/// +/// The refusal of the disjunct read last in key order when every disjunct is +/// refused, as a statement whose every disjunct fails a check made when it is +/// read gets that disjunct's refusal; an error other than a refusal at once. +async fn each_disjunct( + authorizer: Option<&dyn ItemAuthorizer>, + access: &ItemAccess, + analysis: &WhereAnalysis<'_>, + req: &SingleRequest, + value_of: &(dyn Fn(&Operand) -> Option + Sync), +) -> Result { + let mut refused = Refused { + by_disjunct: true, + by_partition: Vec::new(), + }; + let mut allowed = false; + for read in analysis.reads() { + let partition = written_value(read.partition, req); + if refused + .by_partition + .iter() + .any(|(decided, _)| *decided == partition) + { + continue; + } + if refused.by_partition.len() == MAX_DECOMPOSED_READS { + break; + } + let one = ItemAccess { + partition_keys: partition.clone().map(|value| vec![value]), + ..access.clone() + }; + let decision = match authorize_access(authorizer, &one).await { + Ok(()) => { + allowed = true; + None + } + Err(error @ DynamoDbError::AccessDeniedException(_)) => Some(error), + Err(error) => return Err(error), + }; + refused.by_partition.push((partition, decision)); + } + if !allowed + && let Some(error) = analysis + .reads() + .iter() + .filter_map(|read| { + let refusal = refused.decision(read, req).cloned().flatten()?; + Some((read, refusal)) + }) + .max_by(|a, b| read_order(a.0, b.0, value_of)) + .map(|(_, refusal)| refusal) + { + return Err(error); + } + Ok(refused) +} + +/// The key schema of what `stmt` reads or writes: the index's for a read of +/// an index, the table's otherwise. `None` when the table is missing, when +/// the index is missing or a vector index, so that the condition keys read +/// from the key are absent. +/// +/// # Errors +/// +/// A failed read of the index other than a missing index. +async fn target_schema( + ctx: &OperationContext, + stmt: &Statement, + key_info: Option<&TableKeyInfo>, +) -> Result>, DynamoDbError> { + let Some(key_info) = key_info else { + return Ok(None); + }; + let index = match stmt { + Statement::Select(select) | Statement::Exists(select) => select.from.index.as_deref(), + Statement::Insert(_) | Statement::Update(_) | Statement::Delete(_) => None, + }; + let Some(index) = index else { + return Ok(Some(key_info.key_schema.clone())); + }; + if key_info + .vector_indexes + .iter() + .any(|v| v.index_name == index) + { + return Ok(None); + } + match ctx + .storage + .index_info_by_table_id(&key_info.table_id, index) + .await + { + Ok(info) => Ok(Some(info.key_schema)), + Err(StorageError::IndexNotFound(_)) => Ok(None), + Err(e) => Err(storage_err_to_dynamo(e)), + } +} + +/// The key names of a key schema with one partition key and at most one sort +/// key. A multi-attribute key is refused once the statement is authorized, +/// so its condition keys read from the key are absent. +fn single_attribute_keys(schema: &[KeySchemaElement]) -> Option> { + if is_multipart_key_schema(schema) { + return None; + } + let (partition, sort) = common::key_names(schema); + Some(KeyNames { + partition: partition?, + sort, + }) +} + +/// The WHERE of a statement, when it has one. +fn where_clause(stmt: &Statement) -> Option<&WhereExpr> { + match stmt { + Statement::Select(select) | Statement::Exists(select) => select.where_clause.as_ref(), + Statement::Update(update) => update.where_clause.as_ref(), + Statement::Delete(delete) => Some(&delete.where_clause), + Statement::Insert(_) => None, + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/engine/src/partiql/authorize/access.rs b/crates/engine/src/partiql/authorize/access.rs new file mode 100644 index 000000000..6a75cf410 --- /dev/null +++ b/crates/engine/src/partiql/authorize/access.rs @@ -0,0 +1,252 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! What a statement is authorized for: the action, the resource and the +//! condition keys of its [`ItemAccess`]. + +use extenddb_core::partiql::Statement; +use extenddb_core::partiql::ast::{ + Operand, Projection, Returning, SelectStatement, UpdateStatement, WhereExpr, +}; +use extenddb_core::partiql::plan::{KeyEquality, KeyNames, WhereAnalysis, key_equality}; +use extenddb_core::types::AttributeValue; + +use super::Surface; +use crate::ItemAccess; +use crate::partiql::SingleRequest; +use crate::partiql::common; +use crate::partiql::index_read::{operand_roots, where_roots}; + +/// What a statement reads or writes, as far as the catalog knows it. +pub(in crate::partiql) struct Target<'k> { + /// The table's key attribute names, or `None` for a missing table. + pub(in crate::partiql) table_keys: Option>, + /// The key names of what the statement reads or writes: the index's for + /// a read of an index. `None` when they are not known: a missing table + /// or index, a vector index, a key of several attributes. + pub(in crate::partiql) keys: Option>, +} + +/// The value `operand` names as the request wrote it: a literal as the +/// statement writes it, a parameter as the request sends it. A number keeps +/// the text it was written with, which `SingleRequest::parameters` do not. +pub(in crate::partiql) fn written_value( + operand: &Operand, + req: &SingleRequest, +) -> Option { + match operand { + Operand::Literal(value) => Some(value.clone()), + Operand::Parameter(index) => match req.parameter_numbers.get(*index) { + Some(Some(text)) => Some(AttributeValue::N(text.clone())), + _ => req.parameters.get(*index).cloned(), + }, + Operand::Path(_) | Operand::Arith(..) | Operand::Function(..) => None, + } +} + +/// What `stmt`, a member of a request of `surface`, is authorized for, on +/// `target` and with `analysis`, its WHERE read against `target.keys`. For +/// an `ExecuteStatement` SELECT read by key, each disjunct is authorized +/// with its own partition value in place of the ones here (see +/// `each_disjunct`). +/// +/// Measured 2026-11-03: +/// +/// * the resource is the table, or for a read of an index the index; an +/// index read of a missing table is authorized against the table; +/// * `dynamodb:LeadingKeys` holds the partition key values the statement +/// reads or writes, as the request writes them: a number keeps its scale +/// (`10.0` is not `10`, `010` is `10` once the server writes it); +/// * `dynamodb:Attributes` holds the table's key attributes and the +/// top-level names the statement uses; +/// * `dynamodb:ReturnValues` is "NONE" for a write without `RETURNING`; +/// * `dynamodb:FullTableScan` is set for an `ExecuteStatement` read only, +/// and absent when the key of what it reads is not known. +pub(in crate::partiql) fn access( + stmt: &Statement, + target: &Target<'_>, + analysis: Option<&WhereAnalysis<'_>>, + req: &SingleRequest, + surface: Surface, +) -> ItemAccess { + let written = |op: &Operand| written_value(op, req); + let keys = target.keys; + let table_keys = target.table_keys.as_deref().unwrap_or_default(); + let base = ItemAccess { + enclosing_operation: surface.enclosing_operation(), + ..ItemAccess::default() + }; + match stmt { + Statement::Select(select) | Statement::Exists(select) => { + let keyed = analysis.filter(|analysis| analysis.is_keyed()); + ItemAccess { + action: "PartiQLSelect", + table: select.from.table.clone(), + index: target.table_keys.as_ref().and(select.from.index.clone()), + partition_keys: keyed.and_then(|analysis| { + distinct(analysis.reads().iter().map(|read| written(read.partition))) + }), + attributes: names(table_keys, select_names(select)), + select: Some(select_value(select)), + full_table_scan: match surface { + Surface::Statement => keys.map(|_| keyed.is_none()), + Surface::Batch | Surface::Transaction => None, + }, + ..base + } + } + Statement::Insert(insert) => ItemAccess { + action: "PartiQLInsert", + table: insert.table.clone(), + // The value the item stores, the one the write takes its key from. + partition_keys: keys + .and_then(|keys| common::insert_values(insert).get(keys.partition).copied()) + .and_then(written) + .map(|value| vec![value]), + attributes: names( + table_keys, + insert.value.iter().map(|(name, _)| name.as_str()).collect(), + ), + return_values: Some(return_values(None)), + ..base + }, + Statement::Update(update) => ItemAccess { + action: "PartiQLUpdate", + table: update.table.clone(), + partition_keys: keys.zip(update.where_clause.as_ref()).and_then( + |(keys, where_clause)| partition_equalities(where_clause, keys, &written), + ), + attributes: names(table_keys, update_names(update)), + return_values: Some(return_values(update.returning)), + ..base + }, + Statement::Delete(delete) => ItemAccess { + action: "PartiQLDelete", + table: delete.table.clone(), + partition_keys: keys + .and_then(|keys| partition_equalities(&delete.where_clause, keys, &written)), + attributes: names(table_keys, where_names(&delete.where_clause)), + return_values: Some(return_values(delete.returning)), + ..base + }, + } +} + +/// The values of `values` once each, in order, or `None` when one of them +/// does not resolve (a parameter the request does not supply), as a key the +/// classic operations cannot read is left absent. +fn distinct(values: impl Iterator>) -> Option> { + let mut out: Vec = Vec::new(); + for value in values { + let value = value?; + if !out.contains(&value) { + out.push(value); + } + } + (!out.is_empty()).then_some(out) +} + +/// The values a write's WHERE gives the partition key by equality among its +/// top-level `AND` conditions, written either way round, or `None` when it +/// gives none (measured 2026-11-03: UPDATE and DELETE of an item of +/// partition a are allowed under `ForAllValues` `["a"]` and of b refused). +fn partition_equalities( + where_clause: &WhereExpr, + keys: KeyNames<'_>, + written: &dyn Fn(&Operand) -> Option, +) -> Option> { + let mut terms = Vec::new(); + common::flatten_and_lossy(where_clause, &mut terms); + distinct( + terms + .into_iter() + .filter_map(|term| match key_equality(term, keys) { + Some(KeyEquality::Partition(value)) => Some(written(value)), + Some(KeyEquality::Sort(_)) | None => None, + }), + ) +} + +/// `dynamodb:Select` for a SELECT: `SPECIFIC_ATTRIBUTES` for a projection +/// list, `ALL_ATTRIBUTES` for `*` on a table and `ALL_PROJECTED_ATTRIBUTES` +/// for `*` on an index, of any projection, in all three operations +/// (measured 2026-11-03). +fn select_value(select: &SelectStatement) -> &'static str { + match (&select.projection, &select.from.index) { + (Projection::Paths(_), _) => "SPECIFIC_ATTRIBUTES", + (Projection::All, None) => "ALL_ATTRIBUTES", + (Projection::All, Some(_)) => "ALL_PROJECTED_ATTRIBUTES", + } +} + +/// `dynamodb:ReturnValues` for a write's `RETURNING` clause: "NONE" without +/// one, in all three operations, and the classic name of what each mode +/// returns (measured 2026-11-03). +fn return_values(returning: Option) -> &'static str { + match returning { + None => "NONE", + Some(Returning::AllOld) => "ALL_OLD", + Some(Returning::ModifiedOld) => "UPDATED_OLD", + Some(Returning::AllNew) => "ALL_NEW", + Some(Returning::ModifiedNew) => "UPDATED_NEW", + } +} + +/// `dynamodb:Attributes`: the table's key attributes `table_keys`, then the +/// names the statement uses, each once, in the order first named, or absent +/// when there are none. For a read of an index the index's key attributes +/// are in the set only when the statement names them (not measured). +fn names(table_keys: &[&str], used: Vec<&str>) -> Option> { + let mut out: Vec = Vec::new(); + for name in table_keys.iter().copied().chain(used) { + if !out.iter().any(|seen| seen == name) { + out.push(name.to_owned()); + } + } + (!out.is_empty()).then_some(out) +} + +/// The top-level names a SELECT names: its projection list (`*` names none), +/// its WHERE and its ORDER BY (measured 2026-11-03: a name in the WHERE +/// counts as one in the projection does). +fn select_names(select: &SelectStatement) -> Vec<&str> { + let mut out = Vec::new(); + if let Projection::Paths(paths) = &select.projection { + out.extend(paths.iter().map(|path| path.root.as_str())); + } + if let Some(where_clause) = &select.where_clause { + where_roots(where_clause, &mut out); + } + if let Some(order_by) = &select.order_by { + out.extend(order_by.terms.iter().map(|term| term.path.root.as_str())); + } + out +} + +/// The top-level names an UPDATE names: what it sets, the values it sets +/// them to, what it removes, and its WHERE (measured 2026-11-03: `SET g` is +/// refused under `ForAllValues` `[pk, sk, a, q]`, and so are `SET q` with `g` +/// in the WHERE, `SET q = g` and `REMOVE q` without q in the set). +fn update_names(update: &UpdateStatement) -> Vec<&str> { + let mut out = Vec::new(); + for action in &update.sets { + out.push(action.path.root.as_str()); + operand_roots(&action.value, &mut out); + } + for action in &update.set_members { + out.push(action.path.root.as_str()); + operand_roots(&action.members, &mut out); + } + out.extend(update.removes.iter().map(|path| path.root.as_str())); + if let Some(where_clause) = &update.where_clause { + where_roots(where_clause, &mut out); + } + out +} + +/// The top-level names a WHERE names. +fn where_names(where_clause: &WhereExpr) -> Vec<&str> { + let mut out = Vec::new(); + where_roots(where_clause, &mut out); + out +} diff --git a/crates/engine/src/partiql/authorize/gate.rs b/crates/engine/src/partiql/authorize/gate.rs new file mode 100644 index 000000000..c512fe084 --- /dev/null +++ b/crates/engine/src/partiql/authorize/gate.rs @@ -0,0 +1,162 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Where a statement is authorized among its checks. +//! +//! The service checks a statement in one order, and authorizes it at a +//! point in that order that depends on the checks (measured 2026-11-03). +//! Some checks are made before authorization: a caller with no permission +//! gets their refusals. The others are made after it: such a caller gets +//! AccessDenied instead. The executors make their checks in the order the +//! service makes them for a permitted caller, and a [`Gate`] places the +//! authorization in it: +//! +//! * the executor passes the gate after its last check made before +//! authorization and before it reads or writes an item; +//! * when a check fails before that, the refusal is answered at once if +//! the service makes that check before authorization, and otherwise once +//! the statement is authorized, so a caller with no permission gets +//! AccessDenied ([`Gate::settle`]). +//! +//! So a check made before authorization is answered first only when no +//! check that comes ahead of it for a permitted caller fails. The checks +//! made before authorization are listed in [`made_before_authorization`]. + +use extenddb_core::error::DynamoDbError; +use extenddb_core::partiql::Statement; +use extenddb_core::partiql::parser::PARAMETER_COUNT_MISMATCH; +use extenddb_core::partiql::plan::{OVERLAP_MESSAGE, SCALAR_KEY_REFUSAL}; +use extenddb_core::types::TableKeyInfo; + +use super::{Refused, Surface, statement}; +use crate::OperationContext; +use crate::partiql::SingleRequest; +use crate::partiql::batch::PRIMARY_KEY_REQUIRED; +use crate::partiql::next_token::{INVALID_NEXT_TOKEN, TOKEN_MISMATCH}; +use crate::partiql::select::{GSI_STRONG_READ, TOO_MANY_DECOMPOSED_READS}; +use crate::partiql::write::INVALID_RETURNING; + +/// The authorization of one statement, made once, the first time the +/// statement's executor or [`Gate::settle`] needs it. +pub(in crate::partiql) struct Gate<'a> { + ctx: &'a OperationContext, + stmt: &'a Statement, + key_info: Option<&'a TableKeyInfo>, + req: &'a SingleRequest, + surface: Surface, + decision: Option>, +} + +impl<'a> Gate<'a> { + /// The gate of `stmt`, a member of a request of `surface`, whose table + /// has the key info `key_info` (`None` for a missing table). + pub(in crate::partiql) fn new( + ctx: &'a OperationContext, + stmt: &'a Statement, + key_info: Option<&'a TableKeyInfo>, + req: &'a SingleRequest, + surface: Surface, + ) -> Self { + Self { + ctx, + stmt, + key_info, + req, + surface, + decision: None, + } + } + + /// Authorize the statement, once: the disjuncts of a read by key the + /// caller may not read, empty for any other statement. + /// + /// # Errors + /// + /// The statement's refusal (`AccessDeniedException`), or a fault of + /// the authorizer or of a catalog read. + pub(in crate::partiql) async fn pass(&mut self) -> Result<&Refused, DynamoDbError> { + if self.decision.is_none() { + let decision = + statement(self.ctx, self.stmt, self.key_info, self.req, self.surface).await; + self.decision = Some(decision); + } + match self.decision.as_ref() { + Some(Ok(refused)) => Ok(refused), + Some(Err(error)) => Err(error.clone()), + None => unreachable!("decided above"), + } + } + + /// Whether the statement has been authorized, or refused. + pub(in crate::partiql) fn decided(&self) -> bool { + self.decision.is_some() + } + + /// The answer of a statement whose executor returned `result`. A + /// refusal raised before the executor passed the gate is answered as it + /// is when the service makes its check before authorization, and once + /// the statement is authorized otherwise; an answer is never given for + /// a statement that has not been authorized. The outer error is a fault + /// of the authorization, which fails the whole request; a refusal is the + /// inner one. + /// + /// # Errors + /// + /// A fault of the authorizer or of a catalog read. + pub(in crate::partiql) async fn settle( + &mut self, + result: Result, + ) -> Result, DynamoDbError> { + if !self.decided() + && let Err(error) = &result + && made_before_authorization(error, self.stmt) + { + return Ok(result); + } + match self.pass().await { + Ok(_) => Ok(result), + Err(denied @ DynamoDbError::AccessDeniedException(_)) => Ok(Err(denied)), + Err(fault) => Err(fault), + } + } +} + +/// Whether `error` is the refusal of a check the service makes before it +/// authorizes `stmt` (measured 2026-11-03 with a caller that may not run +/// the statement): a statement with fewer parameters than placeholders, +/// `SELECT COUNT(*)`, a strongly consistent read of a global secondary +/// index, a path into a key attribute, overlapping disjuncts, more than 50 +/// disjuncts, a NextToken that is invalid or another statement's, a DELETE +/// with a `RETURNING` clause other than `ALL OLD *`, and a +/// `BatchExecuteStatement` read without the full primary key. The checks of +/// a request, the parse of a statement and the checks that need no catalog +/// are made before any [`Gate`] is made. +pub(in crate::partiql) fn made_before_authorization( + error: &DynamoDbError, + stmt: &Statement, +) -> bool { + let DynamoDbError::ValidationException(message) = error else { + return false; + }; + let projection_error = match stmt { + Statement::Select(select) => select.projection_error.as_deref(), + _ => None, + }; + [ + PARAMETER_COUNT_MISMATCH, + GSI_STRONG_READ, + OVERLAP_MESSAGE, + TOO_MANY_DECOMPOSED_READS, + INVALID_NEXT_TOKEN, + TOKEN_MISMATCH, + PRIMARY_KEY_REQUIRED, + ] + .contains(&message.as_str()) + || [SCALAR_KEY_REFUSAL, INVALID_RETURNING] + .iter() + .any(|prefix| message.starts_with(prefix)) + || projection_error == Some(message.as_str()) +} + +#[cfg(test)] +mod tests; diff --git a/crates/engine/src/partiql/authorize/gate/tests.rs b/crates/engine/src/partiql/authorize/gate/tests.rs new file mode 100644 index 000000000..d8f453548 --- /dev/null +++ b/crates/engine/src/partiql/authorize/gate/tests.rs @@ -0,0 +1,186 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Unit tests for the place of a statement's authorization among its checks. + +use std::sync::Arc; + +use serde_json::json; + +use extenddb_core::partiql::parse; + +use super::*; +use crate::ItemAuthorizer; +use crate::partiql::test_context::{self, Fixed, block_on}; + +const SELECT: &str = "SELECT * FROM \"t\" WHERE pk = 'a'"; + +fn validation(message: &str) -> DynamoDbError { + DynamoDbError::ValidationException(message.to_owned()) +} + +fn not_found() -> DynamoDbError { + DynamoDbError::ResourceNotFoundException("Requested resource not found".to_owned()) +} + +fn refused() -> DynamoDbError { + DynamoDbError::AccessDeniedException("refused".to_owned()) +} + +/// An answer of [`Gate::settle`] with its errors written out, the inner one +/// a refusal and the outer one a fault. +type Shown = Result, String>; + +fn shown(answer: Result, DynamoDbError>) -> Shown { + answer + .map(|inner| inner.map_err(|e| format!("{e:?}"))) + .map_err(|e| format!("{e:?}")) +} + +/// The answer that refuses the statement with `error`. +fn refusal(error: &DynamoDbError) -> Shown { + Ok(Err(format!("{error:?}"))) +} + +/// Run `body` with the gate of `sql`, an `ExecuteStatement` on the missing +/// table "t", whose context authorizes with `authorizer`; and give its +/// answer with the number of accesses `authorizer` was asked about. +fn with_gate( + sql: &str, + authorizer: &Arc, + body: impl AsyncFnOnce(&mut Gate<'_>) -> T, +) -> (T, usize) { + block_on(async { + let shared: Arc = Arc::clone(authorizer) as Arc; + let ctx = test_context::context(Some(shared)).await; + let stmt = parse(sql).unwrap(); + let req = SingleRequest::from_body(&json!({ "Statement": sql })).unwrap(); + let mut gate = Gate::new(&ctx, &stmt, None, &req, Surface::Statement); + let answer = body(&mut gate).await; + (answer, authorizer.asked()) + }) +} + +/// The answer `settle` gives `sql` for an executor that returned `result` +/// without passing the gate. +fn settled(authorizer: &Arc, result: Result<(), DynamoDbError>) -> (Shown, usize) { + let (answer, asked) = with_gate(SELECT, authorizer, async |gate| gate.settle(result).await); + (shown(answer), asked) +} + +#[test] +fn a_refusal_made_before_authorization_is_answered_without_it() { + for message in [ + PARAMETER_COUNT_MISMATCH, + GSI_STRONG_READ, + TOO_MANY_DECOMPOSED_READS, + INVALID_NEXT_TOKEN, + ] { + let (answer, asked) = settled(&Fixed::refusing(), Err(validation(message))); + assert_eq!(answer, refusal(&validation(message)), "{message}"); + assert_eq!(asked, 0, "{message}"); + } +} + +#[test] +fn a_refusal_made_after_authorization_is_answered_once_it_allows() { + let (answer, asked) = settled(&Fixed::allowing(), Err(not_found())); + assert_eq!(answer, refusal(¬_found())); + assert_eq!(asked, 1); + // A caller that may not run the statement gets the refusal of the + // statement instead (measured 2026-11-03: a missing table, a missing + // index, a key of the wrong type, ORDER BY on another attribute). + let (answer, asked) = settled(&Fixed::refusing(), Err(not_found())); + assert_eq!(answer, refusal(&refused())); + assert_eq!(asked, 1); +} + +#[test] +fn no_answer_is_given_for_a_statement_that_is_not_authorized() { + let (answer, _) = settled(&Fixed::refusing(), Ok(())); + assert_eq!(answer, refusal(&refused())); + let (answer, asked) = settled(&Fixed::allowing(), Ok(())); + assert_eq!(answer, Ok(Ok(()))); + assert_eq!(asked, 1); +} + +#[test] +fn a_fault_of_the_authorizer_fails_the_request() { + let (answer, _) = settled(&Fixed::failing(), Err(not_found())); + assert!( + answer + .as_ref() + .is_err_and(|fault| fault.starts_with("InternalServerError")), + "{answer:?}" + ); +} + +#[test] +fn a_statement_is_authorized_once() { + let (answer, asked) = with_gate(SELECT, &Fixed::allowing(), async |gate| { + assert!(!gate.decided()); + assert!(gate.pass().await.is_ok()); + assert!(gate.decided()); + assert!(gate.pass().await.is_ok()); + // A refusal raised once the gate is passed is the executor's + // answer, whatever check made it. + gate.settle(Err(validation(INVALID_NEXT_TOKEN))).await + }); + assert_eq!(shown(answer), refusal(&validation(INVALID_NEXT_TOKEN))); + assert_eq!(asked, 1); + let (answer, asked) = with_gate(SELECT, &Fixed::refusing(), async |gate| { + assert!(gate.pass().await.is_err()); + gate.settle(Err(not_found())).await + }); + assert_eq!(shown(answer), refusal(&refused())); + assert_eq!(asked, 1); +} + +#[test] +fn the_checks_made_before_authorization_are_known_by_their_refusal() { + // Measured 2026-11-03 with a caller that may not run the statement. + let select = parse(SELECT).unwrap(); + for message in [ + PARAMETER_COUNT_MISMATCH, + GSI_STRONG_READ, + OVERLAP_MESSAGE, + TOO_MANY_DECOMPOSED_READS, + INVALID_NEXT_TOKEN, + TOKEN_MISMATCH, + PRIMARY_KEY_REQUIRED, + "Key attributes must be scalars; list random access '[]' and map lookup '.' are not \ + allowed: Key pk", + "Invalid returning clause: RETURNING ALL NEW *. Only RETURNING ALL OLD * is allowed in \ + DELETE statements.", + ] { + assert!( + made_before_authorization(&validation(message), &select), + "{message}" + ); + } + // `SELECT COUNT(*)`, in its statement only. + let count = parse("SELECT COUNT(*) FROM \"t\"").unwrap(); + let projected = "Unexpected path component at 1:8:5"; + assert!(made_before_authorization(&validation(projected), &count)); + assert!(!made_before_authorization(&validation(projected), &select)); +} + +#[test] +fn the_checks_made_after_authorization_are_not() { + let select = parse(SELECT).unwrap(); + for error in [ + validation("The table does not have the specified index"), + validation( + "Filter Expression can only contain non-primary key attributes: Primary key \ + attribute: sk", + ), + validation( + "One or more parameter values were invalid: Size of hashkey has exceeded the \ + maximum size limit of2048 bytes", + ), + not_found(), + DynamoDbError::AccessDeniedException(GSI_STRONG_READ.to_owned()), + ] { + assert!(!made_before_authorization(&error, &select), "{error:?}"); + } +} diff --git a/crates/engine/src/partiql/authorize/tests.rs b/crates/engine/src/partiql/authorize/tests.rs new file mode 100644 index 000000000..dc2146bf1 --- /dev/null +++ b/crates/engine/src/partiql/authorize/tests.rs @@ -0,0 +1,587 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Unit tests for the authorization of PartiQL statements: what each +//! statement is authorized for, and the decision for each disjunct of a read +//! by key. + +use std::sync::Mutex; + +use serde_json::{Value, json}; + +use extenddb_core::partiql::parse; + +use super::*; +use crate::ItemAuthorization; +use crate::partiql::test_context::block_on; + +const KEYS: KeyNames<'static> = KeyNames { + partition: "pk", + sort: Some("sk"), +}; + +const INDEX_KEYS: KeyNames<'static> = KeyNames { + partition: "g", + sort: None, +}; + +/// The key attributes of the table the statements name. +const TABLE_KEYS: [&str; 2] = ["pk", "sk"]; + +fn s(value: &str) -> AttributeValue { + AttributeValue::S(value.to_owned()) +} + +fn n(value: &str) -> AttributeValue { + AttributeValue::N(value.to_owned()) +} + +fn strings(names: &[&str]) -> Option> { + Some(names.iter().map(|n| (*n).to_owned()).collect()) +} + +/// A request for `sql` with the wire-form parameters `params`. +fn request(sql: &str, params: Value) -> SingleRequest { + SingleRequest::from_body(&json!({ "Statement": sql, "Parameters": params })).unwrap() +} + +/// What `sql`, sent with `params` in a request of `surface`, is authorized +/// for on a table keyed on `table_keys` (`None` for a missing table) whose +/// target has the key names `keys`. +fn access_on( + sql: &str, + params: Value, + table_keys: Option<&[&str]>, + keys: Option>, + surface: Surface, +) -> ItemAccess { + let req = request(sql, params); + let stmt = parse(sql).unwrap(); + let value_of = |op: &Operand| bound_value(op, &req.parameters).cloned(); + let analysis = match (where_clause(&stmt), keys) { + (Some(where_clause), Some(keys)) => Some(WhereAnalysis::new(where_clause, keys, &value_of)), + _ => None, + }; + let target = Target { + table_keys: table_keys.map(<[&str]>::to_vec), + keys, + }; + access(&stmt, &target, analysis.as_ref(), &req, surface) +} + +/// What `sql`, sent with `params`, is authorized for in an +/// `ExecuteStatement` on the pk/sk table. +fn access_with(sql: &str, params: Value) -> ItemAccess { + access_on( + sql, + params, + Some(&TABLE_KEYS), + Some(KEYS), + Surface::Statement, + ) +} + +/// What `sql` is authorized for in an `ExecuteStatement` on the pk/sk +/// table. +fn access_of(sql: &str) -> ItemAccess { + access_with(sql, json!([])) +} + +#[test] +fn a_scan_is_a_full_table_scan_and_names_the_table_key() { + // Measured 2026-11-03: `SELECT *` with no WHERE is refused under `Null` + // "true" on dynamodb:Attributes, so the key attributes are in the set. + assert_eq!( + access_of("SELECT * FROM \"t\""), + ItemAccess { + action: "PartiQLSelect", + table: "t".to_owned(), + attributes: strings(&["pk", "sk"]), + select: Some("ALL_ATTRIBUTES"), + full_table_scan: Some(true), + ..ItemAccess::default() + } + ); + let filtered = access_of("SELECT * FROM \"t\" WHERE a = '1'"); + assert_eq!(filtered.full_table_scan, Some(true)); + assert_eq!(filtered.partition_keys, None); + assert_eq!(filtered.attributes, strings(&["pk", "sk", "a"])); +} + +#[test] +fn the_table_key_is_named_whether_or_not_the_statement_names_it() { + // Measured 2026-11-03: `SELECT pk ... WHERE pk = 'a'` is refused under + // ForAllValues [pk, q] and `SELECT pk ... WHERE a = '1'` under [pk, a]. + assert_eq!( + access_of("SELECT pk FROM \"t\" WHERE pk = 'a'").attributes, + strings(&["pk", "sk"]) + ); + assert_eq!( + access_of("SELECT pk FROM \"t\" WHERE a = '1'").attributes, + strings(&["pk", "sk", "a"]) + ); + // A nested path counts by its top-level name. + assert_eq!( + access_of("SELECT m.x FROM \"t\" WHERE pk = 'a' ORDER BY sk DESC").attributes, + strings(&["pk", "sk", "m"]) + ); +} + +#[test] +fn a_read_by_key_carries_its_partition_values_and_no_full_table_scan() { + let access = access_of("SELECT pk, a FROM \"t\" WHERE pk = 'a' AND sk = 'x' AND g = 'ga'"); + assert_eq!(access.full_table_scan, Some(false)); + assert_eq!(access.partition_keys, Some(vec![s("a")])); + assert_eq!(access.attributes, strings(&["pk", "sk", "a", "g"])); + assert_eq!(access.select, Some("SPECIFIC_ATTRIBUTES")); + let disjuncts = access_of("SELECT * FROM \"t\" WHERE pk IN ['a', 'b', 'a']"); + assert_eq!(disjuncts.partition_keys, Some(vec![s("a"), s("b")])); +} + +#[test] +fn a_parameter_gives_its_value_and_a_missing_one_leaves_the_key_absent() { + let sql = "SELECT * FROM \"t\" WHERE pk = ?"; + let bound = access_with(sql, json!([{ "S": "a" }])); + assert_eq!(bound.partition_keys, Some(vec![s("a")])); + let unbound = access_of(sql); + assert_eq!(unbound.partition_keys, None); +} + +#[test] +fn a_number_partition_value_is_given_as_the_request_writes_it() { + // Measured 2026-11-03 under ForAllValues LeadingKeys ["10"]: `pk = 10` + // and `pk = 010` run, and `pk = 10.0`, `pk = 1E1` and the parameter + // {"N": "10.0"} are refused. The server writes the text as Java's + // BigDecimal prints it, so the text reaches it as written. + for literal in ["10", "010", "10.0", "1E1", "-10"] { + let sql = format!("SELECT * FROM \"t\" WHERE pk = {literal}"); + assert_eq!( + access_of(&sql).partition_keys, + Some(vec![n(literal)]), + "{literal}" + ); + } + let sql = "SELECT * FROM \"t\" WHERE pk = ?"; + // The parameters a statement runs with hold the number normalized. + assert_eq!(request(sql, json!([{ "N": "10.0" }])).parameters, [n("10")]); + assert_eq!( + access_with(sql, json!([{ "N": "10.0" }])).partition_keys, + Some(vec![n("10.0")]) + ); + // The same in every statement that names a partition value. + for (sql, params) in [ + ( + "UPDATE \"t\" SET q = 1 WHERE pk = ? AND sk = 'x'", + json!([{ "N": "10.0" }]), + ), + ("DELETE FROM \"t\" WHERE pk = 10.0 AND sk = 'x'", json!([])), + ( + "INSERT INTO \"t\" VALUE {'pk': ?, 'sk': 'x'}", + json!([{ "N": "10.0" }]), + ), + ("INSERT INTO \"t\" VALUE {'pk': 10.0, 'sk': 'x'}", json!([])), + ] { + assert_eq!( + access_with(sql, params).partition_keys, + Some(vec![n("10.0")]), + "{sql}" + ); + } +} + +#[test] +fn an_index_read_names_the_index_and_reads_its_key() { + let on_index = |sql: &str| { + access_on( + sql, + json!([]), + Some(&TABLE_KEYS), + Some(INDEX_KEYS), + Surface::Statement, + ) + }; + let access = on_index("SELECT * FROM \"t\".\"gix\" WHERE g = 'ga'"); + assert_eq!(access.index.as_deref(), Some("gix")); + assert_eq!(access.partition_keys, Some(vec![s("ga")])); + assert_eq!(access.full_table_scan, Some(false)); + assert_eq!(access.select, Some("ALL_PROJECTED_ATTRIBUTES")); + assert_eq!(access.attributes, strings(&["pk", "sk", "g"])); + // The index's key attributes are in the set only when the statement + // names them (not measured). + let scan = on_index("SELECT pk FROM \"t\".\"gix\""); + assert_eq!(scan.full_table_scan, Some(true)); + assert_eq!(scan.select, Some("SPECIFIC_ATTRIBUTES")); + assert_eq!(scan.attributes, strings(&["pk", "sk"])); +} + +#[test] +fn an_index_read_of_a_missing_table_is_authorized_against_the_table() { + // Measured 2026-11-03: the refusal names the table's ARN for a missing + // table, and the index's for an existing table's missing index. Neither + // carries FullTableScan. + let sql = "SELECT * FROM \"t\".\"nix\" WHERE g = 'ga'"; + let missing_table = access_on(sql, json!([]), None, None, Surface::Statement); + assert_eq!(missing_table.index, None); + assert_eq!(missing_table.partition_keys, None); + assert_eq!(missing_table.full_table_scan, None); + assert_eq!(missing_table.attributes, strings(&["g"])); + let missing_index = access_on(sql, json!([]), Some(&TABLE_KEYS), None, Surface::Statement); + assert_eq!(missing_index.index.as_deref(), Some("nix")); + assert_eq!(missing_index.partition_keys, None); + assert_eq!(missing_index.full_table_scan, None); + assert_eq!(missing_index.attributes, strings(&["pk", "sk", "g"])); +} + +#[test] +fn batch_and_transaction_reads_carry_their_operation_and_no_full_table_scan() { + let sql = "SELECT * FROM \"t\" WHERE pk = 'a' AND sk = 'x'"; + let on = |surface| access_on(sql, json!([]), Some(&TABLE_KEYS), Some(KEYS), surface); + let batch = on(Surface::Batch); + assert_eq!(batch.enclosing_operation, Some("BatchExecuteStatement")); + assert_eq!(batch.full_table_scan, None); + assert_eq!(batch.partition_keys, Some(vec![s("a")])); + let transaction = on(Surface::Transaction); + assert_eq!(transaction.enclosing_operation, Some("ExecuteTransaction")); + assert_eq!(transaction.full_table_scan, None); + assert_eq!(on(Surface::Statement).enclosing_operation, None); +} + +#[test] +fn an_exists_check_is_authorized_as_its_select() { + // Measured 2026-11-03 (X3 to X7). + let access = access_on( + "EXISTS(SELECT * FROM \"t\" WHERE pk = 'a' AND sk = 'x' AND a = '1')", + json!([]), + Some(&TABLE_KEYS), + Some(KEYS), + Surface::Transaction, + ); + assert_eq!(access.action, "PartiQLSelect"); + assert_eq!(access.index, None); + assert_eq!(access.partition_keys, Some(vec![s("a")])); + assert_eq!(access.attributes, strings(&["pk", "sk", "a"])); + assert_eq!(access.enclosing_operation, Some("ExecuteTransaction")); +} + +#[test] +fn an_insert_carries_its_item_partition_key_and_names() { + let access = access_with( + "INSERT INTO \"t\" VALUE {'pk': ?, 'sk': 's', 'g': 'gz'}", + json!([{ "S": "b" }]), + ); + assert_eq!( + access, + ItemAccess { + action: "PartiQLInsert", + table: "t".to_owned(), + partition_keys: Some(vec![s("b")]), + attributes: strings(&["pk", "sk", "g"]), + return_values: Some("NONE"), + ..ItemAccess::default() + } + ); + let keyless = access_of("INSERT INTO \"t\" VALUE {'sk': 's'}"); + assert_eq!(keyless.partition_keys, None); + assert_eq!(keyless.attributes, strings(&["pk", "sk"])); +} + +#[test] +fn an_update_carries_the_partition_key_of_its_where_and_every_name() { + let access = access_on( + "UPDATE \"t\" SET q = a + 1 REMOVE r WHERE 'a' = pk AND sk = 'x' AND g = 'ga' \ + RETURNING MODIFIED OLD *", + json!([]), + Some(&TABLE_KEYS), + Some(KEYS), + Surface::Batch, + ); + assert_eq!(access.action, "PartiQLUpdate"); + assert_eq!(access.partition_keys, Some(vec![s("a")])); + assert_eq!( + access.attributes, + strings(&["pk", "sk", "q", "a", "r", "g"]) + ); + assert_eq!(access.return_values, Some("UPDATED_OLD")); + assert_eq!(access.full_table_scan, None); + assert_eq!(access.enclosing_operation, Some("BatchExecuteStatement")); + // Measured 2026-11-03: `SET q = g` is refused under ForAllValues [pk, + // sk, q]. + assert_eq!( + access_of("UPDATE \"t\" SET q = g WHERE pk = 'a' AND sk = 'x'").attributes, + strings(&["pk", "sk", "q", "g"]) + ); + // Without a partition key equality the key is absent. + let partial = access_of("UPDATE \"t\" SET q = 1 WHERE sk = 'x'"); + assert_eq!(partial.partition_keys, None); +} + +#[test] +fn every_returning_mode_has_its_return_values() { + for (mode, value) in [ + ("ALL OLD", "ALL_OLD"), + ("MODIFIED OLD", "UPDATED_OLD"), + ("ALL NEW", "ALL_NEW"), + ("MODIFIED NEW", "UPDATED_NEW"), + ] { + let sql = format!("UPDATE \"t\" SET q = 1 WHERE pk = 'a' AND sk = 'x' RETURNING {mode} *"); + assert_eq!(access_of(&sql).return_values, Some(value), "{mode}"); + } + let delete = access_of("DELETE FROM \"t\" WHERE pk = 'b' AND sk = 'x' RETURNING ALL OLD *"); + assert_eq!(delete.action, "PartiQLDelete"); + assert_eq!(delete.partition_keys, Some(vec![s("b")])); + assert_eq!(delete.attributes, strings(&["pk", "sk"])); + assert_eq!(delete.return_values, Some("ALL_OLD")); +} + +#[test] +fn a_write_without_returning_returns_none_in_every_operation() { + // Measured 2026-11-03: an Allow on StringEquals "NONE" runs an INSERT, + // an UPDATE and a DELETE without RETURNING and one on Null "true" + // refuses them, in all three operations; an Allow on Bool "false" for + // dynamodb:FullTableScan refuses every write. + for sql in [ + "INSERT INTO \"t\" VALUE {'pk': 'a', 'sk': 'x'}", + "UPDATE \"t\" SET q = 1 WHERE pk = 'a' AND sk = 'x'", + "DELETE FROM \"t\" WHERE pk = 'a' AND sk = 'x'", + ] { + for surface in [Surface::Statement, Surface::Batch, Surface::Transaction] { + let access = access_on(sql, json!([]), Some(&TABLE_KEYS), Some(KEYS), surface); + assert_eq!(access.return_values, Some("NONE"), "{sql} {surface:?}"); + assert_eq!(access.full_table_scan, None, "{sql} {surface:?}"); + } + } + // A read carries none. + assert_eq!( + access_of("SELECT * FROM \"t\" WHERE pk = 'a'").return_values, + None + ); +} + +#[test] +fn a_write_with_two_partition_values_carries_both() { + let access = access_of("DELETE FROM \"t\" WHERE pk = 'a' AND pk = 'b' AND sk = 'x'"); + assert_eq!(access.partition_keys, Some(vec![s("a"), s("b")])); +} + +#[test] +fn a_multi_attribute_key_has_no_key_names() { + use extenddb_core::types::KeyType; + let key = |name: &str, key_type| KeySchemaElement { + attribute_name: name.to_owned(), + key_type, + }; + let single = [key("pk", KeyType::Hash), key("sk", KeyType::Range)]; + let names = single_attribute_keys(&single).unwrap(); + assert_eq!((names.partition, names.sort), ("pk", Some("sk"))); + let multi = [key("a", KeyType::Hash), key("b", KeyType::Hash)]; + assert!(single_attribute_keys(&multi).is_none()); +} + +// --- the decisions ----------------------------------------------------- + +/// Allows every access but those whose partition keys include one of +/// `refused`, and records every access it is asked about. +struct Fence { + refused: Vec, + asked: Mutex>, +} + +impl Fence { + fn refusing(values: &[&str]) -> Self { + Self { + refused: values.iter().map(|value| s(value)).collect(), + asked: Mutex::new(Vec::new()), + } + } + + fn asked(&self) -> Vec { + self.asked.lock().unwrap().clone() + } +} + +impl ItemAuthorizer for Fence { + fn authorize<'a>(&'a self, access: &'a ItemAccess) -> ItemAuthorization<'a> { + Box::pin(async move { + self.asked.lock().unwrap().push(access.clone()); + match &access.partition_keys { + Some(values) if values.iter().any(|value| self.refused.contains(value)) => Err( + DynamoDbError::AccessDeniedException(format!("refused {values:?}")), + ), + _ => Ok(()), + } + }) + } +} + +/// Refuses every access, naming its partition values. +struct RefuseAll; + +impl ItemAuthorizer for RefuseAll { + fn authorize<'a>(&'a self, access: &'a ItemAccess) -> ItemAuthorization<'a> { + Box::pin(async move { + Err(DynamoDbError::AccessDeniedException(format!( + "refused {:?}", + access.partition_keys + ))) + }) + } +} + +/// Fails every access with an internal error. +struct Broken; + +impl ItemAuthorizer for Broken { + fn authorize<'a>(&'a self, _access: &'a ItemAccess) -> ItemAuthorization<'a> { + Box::pin(async { + Err(DynamoDbError::InternalServerError( + "Internal error during authorization".to_owned(), + )) + }) + } +} + +/// Decide each disjunct of `sql`, a read by key of the pk/sk table, with +/// `authorizer`: the decision, and the refusal of each disjunct in read +/// order when the statement is not refused. +fn decide( + sql: &str, + authorizer: Option<&dyn ItemAuthorizer>, +) -> (Result, Vec>) { + let req = request(sql, json!([])); + let stmt = parse(sql).unwrap(); + let value_of = |op: &Operand| bound_value(op, &req.parameters).cloned(); + let analysis = WhereAnalysis::new(where_clause(&stmt).unwrap(), KEYS, &value_of); + assert!(analysis.is_keyed(), "{sql}"); + let target = Target { + table_keys: Some(TABLE_KEYS.to_vec()), + keys: Some(KEYS), + }; + let base = access(&stmt, &target, Some(&analysis), &req, Surface::Statement); + let result = block_on(each_disjunct(authorizer, &base, &analysis, &req, &value_of)); + let per_read = match &result { + Ok(refused) => analysis + .reads() + .iter() + .map(|read| refused.refusal(read, &req)) + .collect(), + Err(_) => Vec::new(), + }; + (result, per_read) +} + +fn refused_message(result: &Result) -> Option<&str> { + match result { + Err(DynamoDbError::AccessDeniedException(message)) => Some(message), + _ => None, + } +} + +#[test] +fn each_disjunct_is_authorized_with_its_own_partition() { + let fence = Fence::refusing(&["b"]); + let (result, per_read) = decide("SELECT * FROM \"t\" WHERE pk IN ['a', 'b']", Some(&fence)); + assert!(result.is_ok()); + assert!(per_read[0].is_none(), "a is allowed"); + assert!( + matches!(per_read[1], Some(DynamoDbError::AccessDeniedException(_))), + "b is held back for its refusal" + ); + let asked = fence.asked(); + assert_eq!(asked.len(), 2); + assert_eq!(asked[0].partition_keys, Some(vec![s("a")])); + assert_eq!(asked[1].partition_keys, Some(vec![s("b")])); + for access in &asked { + assert_eq!(access.full_table_scan, Some(false)); + assert_eq!(access.action, "PartiQLSelect"); + assert_eq!(access.attributes, strings(&["pk", "sk"])); + } +} + +#[test] +fn a_partition_is_authorized_once() { + let fence = Fence::refusing(&[]); + let (result, _) = decide( + "SELECT * FROM \"t\" WHERE (pk = 'a' AND sk = 'x') OR (pk = 'a' AND sk = 'y')", + Some(&fence), + ); + assert!(result.is_ok()); + assert_eq!(fence.asked().len(), 1); +} + +#[test] +fn a_statement_whose_every_disjunct_is_refused_is_refused() { + let fence = Fence::refusing(&["b"]); + let (result, _) = decide("SELECT * FROM \"t\" WHERE pk = 'b'", Some(&fence)); + assert!(refused_message(&result).is_some()); +} + +#[test] +fn of_several_refused_disjuncts_the_one_read_last_is_reported() { + let (result, _) = decide( + "SELECT * FROM \"t\" WHERE pk IN ['c', 'a', 'b']", + Some(&RefuseAll), + ); + let want = format!("refused {:?}", Some(vec![s("c")])); + assert_eq!(refused_message(&result), Some(want.as_str())); +} + +/// A read by key of `n` partitions, v00 to v(n-1), in key order. +fn partitions(n: usize) -> String { + let values: Vec = (0..n).map(|i| format!("'v{i:02}'")).collect(); + format!("SELECT * FROM \"t\" WHERE pk IN [{}]", values.join(", ")) +} + +#[test] +fn at_most_50_partitions_are_authorized_and_the_others_are_refused() { + // A statement with more than 50 disjuncts is refused whatever the + // decision, so the partitions past the 50th are not asked about, and + // are refused if they are ever read. + let fence = Fence::refusing(&[]); + let (result, per_read) = decide(&partitions(60), Some(&fence)); + assert!(result.is_ok()); + assert_eq!(fence.asked().len(), MAX_DECOMPOSED_READS); + assert_eq!(per_read.len(), 60); + assert!(per_read[..50].iter().all(Option::is_none)); + assert!( + per_read[50..] + .iter() + .all(|refusal| matches!(refusal, Some(DynamoDbError::AccessDeniedException(_)))) + ); + // When the 50 asked about are refused, the statement gets the refusal + // of the last of them in read order. + let (result, _) = decide(&partitions(60), Some(&RefuseAll)); + let want = format!("refused {:?}", Some(vec![s("v49")])); + assert_eq!(refused_message(&result), Some(want.as_str())); +} + +#[test] +fn a_statement_authorized_as_a_whole_refuses_no_disjunct() { + let sql = "SELECT * FROM \"t\" WHERE pk IN ['a', 'b']"; + let req = request(sql, json!([])); + let stmt = parse(sql).unwrap(); + let value_of = |op: &Operand| bound_value(op, &req.parameters).cloned(); + let analysis = WhereAnalysis::new(where_clause(&stmt).unwrap(), KEYS, &value_of); + let whole = Refused::default(); + for read in analysis.reads() { + assert!(whole.refusal(read, &req).is_none()); + } +} + +#[test] +fn an_authorizer_fault_fails_the_statement() { + let (result, _) = decide("SELECT * FROM \"t\" WHERE pk IN ['a', 'b']", Some(&Broken)); + assert!(matches!(result, Err(DynamoDbError::InternalServerError(_)))); +} + +#[test] +fn without_an_authorizer_every_statement_is_refused() { + let scan = access_of("SELECT * FROM \"t\""); + let result = block_on(authorize_access(None, &scan)); + assert!(matches!( + result, + Err(DynamoDbError::AccessDeniedException(_)) + )); + // A read by key is refused disjunct by disjunct, so as a whole. + let (result, _) = decide("SELECT * FROM \"t\" WHERE pk IN ['a', 'b']", None); + assert!(refused_message(&result).is_some()); +} diff --git a/crates/engine/src/partiql/base_join.rs b/crates/engine/src/partiql/base_join.rs new file mode 100644 index 000000000..fa876ba15 --- /dev/null +++ b/crates/engine/src/partiql/base_join.rs @@ -0,0 +1,244 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The base-table read under an LSI read whose projection names an attribute +//! the index does not project (the rule is described in `index_read`). +//! +//! The index is read without a filter, so every row it walks comes back. Each +//! row's item is then read from the base table and charged on the table arm, +//! one read unit per 4 KB of item (half that when eventually consistent), +//! before the filter is evaluated against the item (measured 2026-11-02: a +//! filter that keeps no row costs what one that keeps every row costs, and a +//! `Limit` that walks fewer rows costs less). + +use serde_json::{Map, Value}; + +use extenddb_core::error::DynamoDbError; +use extenddb_core::expression::{ + Expr, ExpressionKind, ExpressionMaps, evaluate_condition, validate_begins_with_operands, + validate_ordering_operand_types, +}; +use extenddb_core::limits::LimitsConfig; +use extenddb_core::partiql::ast::{SelectStatement, WhereExpr}; +use extenddb_core::types::{ + AttributeValue, IndexInfo, Item, KeySchemaElement, TableKeyInfo, extract_key, item_size_bytes, +}; + +use super::SingleRequest; +use super::expr_bridge::ExprBridge; +use super::index_read; +use crate::capacity_helpers::read_capacity_units; +use crate::create_table::storage_err_to_dynamo; +use crate::expression_helpers::{ + build_expression_maps, parse_optional_filter, prefix_expression_error, +}; +use crate::{DispatchResult, OperationContext}; + +/// The filter of a read that reaches back, evaluated against each base-table +/// item rather than the index row. +pub(super) struct BaseTableJoin { + filter: Option, + maps: ExpressionMaps, +} + +impl BaseTableJoin { + /// The join a read of `info` needs, or `None` when the index serves the + /// read alone. `filter_terms` are a keyed read's filter terms; a scan + /// (`None`) filters on the whole WHERE. The filter is prepared before the + /// index is read, so a parameter or operand error is reported ahead of the + /// read as it is on any other read. + /// + /// # Errors + /// + /// Returns the parameter count mismatch or the filter validation error the + /// same filter gets on a read that does not reach back. + pub(super) fn for_read( + ctx: &OperationContext, + req: &SingleRequest, + stmt: &SelectStatement, + info: &IndexInfo, + base_key_schema: &[KeySchemaElement], + filter_terms: Option<&[&WhereExpr]>, + ) -> Result, DynamoDbError> { + if !index_read::reaches_back(&stmt.projection, info, base_key_schema) { + return Ok(None); + } + let terms: Vec<&WhereExpr> = match filter_terms { + Some(terms) => terms.to_vec(), + None => stmt.where_clause.iter().collect(), + }; + Self::prepare(&ctx.limits, &req.parameters, &terms).map(Some) + } + + fn prepare( + limits: &LimitsConfig, + params: &[AttributeValue], + terms: &[&WhereExpr], + ) -> Result { + if terms.is_empty() { + return Ok(Self { + filter: None, + maps: ExpressionMaps::default(), + }); + } + let mut bridge = ExprBridge::new(params); + let parts: Vec = terms + .iter() + .map(|t| bridge.render_condition(t)) + .collect::>()?; + let maps = build_expression_maps(Some(&bridge.names_map()), Some(&bridge.values_map())); + let filter = parse_optional_filter(Some(&parts.join(" AND ")), limits)?; + if let Some(f) = &filter { + validate_begins_with_operands(f, &maps) + .map_err(|e| prefix_expression_error(e, ExpressionKind::Filter))?; + validate_ordering_operand_types(f, &maps) + .map_err(|e| prefix_expression_error(e, ExpressionKind::Filter))?; + } + Ok(Self { filter, maps }) + } + + /// Replace each row of an unfiltered index read with its base-table item, + /// keep the items the filter passes, and add the base-table reads to the + /// consumed capacity, on the table arm and in the total. + /// + /// # Errors + /// + /// Returns a storage error from a base-table read, or a filter evaluation + /// error. + pub(super) async fn apply( + &self, + ctx: &OperationContext, + key_info: &TableKeyInfo, + strongly_consistent: bool, + result: DispatchResult, + ) -> Result { + let DispatchResult { body, mut metrics } = result; + let mut body = match body { + Value::Object(m) => m, + _ => Map::new(), + }; + let rows = match body.remove("Items") { + Some(Value::Array(rows)) => rows, + _ => Vec::new(), + }; + let mut kept = Vec::new(); + let mut table_units = 0.0; + for row in rows { + let index_row: Item = serde_json::from_value(row).map_err(|e| internal(&e))?; + let key = extract_key(&index_row, &key_info.key_schema); + let base = ctx + .storage + .get_item(key_info, &key) + .await + .map_err(storage_err_to_dynamo)?; + table_units += read_capacity_units( + base.as_ref().map_or(0, item_size_bytes), + strongly_consistent, + ); + // An item deleted since its index row was read is served from the + // row it left behind. + let item = base.unwrap_or(index_row); + if self.passes(&item)? { + kept.push(serde_json::to_value(&item).map_err(|e| internal(&e))?); + } + } + metrics.returned_item_count = kept.len() as u64; + metrics.read_capacity_units += table_units; + body.insert("Items".to_owned(), Value::Array(kept)); + if let Some(Value::Object(cc)) = body.get_mut("ConsumedCapacity") { + add_units(cc, table_units); + if let Some(Value::Object(table)) = cc.get_mut("Table") { + add_units(table, table_units); + } + } + Ok(DispatchResult { + body: Value::Object(body), + metrics, + }) + } + + fn passes(&self, item: &Item) -> Result { + let Some(filter) = &self.filter else { + return Ok(true); + }; + evaluate_condition(filter, item, &self.maps).map_err(|e| { + let msg = e.to_string(); + if msg.starts_with("Invalid ") { + DynamoDbError::ValidationException(msg) + } else { + DynamoDbError::ValidationException(format!("Invalid FilterExpression: {msg}")) + } + }) + } +} + +fn add_units(capacity: &mut Map, units: f64) { + let current = capacity + .get("CapacityUnits") + .and_then(Value::as_f64) + .unwrap_or(0.0); + capacity.insert("CapacityUnits".to_owned(), Value::from(current + units)); +} + +fn internal(e: &serde_json::Error) -> DynamoDbError { + DynamoDbError::InternalServerError(format!("failed to convert an index row: {e}")) +} + +#[cfg(test)] +mod tests { + use super::*; + use extenddb_core::partiql::ast::Statement; + use extenddb_core::partiql::parse; + + fn where_of(sql: &str) -> WhereExpr { + match parse(sql).unwrap() { + Statement::Select(s) => s.where_clause.unwrap(), + _ => panic!("not a select"), + } + } + + fn item(pairs: &[(&str, &str)]) -> Item { + pairs + .iter() + .map(|(k, v)| ((*k).to_owned(), AttributeValue::S((*v).to_owned()))) + .collect() + } + + #[test] + fn the_filter_is_evaluated_against_the_base_item() { + let w = where_of("SELECT a FROM \"t\".\"idx\" WHERE hidden = ? AND kept = 'k'"); + let params = [AttributeValue::S("h".to_owned())]; + let join = BaseTableJoin::prepare(&LimitsConfig::default(), ¶ms, &[&w]).unwrap(); + assert!( + join.passes(&item(&[("hidden", "h"), ("kept", "k")])) + .unwrap() + ); + assert!(!join.passes(&item(&[("kept", "k")])).unwrap()); + } + + #[test] + fn no_terms_keeps_every_item() { + let join = BaseTableJoin::prepare(&LimitsConfig::default(), &[], &[]).unwrap(); + assert!(join.passes(&Item::new()).unwrap()); + } + + #[test] + fn a_missing_parameter_is_reported_when_the_join_is_prepared() { + let w = where_of("SELECT a FROM \"t\".\"idx\" WHERE hidden = ?"); + let err = BaseTableJoin::prepare(&LimitsConfig::default(), &[], &[&w]) + .err() + .unwrap(); + assert_eq!( + err.message(), + "Number of parameters in request and statement don't match." + ); + } + + #[test] + fn units_are_added_to_an_existing_arm() { + let mut arm = Map::new(); + arm.insert("CapacityUnits".to_owned(), Value::from(0.5)); + add_units(&mut arm, 1.5); + assert_eq!(arm["CapacityUnits"], Value::from(2.0)); + } +} diff --git a/crates/engine/src/partiql/batch.rs b/crates/engine/src/partiql/batch.rs new file mode 100644 index 000000000..2fcf66342 --- /dev/null +++ b/crates/engine/src/partiql/batch.rs @@ -0,0 +1,740 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! PartiQL `BatchExecuteStatement` executor. +//! +//! All-read or all-write (mixing is a request-level `ValidationException`). +//! Reads behave like `BatchGetItem` (`Responses[i] = {TableName, Item}`), writes +//! carry per-statement `Error` entries for partial success. 1..=25 statements, +//! no two on the same key. Every behaviour is justified by a measurement +//! against the live service, cited where it is implemented. +//! +//! The executor composes the single-statement building blocks: a read is the +//! point read `handle_get_item` already serves, a write is `write::run` (each +//! statement committing independently, which is what makes partial success +//! real). Nothing here re-implements statement translation. +//! +//! Only the statements that translate count as reads or writes, and only +//! those whose request passes its checks count in the check for two +//! statements on one item (see `translation`). +//! +//! Each statement is authorized at its place among its checks (see +//! `authorize::gate`). Two statements that name one item fail the request +//! before any statement is authorized, as a read without the full primary +//! key is refused before it is authorized (measured 2026-11-03 for that +//! refusal, with a caller that may not run the statement). For two +//! statements on one item the order against authorization is not measured. + +use std::collections::{BTreeMap, HashMap, HashSet}; + +use serde_json::{Map, Value, json}; + +use extenddb_core::error::DynamoDbError; +use extenddb_core::partiql::Statement; +use extenddb_core::partiql::ast::{FromSource, Projection, SelectStatement}; +use extenddb_core::partiql::parser::parse_for_request; +use extenddb_core::types::{Item, TableKeyInfo, item_size_bytes}; +use extenddb_storage::error::StorageError; + +use super::authorize::{Gate, Surface}; +use super::common::{self, StatementError}; +use super::{SingleRequest, write}; +use crate::capacity_helpers; +use crate::create_table::storage_err_to_dynamo; +use crate::{DispatchMetrics, DispatchResult, OperationContext, handle_get_item}; + +use translation::Translation; + +mod translation; + +/// Per-statement request state after parsing. +struct Parsed { + req: SingleRequest, + stmt: Option, + parse_err: Option, +} + +/// Execute a `BatchExecuteStatement` request. +/// +/// # Errors +/// +/// Returns the service's request-level errors (read/write mix, duplicate key, +/// size limit); per-statement failures are carried inside `Responses`. +// The steps follow the service's order; helpers would split that order across functions. +#[allow(clippy::too_many_lines)] +pub(super) async fn run( + ctx: &OperationContext, + body: &Value, +) -> Result { + super::request_checks::check_batch_execute_statement(body)?; + let statements = body + .get("Statements") + .and_then(Value::as_array) + .ok_or_else(|| { + DynamoDbError::ValidationException("Statements must be a non-empty list".to_owned()) + })?; + + let rcc = body.get("ReturnConsumedCapacity").and_then(Value::as_str); + + // Check and parse each statement; a failure is deferred to per-statement + // execution (the batch stays well-formed as long as size/mix/keys are OK). + // A parameter that cannot be converted fails only its own statement + // (measured 2026-11-02). + let mut checked_all = Vec::with_capacity(statements.len()); + for obj in statements { + let req = SingleRequest::from_body(obj)?; + let statement = if super::request_checks::is_blank_statement(&req.statement) { + Err(super::request_checks::EMPTY_BATCH_STATEMENT.to_owned()) + } else { + super::check_parameters(obj.get("Parameters")).and_then(|()| { + parse_for_request(&req.statement, req.parameters.len()).map_err(|e| e.message) + }) + }; + let projected_call = match &statement { + Ok(Statement::Select(select)) => select + .projection_error + .clone() + .map(|message| (select.from.clone(), message)), + _ => None, + }; + // A projection list is checked with the statement: its refusal is the + // statement's own, and the statement counts as neither a read nor a + // write (measured 2026-11-05). + let checked = statement.and_then(|stmt| { + super::check_statement(&stmt, &req.parameters)?; + if let Statement::Select(select) = &stmt { + super::select::check_projection(select)?; + } + Ok(stmt) + }); + let projected = match (&checked, projected_call) { + (Err(message), Some((from, call))) if *message == call => Some(from), + _ => None, + }; + checked_all.push((req, checked, projected)); + } + + // Each table the statements read or write is read from the catalog once, + // and every statement runs against that read. + let tables: Vec<&str> = checked_all + .iter() + .filter_map(|(_, checked, projected)| match (checked, projected) { + (Ok(Statement::Exists(_)), _) | (Err(_), None) => None, + (Ok(stmt), _) => Some(common::statement_table(stmt)), + (Err(_), Some(from)) => Some(from.table.as_str()), + }) + .collect(); + let ctx = &super::snapshot::snapshot_context(ctx, &tables).await?; + + let mut parsed: Vec = Vec::with_capacity(checked_all.len()); + for (req, checked, projected) in checked_all { + let checked = match (checked, projected) { + (Err(message), Some(from)) => Err(projected_call_error(ctx, &from, message).await?), + (checked, _) => checked, + }; + match checked { + // A condition check belongs to a transaction: in a batch it is a + // per-statement refusal and counts as neither a read nor a write. + Ok(Statement::Exists(_)) => parsed.push(Parsed { + req, + stmt: None, + parse_err: Some(write::exists_outside_transaction().message().to_owned()), + }), + Ok(stmt) => parsed.push(Parsed { + req, + stmt: Some(stmt), + parse_err: None, + }), + Err(message) => parsed.push(Parsed { + req, + stmt: None, + parse_err: Some(message), + }), + } + } + + // Each statement's table key schema, from the read above (None when the + // table is unknown; that statement's ResourceNotFound surfaces + // per-statement). + let mut key_infos: Vec> = Vec::with_capacity(parsed.len()); + for p in &parsed { + match &p.stmt { + Some(stmt) => { + let table = common::statement_table(stmt); + key_infos.push(ctx.table_key_info(table).await.ok()); + } + None => key_infos.push(None), + } + } + + // Each statement that parsed is translated into the read or the write it + // runs as (see `translation`). + let mut translations: Vec> = Vec::with_capacity(parsed.len()); + for (p, key_info) in parsed.iter().zip(&key_infos) { + translations.push(match &p.stmt { + Some(Statement::Select(select)) => { + Some(check_read(ctx, select, &p.req, key_info.as_ref()).await?) + } + Some(stmt) => { + Some(translation::translate_write(ctx, stmt, &p.req, key_info.as_ref()).await?) + } + None => None, + }); + } + + // Classify over the statements that translate. A batch is all reads or + // all writes; mixing fails the whole request. + let counted = || { + parsed + .iter() + .zip(&translations) + .filter(|(_, translation)| translation.as_ref().is_some_and(Translation::counts)) + .filter_map(|(p, _)| p.stmt.as_ref()) + }; + let reads = counted().filter(|s| s.is_read()).count(); + let writes = counted().filter(|s| !s.is_read()).count(); + if reads > 0 && writes > 0 { + return Err(DynamoDbError::ValidationException( + "Read and write requests together in the same batch is not supported.".to_owned(), + )); + } + + // Two statements naming the same item fail the whole request. + detect_duplicate_keys(&parsed, &translations)?; + + // Execute positionally. + let mut responses: Vec = Vec::with_capacity(parsed.len()); + let mut capacity = CapacityList::default(); + for ((p, translation), key_info) in parsed.into_iter().zip(translations).zip(&key_infos) { + if let Some(msg) = p.parse_err { + responses.push(StatementError::parse(msg).into_entry(None)); + continue; + } + let stmt = p.stmt.expect("non-parse-error statement is Some"); + let translation = translation.expect("a statement that parsed is translated"); + let table = common::statement_table(&stmt).to_owned(); + let mut req = p.req; + req.return_consumed_capacity = rcc.map(str::to_owned); + + // Each statement is authorized at its place among its checks (see + // `authorize::gate`). A refused statement's entry carries the + // refusal and its table, and the others run (measured 2026-11-03). + let mut gate = Gate::new(ctx, &stmt, key_info.as_ref(), &req, Surface::Batch); + let key = match translation { + Translation::Runs { key } => key, + Translation::Untranslated(error) => { + let answer = gate.settle(Err::(error)).await?; + responses.push(refusal_entry(answer, false, &table)); + continue; + } + Translation::Refused { error, names_table } => { + let answer = gate.settle(Err::(error)).await?; + responses.push(refusal_entry(answer, names_table, &table)); + continue; + } + }; + let result = match &stmt { + Statement::Select(_) => { + let key = key.clone().expect("a read that runs names its item"); + execute_read(ctx, &req, &table, key, &mut gate).await + } + _ => write::run(ctx, &stmt, &req, &mut gate).await, + }; + // Every error of a statement that reached its item names the table: + // a failed condition, a duplicate key, a key value past its size + // limit, an update the item cannot take (measured 2026-11-02 for the + // key size, 2026-11-05 for the others). + let reached_item = result.is_err() && gate.decided(); + let entry = match gate.settle(result).await? { + Ok(result) => { + capacity.add(&result.body); + match &stmt { + Statement::Select(select) => read_entry(select, &table, &result.body), + _ => write_entry(&table, &result.body), + } + } + Err(error) if reached_item => { + if rcc.is_some() + && !stmt.is_read() + && is_charged_failure(&error) + && let (Some(key_info), Some(key)) = (key_info, &key) + { + capacity.add_failed(&table, failed_write_units(ctx, key_info, key).await?); + } + StatementError::from_item_error(&error).into_entry(Some(&table)) + } + Err(error) => StatementError::from_error(&error).into_entry(Some(&table)), + }; + responses.push(entry); + } + + let mut out = Map::new(); + out.insert("Responses".to_owned(), Value::Array(responses)); + if let Some(cc) = capacity.into_value() { + out.insert("ConsumedCapacity".to_owned(), cc); + } + Ok(DispatchResult { + body: Value::Object(out), + metrics: DispatchMetrics::default(), + }) +} + +/// The error a read whose projection is a call (`COUNT(*)`) reports: the +/// call's refusal, on an existing table or a missing one, unless the target's +/// key has more than one HASH or RANGE attribute, whose refusal comes first +/// (measured 2026-11-02 on an index with two HASH attributes). +async fn projected_call_error( + ctx: &OperationContext, + from: &FromSource, + message: String, +) -> Result { + let key_info = match ctx.table_key_info(&from.table).await { + Ok(info) => info, + Err(StorageError::TableNotFound(_)) => return Ok(message), + Err(e) => return Err(storage_err_to_dynamo(e)), + }; + let schema = match &from.index { + None => key_info.key_schema, + Some(index) => match ctx + .storage + .index_info_by_table_id(&key_info.table_id, index) + .await + { + Ok(info) => info.key_schema, + Err(StorageError::IndexNotFound(_)) => return Ok(message), + Err(e) => return Err(storage_err_to_dynamo(e)), + }, + }; + Ok(match common::ensure_single_attribute_keys(&schema) { + Ok(()) => message, + Err(refusal) => refusal.message().to_owned(), + }) +} + +/// The refusal of a `BatchExecuteStatement` read that is not a read of one +/// item by its full primary key. It comes before the statement is +/// authorized (measured 2026-11-03). +pub(super) const PRIMARY_KEY_REQUIRED: &str = "Select statements within BatchExecuteStatement must specify the primary key in the where \ + clause."; + +/// How far the read statement `select` gets (see [`Translation`]), from the +/// checks it gets before its item is read, in the service's order. A read +/// that runs names its item. The outer error is a fault that fails the +/// request. +async fn check_read( + ctx: &OperationContext, + select: &SelectStatement, + req: &SingleRequest, + key_info: Option<&TableKeyInfo>, +) -> Result { + let untranslated = |error| Ok(Translation::Untranslated(error)); + // A read that is not of one item by its full primary key translates, and + // counts as a read (measured 2026-11-05). + let primary_key_required = || { + Ok(Translation::Refused { + error: DynamoDbError::ValidationException(PRIMARY_KEY_REQUIRED.to_owned()), + names_table: false, + }) + }; + // An unknown table is a per-statement ResourceNotFound (the call stays 200). + let Some(key_info) = key_info else { + return untranslated(DynamoDbError::ResourceNotFoundException(String::new())); + }; + + // A named index is resolved before the key check. Measured on the service: + // a missing index is a per-statement ValidationError with the bare index + // message; a multi-attribute index key is the multi-attribute refusal; an + // index that exists (GSI, LSI or vector) can never satisfy the primary-key + // rule and gets the key message whatever the WHERE names. + if let Some(index_name) = &select.from.index { + let is_vector = key_info + .vector_indexes + .iter() + .any(|v| &v.index_name == index_name); + if !is_vector { + match ctx + .storage + .index_info_by_table_id(&key_info.table_id, index_name) + .await + { + Ok(info) => { + if let Err(e) = common::ensure_single_attribute_keys(&info.key_schema) { + return untranslated(e); + } + } + Err(StorageError::IndexNotFound(_)) => { + return untranslated(DynamoDbError::ValidationException( + "The table does not have the specified index".to_owned(), + )); + } + Err(e) => return Err(storage_err_to_dynamo(e)), + } + } + return primary_key_required(); + } + + // A multi-part base table key (this engine's preview extension) is refused + // per statement with the service's multi-attribute message. + if let Err(e) = common::ensure_single_attribute_keys(&key_info.key_schema) { + return untranslated(e); + } + + let (pk, sk) = common::key_names(&key_info.key_schema); + let Some(pk) = pk else { + return Err(DynamoDbError::InternalServerError( + "table has no partition key".to_owned(), + )); + }; + + // A batch SELECT must be a full-primary-key point read; a partial key is a + // per-statement ValidationError. + let Some(key) = common::point_read_key(select, pk, sk, &req.parameters)? else { + return primary_key_required(); + }; + // A key value of the wrong type gets the message a single SELECT gets, + // not the GetItem one (measured 2026-11-02). + if let Err(e) = common::check_key_types(&key, key_info) { + return untranslated(e); + } + Ok(Translation::Runs { key: Some(key) }) +} + +/// Read the item `key` names in `table` for a read statement of `req`, +/// passing `gate` first: the `GetItem` answer, or the statement's refusal. +async fn execute_read( + ctx: &OperationContext, + req: &SingleRequest, + table: &str, + key: Map, + gate: &mut Gate<'_>, +) -> Result { + gate.pass().await?; + let mut get_body = json!({ "TableName": table, "Key": Value::Object(key) }); + // ConsistentRead is carried per member: a consistent member is charged a + // full read unit where an eventual one is charged half (measured + // 2026-11-02). + if req.consistent_read { + get_body["ConsistentRead"] = json!(true); + } + if let Some(rcc) = &req.return_consumed_capacity { + get_body["ReturnConsumedCapacity"] = json!(rcc); + } + handle_get_item(get_body, ctx).await +} + +/// A read member's `Responses[i]` entry: its table, plus the item `body` +/// holds. A batch SELECT with a projection list projects the item, as the +/// service does; SELECT * returns it whole. +fn read_entry(select: &SelectStatement, table: &str, body: &Value) -> Value { + let mut entry = Map::new(); + entry.insert("TableName".to_owned(), Value::String(table.to_owned())); + if let Some(item) = body.get("Item") { + let shaped = match &select.projection { + Projection::All => item.clone(), + Projection::Paths(paths) => super::select::project_item(item, paths), + }; + entry.insert("Item".to_owned(), shaped); + } + Value::Object(entry) +} + +/// The `Responses[i]` entry of a statement refused before it ran: `answer` +/// is the refusal, or the statement's `AccessDenied`. The refusal carries +/// the statement's table when `names_table`. +fn refusal_entry(answer: Result, names_table: bool, table: &str) -> Value { + let error = match answer { + Err(error) => error, + Ok(_) => unreachable!("a refused statement is answered with an error"), + }; + let entry = if names_table { + StatementError::from_item_error(&error) + } else { + StatementError::from_error(&error) + }; + entry.into_entry(Some(table)) +} + +/// Fail the request when two statements that run address the same item. +fn detect_duplicate_keys( + parsed: &[Parsed], + translations: &[Option], +) -> Result<(), DynamoDbError> { + let mut seen: HashSet = HashSet::with_capacity(parsed.len()); + for (p, translation) in parsed.iter().zip(translations) { + let (Some(stmt), Some(key)) = (&p.stmt, translation.as_ref().and_then(Translation::key)) + else { + continue; + }; + let id = common::canonical_identity(common::statement_table(stmt), key); + if !seen.insert(id) { + return Err(DynamoDbError::ValidationException( + "Provided list of item keys contains duplicates".to_owned(), + )); + } + } + Ok(()) +} + +/// Whether a write that failed at its item is charged the write units of +/// that item: a failed condition, a duplicate INSERT and an update that the +/// item cannot take are; a refusal of the request is not (measured +/// 2026-11-05). +fn is_charged_failure(error: &DynamoDbError) -> bool { + match error { + DynamoDbError::ConditionalCheckFailedException(..) + | DynamoDbError::DuplicateItemException(_) => true, + DynamoDbError::ValidationException(message) => UPDATE_FAILURES.contains(&message.as_str()), + _ => false, + } +} + +/// The refusals of an update that the stored item cannot take. +const UPDATE_FAILURES: [&str; 2] = [ + "An operand in the update expression has an incorrect data type", + "The document path provided in the update expression is invalid for update", +]; + +/// The write units a failed write is charged: those of the item stored under +/// `key`, or one when there is none (measured 2026-11-05: a failed condition +/// on a 3 KB item is charged 3 units, on a missing item 1, and a duplicate +/// INSERT of a small item over a 3 KB item 3). +async fn failed_write_units( + ctx: &OperationContext, + key_info: &TableKeyInfo, + key: &Map, +) -> Result { + let key: Item = serde_json::from_value(Value::Object(key.clone())).map_err(|e| { + tracing::error!(internal_error = %e, "failed to read a translated PartiQL key"); + DynamoDbError::InternalServerError("Internal server error".to_owned()) + })?; + let stored = ctx + .storage + .get_item(key_info, &key) + .await + .map_err(storage_err_to_dynamo)?; + Ok(capacity_helpers::write_capacity_units( + stored.as_ref().map_or(0, item_size_bytes), + )) +} + +/// A write member's `Responses[i]` entry: its table, plus the `RETURNING` row +/// as `Item` when the statement returned one. A batch member honours +/// `RETURNING` with the same projection `ExecuteStatement` puts in `Items`; +/// the singular `Item` cannot hold an empty row, so an empty projection leaves +/// the field out (measured 2026-11-02). +fn write_entry(table: &str, body: &Value) -> Value { + let mut entry = Map::new(); + entry.insert("TableName".to_owned(), Value::String(table.to_owned())); + if let Some(item) = body + .get("Items") + .and_then(Value::as_array) + .and_then(|items| items.first()) + { + entry.insert("Item".to_owned(), item.clone()); + } + Value::Object(entry) +} + +/// Accumulates per-table `ConsumedCapacity` objects from each sub-operation into +/// the single list a batch returns, summing `CapacityUnits`, and at INDEXES +/// granularity the nested `Table` units and the units of each global and +/// local secondary index, per table in first-seen order (measured +/// 2026-11-05). +/// +/// The units of a failed write are added to its table's `CapacityUnits` +/// alone, and only when a statement on that table succeeded: a table on +/// which every write failed has no entry (measured 2026-11-05). +#[derive(Default)] +struct CapacityList { + order: Vec, + totals: HashMap, + failed: HashMap, +} + +/// The units summed for one table of a batch. +#[derive(Default)] +struct TableCapacity { + units: f64, + table: Option, + global: BTreeMap, + local: BTreeMap, +} + +impl CapacityList { + /// Fold one sub-operation response's `ConsumedCapacity` object in. + fn add(&mut self, body: &Value) { + let Some(cc) = body.get("ConsumedCapacity") else { + return; + }; + let Some(table) = cc.get("TableName").and_then(Value::as_str) else { + return; + }; + if !self.totals.contains_key(table) { + self.order.push(table.to_owned()); + self.totals + .insert(table.to_owned(), TableCapacity::default()); + } + let entry = self.totals.get_mut(table).expect("inserted above"); + entry.units += units_of(cc).unwrap_or(0.0); + if let Some(units) = cc.get("Table").and_then(units_of) { + entry.table = Some(entry.table.unwrap_or(0.0) + units); + } + add_index_units(&mut entry.global, cc.get("GlobalSecondaryIndexes")); + add_index_units(&mut entry.local, cc.get("LocalSecondaryIndexes")); + } + + /// Count `units` for a write on `table` that failed at its item. + fn add_failed(&mut self, table: &str, units: f64) { + *self.failed.entry(table.to_owned()).or_default() += units; + } + + /// The `ConsumedCapacity` list, or `None` when nothing was accumulated. + fn into_value(self) -> Option { + if self.order.is_empty() { + return None; + } + let list: Vec = self + .order + .iter() + .map(|table| { + let entry = &self.totals[table]; + let failed = self.failed.get(table).copied().unwrap_or(0.0); + let mut obj = Map::new(); + obj.insert("TableName".to_owned(), Value::String(table.clone())); + obj.insert("CapacityUnits".to_owned(), json!(entry.units + failed)); + if let Some(units) = entry.table { + obj.insert("Table".to_owned(), json!({ "CapacityUnits": units })); + } + for (field, indexes) in [ + ("GlobalSecondaryIndexes", &entry.global), + ("LocalSecondaryIndexes", &entry.local), + ] { + if !indexes.is_empty() { + let arms: Map = indexes + .iter() + .map(|(name, units)| (name.clone(), json!({ "CapacityUnits": units }))) + .collect(); + obj.insert(field.to_owned(), Value::Object(arms)); + } + } + Value::Object(obj) + }) + .collect(); + Some(Value::Array(list)) + } +} + +/// The `CapacityUnits` of one arm of a `ConsumedCapacity` object. +fn units_of(arm: &Value) -> Option { + arm.get("CapacityUnits").and_then(Value::as_f64) +} + +/// Add the units of each index of an `INDEXES` breakdown to `totals`. +fn add_index_units(totals: &mut BTreeMap, indexes: Option<&Value>) { + let Some(Value::Object(indexes)) = indexes else { + return; + }; + for (name, arm) in indexes { + *totals.entry(name.clone()).or_default() += units_of(arm).unwrap_or(0.0); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_returned_row_becomes_the_member_item() { + let body = json!({ "Items": [{ "pk": {"S": "k"}, "data": {"S": "new"} }] }); + assert_eq!( + write_entry("t", &body), + json!({ "TableName": "t", "Item": { "pk": {"S": "k"}, "data": {"S": "new"} } }) + ); + } + + #[test] + fn no_returned_row_leaves_the_item_out() { + assert_eq!( + write_entry("t", &json!({ "Items": [] })), + json!({ "TableName": "t" }) + ); + } + + fn capacity(table: &str, units: f64, gsi: f64, lsi: f64) -> Value { + json!({ "ConsumedCapacity": { + "TableName": table, + "CapacityUnits": units, + "Table": { "CapacityUnits": 1.0 }, + "GlobalSecondaryIndexes": { "gsi": { "CapacityUnits": gsi } }, + "LocalSecondaryIndexes": { "lsi": { "CapacityUnits": lsi } }, + }}) + } + + #[test] + fn the_index_arms_are_summed_per_table() { + let mut list = CapacityList::default(); + list.add(&capacity("t", 4.0, 2.0, 1.0)); + list.add(&capacity("u", 2.0, 0.5, 0.5)); + list.add(&capacity("t", 3.0, 1.0, 1.0)); + assert_eq!( + list.into_value(), + Some(json!([ + { + "TableName": "t", + "CapacityUnits": 7.0, + "Table": { "CapacityUnits": 2.0 }, + "GlobalSecondaryIndexes": { "gsi": { "CapacityUnits": 3.0 } }, + "LocalSecondaryIndexes": { "lsi": { "CapacityUnits": 2.0 } }, + }, + { + "TableName": "u", + "CapacityUnits": 2.0, + "Table": { "CapacityUnits": 1.0 }, + "GlobalSecondaryIndexes": { "gsi": { "CapacityUnits": 0.5 } }, + "LocalSecondaryIndexes": { "lsi": { "CapacityUnits": 0.5 } }, + }, + ])) + ); + } + + #[test] + fn a_failed_write_counts_in_the_total_of_a_table_with_an_entry() { + let mut list = CapacityList::default(); + list.add_failed("t", 3.0); + list.add(&json!({ "ConsumedCapacity": { "TableName": "t", "CapacityUnits": 2.0 } })); + list.add_failed("u", 1.0); + assert_eq!( + list.into_value(), + Some(json!([{ "TableName": "t", "CapacityUnits": 5.0 }])) + ); + let mut failed_only = CapacityList::default(); + failed_only.add_failed("t", 1.0); + assert_eq!(failed_only.into_value(), None); + } + + #[test] + fn failures_at_the_item_are_charged() { + for error in [ + DynamoDbError::ConditionalCheckFailedException( + "The conditional request failed".to_owned(), + None, + ), + DynamoDbError::DuplicateItemException( + "Duplicate primary key exists in table".to_owned(), + ), + DynamoDbError::ValidationException(UPDATE_FAILURES[0].to_owned()), + DynamoDbError::ValidationException(UPDATE_FAILURES[1].to_owned()), + ] { + assert!(is_charged_failure(&error), "{error:?}"); + } + let key_update = DynamoDbError::ValidationException( + "One or more parameter values were invalid: Cannot update attribute pk. This \ + attribute is part of the key" + .to_owned(), + ); + assert!(!is_charged_failure(&key_update)); + assert!(!is_charged_failure(&DynamoDbError::AccessDeniedException( + "refused".to_owned() + ))); + } +} diff --git a/crates/engine/src/partiql/batch/translation.rs b/crates/engine/src/partiql/batch/translation.rs new file mode 100644 index 000000000..0ba097eeb --- /dev/null +++ b/crates/engine/src/partiql/batch/translation.rs @@ -0,0 +1,134 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! How far a statement of a `BatchExecuteStatement` gets before it runs. +//! +//! The service translates each statement into the read or the write it runs +//! as. It then sorts the statements that translate into reads and writes, +//! checks the request each one translates into, refuses two statements on +//! one item, and runs each statement (measured 2026-11-05). +//! +//! A statement whose translation fails gets its own error, and counts +//! neither as a read nor as a write, nor in the check for two statements on +//! one item: a missing table or index, a key value of the wrong type, an +//! INSERT without a key attribute, a write whose WHERE does not give the full +//! key or tests a path into a key attribute against values, a `?` with no +//! parameter, and a DELETE returning clause other than `RETURNING ALL OLD *`. +//! +//! A statement that translates and fails the check of its request counts as a +//! read or a write, and not in the check for two statements on one item: a +//! read that is not of one item by its full primary key, an UPDATE that sets +//! or removes a key attribute, a write that gives a secondary index key a +//! value of the wrong type, and, after those two, a write whose WHERE reads a +//! path into a key attribute among its other conditions (measured 2026-11-05 +//! on UPDATE and DELETE). The write refusals name the table, as the errors of +//! a write that reached its item do. + +use serde_json::{Map, Value}; + +use extenddb_core::error::DynamoDbError; +use extenddb_core::partiql::Statement; +use extenddb_core::partiql::ast::WhereExpr; +use extenddb_core::types::{AttributeValue, TableKeyInfo}; + +use crate::OperationContext; +use crate::partiql::{SingleRequest, common, write}; + +/// How far a statement that parsed gets before the batch runs it. +pub(super) enum Translation { + /// The statement runs. `key` is the item it names, when it names one. + Runs { key: Option> }, + /// The statement fails its translation, with this error. + Untranslated(DynamoDbError), + /// The statement translates and fails the check of its request with + /// `error`, which carries the statement's table when `names_table`. + Refused { + error: DynamoDbError, + names_table: bool, + }, +} + +impl Translation { + /// Whether the statement counts as a read or a write. + pub(super) fn counts(&self) -> bool { + !matches!(self, Self::Untranslated(_)) + } + + /// The item the statement names, when it runs and names one. + pub(super) fn key(&self) -> Option<&Map> { + match self { + Self::Runs { key } => key.as_ref(), + Self::Untranslated(_) | Self::Refused { .. } => None, + } + } +} + +/// How far the write statement `stmt` gets (see [`Translation`]). Its +/// translation makes the checks [`write::run`] makes before it authorizes +/// the statement. +/// +/// # Errors +/// +/// A fault of a catalog read, or of reading back a translated map. +pub(super) async fn translate_write( + ctx: &OperationContext, + stmt: &Statement, + req: &SingleRequest, + key_info: Option<&TableKeyInfo>, +) -> Result { + if let Err(error) = write::check_translation(ctx, stmt, &req.parameters).await { + return Ok(Translation::Untranslated(error)); + } + let Some(key_info) = key_info else { + return Ok(Translation::Untranslated( + DynamoDbError::ResourceNotFoundException(String::new()), + )); + }; + let params = &req.parameters; + let refusal = match stmt { + Statement::Update(update) => { + let parts = write::build_update_parts(ctx, update, params).await?; + write::update_refusal(&ctx.limits, &parts, key_info)? + .or_else(|| condition_key_refusal(update.where_clause.as_ref(), key_info, params)) + } + Statement::Insert(insert) => { + let parts = write::build_insert_parts(ctx, insert, params).await?; + write::insert_refusal(&parts, key_info)? + } + Statement::Delete(delete) => { + condition_key_refusal(Some(&delete.where_clause), key_info, params) + } + _ => None, + }; + if let Some(error) = refusal { + return Ok(Translation::Refused { + error, + names_table: true, + }); + } + let (Some(pk), sk) = common::key_names(&key_info.key_schema) else { + return Err(DynamoDbError::InternalServerError( + "table has no partition key".to_owned(), + )); + }; + Ok(Translation::Runs { + key: common::write_key(stmt, pk, sk, &req.parameters)?, + }) +} + +/// The refusal of a write whose WHERE pins its item and reads a path into a +/// key attribute among its other conditions (see +/// `common::condition_key_refusal`). An UPDATE gets it after the refusals +/// of its `UpdateItem` request (measured 2026-11-05: `SET pk = 'z'` and a +/// `SET` of an index key to a value of the wrong type are refused first). +fn condition_key_refusal( + where_clause: Option<&WhereExpr>, + key_info: &TableKeyInfo, + params: &[AttributeValue], +) -> Option { + let (Some(pk), sk) = common::key_names(&key_info.key_schema) else { + return None; + }; + common::condition_key_refusal(where_clause?, pk, sk, params) + .map(DynamoDbError::ValidationException) +} diff --git a/crates/engine/src/partiql/common.rs b/crates/engine/src/partiql/common.rs new file mode 100644 index 000000000..4e73cee80 --- /dev/null +++ b/crates/engine/src/partiql/common.rs @@ -0,0 +1,577 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Shared helpers for the `BatchExecuteStatement` and `ExecuteTransaction` +//! executors: deriving an item's primary key from a parsed statement, and +//! mapping a single-statement error into the short per-statement `Error` shape +//! a batch carries. Every behaviour is justified by a measurement against the +//! live service, cited where it is implemented. + +use std::collections::BTreeMap; + +use serde::Deserialize; +use serde_json::{Map, Value}; + +use extenddb_core::error::DynamoDbError; +use extenddb_core::partiql::Statement; +use extenddb_core::partiql::ast::{ + CompareOp, InsertStatement, Operand, SelectStatement, WhereExpr, +}; +use extenddb_core::partiql::plan::{ + CONDITION_KEY_REFUSAL, KeyEquality, KeyNames, SortCondition, WhereAnalysis, condition_key_path, + key_equality, path_tested_against_values, same_value, +}; +use extenddb_core::types::{ + AttributeValue, CancellationReason, KeySchemaElement, KeyType, ScalarAttributeType, + TableKeyInfo, is_multipart_key_schema, +}; + +use super::write::{resolve_av, to_json_value}; + +/// The service's refusal for any PartiQL statement whose target (table or +/// index) has more than one HASH or more than one RANGE key attribute. +pub(super) const MULTI_ATTRIBUTE_KEY_MESSAGE: &str = "PartiQL statements cannot be executed on \ + tables with multi-attribute HASH keys or multi-attribute RANGE keys."; + +/// Refuse a target whose key schema has more than one HASH or RANGE element. +/// +/// Measured on a GSI with a `HASH, RANGE, RANGE` key: every `ExecuteStatement` +/// form (no WHERE, partial key, full key, filter on a non-key, `ConsistentRead`) +/// returns this `ValidationException`, and a batch carries it as a per-statement +/// `ValidationError`. A multi-part base table exists only under this engine's +/// preview extension; the same refusal applies there. +/// +/// # Errors +/// +/// Returns `ValidationException` with the service's message. +pub(super) fn ensure_single_attribute_keys( + schema: &[KeySchemaElement], +) -> Result<(), DynamoDbError> { + if is_multipart_key_schema(schema) { + return Err(DynamoDbError::ValidationException( + MULTI_ATTRIBUTE_KEY_MESSAGE.to_owned(), + )); + } + Ok(()) +} + +/// The refusal of a key value whose type is not the one the table defines for +/// that key attribute, naming the attribute (measured 2026-11-02 for an INSERT +/// key, for SELECT, UPDATE and DELETE key conditions, and for a +/// `BatchExecuteStatement` SELECT, a number against a string key). +pub(super) fn key_type_error(name: &str) -> DynamoDbError { + DynamoDbError::ValidationException(format!( + "Key attribute's data type should match its data type in table's schema: Key {name}" + )) +} + +/// The reason an `ExecuteTransaction` member reports for a key that does not +/// match the table's key schema: a write member's partial key, or a key value +/// of the wrong type in a read or a write member (measured 2026-11-02). +pub(super) const KEY_SCHEMA_MISMATCH: &str = "The provided key element does not match the schema"; + +/// The reason an `INSERT` member of `ExecuteTransaction` reports when its key +/// is already in the table. +pub(super) fn duplicate_item_reason() -> CancellationReason { + CancellationReason { + code: "DuplicateItem".to_owned(), + message: Some("Duplicate primary key exists in table".to_owned()), + item: None, + } +} + +/// Refuse a key (in wire form) whose value for a key attribute has a type the +/// table does not define for it, the sort key before the partition key (see +/// [`key_type_error`]). For `pk = 1 AND sk = 1` the service names `sk` on +/// SELECT, UPDATE and DELETE and in `BatchExecuteStatement` (measured +/// 2026-11-02); for some other pairs of key names it names the partition +/// key, and no rule was found. +/// +/// # Errors +/// +/// Returns [`key_type_error`] for the attribute. +pub(super) fn check_key_types( + key: &Map, + key_info: &TableKeyInfo, +) -> Result<(), DynamoDbError> { + let sort_first = key_info + .key_schema + .iter() + .filter(|k| k.key_type == KeyType::Range) + .chain( + key_info + .key_schema + .iter() + .filter(|k| k.key_type == KeyType::Hash), + ); + for element in sort_first { + let name = element.attribute_name.as_str(); + if let Some(value) = key.get(name) + && !wire_key_value_matches(key_info, name, value) + { + return Err(key_type_error(name)); + } + } + Ok(()) +} + +/// Whether a wire-form key value has the type the table defines for key +/// attribute `name` (see [`check_key_value_type`]). A value that is no +/// attribute value does not. +pub(super) fn wire_key_value_matches(key_info: &TableKeyInfo, name: &str, value: &Value) -> bool { + AttributeValue::deserialize(value) + .is_ok_and(|value| check_key_value_type(key_info, name, &value).is_ok()) +} + +/// The wire type tag (`S`, `N` or `B`) the table defines for attribute `name`, +/// or `None` when it defines none. +fn defined_key_type(key_info: &TableKeyInfo, name: &str) -> Option<&'static str> { + key_info + .attribute_definitions + .iter() + .find(|d| d.attribute_name == name) + .map(|d| match d.attribute_type { + ScalarAttributeType::S => "S", + ScalarAttributeType::N => "N", + ScalarAttributeType::B => "B", + }) +} + +/// Refuse a key value whose type is not the one the table defines for key +/// attribute `name`; an attribute with no definition takes any value. +/// +/// # Errors +/// +/// Returns [`key_type_error`] for the attribute. +pub(super) fn check_key_value_type( + key_info: &TableKeyInfo, + name: &str, + value: &AttributeValue, +) -> Result<(), DynamoDbError> { + let tag = match value { + AttributeValue::S(_) => "S", + AttributeValue::N(_) => "N", + AttributeValue::B(_) => "B", + _ => "", + }; + match defined_key_type(key_info, name) { + Some(defined) if defined != tag => Err(key_type_error(name)), + _ => Ok(()), + } +} + +/// The partition and optional sort attribute names of a key schema. +pub(super) fn key_names(schema: &[KeySchemaElement]) -> (Option<&str>, Option<&str>) { + let mut partition = None; + let mut sort = None; + for element in schema { + match element.key_type { + KeyType::Hash => partition = Some(element.attribute_name.as_str()), + KeyType::Range => sort = Some(element.attribute_name.as_str()), + } + } + (partition, sort) +} + +/// The target table name of any parsed statement. +pub(super) fn statement_table(stmt: &Statement) -> &str { + match stmt { + Statement::Select(s) | Statement::Exists(s) => &s.from.table, + Statement::Insert(i) => &i.table, + Statement::Update(u) => &u.table, + Statement::Delete(d) => &d.table, + } +} + +/// The wire-form primary key of a `SELECT` that is a point read, or `None` +/// when the WHERE is not one disjunct that reads by key with an equality on +/// every key attribute and nothing else (see [`WhereAnalysis`]). A value on +/// the left is the mirrored equality (measured 2026-11-02: `'a' = pk` reads +/// the item in `BatchExecuteStatement` and `ExecuteTransaction`). A +/// batch/transaction read must be a point read; `None` is what drives the +/// "must specify the primary key" error at the call site (batch +/// per-statement, transaction request-level). +pub(super) fn point_read_key( + stmt: &SelectStatement, + pk: &str, + sk: Option<&str>, + params: &[AttributeValue], +) -> Result>, DynamoDbError> { + let Some(where_clause) = &stmt.where_clause else { + return Ok(None); + }; + let value_of = |op: &Operand| bound_value(op, params).cloned(); + let keys = KeyNames { + partition: pk, + sort: sk, + }; + let analysis = WhereAnalysis::new(where_clause, keys, &value_of); + // A key condition compared with a boolean is not measured in these + // reads: one compared so that the comparison negates it (`= false`, `<> + // true`) is no point read, and one that keeps it is the key condition. + let Some(read) = analysis + .single() + .filter(|read| read.filter.is_empty() && !read.comparisons.iter().any(|c| c.negates())) + else { + return Ok(None); + }; + let sort_value = match read.sort { + None => None, + Some(SortCondition::Compare(CompareOp::Eq, value)) => Some(value), + Some(_) => return Ok(None), + }; + build_key(pk, sk, Some(read.partition), sort_value, params) +} + +/// How a write's WHERE pins its item: the top-level `AND` conditions that are +/// equalities of a key attribute with a value, written either way round, and +/// the other conditions. +pub(super) enum WhereKey<'a> { + /// Every key attribute has one value; `residual` holds the other + /// conditions in statement order. + Pinned { + partition: &'a Operand, + sort: Option<&'a Operand>, + residual: Vec<&'a WhereExpr>, + }, + /// A key attribute has no equality. + Partial, + /// A key attribute has equalities with two different values; `name` is + /// the first such attribute in statement order. `first` is the key the + /// first equality of each key attribute gives, when every one has an + /// equality. + Conflict { + name: String, + first: Option<(&'a Operand, Option<&'a Operand>)>, + }, + /// A condition tests a path into the key attribute `name` against values + /// (see `plan::path_tested_against_values`), ahead of any second value + /// for a key attribute. + IntoKey { name: String }, +} + +/// Read a write's WHERE for the item it pins (see [`WhereKey`]). Two +/// equalities with one value are one (measured 2026-11-02: `pk = 'a' AND +/// pk = 'a'` and `sk = 'x' AND 'x' = sk` update the item). +/// +/// A condition that tests a path into a key attribute against values and a +/// second value for a key attribute are refused in statement order, ahead +/// of a key attribute with no equality and of the key's types (measured +/// 2026-11-05 on UPDATE and DELETE: `pk = 'k' AND pk = 'zz' AND sk = 'b' AND +/// sk.c = 1` names the second value, `pk = 'k' AND sk = 'b' AND pk.c = 1 AND +/// pk = 'zz'` the path, and `pk = 'k' AND pk.c = 1` and `pk = 1 AND sk = 'b' +/// AND pk.c = 1` the path). +pub(super) fn where_key<'a>( + where_clause: &'a WhereExpr, + pk: &str, + sk: Option<&str>, + params: &[AttributeValue], +) -> WhereKey<'a> { + let keys = KeyNames { + partition: pk, + sort: sk, + }; + let value_of = |op: &Operand| bound_value(op, params).cloned(); + let mut terms = Vec::new(); + flatten_and_lossy(where_clause, &mut terms); + let mut pk_value: Option<&Operand> = None; + let mut sk_value: Option<&Operand> = None; + let mut residual = Vec::new(); + let mut conflict: Option<&str> = None; + let mut into_key: Option<&str> = None; + for term in terms { + let (existing, value, name) = match key_equality(term, keys) { + Some(KeyEquality::Partition(value)) => (&mut pk_value, value, pk), + Some(KeyEquality::Sort(value)) => (&mut sk_value, value, sk.unwrap_or_default()), + None => { + if conflict.is_none() && into_key.is_none() { + into_key = path_tested_against_values(term, keys); + } + residual.push(term); + continue; + } + }; + match *existing { + Some(seen) => { + if conflict.is_none() && into_key.is_none() && !same_value(seen, value, &value_of) { + conflict = Some(name); + } + } + None => *existing = Some(value), + } + } + if let Some(name) = into_key { + return WhereKey::IntoKey { + name: name.to_owned(), + }; + } + let first = match (pk_value, sk_value, sk) { + (Some(partition), sort, None) | (Some(partition), sort @ Some(_), Some(_)) => { + Some((partition, sort)) + } + _ => None, + }; + match (conflict, first) { + (Some(name), first) => WhereKey::Conflict { + name: name.to_owned(), + first, + }, + (None, Some((partition, sort))) => WhereKey::Pinned { + partition, + sort, + residual, + }, + (None, None) => WhereKey::Partial, + } +} + +/// The value of a literal, or of a parameter the request supplies. +pub(super) fn bound_value<'a>( + operand: &'a Operand, + params: &'a [AttributeValue], +) -> Option<&'a AttributeValue> { + match operand { + Operand::Literal(value) => Some(value), + Operand::Parameter(index) => params.get(*index), + _ => None, + } +} + +/// The value an `INSERT`'s VALUE gives each attribute it names, by name: the +/// last one given, which is the value the item a write builds holds. The +/// parser refuses a VALUE that names an attribute twice (measured +/// 2026-11-03), so each name has one value. The write, the authorization of +/// the statement and the check for two statements on one item all take the +/// item's values from here, so they agree on its key. +pub(super) fn insert_values(insert: &InsertStatement) -> BTreeMap<&str, &Operand> { + insert + .value + .iter() + .map(|(name, operand)| (name.as_str(), operand)) + .collect() +} + +/// The wire-form primary key a write statement targets (`INSERT` from its VALUE +/// map, see [`insert_values`], `UPDATE`/`DELETE` from the WHERE key equalities), +/// or `None` when the key is not fully derivable. Used only to detect two +/// statements naming the same item within one batch/transaction. +pub(super) fn write_key( + stmt: &Statement, + pk: &str, + sk: Option<&str>, + params: &[AttributeValue], +) -> Result>, DynamoDbError> { + match stmt { + Statement::Insert(insert) => { + let values = insert_values(insert); + let pk_op = values.get(pk).copied(); + let sk_op = sk.and_then(|sk| values.get(sk).copied()); + build_key(pk, sk, pk_op, sk_op, params) + } + Statement::Update(u) => match &u.where_clause { + Some(where_clause) => pinned_key(where_clause, pk, sk, params), + None => Ok(None), + }, + Statement::Delete(d) => pinned_key(&d.where_clause, pk, sk, params), + Statement::Select(_) | Statement::Exists(_) => Ok(None), + } +} + +/// The wire-form key a write's WHERE pins, or `None` when it pins none (see +/// [`where_key`]). +fn pinned_key( + where_clause: &WhereExpr, + pk: &str, + sk: Option<&str>, + params: &[AttributeValue], +) -> Result>, DynamoDbError> { + match where_key(where_clause, pk, sk, params) { + WhereKey::Pinned { + partition, sort, .. + } => build_key(pk, sk, Some(partition), sort, params), + WhereKey::Partial | WhereKey::Conflict { .. } | WhereKey::IntoKey { .. } => Ok(None), + } +} + +/// The refusal of a write, or of an `EXISTS` check, whose WHERE pins its +/// item and whose other conditions read a path into a key attribute in a +/// way [`where_key`] does not refuse, naming the attribute +/// `plan::condition_key_path` gives; `None` otherwise. The service makes +/// this check after the key's types, the refusal of a SET of a key +/// attribute and the refusal of a secondary index key given a value of the +/// wrong type, whether or not the item exists. A batch member's error names +/// its table, and the member counts as a write but not in the check for two +/// statements on one item; a transaction member has it as its cancellation +/// reason (measured 2026-11-05). +pub(super) fn condition_key_refusal( + where_clause: &WhereExpr, + pk: &str, + sk: Option<&str>, + params: &[AttributeValue], +) -> Option { + let WhereKey::Pinned { residual, .. } = where_key(where_clause, pk, sk, params) else { + return None; + }; + let keys = KeyNames { + partition: pk, + sort: sk, + }; + condition_key_path(&residual, keys).map(|key| format!("{CONDITION_KEY_REFUSAL}{key}")) +} + +/// A canonical string identity for an item: its table and wire-form key. Two +/// statements share an identity when they address the same item. +pub(super) fn canonical_identity(table: &str, key: &Map) -> String { + // A JSON map holds its members in the order the build gives it: the order + // they were inserted in when the build includes the MongoDB storage crate + // (any build of the whole workspace), name order otherwise. The key is + // always built pk before sk, so in either order equal (table, key) pairs + // serialise identically. + serde_json::json!({ "t": table, "k": key }).to_string() +} + +/// Assemble the wire-form key map from the resolved pk/sk operands, or `None` +/// when the required key parts are absent. +fn build_key( + pk: &str, + sk: Option<&str>, + pk_val: Option<&Operand>, + sk_val: Option<&Operand>, + params: &[AttributeValue], +) -> Result>, DynamoDbError> { + let Some(pk_val) = pk_val else { + return Ok(None); + }; + if sk.is_some() && sk_val.is_none() { + return Ok(None); + } + let mut key = Map::new(); + key.insert(pk.to_owned(), to_json_value(&resolve_av(pk_val, params)?)?); + if let (Some(sk), Some(v)) = (sk, sk_val) { + key.insert(sk.to_owned(), to_json_value(&resolve_av(v, params)?)?); + } + Ok(Some(key)) +} + +/// Flatten a top-level `AND` chain, treating an `OR`/`NOT` as a single opaque +/// term. An explicit stack keeps a long chain off the call stack. +pub(super) fn flatten_and_lossy<'a>(expr: &'a WhereExpr, out: &mut Vec<&'a WhereExpr>) { + let mut pending = vec![expr]; + while let Some(expr) = pending.pop() { + match expr { + WhereExpr::And(a, b) => { + pending.push(b); + pending.push(a); + } + other => out.push(other), + } + } +} + +/// The short per-statement `Error` a batch carries for a failing statement. +/// +/// Per-statement failures inside a batch surface as short Codes without the +/// `Exception` suffix, and the call itself stays HTTP 200. `TableName` is echoed +/// only for an authorization refusal and the failures of the item operation a +/// statement runs as (see [`StatementError::from_item_error`]); the errors of +/// reading and translating a statement carry the `Error` object alone, +/// matching the service. A conditional failure on a +/// statement that set `ReturnValuesOnConditionCheckFailure` to `ALL_OLD` +/// carries the existing item on the `Error` itself, never on the entry +/// (measured 2026-11-02). +pub(super) struct StatementError { + pub code: &'static str, + pub message: String, + pub include_table_name: bool, + pub item: Option, +} + +impl StatementError { + /// Map a single-statement `DynamoDbError` into the batch per-statement shape. + pub(super) fn from_error(err: &DynamoDbError) -> Self { + match err { + DynamoDbError::DuplicateItemException(m) => Self { + code: "DuplicateItem", + message: m.clone(), + include_table_name: true, + item: None, + }, + DynamoDbError::ConditionalCheckFailedException(m, item) => Self { + code: "ConditionalCheckFailed", + message: m.clone(), + include_table_name: true, + item: item.as_ref().and_then(|i| serde_json::to_value(i).ok()), + }, + DynamoDbError::ResourceNotFoundException(_) => Self { + code: "ResourceNotFound", + message: "Requested resource not found".to_owned(), + include_table_name: false, + item: None, + }, + // A statement the caller may not run carries the refusal and its + // table (measured 2026-11-03). + DynamoDbError::AccessDeniedException(m) => Self { + code: "AccessDenied", + message: m.clone(), + include_table_name: true, + item: None, + }, + // Everything else a well-formed statement can raise per-statement is + // a validation error (partial key, bad value, key-type mismatch). + other => Self { + code: "ValidationError", + message: other.message().to_owned(), + include_table_name: false, + item: None, + }, + } + } + + /// Map an error of the item operation a statement runs as: as + /// [`Self::from_error`], carrying the statement's `TableName`. The service + /// names the table on each of them: a key value past its size limit + /// (measured 2026-11-02 for SELECT, INSERT, UPDATE and DELETE), an + /// UPDATE of a key attribute, a secondary index key of the wrong type, + /// an update the item cannot take, and the refusal of + /// [`condition_key_refusal`] (measured 2026-11-05). The errors of reading + /// and translating a statement name none: a key value of the wrong type, + /// a missing parameter, a WHERE without the full key, a path into a key + /// attribute tested against values. + pub(super) fn from_item_error(err: &DynamoDbError) -> Self { + let mut error = Self::from_error(err); + error.include_table_name = true; + error + } + + /// A parse failure's per-statement error (always `ValidationError`). + pub(super) fn parse(message: String) -> Self { + Self { + code: "ValidationError", + message, + include_table_name: false, + item: None, + } + } + + /// Render this error as a `Responses[i]` entry, attaching `TableName` when + /// the failure reached the item layer and a table is known. + pub(super) fn into_entry(self, table: Option<&str>) -> Value { + let mut error = Map::new(); + error.insert("Code".to_owned(), Value::String(self.code.to_owned())); + error.insert("Message".to_owned(), Value::String(self.message)); + if let Some(item) = self.item { + error.insert("Item".to_owned(), item); + } + let mut entry = Map::new(); + entry.insert("Error".to_owned(), Value::Object(error)); + if self.include_table_name + && let Some(table) = table + { + entry.insert("TableName".to_owned(), Value::String(table.to_owned())); + } + Value::Object(entry) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/engine/src/partiql/common/tests.rs b/crates/engine/src/partiql/common/tests.rs new file mode 100644 index 000000000..6b6617732 --- /dev/null +++ b/crates/engine/src/partiql/common/tests.rs @@ -0,0 +1,263 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Unit tests for the helpers the batch and transaction executors share. + +use super::*; +use serde_json::json; + +#[test] +fn a_conditional_failure_carries_its_item_on_the_error() { + let mut item = extenddb_core::types::Item::new(); + item.insert("pk".to_owned(), AttributeValue::S("k".to_owned())); + let err = DynamoDbError::ConditionalCheckFailedException( + "The conditional request failed".to_owned(), + Some(item), + ); + assert_eq!( + StatementError::from_error(&err).into_entry(Some("t")), + json!({ + "Error": { + "Code": "ConditionalCheckFailed", + "Message": "The conditional request failed", + "Item": { "pk": {"S": "k"} } + }, + "TableName": "t" + }) + ); +} + +fn where_of(condition: &str) -> WhereExpr { + match extenddb_core::partiql::parse(&format!("SELECT * FROM \"t\" WHERE {condition}")) { + Ok(Statement::Select(s)) => s.where_clause.unwrap(), + other => panic!("not a select: {other:?}"), + } +} + +#[test] +fn a_key_with_two_values_keeps_the_first_values() { + let a = Operand::Literal(AttributeValue::S("a".to_owned())); + let x = Operand::Literal(AttributeValue::S("x".to_owned())); + let w = where_of("pk = 'a' AND a = 1 AND pk = 'b'"); + match where_key(&w, "pk", None, &[]) { + WhereKey::Conflict { + name, + first: Some((partition, None)), + } => { + assert_eq!(name, "pk"); + assert_eq!(partition, &a); + } + _ => panic!("expected a conflict with a full key"), + } + let w = where_of("sk = 'x' AND pk = 'a' AND sk = 'y' AND pk = 'b'"); + match where_key(&w, "pk", Some("sk"), &[]) { + WhereKey::Conflict { + name, + first: Some((partition, Some(sort))), + } => { + assert_eq!(name, "sk"); + assert_eq!((partition, sort), (&a, &x)); + } + _ => panic!("expected a conflict with a full key"), + } + let w = where_of("pk = 'a' AND pk = 'b' AND a = 1"); + assert!(matches!( + where_key(&w, "pk", Some("sk"), &[]), + WhereKey::Conflict { first: None, .. } + )); + let w = where_of("pk = 'a' AND 'a' = pk AND a = 1"); + assert!(matches!( + where_key(&w, "pk", None, &[]), + WhereKey::Pinned { .. } + )); +} + +#[test] +fn a_path_into_a_key_tested_against_values_is_refused_in_statement_order() { + let into = |condition: &str| match where_key(&where_of(condition), "pk", Some("sk"), &[]) { + WhereKey::IntoKey { name } => Some(name), + WhereKey::Conflict { name, .. } => Some(format!("conflict {name}")), + WhereKey::Pinned { .. } | WhereKey::Partial => None, + }; + for (condition, want) in [ + ("pk = 'k' AND sk = 'b' AND pk.c = 1", Some("pk")), + ("pk = 'k' AND sk = 'b' AND 1 = sk[0]", Some("sk")), + ("pk = 'k' AND sk = 'b' AND pk.c BETWEEN 1 AND 2", Some("pk")), + ("pk = 'k' AND sk = 'b' AND pk.c IN [1, ?]", Some("pk")), + ( + "pk = 'k' AND sk = 'b' AND begins_with(pk.c, 'x')", + Some("pk"), + ), + ("pk = 'k' AND sk = 'b' AND pk.c IS NULL", Some("pk")), + ("pk = 'k' AND sk = 'b' AND pk.c = 1 = false", Some("pk")), + ("pk = 'k' AND pk.c = 1", Some("pk")), + ( + "pk = 'k' AND sk = 'b' AND sk.c = 1 AND pk.d = 1", + Some("sk"), + ), + ( + "pk = 'k' AND sk = 'b' AND pk.c = 1 AND pk = 'z'", + Some("pk"), + ), + ( + "pk = 'k' AND pk = 'z' AND sk = 'b' AND sk.c = 1", + Some("conflict pk"), + ), + // The other conditions with a path into a key are left to the + // refusal made once the key is checked. + ("pk = 'k' AND sk = 'b' AND NOT pk.c = 1", None), + ("pk = 'k' AND sk = 'b' AND pk.c = a", None), + ("pk = 'k' AND sk = 'b' AND pk.c IS NOT NULL", None), + ("pk = 'k' AND sk = 'b' AND size(pk.c) > 1", None), + ("pk = 'k' AND sk = 'b' AND a.c = 1", None), + ] { + assert_eq!(into(condition).as_deref(), want, "{condition}"); + } +} + +#[test] +fn a_path_into_a_key_among_the_other_conditions_names_the_sort_key_first() { + let refusal = |condition: &str| { + condition_key_refusal(&where_of(condition), "pk", Some("sk"), &[]) + .map(|m| m.strip_prefix(CONDITION_KEY_REFUSAL).unwrap().to_owned()) + }; + for (condition, want) in [ + ("pk = 'k' AND sk = 'b' AND NOT pk.c = 1", Some("pk")), + ("pk = 'k' AND sk = 'b' AND (a = 1 OR pk.c = 1)", Some("pk")), + ( + "pk = 'k' AND sk = 'b' AND attribute_exists(pk.c)", + Some("pk"), + ), + ("pk = 'k' AND sk = 'b' AND pk.c IS MISSING", Some("pk")), + ( + "pk = 'k' AND sk = 'b' AND NOT pk.c = 1 AND NOT sk.d = 1", + Some("sk"), + ), + ( + "pk = 'k' AND sk = 'b' AND NOT sk.d = 1 AND NOT pk.c = 1", + Some("sk"), + ), + ("pk = 'k' AND sk = 'b' AND pk.c = sk.d", Some("sk")), + ( + "pk = 'k' AND sk = 'b' AND (pk.c = 1 OR sk = 'b')", + Some("pk"), + ), + ("pk = 'k' AND sk = 'b' AND NOT a.c = 1", None), + // Not a pinned item: refused before. + ("pk = 'k' AND NOT pk.c = 1", None), + ("pk = 'k' AND sk = 'b' AND pk.c = 1", None), + ] { + assert_eq!(refusal(condition).as_deref(), want, "{condition}"); + } +} + +#[test] +fn an_error_of_the_item_operation_names_the_batch_table() { + let size = DynamoDbError::ValidationException( + "One or more parameter values were invalid: Size of hashkey has exceeded the \ + maximum size limit of2048 bytes" + .to_owned(), + ); + let key_update = DynamoDbError::ValidationException( + "One or more parameter values were invalid: Cannot update attribute pk. This \ + attribute is part of the key" + .to_owned(), + ); + let key_type = DynamoDbError::ValidationException(key_type_error("pk").message().to_owned()); + for error in [&size, &key_update] { + assert_eq!( + StatementError::from_item_error(error).into_entry(Some("t")), + json!({ + "Error": { "Code": "ValidationError", "Message": error.message() }, + "TableName": "t", + }) + ); + } + for error in [&size, &key_type] { + assert!( + StatementError::from_error(error) + .into_entry(Some("t")) + .get("TableName") + .is_none() + ); + } +} + +#[test] +fn a_conditional_failure_without_an_item_has_none() { + let err = DynamoDbError::ConditionalCheckFailedException( + "The conditional request failed".to_owned(), + None, + ); + assert_eq!( + StatementError::from_error(&err).into_entry(Some("t")), + json!({ + "Error": { + "Code": "ConditionalCheckFailed", + "Message": "The conditional request failed" + }, + "TableName": "t" + }) + ); +} + +#[test] +fn a_wire_key_value_has_the_type_the_table_defines() { + use extenddb_core::types::AttributeDefinition; + let key_info = TableKeyInfo { + table_name: "t".to_owned(), + attribute_definitions: vec![AttributeDefinition { + attribute_name: "pk".to_owned(), + attribute_type: ScalarAttributeType::N, + }], + ..TableKeyInfo::default() + }; + assert!(wire_key_value_matches(&key_info, "pk", &json!({"N": "1"}))); + assert!(!wire_key_value_matches(&key_info, "pk", &json!({"S": "1"}))); + // A value that is no attribute value has no type. + assert!(!wire_key_value_matches( + &key_info, + "pk", + &json!({"N": "1", "S": "1"}) + )); + assert!(!wire_key_value_matches(&key_info, "pk", &json!("1"))); + // An attribute the table defines no type for takes any value. + assert!(wire_key_value_matches(&key_info, "sk", &json!({"S": "1"}))); +} + +/// An INSERT whose VALUE gives `pk` the values `first` then `last`, built by +/// hand: the parser refuses such a VALUE. +fn insert_naming_pk_twice(first: &str, last: &str) -> Statement { + let s = |v: &str| Operand::Literal(AttributeValue::S(v.to_owned())); + Statement::Insert(InsertStatement { + table: "t".to_owned(), + value: vec![ + ("pk".to_owned(), s(first)), + ("sk".to_owned(), s("x")), + ("pk".to_owned(), s(last)), + ], + }) +} + +#[test] +fn an_insert_has_one_value_for_each_name_the_last_given() { + let stmt = insert_naming_pk_twice("a", "b"); + let Statement::Insert(insert) = &stmt else { + unreachable!("built as an INSERT"); + }; + let values = insert_values(insert); + assert_eq!(values.len(), 2); + assert_eq!( + values.get("pk"), + Some(&&Operand::Literal(AttributeValue::S("b".to_owned()))) + ); + // The key the duplicate check reads is the one the item stores. + let key = write_key(&stmt, "pk", Some("sk"), &[]).unwrap().unwrap(); + assert_eq!( + key, + json!({"pk": {"S": "b"}, "sk": {"S": "x"}}) + .as_object() + .unwrap() + .clone() + ); +} diff --git a/crates/engine/src/partiql/expr_bridge.rs b/crates/engine/src/partiql/expr_bridge.rs new file mode 100644 index 000000000..660778622 --- /dev/null +++ b/crates/engine/src/partiql/expr_bridge.rs @@ -0,0 +1,357 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! PartiQL operand/condition bridge. +//! +//! Translates a PartiQL [`WhereExpr`] (or an [`Operand`] or [`Path`]) into the +//! engine's existing classic expression forms: a `KeyConditionExpression`, +//! `FilterExpression`, `ConditionExpression`, or `UpdateExpression` fragment +//! plus the `ExpressionAttributeNames` / `ExpressionAttributeValues` maps those +//! expressions reference. Translating into the classic forms reuses the +//! measured classic semantics rather than re-evaluating PartiQL expressions. +//! +//! The SELECT executor (this phase) uses it for key conditions and filters; the +//! write executor reuses the same accumulator for conditions and update +//! expressions. Every attribute reference is emitted as a `#name` placeholder, +//! so a reserved word used as an attribute name never needs special handling. + +use std::collections::{BTreeMap, HashMap}; + +use serde_json::Value; + +use extenddb_core::error::DynamoDbError; +use extenddb_core::partiql::ast::{ + ArithOp, CompareOp, Operand, Path, PathStep, TypeName, WhereExpr, +}; +use extenddb_core::types::AttributeValue; + +use super::bind_parameter; + +/// Accumulates expression-attribute maps and renders classic expression +/// fragments for a PartiQL statement. One bridge is shared across the key +/// condition and filter of a single request so the two fragments draw from one +/// set of `#name` / `:value` placeholders. +pub(crate) struct ExprBridge<'p> { + params: &'p [AttributeValue], + names: BTreeMap, + values: BTreeMap, + name_seq: usize, + value_seq: usize, +} + +impl<'p> ExprBridge<'p> { + /// Create a bridge bound to the request's positional `Parameters`. + pub(crate) fn new(params: &'p [AttributeValue]) -> Self { + Self { + params, + names: BTreeMap::new(), + values: BTreeMap::new(), + name_seq: 0, + value_seq: 0, + } + } + + /// Mint a fresh `#name` placeholder for an attribute name. + pub(crate) fn name_ref(&mut self, attr: &str) -> String { + let placeholder = format!("#n{}", self.name_seq); + self.name_seq += 1; + self.names.insert(placeholder.clone(), attr.to_owned()); + placeholder + } + + /// Mint a fresh `:value` placeholder for a literal attribute value. + pub(crate) fn value_ref(&mut self, value: AttributeValue) -> String { + let placeholder = format!(":v{}", self.value_seq); + self.value_seq += 1; + self.values.insert(placeholder.clone(), value); + placeholder + } + + /// Render a document path as `#n0.#n1[2]...`, one name placeholder per + /// attribute step and a literal `[index]` per list-index step. + pub(crate) fn path(&mut self, path: &Path) -> String { + let mut rendered = self.name_ref(&path.root); + for step in &path.steps { + match step { + PathStep::Attribute(name) => { + rendered.push('.'); + rendered.push_str(&self.name_ref(name)); + } + PathStep::Index(i) => { + rendered.push_str(&format!("[{i}]")); + } + } + } + rendered + } + + /// Render an operand (value, parameter, path, arithmetic, or function). + /// + /// # Errors + /// + /// Returns the service's count-mismatch `ValidationException` when a `?` + /// parameter has no corresponding entry in `Parameters`. + pub(crate) fn operand(&mut self, op: &Operand) -> Result { + Ok(match op { + Operand::Literal(av) => self.value_ref(av.clone()), + Operand::Parameter(i) => { + let av = bind_parameter(self.params, *i)?; + self.value_ref(av) + } + Operand::Path(p) => self.path(p), + Operand::Arith(l, arith, r) => { + let l = self.operand(l)?; + let r = self.operand(r)?; + let sym = match arith { + ArithOp::Add => "+", + ArithOp::Sub => "-", + }; + format!("{l} {sym} {r}") + } + Operand::Function(name, args) => { + let rendered: Vec = args + .iter() + .map(|a| self.operand(a)) + .collect::>()?; + format!("{name}({})", rendered.join(", ")) + } + }) + } + + /// Render a boolean `WHERE` expression as a classic filter / condition + /// expression string, written by [`condition_text`]. An OR at the top is + /// in parentheses, so the text can be joined to others with AND. + /// + /// # Errors + /// + /// Propagates the parameter count-mismatch `ValidationException` from + /// [`ExprBridge::operand`]. + pub(crate) fn render_condition(&mut self, expr: &WhereExpr) -> Result { + condition_text(expr, true, |predicate, negated| { + self.predicate(predicate, negated) + }) + } + + /// Render one predicate of a condition, negated when `negated` is set. + fn predicate(&mut self, expr: &WhereExpr, negated: bool) -> Result { + let text = match expr { + WhereExpr::Compare(l, op, r) => { + let l = self.operand(l)?; + let r = self.operand(r)?; + format!("{l} {} {r}", compare_symbol(*op)) + } + WhereExpr::Between(v, low, high) => { + let v = self.operand(v)?; + let low = self.operand(low)?; + let high = self.operand(high)?; + format!("{v} BETWEEN {low} AND {high}") + } + WhereExpr::In(v, items) => { + let v = self.operand(v)?; + let rendered: Vec = items + .iter() + .map(|i| self.operand(i)) + .collect::>()?; + format!("{v} IN ({})", rendered.join(", ")) + } + WhereExpr::BeginsWith(a, b) => { + let a = self.operand(a)?; + let b = self.operand(b)?; + format!("begins_with({a}, {b})") + } + WhereExpr::Contains(a, b) => { + let a = self.operand(a)?; + let b = self.operand(b)?; + format!("contains({a}, {b})") + } + WhereExpr::AttributeType(a, b) => { + let a = self.operand(a)?; + let b = self.operand(b)?; + format!("attribute_type({a}, {b})") + } + WhereExpr::Function(name, args) => { + let rendered: Vec = args + .iter() + .map(|a| self.operand(a)) + .collect::>()?; + format!("{name}({})", rendered.join(", ")) + } + // `IS MISSING` means the attribute is absent, `IS NOT MISSING` that + // it is present; the classic equivalents are attribute_not_exists / + // attribute_exists, and each is the negation of the other. + WhereExpr::IsMissing(path, is_missing) => { + let p = self.path(path); + return Ok(if *is_missing == negated { + format!("attribute_exists({p})") + } else { + format!("attribute_not_exists({p})") + }); + } + // `IS NULL` is an equality with NULL and `IS ` an + // attribute_type test, which is how the service filters on them + // (measured 2026-11-02: `IS NULL`, `IS NOT NULL`, `IS BOOL` and + // `IS STRING` filter rows). + WhereExpr::IsType(operand, ty, is_not) => { + let value = self.operand(operand)?; + let test = if *ty == TypeName::Null { + let null = self.value_ref(AttributeValue::Null); + format!("{value} = {null}") + } else { + let code = ty.attribute_type().ok_or_else(|| { + DynamoDbError::ValidationException(ty.unsupported_filter_message()) + })?; + let code = self.value_ref(AttributeValue::S(code.to_owned())); + format!("attribute_type({value}, {code})") + }; + return Ok(if *is_not == negated { + test + } else { + format!("NOT {test}") + }); + } + WhereExpr::And(..) + | WhereExpr::Or(..) + | WhereExpr::Not(_) + | WhereExpr::ComparedWithBoolean(..) => { + unreachable!("condition_text writes AND, OR and NOT itself") + } + }; + Ok(if negated { format!("NOT {text}") } else { text }) + } + + /// The accumulated `ExpressionAttributeNames` as a JSON object, or `None` + /// when no attribute placeholders were emitted. + pub(crate) fn names_value(&self) -> Option { + if self.names.is_empty() { + return None; + } + let map: serde_json::Map = self + .names + .iter() + .map(|(k, v)| (k.clone(), Value::String(v.clone()))) + .collect(); + Some(Value::Object(map)) + } + + /// The accumulated name placeholders, for evaluating a rendered expression + /// in this module rather than sending it in a request. + pub(crate) fn names_map(&self) -> HashMap { + self.names + .iter() + .map(|(k, v)| (k.clone(), v.clone())) + .collect() + } + + /// The accumulated value placeholders, for the same use as [`Self::names_map`]. + pub(crate) fn values_map(&self) -> HashMap { + self.values + .iter() + .map(|(k, v)| (k.clone(), v.clone())) + .collect() + } + + /// The accumulated `ExpressionAttributeValues` as a JSON object of wire-form + /// attribute values, or `None` when no value placeholders were emitted. + /// + /// # Errors + /// + /// Returns `InternalServerError` if an accumulated value fails to serialize + /// (not reachable for well-formed attribute values). + pub(crate) fn values_value(&self) -> Result, DynamoDbError> { + if self.values.is_empty() { + return Ok(None); + } + let mut map = serde_json::Map::new(); + for (k, v) in &self.values { + let wire = serde_json::to_value(v).map_err(|e| { + DynamoDbError::InternalServerError(format!("failed to encode parameter: {e}")) + })?; + map.insert(k.clone(), wire); + } + Ok(Some(Value::Object(map))) + } +} + +/// Write a WHERE condition as classic condition text, with `predicate` +/// writing each predicate (anything but AND, OR and NOT), negated when its +/// second argument is set. +/// +/// A condition the service runs can be nested deeper than the classic parser +/// takes (150 levels of parentheses and NOTs, the `max_expression_depth` +/// limit): the service runs any condition within its 300 operators, such as +/// 99 levels of `NOT (a = 1 AND ...)` (measured 2026-11-02). So the text is +/// written with as little nesting as the classic grammar allows: a NOT is +/// moved onto the predicates under it (NOT of an AND is the OR of the +/// negated sides, and NOT of an OR their AND), and parentheses go only around +/// an OR inside an AND, and around an OR at the top when +/// `parenthesize_top_or` is set. A condition of 300 operators then nests at +/// most about 75 levels. The meaning is unchanged: a classic condition is +/// true or false, and the predicates are written, and so evaluated, in the +/// order of the statement. +/// +/// The walk keeps its own stack rather than recursing, so a deep condition +/// takes no more of the thread's stack than a shallow one (a recursive walk +/// took about 9 KB per level on a debug build). +/// +/// # Errors +/// +/// Returns the first error `predicate` returns. +pub(crate) fn condition_text( + expr: &WhereExpr, + parenthesize_top_or: bool, + mut predicate: impl FnMut(&WhereExpr, bool) -> Result, +) -> Result { + enum Step<'e> { + /// A condition to write, whether it is negated, and whether an OR + /// there needs parentheses. + Write(&'e WhereExpr, bool, bool), + Text(&'static str), + } + let mut text = String::new(); + let mut steps = vec![Step::Write(expr, false, parenthesize_top_or)]; + while let Some(step) = steps.pop() { + let (expr, negated, in_and) = match step { + Step::Text(part) => { + text.push_str(part); + continue; + } + Step::Write(expr, negated, in_and) => (expr, negated, in_and), + }; + let (odd, expr) = expr.without_nots(); + let negated = negated != odd; + let (left, right, and) = match expr { + WhereExpr::And(left, right) => (left, right, !negated), + WhereExpr::Or(left, right) => (left, right, negated), + other => { + text.push_str(&predicate(other, negated)?); + continue; + } + }; + let parenthesize = in_and && !and; + if parenthesize { + steps.push(Step::Text(")")); + } + steps.push(Step::Write(right, negated, and)); + steps.push(Step::Text(if and { " AND " } else { " OR " })); + steps.push(Step::Write(left, negated, and)); + if parenthesize { + steps.push(Step::Text("(")); + } + } + Ok(text) +} + +/// The classic comparison symbol for a PartiQL compare operator. +pub(super) fn compare_symbol(op: CompareOp) -> &'static str { + match op { + CompareOp::Eq => "=", + CompareOp::Ne => "<>", + CompareOp::Lt => "<", + CompareOp::Le => "<=", + CompareOp::Gt => ">", + CompareOp::Ge => ">=", + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/engine/src/partiql/expr_bridge/tests.rs b/crates/engine/src/partiql/expr_bridge/tests.rs new file mode 100644 index 000000000..88a1d5586 --- /dev/null +++ b/crates/engine/src/partiql/expr_bridge/tests.rs @@ -0,0 +1,374 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Unit tests for the operand and condition bridge. + +use super::*; +use extenddb_core::partiql::ast::Statement; +use extenddb_core::partiql::parse; + +fn where_of(sql: &str) -> WhereExpr { + match parse(sql).unwrap() { + Statement::Select(s) => s.where_clause.unwrap(), + _ => panic!("not a select"), + } +} + +#[test] +fn renders_equality_with_name_and_value_placeholders() { + let w = where_of("SELECT * FROM \"t\" WHERE pk = 'user1'"); + let mut b = ExprBridge::new(&[]); + let s = b.render_condition(&w).unwrap(); + assert_eq!(s, "#n0 = :v0"); + assert_eq!(b.names_value().unwrap()["#n0"], "pk"); + assert_eq!(b.values_value().unwrap().unwrap()[":v0"]["S"], "user1"); +} + +#[test] +fn renders_nested_path_on_leaf_name_placeholders() { + let w = where_of("SELECT * FROM \"t\" WHERE addr.city = 'NYC'"); + let mut b = ExprBridge::new(&[]); + let s = b.render_condition(&w).unwrap(); + assert_eq!(s, "#n0.#n1 = :v0"); +} + +#[test] +fn renders_begins_with_and_between_and_in() { + let mut b = ExprBridge::new(&[]); + let w = where_of("SELECT * FROM \"t\" WHERE begins_with(\"sk\", 'p')"); + assert_eq!(b.render_condition(&w).unwrap(), "begins_with(#n0, :v0)"); + + let mut b = ExprBridge::new(&[]); + let w = where_of("SELECT * FROM \"t\" WHERE sk BETWEEN 'a' AND 'z'"); + assert_eq!(b.render_condition(&w).unwrap(), "#n0 BETWEEN :v0 AND :v1"); + + let mut b = ExprBridge::new(&[]); + let w = where_of("SELECT * FROM \"t\" WHERE sk IN ['a','b']"); + assert_eq!(b.render_condition(&w).unwrap(), "#n0 IN (:v0, :v1)"); +} + +#[test] +fn renders_is_missing_as_attribute_not_exists() { + let mut b = ExprBridge::new(&[]); + let w = where_of("SELECT * FROM \"t\" WHERE addr IS MISSING"); + assert_eq!(b.render_condition(&w).unwrap(), "attribute_not_exists(#n0)"); + + let mut b = ExprBridge::new(&[]); + let w = where_of("SELECT * FROM \"t\" WHERE addr IS NOT MISSING"); + assert_eq!(b.render_condition(&w).unwrap(), "attribute_exists(#n0)"); +} + +#[test] +fn renders_is_null_as_an_equality_with_null() { + let mut b = ExprBridge::new(&[]); + let w = where_of("SELECT * FROM \"t\" WHERE nul IS NULL"); + assert_eq!(b.render_condition(&w).unwrap(), "#n0 = :v0"); + assert_eq!(b.values_value().unwrap().unwrap()[":v0"]["NULL"], true); + + let mut b = ExprBridge::new(&[]); + let w = where_of("SELECT * FROM \"t\" WHERE s IS NOT NULL"); + assert_eq!(b.render_condition(&w).unwrap(), "NOT #n0 = :v0"); +} + +#[test] +fn renders_is_type_as_attribute_type() { + for (sql, code) in [ + ("SELECT * FROM \"t\" WHERE flag IS BOOL", "BOOL"), + ("SELECT * FROM \"t\" WHERE s IS STRING", "S"), + ("SELECT * FROM \"t\" WHERE n IS NUMERIC", "N"), + ("SELECT * FROM \"t\" WHERE m IS STRUCT", "M"), + ("SELECT * FROM \"t\" WHERE l IS LIST", "L"), + ] { + let mut b = ExprBridge::new(&[]); + let w = where_of(sql); + assert_eq!( + b.render_condition(&w).unwrap(), + "attribute_type(#n0, :v0)", + "{sql}" + ); + assert_eq!( + b.values_value().unwrap().unwrap()[":v0"]["S"], + code, + "{sql}" + ); + } +} + +#[test] +fn a_run_of_nots_renders_as_one_not_or_none() { + for (nots, rendered) in [ + (1, "NOT #n0 = :v0"), + (2, "#n0 = :v0"), + (3, "NOT #n0 = :v0"), + (299, "NOT #n0 = :v0"), + ] { + let mut b = ExprBridge::new(&[]); + let sql = format!("SELECT * FROM \"t\" WHERE {}a = 1", "NOT ".repeat(nots)); + assert_eq!( + b.render_condition(&where_of(&sql)).unwrap(), + rendered, + "{nots}" + ); + } + let mut b = ExprBridge::new(&[]); + let w = where_of("SELECT * FROM \"t\" WHERE NOT NOT (a = 1 OR b = 2)"); + assert_eq!(b.render_condition(&w).unwrap(), "(#n0 = :v0 OR #n1 = :v1)"); +} + +fn rendered(condition: &str) -> String { + let sql = format!("SELECT * FROM \"t\" WHERE {condition}"); + ExprBridge::new(&[]) + .render_condition(&where_of(&sql)) + .unwrap() +} + +#[test] +fn parentheses_go_only_around_an_or_inside_an_and() { + for (condition, text) in [ + ( + "a = 1 AND (b = 1 OR c = 1)", + "#n0 = :v0 AND (#n1 = :v1 OR #n2 = :v2)", + ), + ( + "(a = 1 OR b = 1) AND c = 1", + "(#n0 = :v0 OR #n1 = :v1) AND #n2 = :v2", + ), + ( + "a = 1 OR b = 1 AND c = 1", + "(#n0 = :v0 OR #n1 = :v1 AND #n2 = :v2)", + ), + ( + "a = 1 AND (b = 1 AND c = 1)", + "#n0 = :v0 AND #n1 = :v1 AND #n2 = :v2", + ), + ( + "a = 1 OR (b = 1 OR c = 1)", + "(#n0 = :v0 OR #n1 = :v1 OR #n2 = :v2)", + ), + ("((a = 1))", "#n0 = :v0"), + ] { + assert_eq!(rendered(condition), text, "{condition}"); + } +} + +#[test] +fn a_not_is_moved_onto_the_predicates_under_it() { + for (condition, text) in [ + ("NOT (a = 1 AND b = 1)", "(NOT #n0 = :v0 OR NOT #n1 = :v1)"), + ("NOT (a = 1 OR b = 1)", "NOT #n0 = :v0 AND NOT #n1 = :v1"), + ( + "NOT (a = 1 AND NOT (b = 1 OR c = 1))", + "(NOT #n0 = :v0 OR #n1 = :v1 OR #n2 = :v2)", + ), + ( + "c = 1 AND NOT (a = 1 AND b = 1)", + "#n0 = :v0 AND (NOT #n1 = :v1 OR NOT #n2 = :v2)", + ), + ("NOT a IS MISSING", "attribute_exists(#n0)"), + ("NOT a IS NOT MISSING", "attribute_not_exists(#n0)"), + ("NOT a IS NOT NULL", "#n0 = :v0"), + ("NOT attribute_exists(a) = false", "attribute_exists(#n0)"), + ] { + assert_eq!(rendered(condition), text, "{condition}"); + } +} + +#[test] +fn the_not_forms_render_as_negated_predicates() { + assert_eq!(rendered("a NOT IN [1, 2]"), "NOT #n0 IN (:v0, :v1)"); + assert_eq!(rendered("a NOT IN (1, 2)"), "NOT #n0 IN (:v0, :v1)"); + assert_eq!(rendered("a IN (1, 2)"), "#n0 IN (:v0, :v1)"); + assert_eq!( + rendered("a NOT BETWEEN 1 AND 2"), + "NOT #n0 BETWEEN :v0 AND :v1" + ); + assert_eq!(rendered("NOT a NOT IN [1]"), "#n0 IN (:v0)"); +} + +/// Conditions within the 300 operators the service takes, nested as +/// deep as that allows. +fn deep_conditions() -> Vec { + let alternated = |op: &str, depth: usize| { + format!("NOT (a = 1 {op} ").repeat(depth) + "a = 1" + &")".repeat(depth) + }; + let nested = |innermost: &str| "(a = 1 AND ".repeat(149) + innermost + &")".repeat(149); + vec![ + alternated("AND", 99), + alternated("OR", 99), + nested("a = 1"), + nested("NOT a = 1"), + format!("NOT ({})", vec!["a = 1"; 150].join(" AND ")), + "a = 1 AND (a = 1 OR ".repeat(74) + "a = 1" + &")".repeat(74), + "(a = 1 OR (a = 1 AND ".repeat(74) + "a = 1" + &"))".repeat(74), + ] +} + +#[test] +fn a_deep_condition_within_the_operator_limit_passes_the_classic_parser() { + let limits = extenddb_core::limits::LimitsConfig::default(); + for condition in deep_conditions() { + let sql = format!("SELECT * FROM \"t\" WHERE {condition}"); + let text = ExprBridge::new(&[]) + .render_condition(&where_of(&sql)) + .unwrap(); + let parsed = crate::expression_helpers::parse_optional_filter(Some(&text), &limits); + assert!(parsed.is_ok(), "{condition}: {parsed:?}"); + } +} + +#[test] +fn a_deep_condition_is_written_on_a_small_stack() { + let conditions: Vec = deep_conditions() + .iter() + .map(|c| where_of(&format!("SELECT * FROM \"t\" WHERE {c}"))) + .collect(); + // A recursive walk took about 9 KB a level on a debug build. + std::thread::Builder::new() + .stack_size(128 << 10) + .spawn(move || { + for condition in &conditions { + ExprBridge::new(&[]).render_condition(condition).unwrap(); + } + }) + .unwrap() + .join() + .unwrap(); +} + +/// The condition written the way the bridge wrote it before +/// `condition_text`: every AND, OR and NOT in parentheses where it is. +fn nested_text(expr: &WhereExpr, bridge: &mut ExprBridge<'_>) -> String { + match expr { + WhereExpr::And(a, b) => format!( + "({} AND {})", + nested_text(a, bridge), + nested_text(b, bridge) + ), + WhereExpr::Or(a, b) => { + format!("({} OR {})", nested_text(a, bridge), nested_text(b, bridge)) + } + WhereExpr::Not(inner) => format!("(NOT {})", nested_text(inner, bridge)), + WhereExpr::ComparedWithBoolean(inner, comparison) if comparison.negates() => { + format!("(NOT {})", nested_text(inner, bridge)) + } + WhereExpr::ComparedWithBoolean(inner, _) => nested_text(inner, bridge), + leaf => bridge.predicate(leaf, false).unwrap(), + } +} + +#[test] +fn the_condition_text_means_what_the_condition_says() { + use extenddb_core::expression::evaluate_condition; + let limits = extenddb_core::limits::LimitsConfig::default(); + let evaluate = + |text: &str, bridge: &ExprBridge<'_>, item: &BTreeMap| { + let filter = crate::expression_helpers::parse_optional_filter(Some(text), &limits) + .unwrap() + .unwrap(); + let maps = crate::expression_helpers::build_expression_maps( + Some(&bridge.names_map()), + Some(&bridge.values_map()), + ); + evaluate_condition(&filter, item, &maps).unwrap() + }; + let values = [None, Some("1"), Some("2")]; + let items: Vec> = values + .iter() + .flat_map(|a| values.iter().map(move |b| (a, b))) + .flat_map(|(a, b)| values.iter().map(move |c| (a, b, c))) + .map(|(a, b, c)| { + [("a", a), ("b", b), ("c", c)] + .into_iter() + .filter_map(|(name, v)| { + v.map(|n| (name.to_owned(), AttributeValue::N(n.to_owned()))) + }) + .collect() + }) + .collect(); + for condition in [ + "NOT (a = 1 AND b = 1)", + "NOT (a = 1 OR NOT (b = 2 AND c = 1))", + "a = 1 OR b = 1 AND NOT (c = 1 OR a = 2)", + "NOT ((a = 1 OR b = 2) AND (c = 1 OR NOT a IN [2]))", + "a NOT IN [1, 2] OR b NOT BETWEEN 1 AND 1 AND c <> 2", + "NOT (a IS MISSING OR NOT b IS NOT MISSING) OR NOT c IS NULL", + "NOT NOT (a = 1 AND NOT NOT NOT b = 1)", + "(a = 1 OR b = 2) = false AND NOT c = 1 = false", + ] { + let expr = where_of(&format!("SELECT * FROM \"t\" WHERE {condition}")); + let mut bridge = ExprBridge::new(&[]); + let text = bridge.render_condition(&expr).unwrap(); + let mut reference = ExprBridge::new(&[]); + let nested = nested_text(&expr, &mut reference); + for item in &items { + assert_eq!( + evaluate(&text, &bridge, item), + evaluate(&nested, &reference, item), + "{condition} on {item:?}" + ); + } + } +} + +/// A name of one or two letters for the `n`th attribute, for `n` below +/// 182. The first letters of the two-letter names start no reserved word. +fn short_name(n: usize) -> String { + let letter = |i: usize| char::from(b'a' + u8::try_from(i % 26).unwrap()); + if n < 26 { + letter(n).to_string() + } else { + format!( + "{}{}", + ["j", "k", "q", "w", "x", "z"][n / 26 - 1], + letter(n) + ) + } +} + +#[test] +fn the_text_written_for_a_condition_is_at_most_five_bytes_a_character() { + // The densest conditions the operator limit allows: the shortest + // names and values, the fewest characters between them. + let comparisons: Vec = (0..150).map(|i| format!("{}=1", short_name(i))).collect(); + let members = vec!["1"; 100].join(","); + for condition in [ + comparisons.join(" AND "), + comparisons.join(" OR "), + format!("a IN [{members}]"), + format!("a NOT IN [{members}] OR b IN ({members})"), + format!("NOT ({})", comparisons[..140].join(" AND ")), + ] { + let text = rendered(&condition); + assert!( + text.len() <= 5 * condition.len(), + "{} bytes for {} characters: {condition}", + text.len(), + condition.len() + ); + } +} + +#[test] +fn binds_positional_parameter() { + let params = [AttributeValue::S("p1".to_owned())]; + let mut b = ExprBridge::new(¶ms); + let w = where_of("SELECT * FROM \"t\" WHERE pk = ?"); + assert_eq!(b.render_condition(&w).unwrap(), "#n0 = :v0"); + assert_eq!(b.values_value().unwrap().unwrap()[":v0"]["S"], "p1"); +} + +#[test] +fn too_few_parameters_is_count_mismatch() { + let mut b = ExprBridge::new(&[]); + let w = where_of("SELECT * FROM \"t\" WHERE pk = ?"); + let err = b.render_condition(&w).unwrap_err(); + match err { + DynamoDbError::ValidationException(m) => { + assert_eq!( + m, + "Number of parameters in request and statement don't match." + ); + } + other => panic!("expected ValidationException, got {other:?}"), + } +} diff --git a/crates/engine/src/partiql/index_read.rs b/crates/engine/src/partiql/index_read.rs new file mode 100644 index 000000000..e0d81759d --- /dev/null +++ b/crates/engine/src/partiql/index_read.rs @@ -0,0 +1,391 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! A `SELECT` through a secondary index that does not project every attribute +//! (`KEYS_ONLY` or `INCLUDE`). +//! +//! Three rules, all measured on the service 2026-11-02: +//! +//! * A keyed read (one the service runs as a Query of the index) whose filter +//! names an attribute the index does not project is refused, on a GSI and an +//! LSI alike. Unkeyed, the read is a scan of the index and the filter is +//! evaluated against the index rows. +//! * A GSI read whose projection names such an attribute is refused, keyed or +//! not. The filter refusal is checked first. +//! * An LSI read whose projection names such an attribute reads back every +//! item it walks from the base table and serves the projection and the +//! filter from that item (see `base_join`). + +use std::collections::BTreeSet; + +use extenddb_core::error::DynamoDbError; +use extenddb_core::partiql::ast::{Operand, Projection, SelectStatement, WhereExpr}; +use extenddb_core::types::{IndexInfo, IndexType, KeySchemaElement, ProjectionType}; + +/// Refuse a read of `info` that names an attribute the index does not +/// project, in the filter of a keyed read or in the projection of a GSI read. +/// +/// `keyed_filter` is the filter a keyed read of the index carries (see +/// `WhereAnalysis::shared_filter`): the conditions beside the key conditions +/// at the top of the WHERE, so a condition inside one OR branch is not +/// refused. `None` is a scan of the index, whose filter is evaluated against +/// the index rows. +/// +/// # Errors +/// +/// Returns the service's `ValidationException` naming the index and the +/// attributes it does not project. +pub(super) fn check( + stmt: &SelectStatement, + info: &IndexInfo, + base_key_schema: &[KeySchemaElement], + keyed_filter: Option<&[&WhereExpr]>, +) -> Result<(), DynamoDbError> { + let Some(projected) = projected_attributes(info, base_key_schema) else { + return Ok(()); + }; + if let Some(filter) = keyed_filter { + let mut roots = Vec::new(); + for term in filter { + where_roots(term, &mut roots); + } + let missing = unprojected(roots, &projected); + if !missing.is_empty() { + return Err(DynamoDbError::ValidationException(format!( + "One or more parameter values were invalid: Secondary index {} does not \ + project one or more filter attributes: [{}]", + info.index_name, + missing.join(", ") + ))); + } + } + if info.index_type == IndexType::Gsi + && let Projection::Paths(paths) = &stmt.projection + { + let missing = unprojected(paths.iter().map(|p| p.root.as_str()), &projected); + if !missing.is_empty() { + return Err(DynamoDbError::ValidationException(format!( + "One or more parameter values were invalid: Global secondary index {} does \ + not project [{}]", + info.index_name, + missing.join(", ") + ))); + } + } + Ok(()) +} + +/// Whether a read of `info` must read each walked item back from the base +/// table: an LSI whose projection names an attribute the index does not hold. +/// `SELECT *` is served from the index alone. +pub(super) fn reaches_back( + projection: &Projection, + info: &IndexInfo, + base_key_schema: &[KeySchemaElement], +) -> bool { + if info.index_type != IndexType::Lsi { + return false; + } + match (projection, projected_attributes(info, base_key_schema)) { + (Projection::Paths(paths), Some(projected)) => { + paths.iter().any(|p| !projected.contains(p.root.as_str())) + } + _ => false, + } +} + +/// The attributes an index holds (table key, index key and any `INCLUDE` +/// attributes), or `None` when it projects every attribute. +fn projected_attributes<'a>( + info: &'a IndexInfo, + base_key_schema: &'a [KeySchemaElement], +) -> Option> { + if info.projection.projection_type == ProjectionType::All { + return None; + } + let mut held: BTreeSet<&str> = base_key_schema + .iter() + .chain(&info.key_schema) + .map(|k| k.attribute_name.as_str()) + .collect(); + if let Some(non_key) = &info.projection.non_key_attributes { + held.extend(non_key.iter().map(String::as_str)); + } + Some(held) +} + +/// The names in `names` the index does not hold, deduplicated, in the order +/// the statement first names them. +fn unprojected<'a>( + names: impl IntoIterator, + held: &BTreeSet<&str>, +) -> Vec<&'a str> { + let mut missing: Vec<&str> = Vec::new(); + for name in names { + if !held.contains(name) && !missing.contains(&name) { + missing.push(name); + } + } + missing +} + +/// The top-level attribute name of every path a WHERE term references. +pub(super) fn where_roots<'a>(expr: &'a WhereExpr, out: &mut Vec<&'a str>) { + match expr { + WhereExpr::Compare(a, _, b) + | WhereExpr::BeginsWith(a, b) + | WhereExpr::Contains(a, b) + | WhereExpr::AttributeType(a, b) => { + operand_roots(a, out); + operand_roots(b, out); + } + WhereExpr::Between(a, low, high) => { + operand_roots(a, out); + operand_roots(low, out); + operand_roots(high, out); + } + WhereExpr::In(a, items) => { + operand_roots(a, out); + for item in items { + operand_roots(item, out); + } + } + WhereExpr::Function(_, args) => args.iter().for_each(|a| operand_roots(a, out)), + WhereExpr::IsMissing(p, _) => out.push(p.root.as_str()), + WhereExpr::IsType(a, _, _) => operand_roots(a, out), + WhereExpr::And(a, b) | WhereExpr::Or(a, b) => { + where_roots(a, out); + where_roots(b, out); + } + WhereExpr::Not(a) | WhereExpr::ComparedWithBoolean(a, _) => where_roots(a, out), + } +} + +/// The top-level attribute name of every path an operand references. +pub(super) fn operand_roots<'a>(op: &'a Operand, out: &mut Vec<&'a str>) { + match op { + Operand::Path(p) => out.push(p.root.as_str()), + Operand::Arith(a, _, b) => { + operand_roots(a, out); + operand_roots(b, out); + } + Operand::Function(_, args) => args.iter().for_each(|a| operand_roots(a, out)), + Operand::Literal(_) | Operand::Parameter(_) => {} + } +} + +#[cfg(test)] +mod tests { + use super::*; + use extenddb_core::partiql::ast::Statement; + use extenddb_core::partiql::parse; + use extenddb_core::partiql::plan::{KeyNames, WhereAnalysis}; + use extenddb_core::types::KeyType; + use extenddb_core::types::Projection as IndexProjection; + + fn select(sql: &str) -> SelectStatement { + match parse(sql).unwrap() { + Statement::Select(s) => s, + _ => panic!("not a select"), + } + } + + fn key(name: &str, key_type: KeyType) -> KeySchemaElement { + KeySchemaElement { + attribute_name: name.to_owned(), + key_type, + } + } + + fn base() -> Vec { + vec![key("pk", KeyType::Hash), key("sk", KeyType::Range)] + } + + fn index(index_type: IndexType, keys: Vec, include: &[&str]) -> IndexInfo { + IndexInfo { + index_name: "idx".to_owned(), + index_id: "idx".to_owned(), + index_type, + key_schema: keys, + projection: IndexProjection { + projection_type: if include.is_empty() { + ProjectionType::KeysOnly + } else { + ProjectionType::Include + }, + non_key_attributes: (!include.is_empty()) + .then(|| include.iter().map(|s| (*s).to_owned()).collect()), + }, + } + } + + fn gsi() -> IndexInfo { + index(IndexType::Gsi, vec![key("g", KeyType::Hash)], &["kept"]) + } + + fn lsi() -> IndexInfo { + index( + IndexType::Lsi, + vec![key("pk", KeyType::Hash), key("l", KeyType::Range)], + &[], + ) + } + + /// The refusal of `sql` on `info`, with the keyed filter the WHERE + /// analysis gives it against the index keys. + fn message(sql: &str, info: &IndexInfo) -> Option { + let stmt = select(sql); + let name = |key_type| { + info.key_schema + .iter() + .find(|k| k.key_type == key_type) + .map(|k| k.attribute_name.as_str()) + }; + let keys = KeyNames { + partition: name(KeyType::Hash).unwrap(), + sort: name(KeyType::Range), + }; + let value_of = |op: &Operand| match op { + Operand::Literal(v) => Some(v.clone()), + _ => None, + }; + let filter = stmt + .where_clause + .as_ref() + .and_then(|w| WhereAnalysis::new(w, keys, &value_of).shared_filter()); + check(&stmt, info, &base(), filter.as_deref()) + .err() + .map(|e| e.message().to_owned()) + } + + const FILTER: &str = "One or more parameter values were invalid: Secondary index idx does not \ + project one or more filter attributes: "; + + #[test] + fn keyed_filter_on_an_unprojected_attribute_is_refused_on_both_kinds() { + let m = message( + "SELECT pk FROM \"t\".\"idx\" WHERE g = 'a' AND hidden = 1", + &gsi(), + ); + assert_eq!(m.unwrap(), format!("{FILTER}[hidden]")); + let m = message( + "SELECT pk FROM \"t\".\"idx\" WHERE pk = 'a' AND hidden = 1", + &lsi(), + ); + assert_eq!(m.unwrap(), format!("{FILTER}[hidden]")); + } + + #[test] + fn keyed_shapes_count_in_negation_functions_and_in_lists() { + for where_clause in [ + "g = 'a' AND NOT hidden = 1", + "g = 'a' AND hidden IS MISSING", + "g = 'a' AND begins_with(hidden, 'x')", + "g IN ['a', 'b'] AND hidden = 1", + "g = 'a' AND (hidden = 1 OR kept = 2)", + "g = 'a' AND hidden.child = 1", + ] { + let sql = format!("SELECT pk FROM \"t\".\"idx\" WHERE {where_clause}"); + assert_eq!( + message(&sql, &gsi()), + Some(format!("{FILTER}[hidden]")), + "{sql}" + ); + } + for sort_condition in [ + "l BETWEEN 'a' AND 'b'", + "l > 'a'", + "begins_with(l, 'a')", + "l IN ['a', 'b']", + ] { + let sql = format!( + "SELECT pk FROM \"t\".\"idx\" WHERE pk = 'a' AND {sort_condition} AND hidden = 1" + ); + assert_eq!( + message(&sql, &lsi()), + Some(format!("{FILTER}[hidden]")), + "{sql}" + ); + } + // `<>` cannot be a sort-key condition, so the read is a scan. + let sql = "SELECT pk FROM \"t\".\"idx\" WHERE pk = 'a' AND l <> 'a' AND hidden = 1"; + assert_eq!(message(sql, &lsi()), None); + } + + #[test] + fn unkeyed_reads_filter_on_index_rows() { + for where_clause in [ + "hidden = 1", + "hidden = 1 AND hidden = 2", + "(g = 'a' AND hidden = 1) OR g = 'b'", + "g <> 'a' AND hidden = 1", + "begins_with(g, 'a') AND hidden = 1", + "g = 'a' AND g = 'b' AND hidden = 1", + "g = 'a' AND NOT g = 'b' AND hidden = 1", + ] { + let sql = format!("SELECT pk FROM \"t\".\"idx\" WHERE {where_clause}"); + assert_eq!(message(&sql, &gsi()), None, "{sql}"); + } + } + + #[test] + fn projected_filters_and_projections_pass() { + let sql = + "SELECT pk, sk, g, kept FROM \"t\".\"idx\" WHERE g = 'a' AND kept = 1 AND sk = 's'"; + assert_eq!(message(sql, &gsi()), None); + assert_eq!( + message("SELECT * FROM \"t\".\"idx\" WHERE g = 'a'", &gsi()), + None + ); + } + + #[test] + fn missing_attributes_are_listed_once_in_statement_order() { + let sql = + "SELECT pk FROM \"t\".\"idx\" WHERE g = 'a' AND zeta = 1 AND alpha = 2 AND zeta = 3"; + assert_eq!(message(sql, &gsi()), Some(format!("{FILTER}[zeta, alpha]"))); + } + + #[test] + fn gsi_projection_of_an_unprojected_attribute_is_refused_keyed_or_not() { + let want = "One or more parameter values were invalid: Global secondary index idx does \ + not project [hidden]"; + for sql in [ + "SELECT hidden FROM \"t\".\"idx\" WHERE g = 'a'", + "SELECT hidden FROM \"t\".\"idx\"", + "SELECT pk, hidden.child FROM \"t\".\"idx\" WHERE hidden = 1", + ] { + assert_eq!(message(sql, &gsi()).as_deref(), Some(want), "{sql}"); + } + // On a keyed read the filter refusal is checked first. + let sql = "SELECT hidden FROM \"t\".\"idx\" WHERE g = 'a' AND other = 1"; + assert_eq!(message(sql, &gsi()), Some(format!("{FILTER}[other]"))); + } + + #[test] + fn lsi_projection_reaches_back_instead_of_refusing() { + let s = select("SELECT hidden FROM \"t\".\"idx\" WHERE pk = 'a'"); + assert!(check(&s, &lsi(), &base(), Some(&[])).is_ok()); + assert!(reaches_back(&s.projection, &lsi(), &base())); + let s = select("SELECT pk, l FROM \"t\".\"idx\" WHERE pk = 'a'"); + assert!(!reaches_back(&s.projection, &lsi(), &base())); + let s = select("SELECT * FROM \"t\".\"idx\" WHERE pk = 'a'"); + assert!(!reaches_back(&s.projection, &lsi(), &base())); + let s = select("SELECT hidden FROM \"t\".\"idx\" WHERE g = 'a'"); + assert!(!reaches_back(&s.projection, &gsi(), &base())); + } + + #[test] + fn an_all_projection_index_constrains_nothing() { + let mut all = gsi(); + all.projection.projection_type = ProjectionType::All; + all.projection.non_key_attributes = None; + assert_eq!( + message( + "SELECT hidden FROM \"t\".\"idx\" WHERE g = 'a' AND other = 1", + &all + ), + None + ); + } +} diff --git a/crates/engine/src/partiql/item_depth.rs b/crates/engine/src/partiql/item_depth.rs new file mode 100644 index 000000000..bbfa0d641 --- /dev/null +++ b/crates/engine/src/partiql/item_depth.rs @@ -0,0 +1,404 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The item nesting check on a PartiQL `INSERT` or `UPDATE`. +//! +//! A parameter may hold 32 maps or lists around its leaf; 33 is refused before +//! the statement is read (see `core::partiql::params`). A stored attribute may +//! hold 31, so a 32-level value passes the parameter check and meets the item +//! check, which refuses it with a shorter message than `PutItem` and +//! `UpdateItem` give the same item (measured 2026-11-02). Those operations +//! refuse a 32-level value as part of the request, before the item is read; a +//! PartiQL `UPDATE` reaches the item check only once its condition holds and +//! its update applies, so a missing item fails its condition and a `SET` +//! through a path the item does not have is the update's own error. A `SET` +//! or `REMOVE` of a key attribute is refused ahead of all of these, as it is +//! at any depth (measured 2026-11-02). In `ExecuteTransaction` the item check +//! runs with the members' conditions: two members on one item are refused +//! first, then every member reports its own reason (measured 2026-11-02). + +use std::collections::{HashMap, HashSet}; + +use serde_json::Value; + +use extenddb_core::error::DynamoDbError; +use extenddb_core::expression::{Expr, ExpressionMaps, apply_update_validated, evaluate_condition}; +use extenddb_core::partiql::Statement; +use extenddb_core::partiql::ast::{Operand, UpdateStatement}; +use extenddb_core::partiql::params::NESTING_MESSAGE; +use extenddb_core::types::{ + AttributeValue, CancellationReason, Item, ReturnValuesOnConditionCheckFailure, TableKeyInfo, + extract_key, +}; +use extenddb_core::validation::{ + validate_attribute_values_nesting_depth, validate_item_nesting_depth, validate_key_only, + validate_key_sizes, +}; +use extenddb_storage::error::StorageError; + +use super::SingleRequest; +use super::common; +use super::write::{UpdateParts, build_update_parts}; +use crate::OperationContext; +use crate::create_table::{storage_err_to_dynamo, storage_err_to_dynamo_with_ccf}; +use crate::expression_helpers::{build_expression_maps, parse_condition_expr, parse_update_expr}; +use crate::transact_write_helpers::validate_no_key_updates; + +/// The item check's message on a PartiQL write (measured 2026-11-02). +pub(super) const ITEM_NESTING_MESSAGE: &str = "Nesting Levels have exceeded supported limits"; + +/// Give the engine's item nesting refusal the PartiQL wording. The engine words +/// it as `PutItem` does, and an `INSERT` is put through `PutItem`. +pub(super) fn item_nesting_error(err: DynamoDbError) -> DynamoDbError { + match err { + DynamoDbError::ValidationException(message) if message.ends_with(NESTING_MESSAGE) => { + DynamoDbError::ValidationException(ITEM_NESTING_MESSAGE.to_owned()) + } + other => other, + } +} + +/// Whether a `SET` value nests as deep as an item attribute may not, so the +/// update can never be written. +pub(super) fn sets_a_value_past_the_item_limit( + update: &UpdateStatement, + params: &[AttributeValue], +) -> bool { + update + .sets + .iter() + .any(|action| holds_a_value_past_the_item_limit(&action.value, params)) +} + +/// Whether a write statement stores a value nested past the item limit: an +/// `INSERT` value, or an `UPDATE` `SET` value. +fn stores_a_value_past_the_item_limit(stmt: &Statement, params: &[AttributeValue]) -> bool { + match stmt { + Statement::Insert(insert) => insert + .value + .iter() + .any(|(_, value)| holds_a_value_past_the_item_limit(value, params)), + Statement::Update(update) => sets_a_value_past_the_item_limit(update, params), + _ => false, + } +} + +fn holds_a_value_past_the_item_limit(operand: &Operand, params: &[AttributeValue]) -> bool { + let past = |value: &AttributeValue| { + validate_attribute_values_nesting_depth(std::iter::once(value)).is_err() + }; + match operand { + Operand::Literal(value) => past(value), + Operand::Parameter(index) => params.get(*index).is_some_and(past), + Operand::Arith(left, _, right) => { + holds_a_value_past_the_item_limit(left, params) + || holds_a_value_past_the_item_limit(right, params) + } + Operand::Function(_, args) => args + .iter() + .any(|arg| holds_a_value_past_the_item_limit(arg, params)), + Operand::Path(_) => false, + } +} + +/// The answer to an `UPDATE` that [`sets_a_value_past_the_item_limit`], which +/// `UpdateItem` would refuse before reading the item. A `SET` or `REMOVE` of a +/// key attribute is refused first. The update is then evaluated as the +/// storage layer evaluates any update, against the stored item, and nothing +/// is written: a condition that fails or an update that does not apply is the +/// answer, then the item check on the updated item. +/// +/// `None` when the updated item passes the item check, which happens only +/// when the deep value is not stored, such as `if_not_exists` on an attribute +/// the item has. The caller then sends the update to `UpdateItem`, whose +/// nesting check of every `ExpressionAttributeValue` refuses it, where the +/// service applies the update (measured 2026-11-02). +pub(super) async fn unwritable_update_error( + ctx: &OperationContext, + parts: &UpdateParts, + req: &SingleRequest, +) -> Option { + match evaluate_without_writing(ctx, parts, req).await { + Ok(item) => validate_item_nesting_depth(&item) + .is_err() + .then(|| DynamoDbError::ValidationException(ITEM_NESTING_MESSAGE.to_owned())), + Err(err) => Some(err), + } +} + +/// The item an update would store, or the error it meets on the way. +async fn evaluate_without_writing( + ctx: &OperationContext, + parts: &UpdateParts, + req: &SingleRequest, +) -> Result { + let key_info = ctx + .table_key_info(&parts.table) + .await + .map_err(storage_err_to_dynamo)?; + let key: Item = from_wire(&parts.key)?; + validate_key_only(&key, &key_info.key_schema, &key_info.attribute_definitions)?; + validate_key_sizes(&key, &key_info.key_schema, &ctx.limits)?; + + let names: HashMap = from_wire(&parts.names)?; + let values: HashMap = from_wire(&parts.values)?; + let maps = build_expression_maps(Some(&names), Some(&values)); + let actions = parse_update_expr(&parts.update_expression, &ctx.limits)?; + validate_no_key_updates(&actions, &key_info, &maps)?; + let condition = parse_condition_expr(&parts.condition, &ctx.limits)?; + + // The condition is evaluated against the stored item, or an empty one, and + // the update applied to the stored item, as `update_item` does. + let on_failure = + return_values_on_failure(req.return_values_on_condition_check_failure.as_deref()); + let stored = + stored_item_if_condition_holds(ctx, &key_info, &key, &condition, &maps, on_failure).await?; + let mut item = stored.unwrap_or(key); + apply_update_validated( + &actions, + &mut item, + &maps, + &key_info.vector_indexes, + &key_info.attribute_definitions, + )?; + Ok(item) +} + +/// The stored item, or `None` when there is none, once `condition` holds +/// against it (an absent item is evaluated as an empty one). +/// +/// # Errors +/// +/// Returns the `ConditionalCheckFailedException` a write gets when the +/// condition does not hold, carrying the stored item for `ALL_OLD`. +async fn stored_item_if_condition_holds( + ctx: &OperationContext, + key_info: &TableKeyInfo, + key: &Item, + condition: &Expr, + maps: &ExpressionMaps, + on_failure: ReturnValuesOnConditionCheckFailure, +) -> Result, DynamoDbError> { + let stored = ctx + .storage + .get_item(key_info, key) + .await + .map_err(storage_err_to_dynamo)?; + if evaluate_condition(condition, stored.as_ref().unwrap_or(&Item::new()), maps)? { + Ok(stored) + } else { + Err(storage_err_to_dynamo_with_ccf( + StorageError::ConditionFailed(stored), + on_failure, + )) + } +} + +fn return_values_on_failure(value: Option<&str>) -> ReturnValuesOnConditionCheckFailure { + match value { + Some("ALL_OLD") => ReturnValuesOnConditionCheckFailure::AllOld, + _ => ReturnValuesOnConditionCheckFailure::None, + } +} + +/// Whether a member of a write transaction stores a value nested past the +/// item limit, so the transaction takes [`transaction_reasons`]. +pub(super) fn has_member_past_the_item_limit(members: &[(SingleRequest, Statement)]) -> bool { + members + .iter() + .any(|(req, stmt)| stores_a_value_past_the_item_limit(stmt, &req.parameters)) +} + +/// The reasons a write `ExecuteTransaction` is cancelled with when a member +/// would store a value nested past the item limit, or `None` when no member +/// has a reason. `items` are the members' `TransactWriteItems` translations, +/// and `is_insert` marks the `INSERT`s among them. +/// +/// `TransactWriteItems` refuses such a value as part of the request. The +/// service refuses two members on one item first, then runs every member's +/// condition with the item check: a member whose condition fails reports +/// that (a duplicate key for an `INSERT`), a member whose condition holds and +/// whose value is past the limit reports the item check's refusal, and every +/// other member reports `None` (measured 2026-11-02). +/// +/// # Errors +/// +/// Returns the refusal of two members on one item, or an error the +/// evaluation meets that is not a member's reason. +pub(super) async fn transaction_reasons( + ctx: &OperationContext, + members: &[(SingleRequest, Statement)], + items: &[Value], + is_insert: &[bool], +) -> Result>, DynamoDbError> { + let mut targets = Vec::with_capacity(items.len()); + let mut seen = HashSet::with_capacity(items.len()); + for item in items { + let target = MemberTarget::read(ctx, item).await?; + if !seen.insert(target.identity()) { + return Err(DynamoDbError::ValidationException( + "Transaction request cannot include multiple operations on one item".to_owned(), + )); + } + targets.push(target); + } + let mut reasons = Vec::with_capacity(members.len()); + let mut cancelled = false; + for (((req, stmt), target), insert) in members.iter().zip(&targets).zip(is_insert) { + let reason = member_reason(ctx, stmt, req, target, *insert).await?; + cancelled |= reason.is_some(); + reasons.push(reason.unwrap_or_else(CancellationReason::none)); + } + Ok(cancelled.then_some(reasons)) +} + +/// The reason one transaction member reports: its failed condition, or the +/// item check's refusal of a value it would store, or `None`. +async fn member_reason( + ctx: &OperationContext, + stmt: &Statement, + req: &SingleRequest, + target: &MemberTarget, + insert: bool, +) -> Result, DynamoDbError> { + let err = match target.failed_condition(ctx).await? { + Some(_) if insert => return Ok(Some(common::duplicate_item_reason())), + Some(failed) => failed, + None => match stmt { + Statement::Insert(_) if stores_a_value_past_the_item_limit(stmt, &req.parameters) => { + DynamoDbError::ValidationException(ITEM_NESTING_MESSAGE.to_owned()) + } + Statement::Update(update) + if sets_a_value_past_the_item_limit(update, &req.parameters) => + { + let parts = build_update_parts(ctx, update, &req.parameters).await?; + match unwritable_update_error(ctx, &parts, req).await { + Some(err) => err, + None => return Ok(None), + } + } + _ => return Ok(None), + }, + }; + Ok(Some(match super::bare_expression_error(err) { + DynamoDbError::ConditionalCheckFailedException(_, item) => { + CancellationReason::condition_check_failed_with_item(item) + } + DynamoDbError::ValidationException(message) => { + CancellationReason::validation_error(message) + } + other => return Err(other), + })) +} + +/// The item a transaction member writes or checks and the condition it +/// carries, read back from its `TransactWriteItems` translation. +struct MemberTarget { + key_info: TableKeyInfo, + key: Item, + condition: Option, + names: HashMap, + values: HashMap, + on_failure: ReturnValuesOnConditionCheckFailure, +} + +impl MemberTarget { + async fn read(ctx: &OperationContext, member: &Value) -> Result { + let Some((kind, body)) = member.as_object().and_then(|m| m.iter().next()) else { + return Err(internal("a transaction member without an operation")); + }; + let table = body + .get("TableName") + .and_then(Value::as_str) + .ok_or_else(|| internal("a transaction member without a table"))?; + let key_info = ctx + .table_key_info(table) + .await + .map_err(storage_err_to_dynamo)?; + let key = if kind == "Put" { + let item: Item = from_wire_field(body, "Item")?; + extract_key(&item, &key_info.key_schema) + } else { + from_wire_field(body, "Key")? + }; + Ok(Self { + condition: body + .get("ConditionExpression") + .and_then(Value::as_str) + .map(str::to_owned), + names: from_wire_field(body, "ExpressionAttributeNames")?, + values: from_wire_field(body, "ExpressionAttributeValues")?, + on_failure: return_values_on_failure( + body.get("ReturnValuesOnConditionCheckFailure") + .and_then(Value::as_str), + ), + key_info, + key, + }) + } + + /// The item's identity, as `TransactWriteItems` compares two members. + fn identity(&self) -> String { + format!( + "{}:{}", + self.key_info.table_name, + serde_json::to_string(&self.key).unwrap_or_default() + ) + } + + /// The refusal of this member's condition when it has one that does not + /// hold against the stored item, else `None`. + async fn failed_condition( + &self, + ctx: &OperationContext, + ) -> Result, DynamoDbError> { + let Some(text) = &self.condition else { + return Ok(None); + }; + let maps = build_expression_maps(Some(&self.names), Some(&self.values)); + let condition = parse_condition_expr(text, &ctx.limits)?; + match stored_item_if_condition_holds( + ctx, + &self.key_info, + &self.key, + &condition, + &maps, + self.on_failure, + ) + .await + { + Ok(_) => Ok(None), + Err(err @ DynamoDbError::ConditionalCheckFailedException(..)) => Ok(Some(err)), + Err(other) => Err(other), + } + } +} + +/// Read back a wire-form map the translation built. +fn from_wire( + map: &serde_json::Map, +) -> Result { + serde_json::from_value(Value::Object(map.clone())).map_err(|e| { + tracing::error!(internal_error = %e, "failed to read a translated PartiQL map"); + internal("a translated PartiQL map") + }) +} + +/// Read back one map field of a translated member, empty when it is absent. +fn from_wire_field( + body: &Value, + field: &str, +) -> Result { + match body.get(field) { + Some(Value::Object(map)) => from_wire(map), + _ => Ok(T::default()), + } +} + +fn internal(what: &str) -> DynamoDbError { + tracing::error!(what, "failed to read back a PartiQL translation"); + DynamoDbError::InternalServerError("Internal server error".to_owned()) +} + +#[cfg(test)] +mod tests; diff --git a/crates/engine/src/partiql/item_depth/tests.rs b/crates/engine/src/partiql/item_depth/tests.rs new file mode 100644 index 000000000..54972cd9d --- /dev/null +++ b/crates/engine/src/partiql/item_depth/tests.rs @@ -0,0 +1,118 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Unit tests for the item nesting check on a PartiQL INSERT or UPDATE. + +use super::*; +use extenddb_core::partiql::{Statement, parse}; + +fn deep(levels: usize) -> AttributeValue { + let mut value = AttributeValue::S("leaf".to_owned()); + for _ in 0..levels { + value = AttributeValue::M(std::iter::once(("n".to_owned(), value)).collect()); + } + value +} + +fn update(sql: &str) -> UpdateStatement { + match parse(sql).unwrap() { + Statement::Update(u) => u, + other => panic!("expected an UPDATE, got {other:?}"), + } +} + +#[test] +fn the_engine_item_refusal_gets_the_partiql_wording() { + let mut item = Item::new(); + item.insert("x".to_owned(), deep(32)); + let refused = extenddb_core::validation::validate_item_nesting_depth(&item).unwrap_err(); + match item_nesting_error(refused) { + DynamoDbError::ValidationException(m) => assert_eq!(m, ITEM_NESTING_MESSAGE), + other => panic!("expected ValidationException, got {other:?}"), + } + let prefixed = DynamoDbError::ValidationException(format!( + "1 validation error detected: {NESTING_MESSAGE}" + )); + assert!(matches!( + item_nesting_error(prefixed), + DynamoDbError::ValidationException(m) if m == ITEM_NESTING_MESSAGE + )); +} + +#[test] +fn other_errors_keep_their_wording() { + let key = DynamoDbError::ValidationException( + "One or more parameter values were invalid: Missing the key pk in the item".to_owned(), + ); + assert!(matches!( + item_nesting_error(key), + DynamoDbError::ValidationException(m) if m.starts_with("One or more parameter values") + )); + assert!(matches!( + item_nesting_error(DynamoDbError::DuplicateItemException("d".to_owned())), + DynamoDbError::DuplicateItemException(_) + )); +} + +#[test] +fn a_set_value_of_32_levels_cannot_be_written() { + let stmt = update("UPDATE \"t\" SET x = ?, y = ? WHERE pk = 'p'"); + assert!(!sets_a_value_past_the_item_limit( + &stmt, + &[deep(31), deep(31)] + )); + assert!(sets_a_value_past_the_item_limit( + &stmt, + &[deep(31), deep(32)] + )); +} + +#[test] +fn a_deep_value_in_the_where_clause_is_not_a_set_value() { + let stmt = update("UPDATE \"t\" SET x = ? WHERE pk = 'p' AND y <> ?"); + assert!(!sets_a_value_past_the_item_limit( + &stmt, + &[deep(1), deep(32)] + )); +} + +#[test] +fn a_transaction_takes_the_item_check_path_only_for_a_value_it_stores() { + let insert = parse("INSERT INTO \"t\" VALUE {'pk': 'p', 'x': ?}").unwrap(); + let condition = parse("UPDATE \"t\" SET a = 1 WHERE pk = 'p' AND x <> ?").unwrap(); + let member = |stmt: Statement, levels: usize| { + let req = SingleRequest { + statement: String::new(), + parameters: vec![deep(levels)], + parameter_numbers: vec![None], + consistent_read: false, + limit: None, + next_token: None, + return_consumed_capacity: None, + return_values_on_condition_check_failure: None, + }; + (req, stmt) + }; + assert!(has_member_past_the_item_limit(&[member( + insert.clone(), + 32 + )])); + assert!(!has_member_past_the_item_limit(&[member(insert, 31)])); + assert!(!has_member_past_the_item_limit(&[member(condition, 32)])); +} + +#[test] +fn a_failed_condition_reports_the_stored_item_only_for_all_old() { + assert_eq!( + return_values_on_failure(Some("ALL_OLD")), + ReturnValuesOnConditionCheckFailure::AllOld + ); + assert_eq!( + return_values_on_failure(Some("NONE")), + ReturnValuesOnConditionCheckFailure::None + ); + assert_eq!( + return_values_on_failure(None), + ReturnValuesOnConditionCheckFailure::None + ); +} diff --git a/crates/engine/src/partiql/mod.rs b/crates/engine/src/partiql/mod.rs new file mode 100644 index 000000000..973c9e19f --- /dev/null +++ b/crates/engine/src/partiql/mod.rs @@ -0,0 +1,519 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! PartiQL surface: `ExecuteStatement`, `BatchExecuteStatement`, +//! `ExecuteTransaction`. +//! +//! This module is compiled only with the `partiql` feature and is the whole +//! feature on the engine side. It is a pure translation layer: it parses a +//! statement (via `extenddb_core::partiql`), then drives the operations the +//! engine already has (get/query/scan/put/update/delete and their batch and +//! transact forms). It adds no storage concept. +//! +//! Confinement: the [`PartiqlExecutor`] trait is implemented once, for +//! [`OperationContext`], and nothing else implements or requires it. Removing +//! PartiQL is deleting this module and `core::partiql`, dropping the three +//! `#[cfg(feature = "partiql")]` dispatch arms and `is_known_operation` +//! entries, removing the `DuplicateItemException` variant that only this path +//! uses from `core`'s error enum, and dropping the feature. With the feature +//! off the three operations are unknown and answered with +//! `UnknownOperationException`, exactly as before. +//! +//! The server does not authorize these operations before dispatch: each +//! statement is authorized here, through the request's +//! [`ItemAuthorizer`](crate::ItemAuthorizer), which is not specific to +//! PartiQL (see `authorize`). + +use std::future::Future; +use std::sync::Arc; + +use serde_json::Value; + +use extenddb_core::error::DynamoDbError; +use extenddb_core::limits::LimitsConfig; +use extenddb_core::partiql::number::check_literal_numbers; +use extenddb_core::partiql::operand_types::check_ordering_operands; +use extenddb_core::partiql::params::check_parameter_values; +use extenddb_core::partiql::parser::{MAX_STATEMENT_LENGTH, parse_for_execute_statement}; +use extenddb_core::partiql::{Statement, table_name_refusal}; +use extenddb_core::types::AttributeValue; + +use crate::{DispatchResult, OperationContext}; + +use authorize::{Gate, Surface}; + +mod authorize; +mod base_join; +mod batch; +mod common; +mod expr_bridge; +mod index_read; +mod item_depth; +mod next_token; +mod request_checks; +mod returning; +mod select; +mod snapshot; +#[cfg(test)] +mod test_context; +mod transaction; +mod write; + +/// The confined PartiQL surface, implemented once for [`OperationContext`]. +pub trait PartiqlExecutor { + /// `ExecuteStatement`: one statement, response under `Items`. + fn execute_statement( + &self, + body: Value, + ) -> impl Future> + Send; + + /// `BatchExecuteStatement`: all-read or all-write, response under `Responses`. + fn batch_execute_statement( + &self, + body: Value, + ) -> impl Future> + Send; + + /// `ExecuteTransaction`: all-or-nothing, response under `Responses`. + fn execute_transaction( + &self, + body: Value, + ) -> impl Future> + Send; +} + +impl PartiqlExecutor for OperationContext { + async fn execute_statement(&self, body: Value) -> Result { + let ctx = &translation_context(self); + request_checks::check_execute_statement(&body)?; + let req = SingleRequest::from_body(&body)?; + check_parameters(body.get("Parameters")).map_err(DynamoDbError::ValidationException)?; + // An EXISTS statement is refused by the parser, in its place among + // the refusals of a statement that parses (measured 2026-11-02). + let statement = parse_for_execute_statement(&req.statement, req.parameters.len()) + .map_err(|e| DynamoDbError::ValidationException(e.message))?; + match &statement { + // A projected call (`COUNT(*)`) is reported once the target is + // resolved (see `select::run`), so only the table name rule comes + // ahead of it (measured 2026-11-02). + Statement::Select(select) if select.projection_error.is_some() => { + check_table_name(&statement) + } + _ => check_statement(&statement, &req.parameters), + } + .map_err(DynamoDbError::ValidationException)?; + // The statement is authorized against one read of its table, at its + // place among the checks (see `authorize::gate`). + let (ctx, key_info) = + snapshot::pinned_context(ctx, common::statement_table(&statement)).await?; + let mut gate = Gate::new( + &ctx, + &statement, + key_info.as_ref(), + &req, + Surface::Statement, + ); + let result = match &statement { + Statement::Select(s) => select::run(&ctx, s, &req, &mut gate).await, + other => write::run(&ctx, other, &req, &mut gate).await, + }; + gate.settle(result).await? + } + + async fn batch_execute_statement(&self, body: Value) -> Result { + batch::run(&translation_context(self), &body).await + } + + async fn execute_transaction(&self, body: Value) -> Result { + transaction::run(&translation_context(self), &body).await + } +} + +/// The size, in bytes and in tokens, an expression written for a statement +/// may reach: 16 bytes for each character a statement may hold. +/// +/// The text written for a statement is at most 5 bytes for each of its +/// characters, plus a fixed part under 64 bytes. A name or a value becomes a +/// placeholder of at most 6 bytes (`#n` or `:v` and an index of at most 4 +/// digits, since a statement of 8192 characters holds fewer than 4096 names +/// or values); each is at least 1 character, and two of them have at least +/// one character between them; an operator or a separator gains at most a +/// space on each side. The densest form, `a=1`, becomes `#nNNNN = :vNNNN`, +/// 15 bytes for 3 characters. The fixed part is what a translation adds to +/// what the statement says, such as `attribute_exists(#n0) AND ` on a +/// write's key. +/// +/// The filter of a read of an index by several disjuncts that reaches back +/// to the base table writes each key condition compared with false twice, +/// as the condition and its negation (see `plan::keyed_condition`), so its +/// text is at most twice as long: 10 bytes for each character and twice the +/// fixed part, 82,048 bytes. That is within 16 bytes for each of the 8192 +/// characters, 131,072, and since a token is at least one byte, the same +/// number bounds its tokens. +const TRANSLATED_EXPRESSION_LIMIT: usize = 16 * MAX_STATEMENT_LENGTH; + +/// The limits the requests a statement is translated into run with: `limits` +/// with the size and token limits of an expression raised to +/// [`TRANSLATED_EXPRESSION_LIMIT`]. The classic limits (4096 bytes and 4096 +/// tokens) apply to the expressions a caller writes. The text written for a +/// statement is longer than the statement wherever a name or a value becomes +/// a placeholder, and the service runs a statement within its own limits +/// (8192 characters, 300 operators in its condition) whose text would be past +/// the classic ones (measured 2026-11-02: a condition of 150 `begins_with` +/// calls ANDed, 4275 bytes as classic text). +fn translation_limits(limits: &LimitsConfig) -> LimitsConfig { + LimitsConfig { + max_expression_length_bytes: TRANSLATED_EXPRESSION_LIMIT, + max_expression_tokens: TRANSLATED_EXPRESSION_LIMIT, + ..limits.clone() + } +} + +/// The context the requests a statement is translated into run in: `ctx` +/// with the limits of [`translation_limits`]. +fn translation_context(ctx: &OperationContext) -> OperationContext { + OperationContext { + limits: Arc::new(translation_limits(&ctx.limits)), + ..snapshot::context_with_key_info(ctx, ctx.authorized_key_info.clone()) + } +} + +/// The parsed fields of a single-statement request, shared by the SELECT and +/// write executors. +pub(crate) struct SingleRequest { + pub statement: String, + pub parameters: Vec, + /// The text of each parameter that is a number, as the request sends + /// it. `parameters` hold numbers normalized (`10.0` is `10`), and + /// `dynamodb:LeadingKeys` keeps the text's scale (see `authorize`). + pub parameter_numbers: Vec>, + pub consistent_read: bool, + pub limit: Option, + pub next_token: Option, + pub return_consumed_capacity: Option, + pub return_values_on_condition_check_failure: Option, +} + +impl SingleRequest { + pub(super) fn from_body(body: &Value) -> Result { + let statement = body + .get("Statement") + .and_then(Value::as_str) + .ok_or_else(|| DynamoDbError::ValidationException("Statement is required".to_owned()))? + .to_owned(); + let parameters = parse_parameters(body.get("Parameters"))?; + let parameter_numbers = body + .get("Parameters") + .and_then(Value::as_array) + .map(|values| { + values + .iter() + .map(|value| value.get("N").and_then(Value::as_str).map(str::to_owned)) + .collect() + }) + .unwrap_or_default(); + Ok(Self { + statement, + parameters, + parameter_numbers, + consistent_read: body + .get("ConsistentRead") + .and_then(Value::as_bool) + .unwrap_or(false), + limit: body.get("Limit").and_then(Value::as_i64), + next_token: body + .get("NextToken") + .and_then(Value::as_str) + .map(str::to_owned), + return_consumed_capacity: body + .get("ReturnConsumedCapacity") + .and_then(Value::as_str) + .map(str::to_owned), + return_values_on_condition_check_failure: body + .get("ReturnValuesOnConditionCheckFailure") + .and_then(Value::as_str) + .map(str::to_owned), + }) + } +} + +/// Deserialize a PartiQL `Parameters` array (wire-form AttributeValues). +pub(crate) fn parse_parameters( + value: Option<&Value>, +) -> Result, DynamoDbError> { + let Some(value) = value else { + return Ok(Vec::new()); + }; + let arr = value.as_array().ok_or_else(|| { + DynamoDbError::ValidationException("Parameters must be a list".to_owned()) + })?; + arr.iter() + .map(|v| { + serde_json::from_value::(v.clone()) + .map_err(|e| DynamoDbError::ValidationException(format!("invalid parameter: {e}"))) + }) + .collect() +} + +/// Check a statement's `Parameters` (as the request carries them) for nesting +/// depth and number conversion. The service answers this ahead of the target's +/// key checks, and in `ExecuteTransaction` without a statement-position prefix +/// (measured 2026-11-02), so it runs before the statement is parsed. +/// +/// The walk follows the parsed body: parameters and list elements in request +/// order, the members of a map in the order the parsed request holds them, +/// which is the request's when the build includes the MongoDB storage crate +/// (any build of the whole workspace) and name order otherwise. The service +/// walks map members in request order (measured 2026-11-02), so in name order, +/// when two members of one map both have a problem, the one reported here can +/// differ. +/// +/// # Errors +/// +/// Returns the service's message for the first problem the walk reaches. +pub(crate) fn check_parameters(parameters: Option<&Value>) -> Result<(), String> { + let Some(Value::Array(items)) = parameters else { + return Ok(()); + }; + check_parameter_values(items) +} + +/// The checks a parsed statement gets before its target is resolved: the +/// table name rule, a projected call (`COUNT(*)`), number literals against +/// the item number rules, the operand types of ordering comparisons, then +/// update paths that overlap or conflict. The first four come in that order +/// on all three operations (measured 2026-11-02), except that +/// `ExecuteStatement` reports a projected call once the target is resolved. +/// Where the service checks overlapping or conflicting paths relative to the +/// others was not measured. +/// +/// # Errors +/// +/// Returns the service's message for the first failure. +pub(crate) fn check_statement(stmt: &Statement, params: &[AttributeValue]) -> Result<(), String> { + check_table_name(stmt)?; + if let Statement::Select(select) = stmt + && let Some(message) = &select.projection_error + { + return Err(message.clone()); + } + check_literal_numbers(stmt).map_err(|e| e.message)?; + check_ordering_operands(stmt, params)?; + match stmt { + Statement::Update(update) => write::check_overlapping_paths(update), + _ => Ok(()), + } +} + +/// The table name rule, the first check a parsed statement gets. +/// +/// # Errors +/// +/// Returns the service's refusal of the name. +pub(crate) fn check_table_name(stmt: &Statement) -> Result<(), String> { + match table_name_refusal(stmt) { + Some(refusal) => Err(refusal.to_owned()), + None => Ok(()), + } +} + +/// Drop the expression-kind prefix (`Invalid FilterExpression: ` and the like) +/// from an error raised by a classic handler a statement was translated into. +/// The prefix names an expression the caller never wrote. Measured 2026-11-02: +/// the operand-type messages of a read's filter and of a write's condition +/// ("Incorrect operand type for operator or function; ...") come without it. +/// The messages the update evaluation raises ("The document path provided in +/// the update expression is invalid for update", "An operand in the update +/// expression has an incorrect data type") carry no prefix on either side, and +/// no other prefixed message was measured. +pub(crate) fn bare_expression_error(err: DynamoDbError) -> DynamoDbError { + const PREFIXES: [&str; 5] = [ + "Invalid FilterExpression: ", + "Invalid ConditionExpression: ", + "Invalid KeyConditionExpression: ", + "Invalid UpdateExpression: ", + "Invalid ProjectionExpression: ", + ]; + match err { + DynamoDbError::ValidationException(message) => { + let bare = PREFIXES + .iter() + .find_map(|prefix| message.strip_prefix(prefix)) + .map(str::to_owned); + DynamoDbError::ValidationException(bare.unwrap_or(message)) + } + other => other, + } +} + +/// Bind a positional parameter, returning the service's count-mismatch message +/// when the index is out of range. Supplying more parameters than placeholders +/// is not an error (the extras are ignored); supplying fewer is. +pub(crate) fn bind_parameter( + params: &[AttributeValue], + index: usize, +) -> Result { + params.get(index).cloned().ok_or_else(|| { + DynamoDbError::ValidationException( + "Number of parameters in request and statement don't match.".to_owned(), + ) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use extenddb_core::partiql::parse; + + fn message(err: DynamoDbError) -> String { + match err { + DynamoDbError::ValidationException(m) => m, + other => panic!("expected ValidationException, got {other:?}"), + } + } + + #[test] + fn a_translated_expression_may_reach_sixteen_bytes_a_statement_character() { + let classic = LimitsConfig::default(); + let translated = translation_limits(&classic); + assert_eq!(translated.max_expression_length_bytes, 131_072); + assert_eq!(translated.max_expression_tokens, 131_072); + // The other limits are the caller's. + assert_eq!( + translated.max_expression_depth, + classic.max_expression_depth + ); + assert_eq!( + translated.max_attribute_name_bytes, + classic.max_attribute_name_bytes + ); + } + + #[test] + fn the_expression_prefix_is_dropped() { + let sentence = "Incorrect operand type for operator or function; operator or function: <, \ + operand type: BOOL"; + for kind in [ + "FilterExpression", + "ConditionExpression", + "KeyConditionExpression", + "UpdateExpression", + "ProjectionExpression", + ] { + let err = DynamoDbError::ValidationException(format!("Invalid {kind}: {sentence}")); + assert_eq!(message(bare_expression_error(err)), sentence, "{kind}"); + } + } + + #[test] + fn other_errors_pass_through_unchanged() { + let plain = DynamoDbError::ValidationException( + "One or more parameter values were invalid: Missing the key pk in the item".to_owned(), + ); + assert_eq!( + message(bare_expression_error(plain)), + "One or more parameter values were invalid: Missing the key pk in the item" + ); + let other = bare_expression_error(DynamoDbError::ResourceNotFoundException( + "Requested resource not found".to_owned(), + )); + assert!(matches!(other, DynamoDbError::ResourceNotFoundException(_))); + } + + #[test] + fn parameters_are_checked_as_the_request_carries_them() { + assert_eq!(check_parameters(None), Ok(())); + let body: Value = serde_json::json!({ + "Parameters": [{"S": "a"}, {"L": [{"N": "1"}, {"N": "1e99999999999"}]}] + }); + assert_eq!( + check_parameters(body.get("Parameters")), + Err("The parameter cannot be converted to a numeric value: 1e99999999999".to_owned()) + ); + let body: Value = serde_json::json!({ "Parameters": [{"N": ""}] }); + assert_eq!( + check_parameters(body.get("Parameters")), + Err("The parameter cannot be converted to a numeric value".to_owned()) + ); + } + + #[test] + fn statement_checks_run_the_table_name_rule_first() { + // A number in a condition is refused by the parser; one in a SET + // value is left to these checks. + let stmt = parse("UPDATE \"t\" SET q = 1e126 WHERE pk = 'a'").unwrap(); + assert_eq!( + check_statement(&stmt, &[]), + Err( + "Table name should be within the length [3, 255] and only contain 'a-z', \ + 'A-Z', '0-9', '-', '_', '.'." + .to_owned() + ) + ); + } + + #[test] + fn a_projected_call_is_checked_after_the_name_rule_and_before_literals() { + const PROJECTED_CALL: &str = "Unexpected path component at 1:8:5"; + let stmt = parse("SELECT COUNT(*) FROM \"t\" WHERE pk = 'a'").unwrap(); + assert_eq!( + check_statement(&stmt, &[]), + Err( + "Table name should be within the length [3, 255] and only contain 'a-z', \ + 'A-Z', '0-9', '-', '_', '.'." + .to_owned() + ) + ); + for sql in [ + "SELECT COUNT(*) FROM \"tbl\" WHERE pk = 1e126", + "SELECT COUNT(*) FROM \"tbl\" WHERE pk = 'a' AND v < true", + ] { + let stmt = parse(sql).unwrap(); + assert_eq!( + check_statement(&stmt, &[]), + Err(PROJECTED_CALL.to_owned()), + "{sql}" + ); + } + } + + #[test] + fn statement_checks_run_overlapping_update_paths_last() { + let stmt = parse("UPDATE \"tbl\" SET q = 1, q = 2 WHERE pk = 'a'").unwrap(); + assert_eq!( + check_statement(&stmt, &[]), + Err( + "Two document paths overlap with each other; must remove or rewrite one of \ + these paths; path one: [q], path two: [q]" + .to_owned() + ) + ); + let stmt = parse("UPDATE \"tbl\" SET q = 1e126, q = 2 WHERE pk = 'a'").unwrap(); + assert!( + check_statement(&stmt, &[]) + .unwrap_err() + .starts_with("Number overflow.") + ); + } + + #[test] + fn statement_checks_run_literals_before_operand_types() { + let stmt = parse("UPDATE \"tbl\" SET q = 1e126 WHERE pk = 'a' AND val < ?").unwrap(); + let params = [AttributeValue::Bool(true)]; + assert_eq!( + check_statement(&stmt, ¶ms), + Err( + "Number overflow. Attempting to store a number with magnitude larger than \ + supported range under root" + .to_owned() + ) + ); + let stmt = parse("SELECT * FROM \"tbl\" WHERE pk = 1 AND val < ?").unwrap(); + assert_eq!( + check_statement(&stmt, ¶ms), + Err( + "Incorrect operand type for operator or function; operator or function: <, \ + operand type: BOOL" + .to_owned() + ) + ); + } +} diff --git a/crates/engine/src/partiql/next_token.rs b/crates/engine/src/partiql/next_token.rs new file mode 100644 index 000000000..589ee865c --- /dev/null +++ b/crates/engine/src/partiql/next_token.rs @@ -0,0 +1,167 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The opaque `NextToken` of a paged `SELECT`, bound to the statement and the +//! parameter values that produced it. + +use base64::Engine; +use base64::engine::general_purpose::STANDARD as BASE64; +use serde_json::Value; +use sha2::{Digest, Sha256}; + +use extenddb_core::error::DynamoDbError; +use extenddb_core::types::AttributeValue; + +/// Encode an opaque pagination token binding a `LastEvaluatedKey` to the exact +/// statement and parameter values that produced it. The token holds digests +/// of both, so its length does not grow with the statement: a request +/// refuses a `NextToken` past 32,768 code points, and a statement of 8,192 +/// four-byte characters is 32,768 bytes on its own. +pub(super) fn encode_next_token( + statement: &str, + parameters: &[AttributeValue], + last_evaluated_key: &Value, +) -> String { + let payload = serde_json::json!({ + "s": statement_digest(statement), + "p": parameters_digest(parameters), + "k": last_evaluated_key, + }); + BASE64.encode(payload.to_string().as_bytes()) +} + +/// The refusal of a `NextToken` that does not decode. +pub(super) const INVALID_NEXT_TOKEN: &str = "Invalid NextToken"; + +/// The refusal of a `NextToken` another statement, or other parameter +/// values, gave. +pub(super) const TOKEN_MISMATCH: &str = "NextToken does not match request"; + +/// Decode a pagination token and return its `LastEvaluatedKey`. A token that +/// does not decode is the service's "Invalid NextToken" (measured 2026-11-02); +/// one whose embedded statement or parameter values differ from the current +/// ones is "NextToken does not match request" (measured 2026-11-02: a token +/// from `pk = ?` with 'p' is refused with 'q' and continues with 'p'). Both +/// are refused before the statement is authorized (measured 2026-11-03 for a +/// token that does not decode). +pub(super) fn decode_next_token( + token: &str, + statement: &str, + parameters: &[AttributeValue], +) -> Result { + let invalid = || DynamoDbError::ValidationException(INVALID_NEXT_TOKEN.to_owned()); + let bytes = BASE64.decode(token).map_err(|_| invalid())?; + let decoded: Value = serde_json::from_slice(&bytes).map_err(|_| invalid())?; + let (Some(issued_for), Some(key)) = + (decoded.get("s").and_then(Value::as_str), decoded.get("k")) + else { + return Err(invalid()); + }; + if issued_for != statement_digest(statement) + || decoded.get("p").and_then(Value::as_str) != Some(¶meters_digest(parameters)) + { + return Err(DynamoDbError::ValidationException( + TOKEN_MISMATCH.to_owned(), + )); + } + Ok(key.clone()) +} + +/// A digest of the statement text. +fn statement_digest(statement: &str) -> String { + hex::encode(Sha256::digest(statement.as_bytes())) +} + +/// A digest of the parameter values in wire form. Maps and sets serialize in +/// a fixed order, so equal values give equal digests. +fn parameters_digest(parameters: &[AttributeValue]) -> String { + let wire = serde_json::to_vec(parameters).unwrap_or_default(); + hex::encode(Sha256::digest(&wire)) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn message(err: DynamoDbError) -> String { + match err { + DynamoDbError::ValidationException(m) => m, + other => panic!("expected ValidationException, got {other:?}"), + } + } + + fn refused(result: Result) -> bool { + matches!( + result, + Err(DynamoDbError::ValidationException(m)) if m == "NextToken does not match request" + ) + } + + #[test] + fn next_token_round_trips_and_binds_to_statement() { + let lek = serde_json::json!({"pk": {"S": "p1"}, "sk": {"S": "s02"}}); + let token = encode_next_token("SELECT * FROM \"t\" WHERE pk='p1'", &[], &lek); + let decoded = decode_next_token(&token, "SELECT * FROM \"t\" WHERE pk='p1'", &[]).unwrap(); + assert_eq!(decoded, lek); + assert!(refused(decode_next_token( + &token, + "SELECT * FROM \"t\" WHERE pk='p2'", + &[] + ))); + } + + #[test] + fn a_token_that_does_not_decode_is_invalid() { + let statement = "SELECT * FROM \"t\" WHERE pk='p1'"; + let not_json = BASE64.encode(b"not-a-token"); + let no_key = BASE64.encode(serde_json::json!({ "s": statement }).to_string()); + let no_statement = BASE64.encode(serde_json::json!({ "k": {} }).to_string()); + for token in [ + "n".repeat(32768), + "x".to_owned(), + "a b!".to_owned(), + not_json, + no_key, + no_statement, + ] { + let err = decode_next_token(&token, statement, &[]).unwrap_err(); + assert_eq!(message(err), "Invalid NextToken", "{token}"); + } + } + + #[test] + fn a_token_for_the_longest_statement_fits_the_token_limit() { + // 8,192 code points of four bytes each, the longest statement a + // request carries, and a key of the largest partition and sort key + // values in four-byte characters. + let statement = "\u{1F600}".repeat(8192); + let lek = serde_json::json!({ + "pk": {"S": "\u{1F600}".repeat(512)}, + "sk": {"S": "\u{1F600}".repeat(256)} + }); + let token = encode_next_token(&statement, &[AttributeValue::S("p".to_owned())], &lek); + assert!(token.chars().count() <= 32768, "{}", token.len()); + assert_eq!( + decode_next_token(&token, &statement, &[AttributeValue::S("p".to_owned())]).unwrap(), + lek + ); + let other = format!("{statement}x"); + assert!(refused(decode_next_token( + &token, + &other, + &[AttributeValue::S("p".to_owned())] + ))); + } + + #[test] + fn next_token_binds_to_the_parameter_values() { + let statement = "SELECT * FROM \"t\" WHERE pk = ?"; + let p = [AttributeValue::S("p".to_owned())]; + let q = [AttributeValue::S("q".to_owned())]; + let lek = serde_json::json!({"pk": {"S": "p"}, "sk": {"S": "s1"}}); + let token = encode_next_token(statement, &p, &lek); + assert_eq!(decode_next_token(&token, statement, &p).unwrap(), lek); + assert!(refused(decode_next_token(&token, statement, &q))); + assert!(refused(decode_next_token(&token, statement, &[]))); + } +} diff --git a/crates/engine/src/partiql/request_checks.rs b/crates/engine/src/partiql/request_checks.rs new file mode 100644 index 000000000..065559219 --- /dev/null +++ b/crates/engine/src/partiql/request_checks.rs @@ -0,0 +1,332 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The constraints the request model puts on the fields of the three PartiQL +//! requests, checked before anything else in a request is read. +//! +//! The service reports every failed constraint of a request in one +//! `ValidationException`: "1 validation error detected: " for one and +//! "N validation errors detected: ; " for more, each part worded +//! as it is on its own, at most ten of them. A part echoes the value as it +//! was sent, or `null` for a member the request does not carry. The parts +//! come in this order (measured 2026-11-02): +//! +//! - `ExecuteStatement`: `returnValuesOnConditionCheckFailure`, `nextToken`, +//! `limit`, `statement`, `returnConsumedCapacity`, `parameters`. +//! - `BatchExecuteStatement`: `returnConsumedCapacity`, `statements` (the +//! count), then each statement in order: its +//! `returnValuesOnConditionCheckFailure`, `statement` and `parameters`. +//! - `ExecuteTransaction`: `clientRequestToken`, `transactStatements` (the +//! count), each statement as in a batch, then `returnConsumedCapacity`. +//! +//! A statement holds 1 to 8192 code points, a `Parameters` list at least one +//! value, a `NextToken` 1 to 32768 code points, a `ClientRequestToken` 1 to +//! 36, a batch 1 to 25 statements and a transaction 1 to 100. The two enum +//! members take the values the service lists in its refusal. A statement +//! list past its limit is echoed as the service prints its request objects, +//! up to a fixed size (see [`echo`]). + +use serde_json::Value; + +use extenddb_core::error::DynamoDbError; +use extenddb_core::partiql::statement_length_refusal; + +use echo::{Echo, echo_list, write_batch_member, write_transaction_member}; + +mod echo; + +/// The most statements a `BatchExecuteStatement` holds. +const MAX_BATCH_STATEMENTS: usize = 25; + +/// The most statements an `ExecuteTransaction` holds. +const MAX_TRANSACTION_STATEMENTS: usize = 100; + +/// The longest `ClientRequestToken`, in code points. +const MAX_CLIENT_REQUEST_TOKEN: usize = 36; + +/// The longest `NextToken`, in code points. +const MAX_NEXT_TOKEN: usize = 32768; + +/// The most failed constraints one refusal lists; its count then says 10 too +/// (measured 2026-11-02). +const MAX_PARTS: usize = 10; + +/// The constraint an empty statement or list fails. +const AT_LEAST_ONE: &str = "Member must have length greater than or equal to 1"; + +/// The constraint a member the request does not carry fails. +const NOT_NULL: &str = "Member must not be null"; + +/// The values of `ReturnValuesOnConditionCheckFailure`, as the service lists +/// them. +const ON_FAILURE_VALUES: &[&str] = &["ALL_OLD", "NONE"]; + +/// The values of `ReturnConsumedCapacity`, as the service lists them. +const CONSUMED_CAPACITY_VALUES: &[&str] = &["INDEXES", "TOTAL", "NONE"]; + +/// The refusal a `BatchExecuteStatement` member gets for a statement that +/// holds only whitespace (see [`is_blank_statement`]). +pub(super) const EMPTY_BATCH_STATEMENT: &str = "The statement was empty."; + +/// The refusal of an `ExecuteTransaction` with a member whose statement holds +/// only whitespace (see [`is_blank_statement`]). It names no member. +pub(super) const EMPTY_TRANSACTION_STATEMENT: &str = + "ExecuteTransaction can not have empty statement."; + +/// Whether a statement that passed the length constraint holds only +/// whitespace. `BatchExecuteStatement` refuses such a member and +/// `ExecuteTransaction` the whole request, each ahead of the member's +/// parameters and of the members after it, after the members before it; +/// `ExecuteStatement` reads it as a statement with nothing in it (measured +/// 2026-11-05). Measured as whitespace: space, tab, line feed, the vertical +/// tab, form feed, carriage return, U+001F, U+2028 and U+3000. Measured as +/// not: U+0000, U+0001, U+0085, U+00A0 and a comment. The other information +/// separators (U+001C to U+001E), the paragraph separator and the Unicode +/// spaces other than the no-break ones are taken as whitespace with them. +pub(super) fn is_blank_statement(statement: &str) -> bool { + statement.chars().all(|c| { + matches!( + c, + '\t'..='\r' + | '\u{1c}'..='\u{20}' + | '\u{1680}' + | '\u{2000}'..='\u{2006}' + | '\u{2008}'..='\u{200a}' + | '\u{2028}' + | '\u{2029}' + | '\u{205f}' + | '\u{3000}' + ) + }) +} + +/// Check an `ExecuteStatement` request. +/// +/// # Errors +/// +/// Returns the service's refusal naming every failed constraint. +pub(super) fn check_execute_statement(body: &Value) -> Result<(), DynamoDbError> { + let mut found = Found::default(); + found.on_failure(body, "returnValuesOnConditionCheckFailure"); + found.length(body.get("NextToken"), "nextToken", MAX_NEXT_TOKEN); + if let Some(limit) = body.get("Limit").and_then(Value::as_i64) + && limit < 1 + { + found.add( + &limit.to_string(), + "limit", + "Member must have value greater than or equal to 1", + ); + } + found.statement(body, "statement"); + found.consumed_capacity(body); + found.parameters(body, "parameters"); + found.into_result() +} + +/// Check a `BatchExecuteStatement` request. +/// +/// # Errors +/// +/// Returns the service's refusal naming every failed constraint. +pub(super) fn check_batch_execute_statement(body: &Value) -> Result<(), DynamoDbError> { + let mut found = Found::default(); + found.consumed_capacity(body); + found.statements( + body.get("Statements"), + "statements", + MAX_BATCH_STATEMENTS, + write_batch_member, + ); + found.into_result() +} + +/// Check an `ExecuteTransaction` request. +/// +/// # Errors +/// +/// Returns the service's refusal naming every failed constraint. +pub(super) fn check_execute_transaction(body: &Value) -> Result<(), DynamoDbError> { + let mut found = Found::default(); + found.length( + body.get("ClientRequestToken"), + "clientRequestToken", + MAX_CLIENT_REQUEST_TOKEN, + ); + found.statements( + body.get("TransactStatements"), + "transactStatements", + MAX_TRANSACTION_STATEMENTS, + write_transaction_member, + ); + found.consumed_capacity(body); + found.into_result() +} + +/// The failed constraints of one request, each worded as the service words +/// it, in the order the service reports them. +#[derive(Default)] +struct Found(Vec); + +impl Found { + /// A value that fails `constraint`, echoed as it was sent. + fn add(&mut self, value: &str, path: &str, constraint: &str) { + self.0.push(format!( + "Value '{value}' at '{path}' failed to satisfy constraint: {constraint}" + )); + } + + /// A member the request does not carry. + fn null(&mut self, path: &str) { + self.0.push(format!( + "Value null at '{path}' failed to satisfy constraint: {NOT_NULL}" + )); + } + + /// A token of 1 to `max` code points. + fn length(&mut self, value: Option<&Value>, path: &str, max: usize) { + let Some(Value::String(token)) = value else { + return; + }; + let length = token.chars().count(); + if length == 0 { + self.add(token, path, AT_LEAST_ONE); + } else if length > max { + self.add( + token, + path, + &format!("Member must have length less than or equal to {max}"), + ); + } + } + + /// An enum member's value, which must be one of `values`. + fn one_of(&mut self, value: Option<&Value>, path: &str, values: &[&str]) { + if let Some(Value::String(value)) = value + && !values.contains(&value.as_str()) + { + self.add( + value, + path, + &format!( + "Member must satisfy enum value set: [{}]", + values.join(", ") + ), + ); + } + } + + /// A request object's `ReturnValuesOnConditionCheckFailure`. + fn on_failure(&mut self, request: &Value, path: &str) { + self.one_of( + request.get("ReturnValuesOnConditionCheckFailure"), + path, + ON_FAILURE_VALUES, + ); + } + + /// The request's `ReturnConsumedCapacity`. + fn consumed_capacity(&mut self, body: &Value) { + self.one_of( + body.get("ReturnConsumedCapacity"), + "returnConsumedCapacity", + CONSUMED_CAPACITY_VALUES, + ); + } + + /// A request object's `Statement`: present, and 1 to 8192 code points. + fn statement(&mut self, request: &Value, path: &str) { + let statement = match request.get("Statement") { + None | Some(Value::Null) => { + self.null(path); + return; + } + Some(Value::String(statement)) => statement, + // A statement of another type is left to the executors. + Some(_) => return, + }; + let constraint = if statement.is_empty() { + Some(AT_LEAST_ONE.to_owned()) + } else { + statement_length_refusal(statement).map(|refusal| constraint_of(&refusal)) + }; + if let Some(constraint) = constraint { + self.add(statement, path, &constraint); + } + } + + /// The length of a request object's `Parameters` list. + fn parameters(&mut self, request: &Value, path: &str) { + if let Some(Value::Array(values)) = request.get("Parameters") + && values.is_empty() + { + self.add("[]", path, AT_LEAST_ONE); + } + } + + /// The count of a statement list, then each of its statements. A list + /// past `max` is echoed with `member` writing each of its statements, at + /// most [`echo::MAX_ECHO_BYTES`] of it. + fn statements( + &mut self, + list: Option<&Value>, + path: &str, + max: usize, + member: fn(&mut Echo, &Value), + ) { + let members = match list { + None | Some(Value::Null) => { + self.null(path); + return; + } + Some(Value::Array(members)) => members, + // A list of another type is left to the executors. + Some(_) => return, + }; + if members.is_empty() { + self.add("[]", path, AT_LEAST_ONE); + } else if members.len() > max { + self.add( + &echo_list(list, member), + path, + &format!("Member must have length less than or equal to {max}"), + ); + } + for (i, member) in members.iter().enumerate() { + let at = format!("{path}.{}.member", i + 1); + self.on_failure(member, &format!("{at}.returnValuesOnConditionCheckFailure")); + self.statement(member, &format!("{at}.statement")); + self.parameters(member, &format!("{at}.parameters")); + } + } + + /// The refusal, listing the first [`MAX_PARTS`] failed constraints. + fn into_result(mut self) -> Result<(), DynamoDbError> { + self.0.truncate(MAX_PARTS); + let message = match self.0.as_slice() { + [] => return Ok(()), + [part] => format!("1 validation error detected: {part}"), + parts => format!( + "{} validation errors detected: {}", + parts.len(), + parts.join("; ") + ), + }; + Err(DynamoDbError::ValidationException(message)) + } +} + +/// The constraint named at the end of a single-error refusal ("... failed to +/// satisfy constraint: "), so the statement length rule keeps the +/// parser's limit. +fn constraint_of(refusal: &str) -> String { + refusal + .rsplit_once(" failed to satisfy constraint: ") + .map_or_else( + || refusal.to_owned(), + |(_, constraint)| constraint.to_owned(), + ) +} + +#[cfg(test)] +mod tests; diff --git a/crates/engine/src/partiql/request_checks/echo.rs b/crates/engine/src/partiql/request_checks/echo.rs new file mode 100644 index 000000000..89be6da8f --- /dev/null +++ b/crates/engine/src/partiql/request_checks/echo.rs @@ -0,0 +1,222 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The echo of a statement list past its limit: the service prints its +//! request objects with every field (measured 2026-11-02). + +use base64::Engine; +use base64::engine::general_purpose::STANDARD as BASE64; +use serde_json::{Map, Value}; + +/// The longest echo of a statement list, in bytes. Every echo measured is +/// shorter. A list can echo to about 13 times its size on the wire (a +/// `{"NULL": true}` parameter prints in 186 characters), so the echo stops +/// here rather than render a 16 MB request into hundreds of MB. The +/// service's text for an echo this long is not measured. +pub(super) const MAX_ECHO_BYTES: usize = 1 << 20; + +/// The text of an echo, which stops growing at [`MAX_ECHO_BYTES`]. +pub(super) struct Echo(String); + +impl Echo { + fn new() -> Self { + Self(String::new()) + } + + /// Whether the echo has reached [`MAX_ECHO_BYTES`], so nothing more is + /// written. + fn full(&self) -> bool { + self.0.len() >= MAX_ECHO_BYTES + } + + /// Append `text`, or as much of it as fits, cut at a character boundary. + fn push(&mut self, text: &str) { + let room = MAX_ECHO_BYTES.saturating_sub(self.0.len()); + if text.len() <= room { + self.0.push_str(text); + return; + } + let mut end = room; + while !text.is_char_boundary(end) { + end -= 1; + } + self.0.push_str(&text[..end]); + } +} + +/// A statement list as the service echoes it, each member written by +/// `member`, at most [`MAX_ECHO_BYTES`] of it. +pub(super) fn echo_list(list: Option<&Value>, member: fn(&mut Echo, &Value)) -> String { + let mut echo = Echo::new(); + write_list(&mut echo, list, member); + echo.0 +} + +/// A `BatchExecuteStatement` member as the service echoes it (measured +/// 2026-11-02): a field it does not carry prints as `null`. +pub(super) fn write_batch_member(echo: &mut Echo, member: &Value) { + echo.push("BatchStatementRequest(statement="); + write_text(echo, member.get("Statement")); + echo.push(", parameters="); + write_list(echo, member.get("Parameters"), write_attribute_value); + echo.push(", consistentRead="); + write_flag(echo, member.get("ConsistentRead")); + echo.push(", returnValuesOnConditionCheckFailure="); + write_text(echo, member.get("ReturnValuesOnConditionCheckFailure")); + echo.push(")"); +} + +/// An `ExecuteTransaction` member as the service echoes it (measured +/// 2026-11-02). +pub(super) fn write_transaction_member(echo: &mut Echo, member: &Value) { + echo.push("ParameterizedStatement(statement="); + write_text(echo, member.get("Statement")); + echo.push(", parameters="); + write_list(echo, member.get("Parameters"), write_attribute_value); + echo.push(", returnValuesOnConditionCheckFailure="); + write_text(echo, member.get("ReturnValuesOnConditionCheckFailure")); + echo.push(")"); +} + +/// A parameter as the service echoes it (measured 2026-11-02): every member +/// of its value class, the ones the wire form cannot carry always `null`. +fn write_attribute_value(echo: &mut Echo, value: &Value) { + let field = |name: &str| value.get(name); + echo.push("AttributeValue(s="); + write_text(echo, field("S")); + echo.push(", n="); + write_text(echo, field("N")); + echo.push(", b="); + write_binary(echo, field("B")); + echo.push(", iNT=null, dECIMAL=null, dOUBLE=null, sS="); + write_list(echo, field("SS"), |echo, v| write_text(echo, Some(v))); + echo.push(", nS="); + write_list(echo, field("NS"), |echo, v| write_text(echo, Some(v))); + echo.push(", bS="); + write_list(echo, field("BS"), |echo, v| write_binary(echo, Some(v))); + echo.push(", iNTSET=null, dECIMALSET=null, dOUBLESET=null, m="); + write_map(echo, field("M")); + echo.push(", l="); + write_list(echo, field("L"), write_attribute_value); + echo.push(", nULL="); + write_flag(echo, field("NULL")); + echo.push(", bOOL="); + write_flag(echo, field("BOOL")); + echo.push(")"); +} + +/// A string as it was sent, without quotes. +fn write_text(echo: &mut Echo, value: Option<&Value>) { + match value { + Some(Value::String(text)) => echo.push(text), + _ => echo.push("null"), + } +} + +/// A boolean as `true` or `false`. +fn write_flag(echo: &mut Echo, value: Option<&Value>) { + match value { + Some(Value::Bool(true)) => echo.push("true"), + Some(Value::Bool(false)) => echo.push("false"), + _ => echo.push("null"), + } +} + +/// A binary value as the buffer holding its decoded bytes. A value that is +/// not base64 is not a binary the request can carry, and prints as `null`. +fn write_binary(echo: &mut Echo, value: Option<&Value>) { + match value + .and_then(Value::as_str) + .map(|text| BASE64.decode(text)) + { + Some(Ok(bytes)) => echo.push(&format!( + "java.nio.HeapByteBuffer[pos=0 lim={0} cap={0}]", + bytes.len() + )), + _ => echo.push("null"), + } +} + +/// A list as `[a, b]`. Writing stops at the first member past +/// [`MAX_ECHO_BYTES`]. +fn write_list(echo: &mut Echo, value: Option<&Value>, write: impl Fn(&mut Echo, &Value)) { + let Some(Value::Array(items)) = value else { + echo.push("null"); + return; + }; + echo.push("["); + for (index, item) in items.iter().enumerate() { + if echo.full() { + return; + } + if index > 0 { + echo.push(", "); + } + write(echo, item); + } + echo.push("]"); +} + +/// A map as `{name=value, ...}`, its members in the order the service prints +/// them (see [`print_order`]). Writing stops at the first member past +/// [`MAX_ECHO_BYTES`]. +fn write_map(echo: &mut Echo, value: Option<&Value>) { + let Some(Value::Object(members)) = value else { + echo.push("null"); + return; + }; + echo.push("{"); + for (index, (name, member)) in print_order(members).into_iter().enumerate() { + if echo.full() { + return; + } + if index > 0 { + echo.push(", "); + } + echo.push(name); + echo.push("="); + write_attribute_value(echo, member); + } + echo.push("}"); +} + +/// The members of a map in the order the service prints them (measured +/// 2026-11-02 on maps of 2, 3, 4, 6, 12, 13, 24 and 48 members): the +/// iteration order of a Java `HashMap` filled from the request's map. Its +/// table has the smallest power of two of buckets at or above `size / 0.75` +/// rounded up. A key +/// sits in the bucket its Java `String.hashCode` picks once the hash's high +/// 16 bits are folded into its low 16 by exclusive or. The buckets print in +/// ascending order, and the keys of one bucket in the order the request sends +/// them. +/// +/// Here the keys of one bucket come in the order the parsed request holds +/// them: the request's when the build includes the MongoDB storage crate, +/// whose JSON dependency makes every JSON map keep it (any build of the whole +/// workspace), and name order otherwise. In name order, two keys that share +/// a bucket print sorted by name, which differs from the service when the +/// request sends them in another order. +fn print_order(members: &Map) -> Vec<(&String, &Value)> { + // `size / 0.75` rounded up, in whole numbers. + let buckets = (members.len() * 4).div_ceil(3).next_power_of_two(); + let mut ordered: Vec<(usize, (&String, &Value))> = members + .iter() + .map(|member| (bucket(member.0, buckets), member)) + .collect(); + // The sort is stable, so the keys of one bucket keep their order. + ordered.sort_by_key(|(bucket, _)| *bucket); + ordered.into_iter().map(|(_, member)| member).collect() +} + +/// The bucket of `key` in a Java `HashMap` table of `buckets` buckets. +fn bucket(key: &str, buckets: usize) -> usize { + // Java's `String.hashCode`: 31 times the hash plus each UTF-16 unit, + // wrapping at 32 bits. + let hash = key.encode_utf16().fold(0_u32, |hash, unit| { + hash.wrapping_mul(31).wrapping_add(u32::from(unit)) + }); + (hash ^ (hash >> 16)) as usize & (buckets - 1) +} + +#[cfg(test)] +mod tests; diff --git a/crates/engine/src/partiql/request_checks/echo/tests.rs b/crates/engine/src/partiql/request_checks/echo/tests.rs new file mode 100644 index 000000000..c05e25c45 --- /dev/null +++ b/crates/engine/src/partiql/request_checks/echo/tests.rs @@ -0,0 +1,248 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Unit tests for the echo of a statement list. + +use serde_json::json; + +use super::*; + +/// What `write` writes for `value` into a fresh echo. +fn written(write: fn(&mut Echo, &Value), value: &Value) -> String { + let mut echo = Echo::new(); + write(&mut echo, value); + echo.0 +} + +fn binary(value: Option<&Value>) -> String { + let mut echo = Echo::new(); + write_binary(&mut echo, value); + echo.0 +} + +/// A parameter as echoed, `set` giving the members that are not null. +fn av(set: &[(&str, &str)]) -> String { + const FIELDS: [&str; 16] = [ + "s", + "n", + "b", + "iNT", + "dECIMAL", + "dOUBLE", + "sS", + "nS", + "bS", + "iNTSET", + "dECIMALSET", + "dOUBLESET", + "m", + "l", + "nULL", + "bOOL", + ]; + let fields: Vec = FIELDS + .iter() + .map(|name| { + let value = set + .iter() + .find(|(field, _)| field == name) + .map_or("null", |(_, value)| value); + format!("{name}={value}") + }) + .collect(); + format!("AttributeValue({})", fields.join(", ")) +} + +#[test] +fn a_parameter_is_echoed_with_every_member_of_its_class() { + const HEAD: &str = "SELECT * FROM \"t\" WHERE pk = 'a'"; + // The measured text of a string parameter. + assert_eq!( + av(&[("s", "a")]), + "AttributeValue(s=a, n=null, b=null, iNT=null, dECIMAL=null, dOUBLE=null, \ + sS=null, nS=null, bS=null, iNTSET=null, dECIMALSET=null, dOUBLESET=null, m=null, \ + l=null, nULL=null, bOOL=null)" + ); + let buffer = "java.nio.HeapByteBuffer[pos=0 lim=1 cap=1]"; + let member = json!({ + "Statement": "SELECT * FROM \"t\" WHERE pk = ?", + "Parameters": [ + {"S": "a"}, {"N": "1"}, {"B": "eA=="}, {"BOOL": true}, {"NULL": true}, + {"L": [{"S": "x"}]}, {"M": {"k": {"S": "v"}}}, {"SS": ["a", "b"]}, + {"NS": ["1"]}, {"BS": ["eA=="]} + ], + "ConsistentRead": true, + "ReturnValuesOnConditionCheckFailure": "ALL_OLD" + }); + let parameters = [ + av(&[("s", "a")]), + av(&[("n", "1")]), + av(&[("b", buffer)]), + av(&[("bOOL", "true")]), + av(&[("nULL", "true")]), + av(&[("l", &format!("[{}]", av(&[("s", "x")])))]), + av(&[("m", &format!("{{k={}}}", av(&[("s", "v")])))]), + av(&[("sS", "[a, b]")]), + av(&[("nS", "[1]")]), + av(&[("bS", &format!("[{buffer}]"))]), + ]; + assert_eq!( + written(write_batch_member, &member), + format!( + "BatchStatementRequest(statement=SELECT * FROM \"t\" WHERE pk = ?, \ + parameters=[{}], consistentRead=true, \ + returnValuesOnConditionCheckFailure=ALL_OLD)", + parameters.join(", ") + ) + ); + assert_eq!( + written(write_transaction_member, &member), + format!( + "ParameterizedStatement(statement=SELECT * FROM \"t\" WHERE pk = ?, \ + parameters=[{}], returnValuesOnConditionCheckFailure=ALL_OLD)", + parameters.join(", ") + ) + ); + let member = json!({ "Statement": HEAD, "ConsistentRead": false }); + assert_eq!( + written(write_batch_member, &member), + format!( + "BatchStatementRequest(statement={HEAD}, parameters=null, consistentRead=false, \ + returnValuesOnConditionCheckFailure=null)" + ) + ); + let list = json!([member, member]); + let one = written(write_batch_member, &member); + assert_eq!( + echo_list(Some(&list), write_batch_member), + format!("[{one}, {one}]") + ); + assert_eq!(echo_list(None, write_batch_member), "null"); +} + +#[test] +fn an_echo_stops_at_its_size_limit() { + // A parameter list of NULLs echoes to about 13 times its size on the + // wire; the echo stops at the limit instead. + let nulls: Vec = (0..20_000).map(|_| json!({ "NULL": true })).collect(); + let mut members = vec![json!({ "Statement": "SELECT * FROM \"t\"" }); 25]; + members.push(json!({ "Statement": "SELECT * FROM \"t\" WHERE pk = ?", "Parameters": nulls })); + let list = Value::Array(members); + let echo = echo_list(Some(&list), write_batch_member); + // The echo is the start of the whole text. + let small = written( + write_batch_member, + &json!({ "Statement": "SELECT * FROM \"t\"" }), + ); + let null = av(&[("nULL", "true")]); + let whole = format!( + "[{}, BatchStatementRequest(statement=SELECT * FROM \"t\" WHERE pk = ?, parameters=[{}], \ + consistentRead=null, returnValuesOnConditionCheckFailure=null)]", + vec![small; 25].join(", "), + vec![null; 20_000].join(", ") + ); + assert!(whole.len() > 3 * MAX_ECHO_BYTES); + assert_eq!(echo, whole[..MAX_ECHO_BYTES]); + // A map parameter stops the same way. + let members: Map = (0..20_000) + .map(|i| (format!("k{i}"), json!({ "NULL": true }))) + .collect(); + let list = json!([{ "Statement": "s", "Parameters": [{ "M": members }] }]); + assert_eq!( + echo_list(Some(&list), write_transaction_member).len(), + MAX_ECHO_BYTES + ); +} + +#[test] +fn an_echo_is_cut_at_a_character_boundary() { + let mut echo = Echo::new(); + echo.push(&"a".repeat(MAX_ECHO_BYTES - 1)); + // One byte is left, and the two-byte character does not fit in it. + echo.push("\u{e9}\u{e9}"); + assert_eq!(echo.0.len(), MAX_ECHO_BYTES - 1); + assert!(!echo.full()); + echo.push("b"); + assert_eq!(echo.0.len(), MAX_ECHO_BYTES); + assert!(echo.full()); + echo.push("c"); + assert_eq!(echo.0.len(), MAX_ECHO_BYTES); +} + +fn printed(map: &Value) -> Vec<&str> { + print_order(map.as_object().unwrap()) + .into_iter() + .map(|(name, _)| name.as_str()) + .collect() +} + +#[test] +fn map_members_print_in_the_measured_order() { + let map = json!({ "c": {"S": "1"}, "a": {"S": "2"}, "p": {"S": "3"} }); + assert_eq!(printed(&map), ["p", "a", "c"]); + let map = json!({ "zeta": {}, "alpha": {}, "mid": {}, "beta": {} }); + assert_eq!(printed(&map), ["zeta", "mid", "beta", "alpha"]); + let members: Map = (0..=12) + .rev() + .map(|i| (format!("k{i}"), json!({ "N": i.to_string() }))) + .collect(); + assert_eq!( + printed(&Value::Object(members)), + [ + "k0", "k1", "k2", "k3", "k4", "k11", "k5", "k10", "k6", "k7", "k12", "k8", "k9" + ] + ); +} + +#[test] +fn the_table_holds_size_over_three_quarters_rounded_up() { + // 3 members take 4 buckets and 6 take 8, not 8 and 16. + let map = json!({ "m0": {}, "m1": {}, "m2": {} }); + assert_eq!(printed(&map), ["m1", "m2", "m0"]); + let members: Map = (0..6).map(|i| (format!("k{i}"), json!({}))).collect(); + assert_eq!( + printed(&Value::Object(members)), + ["k3", "k4", "k5", "k0", "k1", "k2"] + ); +} + +#[test] +fn keys_that_share_a_bucket_keep_the_order_the_map_holds_them_in() { + // q and a share bucket 1 of 4. + assert_eq!(bucket("q", 4), bucket("a", 4)); + let map = json!({ "q": {"S": "1"}, "a": {"S": "2"} }); + let held: Vec<&str> = map + .as_object() + .unwrap() + .keys() + .map(String::as_str) + .collect(); + assert_eq!(printed(&map), held); +} + +#[test] +fn a_key_hashes_over_its_utf16_units() { + // 0xD83D * 31 + 0xDE00 = 1772899, whose folded low five bits are 24. + assert_eq!(bucket("\u{1F600}", 32), 24); + assert_eq!(bucket("a", 1), 0); +} + +#[test] +fn a_nested_map_is_echoed_in_the_same_order() { + let value = json!({ "L": [{ "M": { "c": {"S": "1"}, "a": {"S": "2"}, "p": {"S": "3"} } }] }); + let rendered = written(write_attribute_value, &value); + let p = rendered.find("p=").unwrap(); + let a = rendered.find("a=").unwrap(); + let c = rendered.find("c=").unwrap(); + assert!(p < a && a < c, "{rendered}"); +} + +#[test] +fn a_binary_is_echoed_as_the_buffer_of_its_bytes() { + assert_eq!( + binary(Some(&json!("AAECAwQ="))), + "java.nio.HeapByteBuffer[pos=0 lim=5 cap=5]" + ); + assert_eq!(binary(Some(&json!("not base64!"))), "null"); + assert_eq!(binary(None), "null"); +} diff --git a/crates/engine/src/partiql/request_checks/tests.rs b/crates/engine/src/partiql/request_checks/tests.rs new file mode 100644 index 000000000..035ff5b6f --- /dev/null +++ b/crates/engine/src/partiql/request_checks/tests.rs @@ -0,0 +1,594 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Unit tests for the request constraint checks. + +use serde_json::json; + +use super::*; + +const HEAD: &str = "SELECT * FROM \"t\" WHERE pk = 'a'"; +const TOO_LONG: &str = "Member must have length less than or equal to 8192"; +const AT_LEAST_ONE_VALUE: &str = "Member must have value greater than or equal to 1"; +const ON_FAILURE_SET: &str = "Member must satisfy enum value set: [ALL_OLD, NONE]"; +const CAPACITY_SET: &str = "Member must satisfy enum value set: [INDEXES, TOTAL, NONE]"; + +/// A statement of `length` code points. +fn statement(length: usize) -> String { + format!("{HEAD}{}", " ".repeat(length - HEAD.len())) +} + +fn message(result: Result<(), DynamoDbError>) -> String { + match result { + Err(DynamoDbError::ValidationException(message)) => message, + other => panic!("expected a ValidationException, got {other:?}"), + } +} + +fn part(value: &str, path: &str, constraint: &str) -> String { + format!("Value '{value}' at '{path}' failed to satisfy constraint: {constraint}") +} + +fn null(path: &str) -> String { + format!("Value null at '{path}' failed to satisfy constraint: Member must not be null") +} + +/// The refusal that lists `parts`. +fn errors(parts: &[String]) -> String { + match parts { + [part] => format!("1 validation error detected: {part}"), + parts => format!( + "{} validation errors detected: {}", + parts.len(), + parts.join("; ") + ), + } +} + +fn plain_batch_member() -> String { + format!( + "BatchStatementRequest(statement={HEAD}, parameters=null, consistentRead=null, \ + returnValuesOnConditionCheckFailure=null)" + ) +} + +fn plain_transaction_member() -> String { + format!( + "ParameterizedStatement(statement={HEAD}, parameters=null, \ + returnValuesOnConditionCheckFailure=null)" + ) +} + +#[test] +fn a_request_within_its_constraints_passes() { + let members: Vec = (0..25).map(|_| json!({ "Statement": HEAD })).collect(); + assert!(check_batch_execute_statement(&json!({ "Statements": members })).is_ok()); + let members: Vec = (0..100) + .map(|_| json!({ "Statement": statement(8192), "Parameters": [{"S": "a"}] })) + .collect(); + let body = json!({ "TransactStatements": members, "ClientRequestToken": "t".repeat(36) }); + assert!(check_execute_transaction(&body).is_ok()); + let body = json!({ "Statement": HEAD, "Limit": 1, "Parameters": [{"N": "1"}] }); + assert!(check_execute_statement(&body).is_ok()); + // A field of another type than the model's is left to the executors. + assert!(check_execute_statement(&json!({ "Statement": 5 })).is_ok()); + assert!(check_batch_execute_statement(&json!({ "Statements": "x" })).is_ok()); + assert!(check_execute_transaction(&json!({ "TransactStatements": {} })).is_ok()); +} + +#[test] +fn a_missing_member_is_reported_as_null() { + assert_eq!( + message(check_execute_statement(&json!({}))), + errors(&[null("statement")]) + ); + assert_eq!( + message(check_execute_statement(&json!({ "Statement": null }))), + errors(&[null("statement")]) + ); + assert_eq!( + message(check_batch_execute_statement(&json!({}))), + errors(&[null("statements")]) + ); + assert_eq!( + message(check_execute_transaction(&json!({}))), + errors(&[null("transactStatements")]) + ); + let body = json!({ "Statements": [{ "Parameters": [{"S": "a"}] }] }); + assert_eq!( + message(check_batch_execute_statement(&body)), + errors(&[null("statements.1.member.statement")]) + ); + let body = json!({ "TransactStatements": [{ "Parameters": [{"S": "a"}] }] }); + assert_eq!( + message(check_execute_transaction(&body)), + errors(&[null("transactStatements.1.member.statement")]) + ); +} + +#[test] +fn an_enum_value_outside_its_set_is_refused() { + let capacity = || part("X", "returnConsumedCapacity", CAPACITY_SET); + let body = json!({ "Statement": HEAD, "ReturnValuesOnConditionCheckFailure": "X" }); + assert_eq!( + message(check_execute_statement(&body)), + errors(&[part( + "X", + "returnValuesOnConditionCheckFailure", + ON_FAILURE_SET + )]) + ); + let body = json!({ "Statement": HEAD, "ReturnConsumedCapacity": "X" }); + assert_eq!( + message(check_execute_statement(&body)), + errors(&[capacity()]) + ); + let member = json!({ "Statement": HEAD, "ReturnValuesOnConditionCheckFailure": "X" }); + assert_eq!( + message(check_batch_execute_statement( + &json!({ "Statements": [member] }) + )), + errors(&[part( + "X", + "statements.1.member.returnValuesOnConditionCheckFailure", + ON_FAILURE_SET + )]) + ); + assert_eq!( + message(check_execute_transaction( + &json!({ "TransactStatements": [member] }) + )), + errors(&[part( + "X", + "transactStatements.1.member.returnValuesOnConditionCheckFailure", + ON_FAILURE_SET + )]) + ); + let body = json!({ "Statements": [{ "Statement": HEAD }], "ReturnConsumedCapacity": "X" }); + assert_eq!( + message(check_batch_execute_statement(&body)), + errors(&[capacity()]) + ); + let body = + json!({ "TransactStatements": [{ "Statement": HEAD }], "ReturnConsumedCapacity": "X" }); + assert_eq!( + message(check_execute_transaction(&body)), + errors(&[capacity()]) + ); +} + +#[test] +fn every_value_in_an_enum_set_passes() { + for value in ["ALL_OLD", "NONE"] { + let member = json!({ "Statement": HEAD, "ReturnValuesOnConditionCheckFailure": value }); + assert!(check_execute_statement(&member).is_ok(), "{value}"); + assert!(check_batch_execute_statement(&json!({ "Statements": [member] })).is_ok()); + assert!(check_execute_transaction(&json!({ "TransactStatements": [member] })).is_ok()); + } + for value in ["INDEXES", "TOTAL", "NONE"] { + let body = json!({ "Statement": HEAD, "ReturnConsumedCapacity": value }); + assert!(check_execute_statement(&body).is_ok(), "{value}"); + let body = + json!({ "Statements": [{ "Statement": HEAD }], "ReturnConsumedCapacity": value }); + assert!(check_batch_execute_statement(&body).is_ok(), "{value}"); + let body = json!({ + "TransactStatements": [{ "Statement": HEAD }], + "ReturnConsumedCapacity": value + }); + assert!(check_execute_transaction(&body).is_ok(), "{value}"); + } +} + +#[test] +fn execute_statement_reports_its_fields_in_the_measured_order() { + let over = statement(8193); + let long_token = "n".repeat(32769); + for (token, constraint) in [ + ("", AT_LEAST_ONE), + ( + long_token.as_str(), + "Member must have length less than or equal to 32768", + ), + ] { + let body = json!({ + "Statement": over, + "Parameters": [], + "Limit": 0, + "NextToken": token, + "ReturnConsumedCapacity": "X", + "ReturnValuesOnConditionCheckFailure": "X", + "ConsistentRead": true + }); + assert_eq!( + message(check_execute_statement(&body)), + errors(&[ + part("X", "returnValuesOnConditionCheckFailure", ON_FAILURE_SET), + part(token, "nextToken", constraint), + part("0", "limit", AT_LEAST_ONE_VALUE), + part(&over, "statement", TOO_LONG), + part("X", "returnConsumedCapacity", CAPACITY_SET), + part("[]", "parameters", AT_LEAST_ONE), + ]) + ); + } +} + +#[test] +fn a_token_holds_one_code_point_up_to_its_limit() { + for token in ["n".to_owned(), "n".repeat(32768), "\u{1F600}".repeat(32768)] { + let body = json!({ "Statement": HEAD, "NextToken": token }); + assert!(check_execute_statement(&body).is_ok()); + } + let body = json!({ "Statement": HEAD, "NextToken": "" }); + assert_eq!( + message(check_execute_statement(&body)), + errors(&[part("", "nextToken", AT_LEAST_ONE)]) + ); + for token in ["t".to_owned(), "t".repeat(36), "\u{e9}".repeat(36)] { + let body = + json!({ "TransactStatements": [{ "Statement": HEAD }], "ClientRequestToken": token }); + assert!(check_execute_transaction(&body).is_ok()); + } + let body = json!({ "TransactStatements": [{ "Statement": HEAD }], "ClientRequestToken": "" }); + assert_eq!( + message(check_execute_transaction(&body)), + errors(&[part("", "clientRequestToken", AT_LEAST_ONE)]) + ); +} + +#[test] +fn a_batch_reports_its_capacity_then_its_count_then_each_member() { + let over = statement(8193); + let mut members = vec![json!({ + "Statement": over, + "Parameters": [], + "ConsistentRead": true, + "ReturnValuesOnConditionCheckFailure": "X" + })]; + members.extend((0..25).map(|_| json!({ "Statement": HEAD }))); + let echo = format!( + "[BatchStatementRequest(statement={over}, parameters=[], consistentRead=true, \ + returnValuesOnConditionCheckFailure=X), {}]", + vec![plain_batch_member(); 25].join(", ") + ); + let body = json!({ "Statements": members, "ReturnConsumedCapacity": "X" }); + assert_eq!( + message(check_batch_execute_statement(&body)), + errors(&[ + part("X", "returnConsumedCapacity", CAPACITY_SET), + part( + &echo, + "statements", + "Member must have length less than or equal to 25" + ), + part( + "X", + "statements.1.member.returnValuesOnConditionCheckFailure", + ON_FAILURE_SET + ), + part(&over, "statements.1.member.statement", TOO_LONG), + part("[]", "statements.1.member.parameters", AT_LEAST_ONE), + ]) + ); + let body = json!({ "ReturnConsumedCapacity": "X" }); + assert_eq!( + message(check_batch_execute_statement(&body)), + errors(&[ + part("X", "returnConsumedCapacity", CAPACITY_SET), + null("statements") + ]) + ); +} + +#[test] +fn a_transaction_reports_its_capacity_after_its_members() { + let over = statement(8193); + let mut members = vec![json!({ + "Statement": over, + "Parameters": [], + "ReturnValuesOnConditionCheckFailure": "X" + })]; + members.extend((0..100).map(|_| json!({ "Statement": HEAD }))); + let token = "t".repeat(37); + let echo = format!( + "[ParameterizedStatement(statement={over}, parameters=[], \ + returnValuesOnConditionCheckFailure=X), {}]", + vec![plain_transaction_member(); 100].join(", ") + ); + let body = json!({ + "TransactStatements": members, + "ClientRequestToken": token, + "ReturnConsumedCapacity": "X" + }); + assert_eq!( + message(check_execute_transaction(&body)), + errors(&[ + part( + &token, + "clientRequestToken", + "Member must have length less than or equal to 36" + ), + part( + &echo, + "transactStatements", + "Member must have length less than or equal to 100" + ), + part( + "X", + "transactStatements.1.member.returnValuesOnConditionCheckFailure", + ON_FAILURE_SET + ), + part(&over, "transactStatements.1.member.statement", TOO_LONG), + part("[]", "transactStatements.1.member.parameters", AT_LEAST_ONE), + part("X", "returnConsumedCapacity", CAPACITY_SET), + ]) + ); +} + +#[test] +fn a_refusal_lists_the_first_ten_failed_constraints() { + let members: Vec = (0..25) + .map(|_| json!({ "Statement": "", "Parameters": [] })) + .collect(); + let first_ten: Vec = (1..=5) + .flat_map(|i| { + [ + part( + "", + &format!("statements.{i}.member.statement"), + AT_LEAST_ONE, + ), + part( + "[]", + &format!("statements.{i}.member.parameters"), + AT_LEAST_ONE, + ), + ] + }) + .collect(); + let refusal = message(check_batch_execute_statement( + &json!({ "Statements": members }), + )); + assert_eq!(refusal, errors(&first_ten)); + assert!(refusal.starts_with("10 validation errors detected: ")); + // Five such statements fail ten constraints, all listed. + let members: Vec = (0..5) + .map(|_| json!({ "Statement": "", "Parameters": [] })) + .collect(); + assert_eq!( + message(check_batch_execute_statement( + &json!({ "Statements": members }) + )), + errors(&first_ten) + ); +} + +#[test] +fn a_null_member_joins_the_other_parts() { + let body = json!({ "Limit": 0, "Parameters": [] }); + assert_eq!( + message(check_execute_statement(&body)), + errors(&[ + part("0", "limit", AT_LEAST_ONE_VALUE), + null("statement"), + part("[]", "parameters", AT_LEAST_ONE), + ]) + ); + let body = json!({ "Statements": [{ "Parameters": [] }] }); + assert_eq!( + message(check_batch_execute_statement(&body)), + errors(&[ + null("statements.1.member.statement"), + part("[]", "statements.1.member.parameters", AT_LEAST_ONE), + ]) + ); + let token = "t".repeat(37); + assert_eq!( + message(check_execute_transaction( + &json!({ "ClientRequestToken": token }) + )), + errors(&[ + part( + &token, + "clientRequestToken", + "Member must have length less than or equal to 36" + ), + null("transactStatements"), + ]) + ); +} + +#[test] +fn a_long_statement_is_refused_as_the_parser_words_it() { + let over = statement(8193); + assert_eq!( + message(check_execute_statement(&json!({ "Statement": over }))), + statement_length_refusal(&over).unwrap() + ); + assert_eq!( + constraint_of(&statement_length_refusal(&over).unwrap()), + TOO_LONG + ); +} + +#[test] +fn the_length_counts_code_points() { + let head = "SELECT * FROM \"t\" WHERE a = '"; + let at_limit = format!("{head}{}'", "\u{1F600}".repeat(8192 - head.len() - 1)); + assert!(check_execute_statement(&json!({ "Statement": at_limit })).is_ok()); + let over = format!("{head}{}'", "\u{1F600}".repeat(8192 - head.len())); + assert_eq!( + message(check_execute_statement(&json!({ "Statement": over }))), + format!( + "1 validation error detected: {}", + part(&over, "statement", TOO_LONG) + ) + ); +} + +#[test] +fn an_empty_statement_or_parameter_list_is_too_short() { + assert_eq!( + message(check_execute_statement(&json!({ "Statement": "" }))), + format!( + "1 validation error detected: {}", + part("", "statement", AT_LEAST_ONE) + ) + ); + let body = json!({ "Statement": HEAD, "Parameters": [] }); + assert_eq!( + message(check_execute_statement(&body)), + format!( + "1 validation error detected: {}", + part("[]", "parameters", AT_LEAST_ONE) + ) + ); + assert_eq!( + message(check_batch_execute_statement(&json!({ "Statements": [] }))), + "1 validation error detected: Value '[]' at 'statements' failed to satisfy \ + constraint: Member must have length greater than or equal to 1" + ); + assert_eq!( + message(check_execute_transaction( + &json!({ "TransactStatements": [] }) + )), + "1 validation error detected: Value '[]' at 'transactStatements' failed to \ + satisfy constraint: Member must have length greater than or equal to 1" + ); +} + +#[test] +fn execute_statement_reports_the_limit_then_the_statement_then_the_parameters() { + let over = statement(8193); + let body = json!({ "Statement": over, "Limit": 0, "Parameters": [] }); + assert_eq!( + message(check_execute_statement(&body)), + format!( + "3 validation errors detected: {}; {}; {}", + part( + "0", + "limit", + "Member must have value greater than or equal to 1" + ), + part(&over, "statement", TOO_LONG), + part("[]", "parameters", AT_LEAST_ONE) + ) + ); + let body = json!({ "Statement": HEAD, "Limit": -3 }); + assert_eq!( + message(check_execute_statement(&body)), + format!( + "1 validation error detected: {}", + part( + "-3", + "limit", + "Member must have value greater than or equal to 1" + ) + ) + ); +} + +#[test] +fn a_batch_reports_each_statement_in_order() { + let over = statement(8193); + let body = json!({ "Statements": [ + { "Statement": "" }, + { "Statement": HEAD }, + { "Statement": over, "Parameters": [] }, + ]}); + assert_eq!( + message(check_batch_execute_statement(&body)), + format!( + "3 validation errors detected: {}; {}; {}", + part("", "statements.1.member.statement", AT_LEAST_ONE), + part(&over, "statements.3.member.statement", TOO_LONG), + part("[]", "statements.3.member.parameters", AT_LEAST_ONE) + ) + ); +} + +#[test] +fn a_batch_past_its_limit_is_echoed_before_its_statements() { + let over = statement(8193); + let mut members: Vec = (0..25).map(|_| json!({ "Statement": HEAD })).collect(); + members.push(json!({ "Statement": over })); + let echo = format!( + "[{}, BatchStatementRequest(statement={over}, parameters=null, consistentRead=null, \ + returnValuesOnConditionCheckFailure=null)]", + vec![plain_batch_member(); 25].join(", ") + ); + assert_eq!( + message(check_batch_execute_statement( + &json!({ "Statements": members }) + )), + format!( + "2 validation errors detected: {}; {}", + part( + &echo, + "statements", + "Member must have length less than or equal to 25" + ), + part(&over, "statements.26.member.statement", TOO_LONG) + ) + ); +} + +#[test] +fn a_transaction_reports_the_token_then_the_count_then_each_statement() { + let over = statement(8193); + let mut members: Vec = (0..100).map(|_| json!({ "Statement": HEAD })).collect(); + members.push(json!({ "Statement": over })); + let token = "t".repeat(37); + let body = json!({ "TransactStatements": members, "ClientRequestToken": token }); + let echo = format!( + "[{}, ParameterizedStatement(statement={over}, parameters=null, \ + returnValuesOnConditionCheckFailure=null)]", + vec![plain_transaction_member(); 100].join(", ") + ); + assert_eq!( + message(check_execute_transaction(&body)), + format!( + "3 validation errors detected: {}; {}; {}", + part( + &token, + "clientRequestToken", + "Member must have length less than or equal to 36" + ), + part( + &echo, + "transactStatements", + "Member must have length less than or equal to 100" + ), + part(&over, "transactStatements.101.member.statement", TOO_LONG) + ) + ); + let body = json!({ "TransactStatements": [{ "Statement": HEAD, "Parameters": [] }] }); + assert_eq!( + message(check_execute_transaction(&body)), + format!( + "1 validation error detected: {}", + part("[]", "transactStatements.1.member.parameters", AT_LEAST_ONE) + ) + ); +} + +#[test] +fn a_statement_of_whitespace_alone_is_blank() { + for blank in [ + " ", " ", "\t\n ", "\r", "\u{b}", "\u{c}", "\u{1f}", "\u{2028}", "\u{3000}", + ] { + assert!(is_blank_statement(blank), "{blank:?}"); + } + for statement in [ + "\u{0}", + "\u{1}", + "\u{85}", + "\u{a0}", + "-- a note", + "/* a note */", + " ;", + HEAD, + ] { + assert!(!is_blank_statement(statement), "{statement:?}"); + } +} diff --git a/crates/engine/src/partiql/returning.rs b/crates/engine/src/partiql/returning.rs new file mode 100644 index 000000000..a6a22bb9d --- /dev/null +++ b/crates/engine/src/partiql/returning.rs @@ -0,0 +1,275 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The `MODIFIED` forms of a PartiQL `UPDATE ... RETURNING` clause. +//! +//! `RETURNING MODIFIED OLD *` and `RETURNING MODIFIED NEW *` project every +//! path the statement names (each `SET` target, including the attribute a +//! `set_add` or `set_delete` assigns, and each `REMOVE` path) against the item +//! as it was before the write (`OLD`) or after it (`NEW`). A path +//! that resolves contributes its value, a path that does not resolve +//! contributes nothing, and list elements picked by index come back packed +//! into a dense list in ascending index order, whatever order the statement +//! names them in (measured 2026-11-02). That is the rule a +//! `ProjectionExpression` follows, so the shared projection does the shaping: +//! the update handler is asked for the whole item on the matching side and the +//! statement's paths are projected over it here. + +use std::collections::HashMap; + +use serde_json::Value; + +use extenddb_core::error::DynamoDbError; +use extenddb_core::expression::{ExpressionMaps, PathElement, Projection}; +use extenddb_core::partiql::ast::{Path, PathStep, Returning, UpdateStatement}; +use extenddb_core::types::Item; + +/// Replace the handler's whole-item `Attributes` with the statement's +/// `MODIFIED` projection. An empty projection removes `Attributes`, which +/// leaves the PartiQL `Items` list empty. The `ALL` forms and a statement with +/// no `RETURNING` clause leave the body unchanged. +/// +/// # Errors +/// +/// Returns `InternalServerError` when the handler's item does not decode. +pub(super) fn project_modified( + body: &mut Value, + update: &UpdateStatement, +) -> Result<(), DynamoDbError> { + if !matches!( + update.returning, + Some(Returning::ModifiedOld | Returning::ModifiedNew) + ) { + return Ok(()); + } + let Some(fields) = body.as_object_mut() else { + return Ok(()); + }; + let Some(attributes) = fields.remove("Attributes") else { + return Ok(()); + }; + let item: Item = serde_json::from_value(attributes).map_err(|e| internal(&e))?; + let projected = modified_paths(&item, update)?; + if !projected.is_empty() { + fields.insert( + "Attributes".to_owned(), + serde_json::to_value(projected).map_err(|e| internal(&e))?, + ); + } + Ok(()) +} + +/// Project the paths an `UPDATE` names onto `item`. +fn modified_paths(item: &Item, update: &UpdateStatement) -> Result { + let mut names = HashMap::new(); + let paths: Vec> = update + .sets + .iter() + .map(|action| &action.path) + .chain(&update.removes) + .chain(update.set_members.iter().map(|action| &action.path)) + .map(|path| placeholder_path(path, &mut names)) + .collect(); + let maps = ExpressionMaps::new(names, HashMap::new()); + // A statement whose paths overlap was refused before it ran (see + // `write::check_overlapping_paths`), so the projection does not check + // them again. + Ok(Projection::compile(&paths, &maps, false)?.apply(item)) +} + +/// A statement path in projection form, each attribute name carried through a +/// `#` placeholder so a name that itself starts with `#` is not read as one. +fn placeholder_path(path: &Path, names: &mut HashMap) -> Vec { + let mut elements = Vec::with_capacity(path.steps.len() + 1); + elements.push(placeholder(&path.root, names)); + for step in &path.steps { + elements.push(match step { + PathStep::Attribute(name) => placeholder(name, names), + PathStep::Index(index) => PathElement::Index(*index), + }); + } + elements +} + +fn placeholder(name: &str, names: &mut HashMap) -> PathElement { + let key = format!("p{}", names.len()); + names.insert(key.clone(), name.to_owned()); + PathElement::Attribute(format!("#{key}")) +} + +fn internal(e: &serde_json::Error) -> DynamoDbError { + tracing::error!(internal_error = %e, "failed to shape a PartiQL RETURNING item"); + DynamoDbError::InternalServerError("Internal server error".to_owned()) +} + +#[cfg(test)] +mod tests { + use super::*; + use extenddb_core::partiql::{Statement, parse}; + use serde_json::json; + + fn update(sql: &str) -> UpdateStatement { + match parse(sql).unwrap() { + Statement::Update(u) => u, + other => panic!("not an update: {other:?}"), + } + } + + fn item(value: Value) -> Item { + serde_json::from_value(value).unwrap() + } + + fn shaped(sql: &str, attributes: Value) -> Value { + let mut body = json!({ "Attributes": attributes }); + project_modified(&mut body, &update(sql)).unwrap(); + body + } + + fn tags(letters: &[&str]) -> Value { + let list: Vec = letters.iter().map(|s| json!({ "S": s })).collect(); + json!({ "L": list }) + } + + #[test] + fn several_list_indices_pack_densely_in_ascending_index_order() { + let new_item = json!({ "pk": {"S": "k"}, "tags": tags(&["x", "b", "y"]) }); + for sql in [ + "UPDATE \"t\" SET tags[0] = 'x', tags[2] = 'y' WHERE pk = 'k' RETURNING MODIFIED NEW *", + "UPDATE \"t\" SET tags[2] = 'y', tags[0] = 'x' WHERE pk = 'k' RETURNING MODIFIED NEW *", + ] { + let body = shaped(sql, new_item.clone()); + assert_eq!( + body, + json!({ "Attributes": { "tags": tags(&["x", "y"]) } }), + "{sql}" + ); + } + } + + #[test] + fn modified_old_packs_the_prior_elements_the_same_way() { + let old_item = json!({ "pk": {"S": "k"}, "tags": tags(&["a", "b", "c"]) }); + let body = shaped( + "UPDATE \"t\" SET tags[2] = 'y', tags[0] = 'x' WHERE pk = 'k' RETURNING MODIFIED OLD *", + old_item, + ); + assert_eq!(body, json!({ "Attributes": { "tags": tags(&["a", "c"]) } })); + } + + #[test] + fn nested_paths_keep_only_the_named_leaves() { + let new_item = json!({ + "pk": {"S": "k"}, + "m": {"M": {"a": {"N": "1"}, "b": {"N": "2"}, "keep": {"S": "k"}}} + }); + let body = shaped( + "UPDATE \"t\" SET m.a = 1, m.b = 2 WHERE pk = 'k' RETURNING MODIFIED NEW *", + new_item, + ); + assert_eq!( + body, + json!({ "Attributes": { "m": {"M": {"a": {"N": "1"}, "b": {"N": "2"}}} } }) + ); + } + + #[test] + fn a_path_that_no_longer_resolves_projects_nothing() { + // REMOVE data, then project the new item: data is gone, so there is no row. + let body = shaped( + "UPDATE \"t\" REMOVE data WHERE pk = 'k' RETURNING MODIFIED NEW *", + json!({ "pk": {"S": "k"} }), + ); + assert_eq!(body, json!({})); + } + + #[test] + fn set_and_remove_paths_are_both_projected() { + let old_item = json!({ + "pk": {"S": "k"}, + "a": {"S": "old"}, + "gone": {"S": "g"}, + "other": {"S": "o"} + }); + let body = shaped( + "UPDATE \"t\" SET a = 'new' REMOVE gone WHERE pk = 'k' RETURNING MODIFIED OLD *", + old_item, + ); + assert_eq!( + body, + json!({ "Attributes": { "a": {"S": "old"}, "gone": {"S": "g"} } }) + ); + } + + #[test] + fn set_add_and_set_delete_targets_are_projected() { + let new_item = json!({ + "pk": {"S": "s"}, + "ss": {"SS": ["a", "b"]}, + "ns": {"NS": ["2"]}, + "other": {"N": "6"} + }); + let body = shaped( + "UPDATE \"t\" SET ss = set_add(ss, <<'b'>>) SET other = 6 WHERE pk = 's' \ + RETURNING MODIFIED NEW *", + new_item.clone(), + ); + assert_eq!( + body, + json!({ "Attributes": { "ss": {"SS": ["a", "b"]}, "other": {"N": "6"} } }) + ); + let body = shaped( + "UPDATE \"t\" SET ns = set_delete(ns, <<1>>) WHERE pk = 's' RETURNING MODIFIED NEW *", + new_item, + ); + assert_eq!(body, json!({ "Attributes": { "ns": {"NS": ["2"]} } })); + } + + #[test] + fn a_set_the_update_empties_or_adds_projects_nothing_on_that_side() { + // set_delete of every member removes the attribute from the new item. + let body = shaped( + "UPDATE \"t\" SET ss = set_delete(ss, <<'a'>>) WHERE pk = 's' RETURNING MODIFIED NEW *", + json!({ "pk": {"S": "s"}, "other": {"N": "5"} }), + ); + assert_eq!(body, json!({})); + // set_add to an absent attribute: the old item does not have it. + let body = shaped( + "UPDATE \"t\" SET zz = set_add(zz, <<'q'>>) WHERE pk = 's' RETURNING MODIFIED OLD *", + json!({ "pk": {"S": "s"}, "other": {"N": "5"} }), + ); + assert_eq!(body, json!({})); + } + + #[test] + fn a_name_starting_with_a_hash_is_taken_literally() { + let body = shaped( + "UPDATE \"t\" SET \"#odd\" = 'v' WHERE pk = 'k' RETURNING MODIFIED NEW *", + json!({ "pk": {"S": "k"}, "#odd": {"S": "v"} }), + ); + assert_eq!(body, json!({ "Attributes": { "#odd": {"S": "v"} } })); + } + + #[test] + fn all_forms_pass_the_whole_item_through() { + let whole = json!({ "pk": {"S": "k"}, "tags": tags(&["x", "b", "y"]) }); + let body = shaped( + "UPDATE \"t\" SET tags[0] = 'x' WHERE pk = 'k' RETURNING ALL NEW *", + whole.clone(), + ); + assert_eq!(body, json!({ "Attributes": whole })); + } + + #[test] + fn projection_reads_the_item_it_is_given() { + let projected = modified_paths( + &item(json!({ "tags": tags(&["a", "c"]) })), + &update("UPDATE \"t\" REMOVE tags[1] WHERE pk = 'k' RETURNING MODIFIED NEW *"), + ) + .unwrap(); + // The removed index now holds the shifted tail element. + assert_eq!( + serde_json::to_value(projected).unwrap(), + json!({ "tags": tags(&["c"]) }) + ); + } +} diff --git a/crates/engine/src/partiql/select.rs b/crates/engine/src/partiql/select.rs new file mode 100644 index 000000000..6908dd046 --- /dev/null +++ b/crates/engine/src/partiql/select.rs @@ -0,0 +1,403 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! PartiQL `SELECT` executor. +//! +//! Reads the WHERE clause with [`WhereAnalysis`] (which disjuncts read by +//! key, their key conditions and filters), checks it the way the service +//! does, composes the classic request the matching data-plane handler +//! already understands, drives [`crate::handle_query`] / +//! [`crate::handle_scan`], and shapes the response under `Items` with an +//! opaque, statement-bound `NextToken`. Every behaviour is justified by a +//! measurement against the live service, cited where it is implemented. +//! +//! A read of a table or an index by several disjuncts, or of a table by one +//! that names a whole item, takes the disjuncts one after another, the +//! first page in key order and later pages in the order the visit rules +//! give (see [`read_steps`]). A disjunct that fails a check made when it is +//! read, or that the caller may not read, is left until the others have been +//! read. + +use serde_json::Value; + +use extenddb_core::error::DynamoDbError; +use extenddb_core::partiql::ast::{Operand, Projection, SelectStatement, WhereExpr}; +use extenddb_core::partiql::plan::{KeyNames, WhereAnalysis, keyed_condition}; +use extenddb_core::types::{IndexType, KeySchemaElement}; +use extenddb_storage::error::StorageError; + +use super::SingleRequest; +use super::authorize::Gate; +use super::base_join::BaseTableJoin; +use super::common::{self, bound_value}; +use super::expr_bridge::ExprBridge; +use super::index_read; +use super::next_token::{TOKEN_MISMATCH, decode_next_token, encode_next_token}; +use crate::create_table::storage_err_to_dynamo; +use crate::{DispatchResult, OperationContext}; + +use order::direction; +use read_plan::{ReadPlan, check_keys, check_reads}; +use request::{Paging, key_condition, query_request, render_filter, scan_request}; +use response::{output_key, shape_response}; +use steps::{StepTarget, read_steps}; + +pub(super) use read_plan::{MAX_DECOMPOSED_READS, TOO_MANY_DECOMPOSED_READS}; +pub(super) use response::project_item; + +mod order; +mod read_plan; +mod request; +mod response; +mod steps; +mod visits; + +/// Refuse a projection list that names one output identifier twice, or +/// whose paths overlap. The service refuses two projected paths with one +/// identifier rather than collapsing them, ahead of two paths that overlap +/// (measured 2026-11-05: `a, a`, `m.k, k` and `m.k, m` on all three +/// operations). +/// +/// # Errors +/// +/// Returns the service's message for the first refusal. +pub(super) fn check_projection(stmt: &SelectStatement) -> Result<(), String> { + let Projection::Paths(paths) = &stmt.projection else { + return Ok(()); + }; + let mut seen = std::collections::HashSet::new(); + for path in paths { + let key = output_key(path); + if !seen.insert(key.clone()) { + return Err(format!("Duplicate identifiers in select clause: {key}")); + } + } + super::write::first_overlap(&paths.iter().collect::>()) +} + +/// Execute a parsed `SELECT`, passing `gate` once the checks the service +/// makes before authorization are done, ahead of the checks made when a +/// disjunct is read (see `authorize::gate`). The disjuncts the caller may +/// not read are held back as the disjuncts that fail such a check are. +/// +/// # Errors +/// +/// Returns the service's `ValidationException` / `ResourceNotFoundException` +/// for an invalid target, missing table/index, a GSI strong read, a parameter +/// count mismatch, or a `NextToken` that does not match the statement, and +/// the statement's refusal. +// The checks follow the service's order; helpers would split that order across functions. +#[allow(clippy::too_many_lines)] +pub(super) async fn run( + ctx: &OperationContext, + stmt: &SelectStatement, + req: &SingleRequest, + gate: &mut Gate<'_>, +) -> Result { + // Resolve the target table's key metadata. A missing table surfaces as the + // service's ResourceNotFoundException ("Requested resource not found"), + // except that a projected call (`COUNT(*)`) is reported ahead of it + // (measured 2026-11-02). + let key_info = match ctx.table_key_info(&stmt.from.table).await { + Ok(info) => info, + Err(e @ StorageError::TableNotFound(_)) => { + return Err(match &stmt.projection_error { + Some(message) => DynamoDbError::ValidationException(message.clone()), + None => storage_err_to_dynamo(e), + }); + } + Err(e) => return Err(storage_err_to_dynamo(e)), + }; + + check_projection(stmt).map_err(DynamoDbError::ValidationException)?; + + // Resolve the key schema of the target: the index's when one is named, + // otherwise the base table's. A named index that does not exist is the + // PartiQL-specific bare message (the classic path appends the index name). + // + // Measured order of the index checks on the service: a multi-attribute + // key wins over everything (including `ConsistentRead`); on a vector index + // the strong-read refusal wins over the vector refusal. + let (target_schema, index_info) = if let Some(index_name) = &stmt.from.index { + let is_vector = key_info + .vector_indexes + .iter() + .any(|v| &v.index_name == index_name); + if is_vector { + if req.consistent_read { + return Err(gsi_strong_read_error()); + } + // A vector index is not reachable by Scan or Query; the service + // refuses a PartiQL read of one with this exact message. + return Err(DynamoDbError::ValidationException( + "Scan operation not supported on this index type".to_owned(), + )); + } + match ctx + .storage + .index_info_by_table_id(&key_info.table_id, index_name) + .await + { + Ok(info) => { + common::ensure_single_attribute_keys(&info.key_schema)?; + (info.key_schema.clone(), Some(info)) + } + // A projected call (`COUNT(*)`) is refused ahead of the index + // lookup (measured 2026-11-02, with and without a WHERE). + Err(StorageError::IndexNotFound(_)) => { + return Err(DynamoDbError::ValidationException( + stmt.projection_error.clone().unwrap_or_else(|| { + "The table does not have the specified index".to_owned() + }), + )); + } + Err(e) => return Err(storage_err_to_dynamo(e)), + } + } else { + common::ensure_single_attribute_keys(&key_info.key_schema)?; + (key_info.key_schema.clone(), None) + }; + let is_gsi = index_info + .as_ref() + .is_some_and(|info| info.index_type == IndexType::Gsi); + + // On a table that exists, a projected call (`COUNT(*)`) is refused once the + // target is resolved, so a multi-attribute key is reported ahead of it + // (measured 2026-11-02). + if let Some(message) = &stmt.projection_error { + return Err(DynamoDbError::ValidationException(message.clone())); + } + + // A strongly consistent read of a GSI is rejected before the data path. + // The PartiQL message differs from the classic Query/Scan wording, so it is + // emitted here rather than left to the underlying handler. + if req.consistent_read && is_gsi { + return Err(gsi_strong_read_error()); + } + + let (partition, sort) = common::key_names(&target_schema); + let keys = KeyNames { + partition: partition.unwrap_or_default(), + sort, + }; + + // Read the WHERE against the target's keys, and refuse what the service + // refuses about it and about the ORDER BY before anything is read, in + // its order: a path into a key attribute, the ORDER BY terms, the key + // values, the WHERE an ORDER BY reads, overlapping disjuncts and the + // number of disjuncts (see `check_keys`). The checks a disjunct gets + // when it is read come last (measured 2026-11-03: `pk IN [51 values] + // AND sk = a` gets the refusal of too many disjuncts). A read of a + // table refuses a path into a key attribute wherever the WHERE has one + // (see `WhereAnalysis::path_into_key_refusal`), a read of an index only + // in a disjunct read by key; a scan of an index with one is not + // measured. + let value_of = |op: &Operand| bound_value(op, &req.parameters).cloned(); + let analysis = stmt + .where_clause + .as_ref() + .map(|w| WhereAnalysis::new(w, keys, &value_of)); + check_keys( + analysis.as_ref(), + stmt.order_by.as_ref(), + keys, + &key_info, + index_info.is_none(), + &req.parameters, + &value_of, + )?; + + // A NextToken is opaque and bound to the exact statement that produced + // it. It is checked before the statement is authorized (measured + // 2026-11-03), so ahead of the checks a disjunct gets when it is read. + let start_key = req + .next_token + .as_deref() + .map(|token| decode_next_token(token, &req.statement, &req.parameters)) + .transpose()?; + + // The checks made before authorization are done: authorize, each + // disjunct of a read by key by itself, and hold back the ones the caller + // may not read. + let refused = gate.pass().await?; + let checked = match &analysis { + Some(analysis) => Some(check_reads( + analysis, + keys, + &target_schema, + &ctx.limits, + &req.parameters, + &value_of, + &|read| refused.refusal(read, req), + )?), + None => None, + }; + + // An index that does not project every attribute refuses some reads and + // serves others from the base table (see `index_read`). + if let Some(info) = &index_info { + let shared_filter = checked.as_ref().and_then(|c| c.shared_filter.as_deref()); + index_read::check(stmt, info, &key_info.key_schema, shared_filter)?; + } + + let direction = direction(stmt.order_by.as_ref(), keys); + let plan = ReadPlan::of( + checked.as_ref(), + keys, + index_info.is_none(), + direction, + &value_of, + ); + + // A read that reaches back filters the base-table items, so the index read + // under it carries no filter: a read of one disjunct filters on that + // disjunct's conditions, a read of several on the WHERE with each key + // condition compared with a boolean written for what it reads (see + // `keyed_condition`), and a scan on the WHERE. Its errors lose the + // expression prefix as every other read's do. + let keyed_where = match (&plan, &stmt.where_clause) { + (ReadPlan::Steps(_), Some(where_clause)) if index_info.is_some() => { + Some(keyed_condition(where_clause, keys)) + } + _ => None, + }; + let join = match &index_info { + Some(info) => { + let where_terms: Vec<&WhereExpr> = keyed_where + .as_ref() + .or(stmt.where_clause.as_ref()) + .into_iter() + .collect(); + let filter_terms = match &plan { + ReadPlan::Key(read) => read.filter.as_slice(), + ReadPlan::Steps(_) | ReadPlan::Scan => where_terms.as_slice(), + }; + BaseTableJoin::for_read( + ctx, + req, + stmt, + info, + &key_info.key_schema, + Some(filter_terms), + ) + .map_err(super::bare_expression_error)? + } + None => None, + }; + + let paging = Paging { + start: start_key.as_ref(), + limit: req.limit, + descending: direction.items_descending, + }; + let mut resume_after = None; + let mut last_key = None; + let dispatched = match &plan { + ReadPlan::Key(read) => { + let mut bridge = ExprBridge::new(&req.parameters); + let kce = key_condition(&mut bridge, keys, read.partition, read.sort.as_ref())?; + let filter = if join.is_some() { + None + } else { + render_filter(&mut bridge, &read.filter)? + }; + let result = query_request(ctx, stmt, req, paging, &bridge, kce, filter).await?; + if read.returns_nothing() { + without_items(result) + } else { + result + } + } + ReadPlan::Steps(steps) => { + let target = StepTarget { + keys, + last_key: last_key_names(keys, &key_info.key_schema), + filtered: join.is_none(), + refusal: checked.as_ref().and_then(|c| c.deferred.as_ref()), + }; + let steps = read_steps(ctx, stmt, req, &target, steps, paging).await?; + resume_after = steps.next; + last_key = steps.last_key; + steps.page + } + ReadPlan::Scan => { + let filter = stmt.where_clause.as_ref().filter(|_| join.is_none()); + scan_request(ctx, stmt, req, paging, filter).await? + } + }; + let dispatched = match &join { + Some(join) => join + .apply(ctx, &key_info, req.consistent_read, dispatched) + .await + .map_err(super::bare_expression_error)?, + None => dispatched, + }; + + let mut response = shape_response(dispatched, req, &stmt.projection); + if let (Some(key), Value::Object(body)) = (last_key, &mut response.body) { + body.insert("LastEvaluatedKey".to_owned(), key); + } + // A read of disjuncts one after another names where it goes on from, + // the disjuncts held back for a refusal included: the request whose page + // visits only them gets the refusal (see `read_steps`; measured + // 2026-11-02: the first page of `pk IN ['a', <2049 bytes>]` holds item + // 'a' and a NextToken, and the request with that token gets the key size + // refusal). + if let (Some(position), Value::Object(body)) = (resume_after, &mut response.body) + && !body.contains_key("NextToken") + { + body.insert( + "NextToken".to_owned(), + Value::String(encode_next_token( + &req.statement, + &req.parameters, + &position, + )), + ); + } + Ok(response) +} + +/// The attributes of the LastEvaluatedKey of a page of a read of steps: the +/// target's key, and for a read of an index the table's key after it, as a +/// Query of an index names them. +fn last_key_names<'k>(keys: KeyNames<'k>, table: &'k [KeySchemaElement]) -> Vec<&'k str> { + let (partition, sort) = common::key_names(table); + let mut names: Vec<&str> = std::iter::once(keys.partition).chain(keys.sort).collect(); + for name in partition.into_iter().chain(sort) { + if !names.contains(&name) { + names.push(name); + } + } + names +} + +/// A read's result with its items left out, for a disjunct that returns +/// none of the items it reads (see `KeyRead::returns_nothing`). What it read +/// still counts, as a filter that keeps no item counts. +fn without_items(mut result: DispatchResult) -> DispatchResult { + if let Value::Object(body) = &mut result.body { + body.insert("Items".to_owned(), Value::Array(Vec::new())); + body.insert("Count".to_owned(), Value::from(0)); + } + result.metrics.returned_item_count = 0; + result +} + +/// The service's refusal of a NextToken that is not one this statement gave. +fn token_mismatch() -> DynamoDbError { + DynamoDbError::ValidationException(TOKEN_MISMATCH.to_owned()) +} + +/// The PartiQL wording for a `ConsistentRead` against a GSI, also returned +/// for a vector index (measured on the service); it comes before the +/// statement is authorized (measured 2026-11-03). +pub(super) const GSI_STRONG_READ: &str = + "Strongly consistent read is not supported on Global Secondary Indexes"; + +fn gsi_strong_read_error() -> DynamoDbError { + DynamoDbError::ValidationException(GSI_STRONG_READ.to_owned()) +} + +#[cfg(test)] +mod tests; diff --git a/crates/engine/src/partiql/select/order.rs b/crates/engine/src/partiql/select/order.rs new file mode 100644 index 000000000..17e471d47 --- /dev/null +++ b/crates/engine/src/partiql/select/order.rs @@ -0,0 +1,146 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The ORDER BY of a SELECT: the refusals the service gives for it, and the +//! order a read takes under it. + +use extenddb_core::error::DynamoDbError; +use extenddb_core::partiql::ast::{Operand, OrderBy}; +use extenddb_core::partiql::plan::{Direction, KeyNames, KeyRead, WhereAnalysis, same_value}; +use extenddb_core::types::AttributeValue; + +/// The directions a read takes under `order` (measured 2026-11-03): the +/// partitions ascend unless the term that names the partition key says +/// DESC, and the items of a partition follow the term that names the sort +/// key, descending when no term names it. `ORDER BY pk`, `pk ASC`, `pk +/// DESC` and `pk, pk` read the items of a partition in descending order; +/// `pk, sk`, `sk` and `sk, pk` in ascending order; `sk DESC` and `pk, sk +/// DESC` in descending order; `pk IN ['a', 'b'] ORDER BY pk DESC, sk` reads +/// partition b first. On an index the index's keys take those parts +/// (measured 2026-11-03: `ORDER BY g` reads the index sort key in descending +/// order). Without an ORDER BY a read takes key order. +pub(super) fn direction(order: Option<&OrderBy>, keys: KeyNames<'_>) -> Direction { + let Some(order) = order else { + return Direction::default(); + }; + Direction { + partitions_descending: order + .term_for(keys.partition) + .is_some_and(|term| term.descending), + items_descending: keys + .sort + .and_then(|sort| order.term_for(sort)) + .is_none_or(|term| term.descending), + } +} + +/// Refuse an ORDER BY of a statement with no WHERE, then the first term of +/// its list that is a path with steps or a name that is not a key attribute +/// of what is read, the table's key or the index's. The service makes these +/// checks after the one of a path into a key attribute and ahead of the key +/// values (see `read_plan::check_keys`). +/// +/// * No WHERE: "Must have WHERE clause in the statement when using ORDER BY +/// clause.", ahead of the list (measured 2026-11-05: `ORDER BY n`, `ORDER +/// BY m.x` and `ORDER BY pk.x` with no WHERE, on a table and on an index). +/// * A path with steps: "Argument to ORDER BY must be of type +/// VariableReference. Type: PATH at" the path's first token (measured +/// 2026-11-05: `m.x`, `m[0]`, `m['k']`, `pk.x` and `"m"."x"`, on a table +/// and on an index, ahead of a key value of the wrong type and of a WHERE +/// that does not read by key). +/// * A name that is not a key attribute: "Attribute n in ORDER BY clause +/// must be part of the primary key" (measured 2026-11-03 for `n`, for +/// `PK` on a key named `pk`, and for `pk` on an index keyed on other +/// attributes). +/// +/// The terms are checked in list order (measured 2026-11-05: `ORDER BY n, +/// m.x` names `n`, `ORDER BY m.x, n` refuses the path). +pub(super) fn check_order_by_terms( + order: Option<&OrderBy>, + has_where: bool, + keys: KeyNames<'_>, +) -> Result<(), DynamoDbError> { + let Some(order) = order else { + return Ok(()); + }; + let refuse = |message: String| Err(DynamoDbError::ValidationException(message)); + if !has_where { + return refuse( + "Must have WHERE clause in the statement when using ORDER BY clause.".to_owned(), + ); + } + for term in &order.terms { + match term.path.as_simple() { + None => { + return refuse(format!( + "Argument to ORDER BY must be of type VariableReference. Type: PATH at {}", + term.at + )); + } + Some(name) if name != keys.partition && Some(name) != keys.sort => { + return refuse(format!( + "Attribute {name} in ORDER BY clause must be part of the primary key" + )); + } + Some(_) => {} + } + } + Ok(()) +} + +/// Refuse the WHERE an ORDER BY reads, in the order the service checks +/// (measured 2026-11-03 on tables and 2026-11-05 on indexes), after the key +/// values and ahead of overlapping disjuncts (see `read_plan::check_keys`): +/// +/// * a WHERE that does not read by key: `n = 1`, `pk > 'a'`, `pk = 'a' OR n +/// = 1`, and a sort key predicate that is not one key condition, such as +/// `sk <> 'x'`, `contains(sk, 'x')` or `sk > 'a' AND sk < 'z'`; +/// * a read of more than one partition key value whose list does not name +/// the partition key (`pk IN ['a', 'b'] ORDER BY sk`; `ORDER BY sk, pk` +/// runs). +/// +/// A statement with no WHERE was refused by [`check_order_by_terms`]. +pub(super) fn check_order_by_reads( + order: Option<&OrderBy>, + analysis: Option<&WhereAnalysis<'_>>, + keys: KeyNames<'_>, + value_of: &dyn Fn(&Operand) -> Option, +) -> Result<(), DynamoDbError> { + let (Some(order), Some(analysis)) = (order, analysis) else { + return Ok(()); + }; + let refuse = |message: &str| Err(DynamoDbError::ValidationException(message.to_owned())); + if !analysis.is_keyed() { + return refuse( + "Must have at least one non-optional hash key condition in WHERE clause when using \ + ORDER BY clause.", + ); + } + if order.term_for(keys.partition).is_none() && partition_count(analysis.reads(), value_of) > 1 { + return refuse( + "Must have hash key in ORDER BY clause when more than one hash key condition \ + specified in WHERE clause.", + ); + } + Ok(()) +} + +/// How many distinct partition key values the disjuncts `reads` read. +fn partition_count( + reads: &[KeyRead<'_>], + value_of: &dyn Fn(&Operand) -> Option, +) -> usize { + let mut seen: Vec<&Operand> = Vec::new(); + for read in reads { + if !seen + .iter() + .any(|value| same_value(value, read.partition, value_of)) + { + seen.push(read.partition); + } + } + seen.len() +} + +#[cfg(test)] +mod tests; diff --git a/crates/engine/src/partiql/select/order/tests.rs b/crates/engine/src/partiql/select/order/tests.rs new file mode 100644 index 000000000..c71f5f25b --- /dev/null +++ b/crates/engine/src/partiql/select/order/tests.rs @@ -0,0 +1,237 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Unit tests for the ORDER BY of a SELECT. + +use super::super::bound_value; +use super::super::tests::{HASH_ONLY, KEYS, select}; +use super::*; + +/// The refusal an ORDER BY gets on a target keyed on `keys`: of its terms, +/// then of the WHERE it reads. A table and an index get the same checks. +fn refusal(sql: &str, keys: KeyNames<'_>) -> Option { + let s = select(sql); + let value_of = |op: &Operand| bound_value(op, &[]).cloned(); + let analysis = s + .where_clause + .as_ref() + .map(|w| WhereAnalysis::new(w, keys, &value_of)); + check_order_by_terms(s.order_by.as_ref(), analysis.is_some(), keys) + .and_then(|()| { + check_order_by_reads(s.order_by.as_ref(), analysis.as_ref(), keys, &value_of) + }) + .err() + .map(|e| e.message().to_owned()) +} + +/// The index of `an_order_by_names_key_attributes_of_what_is_read`. +const INDEX: KeyNames<'static> = KeyNames { + partition: "g", + sort: Some("gs"), +}; + +const ATTRIBUTE: &str = "in ORDER BY clause must be part of the primary key"; +const NO_WHERE: &str = "Must have WHERE clause in the statement when using ORDER BY clause."; +const NOT_KEYED: &str = "Must have at least one non-optional hash key condition in WHERE clause \ + when using ORDER BY clause."; +const HASH_KEY: &str = "Must have hash key in ORDER BY clause when more than one hash key \ + condition specified in WHERE clause."; + +#[test] +fn an_order_by_names_key_attributes_of_what_is_read() { + let attribute = |name: &str| Some(format!("Attribute {name} {ATTRIBUTE}")); + for (sql, want) in [ + ( + "SELECT * FROM \"t\" WHERE pk = 'a' ORDER BY n", + attribute("n"), + ), + ( + "SELECT * FROM \"t\" WHERE pk = 'a' ORDER BY PK", + attribute("PK"), + ), + ( + "SELECT * FROM \"t\" WHERE pk = 'a' ORDER BY pk, n", + attribute("n"), + ), + // Ahead of the refusal of a WHERE that does not read by key. + ( + "SELECT * FROM \"t\" WHERE pk = 'a' AND sk <> 5 ORDER BY n", + attribute("n"), + ), + ("SELECT * FROM \"t\" WHERE pk = 'a' ORDER BY \"pk\"", None), + ("SELECT * FROM \"t\" WHERE pk = 'a' ORDER BY sk, sk", None), + ] { + assert_eq!(refusal(sql, KEYS), want, "{sql}"); + } + // On an index, its own keys; the table's partition key is not one. + assert_eq!( + refusal( + "SELECT * FROM \"t\".\"gix\" WHERE g = 'a' ORDER BY pk", + INDEX + ), + attribute("pk") + ); + assert_eq!( + refusal( + "SELECT * FROM \"t\".\"gix\" WHERE g = 'a' ORDER BY g, gs", + INDEX + ), + None + ); +} + +#[test] +fn a_path_in_the_list_is_refused_at_its_first_token() { + let path = |at: &str| { + Some(format!( + "Argument to ORDER BY must be of type VariableReference. Type: PATH at {at}" + )) + }; + for (sql, want) in [ + ( + "SELECT * FROM \"t\" WHERE pk = 'a' ORDER BY m.x", + path("1:43:1"), + ), + ( + "SELECT * FROM \"t\" WHERE pk = 'a' ORDER BY m[0]", + path("1:43:1"), + ), + ( + "SELECT * FROM \"t\" WHERE pk = 'a' ORDER BY pk.x", + path("1:43:2"), + ), + ( + "SELECT * FROM \"t\" WHERE pk = 'a' ORDER BY \"m\".\"x\"", + path("1:43:3"), + ), + ( + "SELECT * FROM \"t\" WHERE pk = 'a' ORDER BY sk, m.x", + path("1:47:1"), + ), + ( + "SELECT * FROM \"t\" WHERE pk = 'a' ORDER BY m.x, n", + path("1:43:1"), + ), + // In list order with a name that is not a key attribute. + ( + "SELECT * FROM \"t\" WHERE pk = 'a' ORDER BY n, m.x", + Some(format!("Attribute n {ATTRIBUTE}")), + ), + // Ahead of the refusal of a WHERE that does not read by key. + ( + "SELECT * FROM \"t\" WHERE pk > 'a' ORDER BY m.x", + path("1:43:1"), + ), + ] { + assert_eq!(refusal(sql, KEYS), want, "{sql}"); + } +} + +#[test] +fn an_order_by_needs_a_where_read_by_key() { + // With no WHERE, ahead of the list. + for list in ["pk", "n", "m.x"] { + let sql = format!("SELECT * FROM \"t\" ORDER BY {list}"); + assert_eq!(refusal(&sql, KEYS).as_deref(), Some(NO_WHERE), "{list}"); + } + for condition in [ + "n = 1", + "pk > 'a'", + "pk = 'a' OR n = 1", + "pk = 'a' AND sk <> 'x'", + "pk = 'a' AND NOT sk = 'x'", + "pk = 'a' AND contains(sk, 'x')", + "pk = 'a' AND sk IS NOT MISSING", + "pk = 'a' AND size(sk) = 1", + "pk = 'a' AND sk = 'x' AND sk = 'y'", + "pk = 'a' AND sk > 'a' AND sk < 'z'", + "pk IN ['a', 'b'] AND sk <> 'x'", + ] { + let sql = format!("SELECT * FROM \"t\" WHERE {condition} ORDER BY sk"); + assert_eq!( + refusal(&sql, KEYS).as_deref(), + Some(NOT_KEYED), + "{condition}" + ); + } + let sql = "SELECT * FROM \"t\" WHERE n = 1 ORDER BY pk"; + assert_eq!(refusal(sql, HASH_ONLY).as_deref(), Some(NOT_KEYED)); + for condition in [ + "pk = 'a' AND sk IN ['x', 'y']", + "pk = 'a' AND (sk = 'x' OR sk = 'y')", + "pk = 'a' AND m = 'x'", + "pk = 'a' AND NOT m = 'y'", + ] { + let sql = format!("SELECT * FROM \"t\" WHERE {condition} ORDER BY sk"); + assert_eq!(refusal(&sql, KEYS), None, "{condition}"); + } + // An index read gets the same refusals on its own keys (measured + // 2026-11-05). + for (condition, want) in [ + ("g > 'a' ORDER BY g", NOT_KEYED), + ("g = 'a' OR n = 1 ORDER BY gs", NOT_KEYED), + ("gs = 'x' ORDER BY gs", NOT_KEYED), + ("g = 'a' AND gs <> 'x' ORDER BY gs", NOT_KEYED), + ("g IN ['a', 'b'] ORDER BY gs", HASH_KEY), + ] { + let sql = format!("SELECT * FROM \"t\".\"gix\" WHERE {condition}"); + assert_eq!(refusal(&sql, INDEX).as_deref(), Some(want), "{condition}"); + } + let sql = "SELECT * FROM \"t\".\"gix\" WHERE g IN ['a', 'b'] ORDER BY g, gs"; + assert_eq!(refusal(sql, INDEX), None); +} + +#[test] +fn an_order_by_over_several_partitions_names_the_partition_key() { + for (condition, want) in [ + ("pk IN ['a', 'b'] ORDER BY sk DESC", Some(HASH_KEY)), + ( + "(pk = 'a' AND sk = 'x') OR pk = 'b' ORDER BY sk DESC", + Some(HASH_KEY), + ), + ( + "(pk = 'a' AND sk = 'x') OR (pk = 'a' AND sk > 'y') ORDER BY sk DESC", + None, + ), + ("pk = 'a' AND sk IN ['x', 'z'] ORDER BY sk DESC", None), + ("pk IN ['a', 'b']", None), + // The partition key may be anywhere in the list. + ("pk IN ['a', 'b'] ORDER BY pk, sk", None), + ("pk IN ['a', 'b'] ORDER BY sk, pk", None), + ("(pk = 'a' AND sk = 'x') OR pk = 'b' ORDER BY pk, sk", None), + ] { + let sql = format!("SELECT * FROM \"t\" WHERE {condition}"); + assert_eq!(refusal(&sql, KEYS).as_deref(), want, "{condition}"); + } +} + +#[test] +fn an_order_by_sets_the_direction_of_partitions_and_of_their_items() { + let order = |list: &str, keys: KeyNames<'_>| { + let s = select(&format!( + "SELECT * FROM \"t\" WHERE pk = 'a' ORDER BY {list}" + )); + let d = direction(s.order_by.as_ref(), keys); + (d.partitions_descending, d.items_descending) + }; + for (list, partitions, items) in [ + ("pk", false, true), + ("pk ASC", false, true), + ("pk DESC", true, true), + ("pk, pk", false, true), + ("\"pk\"", false, true), + ("pk, sk", false, false), + ("sk", false, false), + ("sk, pk", false, false), + ("sk, sk", false, false), + ("sk DESC", false, true), + ("pk, sk DESC", false, true), + ("sk DESC, pk", false, true), + ("pk DESC, sk", true, false), + ("pk DESC, sk DESC", true, true), + ] { + assert_eq!(order(list, KEYS), (partitions, items), "{list}"); + } + assert!(order("pk DESC", HASH_ONLY).0); + assert_eq!(direction(None, KEYS), Direction::default()); +} diff --git a/crates/engine/src/partiql/select/read_plan.rs b/crates/engine/src/partiql/select/read_plan.rs new file mode 100644 index 000000000..b47ba845a --- /dev/null +++ b/crates/engine/src/partiql/select/read_plan.rs @@ -0,0 +1,451 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The WHERE checks a SELECT gets before anything is read, and the read plan they lead to. +//! +//! The disjuncts held back for a refusal made when they are read, the +//! caller's authorization refusal among them, share one refusal (see +//! [`check_reads`]): the last refusal of a check in read order, else an +//! authorization refusal. A request whose page visits only held-back +//! disjuncts gets it (measured 2026-11-03 when no disjunct is read, and +//! when every held-back disjunct is refused for authorization). A page that +//! visits only a disjunct the caller may not read, while another held-back +//! disjunct fails a check, gets that check's refusal; that is not measured. + +use serde_json::{Map, Value}; + +use extenddb_core::error::DynamoDbError; +use extenddb_core::limits::LimitsConfig; +use extenddb_core::partiql::ast::{CompareOp, Operand, OrderBy, WhereExpr}; +use extenddb_core::partiql::plan::{ + Direction, KeyNames, KeyRead, SortCondition, WhereAnalysis, filter_key_refusal, read_order, + read_order_in, scalar_key_refusal, +}; +use extenddb_core::types::{AttributeValue, Item, KeySchemaElement, KeyType, TableKeyInfo}; +use extenddb_core::validation::validate_key_size_limits; + +use super::bound_value; +use super::order::{check_order_by_reads, check_order_by_terms}; +use crate::partiql::common; + +/// How a `SELECT` reaches its rows. +pub(super) enum ReadPlan<'a, 'b> { + /// One disjunct read by key: a Query with its key condition and filter. + Key(&'b KeyRead<'a>), + /// Disjuncts read one after another in key order (see `read_steps`). + Steps(Vec>), + /// Everything else: a Scan with the WHERE as its filter. + Scan, +} + +impl<'a, 'b> ReadPlan<'a, 'b> { + /// The read of a checked WHERE on a target with key names `keys`, a + /// table when `table` is set. + /// + /// An index is read by one Query for each disjunct, also when several + /// name one partition of it (measured 2026-11-03: on a GSI keyed on g + /// and gs, `g = 'a' AND gs IN ['w000', 'w001', 'w002']` reports 1.5 + /// capacity units, 0.5 for each Query, and `g = 'a' AND gs IN [...]` + /// with 11 values gives 10 items and then 1). A disjunct of an index + /// never names one item, since an index key does not. + pub(super) fn of( + checked: Option<&Checked<'a, 'b>>, + keys: KeyNames<'_>, + table: bool, + direction: Direction, + value_of: &dyn Fn(&Operand) -> Option, + ) -> Self { + let Some(checked) = checked.filter(|c| c.keyed) else { + return Self::Scan; + }; + let points = table.then_some(keys); + match checked.reads.as_slice() { + [read] + if checked.refused.is_empty() + && points.is_none_or(|keys| Step::wire_key(read, keys, value_of).is_none()) => + { + Self::Key(read) + } + reads => Self::Steps(Step::all( + reads, + &checked.refused, + points, + direction, + value_of, + )), + } + } +} + +/// One disjunct of a read of several, in the order it is read. +pub(super) struct Step<'a, 'b> { + pub(super) read: &'b KeyRead<'a>, + /// The item's key in wire form when the disjunct names one item of the + /// table (an equality on every key attribute). + pub(super) point: Option, + /// Whether the disjunct is held back for the refusal it gets when it is + /// read: it takes its place in a page and reads nothing (see + /// `read_steps`). + pub(super) held_back: bool, +} + +impl<'a, 'b> Step<'a, 'b> { + /// The disjuncts `reads`, and those `held_back` for a refusal, in the + /// order they are read: key order (see [`read_order`]), its partitions + /// and the ranges of a partition each in the direction an ORDER BY gives + /// (measured 2026-11-02: `pk IN ['c', 'a', 'b']` reads a, b then c, and + /// `pk = 'a' AND sk IN ['x', 'y'] ORDER BY sk DESC` reads a/y first; + /// measured 2026-11-03: `pk IN ['a', 'b'] ORDER BY pk DESC, sk` reads + /// b/x, a/x then a/y). A disjunct held back takes its place among the + /// others (measured 2026-11-03: the first page of 10 disjuncts of `pk + /// IN [v000..v010, <2049 bytes that start with v005>]` holds v000 to + /// v008). + /// + /// `points` are the key names of a table, whose disjuncts may name one + /// item each; a read of an index has none. + fn all( + reads: &[&'b KeyRead<'a>], + held_back: &[&'b KeyRead<'a>], + points: Option>, + direction: Direction, + value_of: &dyn Fn(&Operand) -> Option, + ) -> Vec { + let mut steps: Vec = reads + .iter() + .map(|&read| Self { + read, + point: points.and_then(|keys| Self::wire_key(read, keys, value_of)), + held_back: false, + }) + .chain(held_back.iter().map(|&read| Self { + read, + point: None, + held_back: true, + })) + .collect(); + steps.sort_by(|a, b| read_order_in(a.read, b.read, direction, value_of)); + steps + } + + /// The key of the one item a disjunct names, in wire form, or `None` + /// when it does not name one. + fn wire_key( + read: &KeyRead<'_>, + keys: KeyNames<'_>, + value_of: &dyn Fn(&Operand) -> Option, + ) -> Option { + let mut wire = Map::new(); + let partition = value_of(read.partition)?; + wire.insert( + keys.partition.to_owned(), + serde_json::to_value(&partition).ok()?, + ); + match (keys.sort, &read.sort) { + (None, _) => {} + (Some(name), Some(SortCondition::Compare(CompareOp::Eq, value))) => { + wire.insert( + name.to_owned(), + serde_json::to_value(value_of(value)?).ok()?, + ); + } + (Some(_), _) => return None, + } + Some(Value::Object(wire)) + } +} + +/// A WHERE as the service checks it before anything is read. +pub(super) struct Checked<'a, 'b> { + /// Whether every disjunct reads by key, so the statement is not a scan. + pub(super) keyed: bool, + /// The disjuncts read by key that are read, in statement order: every + /// one, unless some fail a check made when they are read. + pub(super) reads: Vec<&'b KeyRead<'a>>, + /// The conditions every disjunct shares, when the statement reads by key. + pub(super) shared_filter: Option>, + /// The refusal of the disjuncts that fail a check made when they are + /// read, which a request whose page visits only them gets (see + /// `read_steps`). + pub(super) deferred: Option, + /// The disjuncts that fail a check made when they are read, in statement + /// order. + pub(super) refused: Vec<&'b KeyRead<'a>>, +} + +/// The most disjuncts a SELECT read by key may split into (measured +/// 2026-11-03: `pk IN [...]` with 50 values runs and with 51 values is +/// refused, on a table and on an index). It is also the most partition +/// values the authorization of a read by key asks about +/// (`authorize::each_disjunct`), which is safe because a statement with +/// more disjuncts is refused before it is authorized. +pub(in crate::partiql) const MAX_DECOMPOSED_READS: usize = 50; + +/// Check the keys of a WHERE and the ORDER BY of a SELECT the way the +/// service does, in its order, before anything is read (measured 2026-11-03 +/// for a permitted caller): +/// +/// 1. a path into a key attribute ("Key attributes must be scalars"): on a +/// table anywhere in the WHERE, on an index in the filter of a disjunct +/// read by key; +/// 2. an ORDER BY with no WHERE, then an ORDER BY term that is a path or +/// is not a key attribute (see `order::check_order_by_terms`); +/// 3. the key values of each disjunct read by key, in statement order; +/// 4. the WHERE an ORDER BY reads (see `order::check_order_by_reads`), +/// with the refusal of more than one partition key value without the +/// partition key in the list; +/// 5. overlapping disjuncts; +/// 6. more than [`MAX_DECOMPOSED_READS`] disjuncts (see +/// [`check_decomposed`]). +/// +/// Measured: with `pk.x = 1` the scalar refusal whatever the ORDER BY; +/// `pk = 1`, `pk IN ['a', 1]` and `pk = 'a' AND sk = 1` with `ORDER BY n` +/// the term refusal, with `ORDER BY sk` the key type refusal; `(pk = 'a' OR +/// pk = 'a')` with `ORDER BY n` the term refusal, with `ORDER BY sk` the +/// overlap refusal; `(pk = 'a' OR pk = 'b' OR pk = 'a') ORDER BY sk` the +/// refusal of more than one partition key value; `pk IN [51 values]` the +/// ORDER BY refusals. Whether a WHERE that does not read by key is refused +/// ahead of a key value of the wrong type is not measured. +/// +/// A path into a key attribute is refused at once, naming the first such +/// disjunct in statement order, ahead of the other refusals and when a later +/// disjunct makes the statement a scan (measured 2026-11-02: `(pk = 'a' AND +/// sk.x = 1) OR pk = 'b'` gets the scalar refusal on the first request, as +/// do `(pk = 'a' AND sk.x = 1) OR (pk = 'a' AND sk = 'y')`, `pk = 1 OR (pk = +/// 'a' AND sk.x = 1)` and `(pk = 'a' AND sk.x = 1) OR x = 1`). The +/// statement is authorized after these checks: 1, 5 and 6 are made before +/// authorization and 2, 3 and 4 after it (see `authorize::gate`). +pub(super) fn check_keys( + analysis: Option<&WhereAnalysis<'_>>, + order: Option<&OrderBy>, + keys: KeyNames<'_>, + key_info: &TableKeyInfo, + table: bool, + params: &[AttributeValue], + value_of: &dyn Fn(&Operand) -> Option, +) -> Result<(), DynamoDbError> { + let path_into_key = analysis.and_then(|analysis| { + if table { + analysis.path_into_key_refusal(keys) + } else { + analysis + .reads() + .iter() + .find_map(|read| scalar_key_refusal(&read.filter, keys)) + } + }); + if let Some(message) = path_into_key { + return Err(DynamoDbError::ValidationException(message)); + } + check_order_by_terms(order, analysis.is_some(), keys)?; + for read in analysis.map(WhereAnalysis::reads).unwrap_or_default() { + check_key_read(read, keys, key_info, params)?; + } + check_order_by_reads(order, analysis, keys, value_of)?; + let Some(analysis) = analysis else { + return Ok(()); + }; + analysis + .check_overlap(value_of) + .map_err(|m| DynamoDbError::ValidationException(m.to_owned()))?; + check_decomposed(analysis) +} + +/// Refuse a statement read by key whose WHERE splits into more than +/// [`MAX_DECOMPOSED_READS`] disjuncts (see +/// `WhereAnalysis::disjunct_count`). Measured 2026-11-03, on tables and on +/// indexes: the service counts each value of an IN on a key attribute, each +/// side of an OR of key conditions, the product of ANDed lists (`pk IN [25 +/// values] AND sk IN [2 values]` runs, `pk IN [17 values] AND sk IN [3 +/// values]` is refused), parameters, and the disjuncts held back for a +/// refusal made when they are read; it does not count the conditions of a +/// filter (`pk IN [50 values]` beside 99 comparisons runs), and a statement +/// with a disjunct that is a scan runs as a scan (`pk IN [51 values] OR x = +/// 1`). The refusal comes last of those of [`check_keys`], and before the +/// checks a disjunct gets when it is read (see [`check_reads`]): `pk IN [50 +/// values, <2049 bytes>]` and `pk IN [51 values] AND sk = a` get it. +fn check_decomposed(analysis: &WhereAnalysis<'_>) -> Result<(), DynamoDbError> { + if analysis.is_keyed() && analysis.disjunct_count() > MAX_DECOMPOSED_READS as u64 { + return Err(DynamoDbError::ValidationException( + TOO_MANY_DECOMPOSED_READS.to_owned(), + )); + } + Ok(()) +} + +/// The refusal of a statement read by key with more than +/// [`MAX_DECOMPOSED_READS`] disjuncts, which comes before the statement is +/// authorized (measured 2026-11-03). +pub(in crate::partiql) const TOO_MANY_DECOMPOSED_READS: &str = + "Too many decomposed read operations for a given query."; + +/// The checks each disjunct of a statement read by key gets when it is read +/// (see [`read_refusal`]), made once the WHERE has passed [`check_keys`] and +/// [`check_decomposed`]. +/// +/// A disjunct that fails them does not fail the statement while another +/// disjunct passes: it is held back, and the request whose page visits only +/// disjuncts held back gets the refusal (see `read_steps`; measured +/// 2026-11-02: `(pk = 'a' AND sk = a) OR pk = 'b'` returns partition 'b' +/// with a NextToken, and the request with that token gets the filter +/// refusal; `pk IN ['a', <2049 bytes>]` the same with the key size +/// refusal). When every disjunct fails, the statement is refused at once. +/// Of several failing disjuncts, the one read last in key order is reported +/// (measured 2026-11-02: `pk = <2049 bytes of z> OR (pk = 'a' AND sk = a) +/// OR pk = 'b'` gets the key size refusal and `(pk = 'b' AND sk = a) OR (pk +/// = 'a' AND pk = b) OR pk = 'c'` the filter refusal naming sk). +/// +/// `denied` gives the authorization refusal of a disjunct the caller may not +/// read (see `authorize`), which is held back the same way, its refusal +/// taking the place of the checks here (measured 2026-11-03: a disjunct the +/// policy refuses that is past the key size limit, or has a filter on a key +/// attribute, gets AccessDenied). Across disjuncts it ranks below the +/// refusals of the checks. +#[allow(clippy::too_many_arguments)] +pub(super) fn check_reads<'a, 'b>( + analysis: &'b WhereAnalysis<'a>, + keys: KeyNames<'_>, + target_schema: &[KeySchemaElement], + limits: &LimitsConfig, + params: &[AttributeValue], + value_of: &dyn Fn(&Operand) -> Option, + denied: &dyn Fn(&KeyRead<'a>) -> Option, +) -> Result, DynamoDbError> { + let checked = Checked { + keyed: analysis.is_keyed(), + reads: analysis.reads().iter().collect(), + shared_filter: analysis.shared_filter(), + deferred: None, + refused: Vec::new(), + }; + if !checked.keyed { + return Ok(checked); + } + // Each failing disjunct, its refusal, and whether the refusal is the + // authorization's. + let mut failing: Vec<(&'b KeyRead<'a>, DynamoDbError, bool)> = Vec::new(); + let mut reads = Vec::with_capacity(checked.reads.len()); + for &read in &checked.reads { + if let Some(error) = denied(read) { + failing.push((read, error, true)); + } else if let Some(error) = read_refusal(read, keys, target_schema, limits, params) { + failing.push((read, error, false)); + } else { + reads.push(read); + } + } + let refused: Vec<&'b KeyRead<'a>> = failing.iter().map(|(read, ..)| *read).collect(); + // An authorization refusal ranks below the others: the one reported is + // the last in read order of the refusals made by the checks, and an + // authorization refusal only when every failing disjunct has one + // (measured 2026-11-03: under `ForAllValues` LeadingKeys ["a"], `(pk = + // 'a' AND sk = a) OR pk = 'b'`, either way round, is refused at once + // with the filter refusal of the disjunct the policy allows). + let by_checks = failing.iter().any(|(_, _, denied)| !denied); + // `max_by` keeps the last of equal elements, so disjuncts whose order is + // unknown are taken in statement order. + let error = failing + .into_iter() + .filter(|(_, _, denied)| !by_checks || !denied) + .max_by(|a, b| read_order(a.0, b.0, value_of)) + .map(|(_, error, _)| error); + match error { + None => Ok(checked), + Some(error) if reads.is_empty() => Err(error), + Some(error) => Ok(Checked { + reads, + deferred: Some(error), + refused, + ..checked + }), + } +} + +/// The refusal a disjunct read by key gets when it is read: a key value +/// longer than the key may be (see [`check_key_sizes`]), else a key +/// attribute compared with another path in its filter. +fn read_refusal( + read: &KeyRead<'_>, + keys: KeyNames<'_>, + schema: &[KeySchemaElement], + limits: &LimitsConfig, + params: &[AttributeValue], +) -> Option { + if let Err(error) = check_key_sizes(read, schema, limits, params) { + return Some(error); + } + filter_key_refusal(&read.filter, keys).map(DynamoDbError::ValidationException) +} + +/// Refuse a key read whose values do not fit the key: on the sort key a +/// `begins_with` prefix that is not a string or binary (the operand-type +/// message a filter gets for it) or a value of another type than the key's, +/// then a partition value of another type than the key's. A parameter the +/// request does not supply is left to the parameter count check. +/// +/// The sort key is checked first: for `pk = 1 AND sk = 1` the service names +/// `sk`, in either order, and it does so for the names `PK`/`SK`, `id`/`sk` +/// and `id`/`ts` too (measured 2026-11-02). For some other pairs of names it +/// names the partition key; no rule was found. +fn check_key_read( + read: &KeyRead<'_>, + keys: KeyNames<'_>, + key_info: &TableKeyInfo, + params: &[AttributeValue], +) -> Result<(), DynamoDbError> { + if let (Some(sort), Some(condition)) = (keys.sort, &read.sort) { + if let SortCondition::BeginsWith(prefix) = condition + && let Some(value) = bound_value(prefix, params) + && !matches!(value, AttributeValue::S(_) | AttributeValue::B(_)) + { + return Err(DynamoDbError::ValidationException(format!( + "Incorrect operand type for operator or function; operator or function: \ + begins_with, operand type: {}", + crate::partiql::write::av_type_label(value) + ))); + } + for operand in condition.values() { + if let Some(value) = bound_value(operand, params) { + common::check_key_value_type(key_info, sort, value)?; + } + } + } + match bound_value(read.partition, params) { + Some(value) => common::check_key_value_type(key_info, keys.partition, value), + None => Ok(()), + } +} + +/// Refuse a disjunct whose partition value is longer than a partition key may +/// be, then one whose sort-key condition names a value longer than a sort key +/// may be, with the messages a GetItem of such a key gets (measured +/// 2026-11-02 for an equality, `>`, the upper bound of `BETWEEN` and a +/// `begins_with` prefix, as literals and parameters). +fn check_key_sizes( + read: &KeyRead<'_>, + schema: &[KeySchemaElement], + limits: &LimitsConfig, + params: &[AttributeValue], +) -> Result<(), DynamoDbError> { + let check = |key_type: KeyType, operand: &Operand| { + let (Some(element), Some(value)) = ( + schema.iter().find(|k| k.key_type == key_type), + bound_value(operand, params), + ) else { + return Ok(()); + }; + let key: Item = [(element.attribute_name.clone(), value.clone())] + .into_iter() + .collect(); + validate_key_size_limits(&key, std::slice::from_ref(element), limits) + }; + check(KeyType::Hash, read.partition)?; + if let Some(condition) = &read.sort { + for operand in condition.values() { + check(KeyType::Range, operand)?; + } + } + Ok(()) +} + +#[cfg(test)] +mod tests; diff --git a/crates/engine/src/partiql/select/read_plan/tests.rs b/crates/engine/src/partiql/select/read_plan/tests.rs new file mode 100644 index 000000000..9a6bf0e23 --- /dev/null +++ b/crates/engine/src/partiql/select/read_plan/tests.rs @@ -0,0 +1,590 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Unit tests for the WHERE checks and the read plan of a SELECT. + +use super::super::tests::{HASH_ONLY, KEYS, select}; +use super::*; + +mod decomposed; +mod held_back; +mod key_order; + +/// The checks a WHERE of a table read without an ORDER BY gets before +/// anything is read, in the order `run` makes them. +fn check_where<'a, 'b>( + analysis: &'b WhereAnalysis<'a>, + keys: KeyNames<'_>, + target_schema: &[KeySchemaElement], + key_info: &TableKeyInfo, + limits: &LimitsConfig, + params: &[AttributeValue], + value_of: &dyn Fn(&Operand) -> Option, +) -> Result, DynamoDbError> { + check_keys(Some(analysis), None, keys, key_info, true, params, value_of)?; + check_reads( + analysis, + keys, + target_schema, + limits, + params, + value_of, + &|_| None, + ) +} + +/// A pk/sk table whose sort key has type `sort`. +fn table(sort: extenddb_core::types::ScalarAttributeType) -> TableKeyInfo { + use extenddb_core::types::{AttributeDefinition, ScalarAttributeType}; + let key = |name: &str, key_type| KeySchemaElement { + attribute_name: name.to_owned(), + key_type, + }; + let definition = |name: &str, attribute_type| AttributeDefinition { + attribute_name: name.to_owned(), + attribute_type, + }; + TableKeyInfo { + table_name: "t".to_owned(), + key_schema: vec![key("pk", KeyType::Hash), key("sk", KeyType::Range)], + attribute_definitions: vec![ + definition("pk", ScalarAttributeType::S), + definition("sk", sort), + ], + ..TableKeyInfo::default() + } +} + +/// The refusal `check_where` gives a statement on a table whose sort key +/// has type `sort`, or `None`. +fn answer( + sort: extenddb_core::types::ScalarAttributeType, + condition: &str, + params: &[AttributeValue], +) -> Option { + let s = select(&format!("SELECT * FROM \"t\" WHERE {condition}")); + let info = table(sort); + let value_of = |op: &Operand| bound_value(op, params).cloned(); + let analysis = WhereAnalysis::new(s.where_clause.as_ref().unwrap(), KEYS, &value_of); + check_where( + &analysis, + KEYS, + &info.key_schema, + &info, + &LimitsConfig::default(), + params, + &value_of, + ) + .err() + .map(|e| e.message().to_owned()) +} + +fn numbered(condition: &str, params: &[AttributeValue]) -> Option { + answer( + extenddb_core::types::ScalarAttributeType::N, + condition, + params, + ) +} + +fn key_type(name: &str) -> Option { + Some(format!( + "Key attribute's data type should match its data type in table's schema: Key {name}" + )) +} + +#[test] +fn each_disjunct_read_by_key_has_its_values_checked_in_order() { + for (condition, name) in [ + ("pk = 1", "pk"), + ("1 = pk", "pk"), + ("pk = 'a' AND sk = 'b'", "sk"), + ("pk = 'a' AND 'b' = sk", "sk"), + ("pk = 'a' AND 'b' < sk", "sk"), + ("pk = 'a' AND sk BETWEEN 1 AND 'z'", "sk"), + ("pk = 'a' AND begins_with(sk, 'x')", "sk"), + ("pk IN [1, 2]", "pk"), + ("pk = 1 OR a = 'x'", "pk"), + ("pk = 'a' OR pk = 1 OR a = 1", "pk"), + ("(pk = 1 AND sk = 1) OR (pk = 'b' AND sk = 'x')", "pk"), + ("(pk = 'b' AND sk = 'x') OR (pk = 1 AND sk = 1)", "sk"), + // The sort key value is checked before the partition key value. + ("pk = 1 AND sk = 'b'", "sk"), + ("sk = 'b' AND pk = 1", "sk"), + ("pk IN [1, 'b'] AND sk = 'x'", "sk"), + // Key values come before the overlap refusal. + ("pk = 'a' AND (a = 1 OR sk = 'x')", "sk"), + ] { + assert_eq!(numbered(condition, &[]), key_type(name), "{condition}"); + } + assert_eq!( + numbered("pk = ?", &[AttributeValue::N("1".to_owned())]), + key_type("pk") + ); +} + +#[test] +fn values_outside_a_disjunct_read_by_key_are_not_checked() { + for condition in [ + "pk = 'a' AND sk > 1", + "sk = 'b'", + "sk > 'b' AND a = 1", + "NOT pk = 1", + "pk <> 1", + "pk = 'a' AND sk <> 'b'", + "pk = 'a' AND NOT sk = 'b'", + "pk = 'a' AND sk = 'b' AND sk = 'c'", + "pk = 'a' AND sk = 1 AND contains(sk, 'x')", + "pk = 1 AND pk = 'a'", + "a = 1 OR pk = 1", + "a = 'x' OR (b = 1 AND pk = 1)", + "pk = 'a' OR sk = 'b'", + "pk = 'a' AND (pk = 1 OR a = 1)", + ] { + assert_eq!(numbered(condition, &[]), None, "{condition}"); + } + // A parameter the request does not supply is left to the count check. + assert_eq!(numbered("pk = ?", &[]), None); +} + +#[test] +fn a_begins_with_key_condition_takes_a_prefix_of_the_key_type() { + use extenddb_core::types::ScalarAttributeType::{B, N, S}; + let operand = |label: &str| { + Some(format!( + "Incorrect operand type for operator or function; operator or function: \ + begins_with, operand type: {label}" + )) + }; + let binary = AttributeValue::B(b"x".to_vec()); + let string = AttributeValue::S("x".to_owned()); + for (sort, prefix, want) in [ + (S, AttributeValue::N("1".to_owned()), operand("N")), + (S, AttributeValue::Bool(true), operand("BOOL")), + (N, AttributeValue::N("1".to_owned()), operand("N")), + (S, binary.clone(), key_type("sk")), + (N, string.clone(), key_type("sk")), + (N, binary.clone(), key_type("sk")), + (B, string.clone(), key_type("sk")), + (S, string, None), + (B, binary, None), + ] { + let params = [prefix]; + assert_eq!( + answer(sort, "pk = 'a' AND begins_with(sk, ?)", ¶ms), + want, + "{sort:?} {:?}", + params[0] + ); + } +} + +#[test] +fn a_read_of_one_disjunct_has_its_key_sizes_checked() { + use extenddb_core::types::ScalarAttributeType::S; + let hash = Some( + "One or more parameter values were invalid: Size of hashkey has exceeded the \ + maximum size limit of2048 bytes" + .to_owned(), + ); + let range = Some( + "One or more parameter values were invalid: Aggregated size of all range keys has \ + exceeded the size limit of 1024 bytes" + .to_owned(), + ); + let (h_over, h_max) = ("a".repeat(2049), "a".repeat(2048)); + let (r_over, r_max) = ("a".repeat(1025), "a".repeat(1024)); + for (condition, want) in [ + (format!("pk = '{h_over}'"), hash.clone()), + (format!("pk = '{}'", "\u{e9}".repeat(1025)), hash.clone()), + (format!("pk = '{h_max}'"), None), + (format!("pk = 'a' AND sk = '{r_over}'"), range.clone()), + (format!("pk = 'a' AND sk > '{r_over}'"), range.clone()), + ( + format!("pk = 'a' AND sk BETWEEN 'a' AND '{r_over}'"), + range.clone(), + ), + ( + format!("pk = 'a' AND begins_with(sk, '{r_over}')"), + range.clone(), + ), + (format!("pk = 'a' AND sk = '{r_max}'"), None), + (format!("pk = '{h_over}' AND sk = '{r_over}'"), hash.clone()), + // Every disjunct fails, so the statement is refused at once. + (format!("pk IN ['{h_over}']"), hash.clone()), + (format!("pk = '{h_over}' OR pk = 'b{h_over}'"), hash.clone()), + // A disjunct that is fine leaves the others for later, and a + // scan is not checked. + (format!("pk IN ['a', '{h_over}']"), None), + (format!("pk = 'a' OR pk = '{h_over}'"), None), + (format!("pk > '{h_over}'"), None), + (format!("sk = '{r_over}'"), None), + ] { + let shown: String = condition.chars().take(40).collect(); + assert_eq!(answer(S, &condition, &[]), want, "{shown}"); + } + let params = [AttributeValue::S(h_over)]; + assert_eq!(answer(S, "pk = ?", ¶ms), hash); +} + +#[test] +fn a_keyed_filter_with_a_key_attribute_is_refused_after_the_overlap_check() { + use extenddb_core::types::ScalarAttributeType::S; + assert_eq!( + answer(S, "pk = 'a' AND sk = a", &[]), + Some( + "Filter Expression can only contain non-primary key attributes: Primary key \ + attribute: sk" + .to_owned() + ) + ); + assert_eq!( + answer(S, "pk = 'a' AND (sk = a OR a = 'x')", &[]), + Some("Overlapping conditions with range keys are not supported in where clause".to_owned()) + ); + // A key value of the wrong type is reported first. + assert_eq!( + numbered("pk = 'a' AND sk = 'x' AND sk = a", &[]), + key_type("sk") + ); + // In a scan the filter may name a key attribute. + assert_eq!(answer(S, "sk = a", &[]), None); + assert_eq!(answer(S, "pk = 'a' AND size(sk) = 1", &[]), None); +} + +/// A table keyed on a string pk alone. +fn hash_table() -> TableKeyInfo { + let mut info = table(extenddb_core::types::ScalarAttributeType::S); + info.key_schema.truncate(1); + info.attribute_definitions.truncate(1); + info +} + +/// How `check_where` leaves a statement on a string-keyed table: whether +/// it reads by key, how many disjuncts are read, and the refusal left +/// for later; or the refusal it gets at once. +fn deferral(condition: &str) -> Result<(bool, usize, Option), String> { + let s = select(&format!("SELECT * FROM \"t\" WHERE {condition}")); + let info = table(extenddb_core::types::ScalarAttributeType::S); + let value_of = |op: &Operand| bound_value(op, &[]).cloned(); + let analysis = WhereAnalysis::new(s.where_clause.as_ref().unwrap(), KEYS, &value_of); + check_where( + &analysis, + KEYS, + &info.key_schema, + &info, + &LimitsConfig::default(), + &[], + &value_of, + ) + .map(|c| { + ( + c.keyed, + c.reads.len(), + c.deferred.map(|e| e.message().to_owned()), + ) + }) + .map_err(|e| e.message().to_owned()) +} + +#[test] +fn a_disjunct_that_fails_when_read_is_left_for_later() { + let hash = "One or more parameter values were invalid: Size of hashkey has exceeded the \ + maximum size limit of2048 bytes"; + let range = "One or more parameter values were invalid: Aggregated size of all range \ + keys has exceeded the size limit of 1024 bytes"; + let filter = "Filter Expression can only contain non-primary key attributes: Primary \ + key attribute: sk"; + let scalar = "Key attributes must be scalars; list random access '[]' and map lookup \ + '.' are not allowed: Key sk"; + let (h, r) = ("z".repeat(2049), "z".repeat(1025)); + let later = |reads: usize, message: &str| Ok((true, reads, Some(message.to_owned()))); + for (condition, want) in [ + (format!("pk IN ['a', '{h}']"), later(1, hash)), + (format!("pk IN ['{h}', 'a']"), later(1, hash)), + ( + format!("pk = 'a' OR pk = '{h}' OR pk = 'b'"), + later(2, hash), + ), + ( + format!("(pk = 'a' AND sk = 'x') OR (pk = 'b' AND sk = '{r}')"), + later(1, range), + ), + (format!("pk = 'a' AND sk IN ['x', '{r}']"), later(1, range)), + ( + "(pk = 'a' AND sk = a) OR pk = 'b'".to_owned(), + later(1, filter), + ), + ( + "pk = 'b' OR (pk = 'a' AND sk = a)".to_owned(), + later(1, filter), + ), + ( + "(pk = 'a' AND sk = a) OR pk = 'b' OR pk = 'c'".to_owned(), + later(2, filter), + ), + // A path into a key attribute is refused at once, ahead of the + // overlap and key type refusals, and when a later disjunct makes + // the statement a scan. + ( + "(pk = 'a' AND sk.x = 1) OR pk = 'b'".to_owned(), + Err(scalar.to_owned()), + ), + ( + "(pk = 'a' AND sk.x = 1) OR (pk = 'a' AND sk = 'y')".to_owned(), + Err(scalar.to_owned()), + ), + ( + "(pk = 'a' AND sk = 'y') OR (pk = 'a' AND sk.x = 1)".to_owned(), + Err(scalar.to_owned()), + ), + ( + "(pk = 'a' AND sk.x = 1) OR pk = 1".to_owned(), + Err(scalar.to_owned()), + ), + ( + "pk = 1 OR (pk = 'a' AND sk.x = 1)".to_owned(), + Err(scalar.to_owned()), + ), + ( + "(pk = 'a' AND sk.x = 1) OR x = 1".to_owned(), + Err(scalar.to_owned()), + ), + // Of several failing disjuncts, the one read last is reported. + ( + format!("(pk = 'a' AND sk = a) OR pk = '{h}' OR pk = 'b'"), + later(1, hash), + ), + ( + format!("pk = '{h}' OR (pk = 'a' AND sk = a) OR pk = 'b'"), + later(1, hash), + ), + ( + format!("(pk = 'b' AND sk = '{r}') OR pk = '{h}' OR pk = 'a'"), + later(1, hash), + ), + ( + "(pk = 'b' AND sk = a) OR (pk = 'a' AND pk = b) OR pk = 'c'".to_owned(), + later(1, filter), + ), + ( + "(pk = 'a' AND pk = b) OR (pk = 'b' AND sk = a) OR pk = 'c'".to_owned(), + later(1, filter), + ), + // Every disjunct fails: refused at once. + ( + "pk IN ['a', 'b'] AND sk = a".to_owned(), + Err(filter.to_owned()), + ), + ( + "(pk = 'a' AND sk = a) OR (pk = 'b' AND sk = b)".to_owned(), + Err(filter.to_owned()), + ), + // Nothing fails, or the statement is a scan. + ("pk IN ['a', 'b']".to_owned(), Ok((true, 2, None))), + ( + "(pk = 'a' AND sk = a) OR x = 1".to_owned(), + Ok((false, 1, None)), + ), + ] { + let shown: String = condition.chars().take(60).collect(); + assert_eq!(deferral(&condition), want, "{shown}"); + } +} + +#[test] +fn the_read_follows_the_disjuncts() { + let plan = |condition: &str, info: &TableKeyInfo, on_table: bool| { + let keys = if info.key_schema.len() == 1 { + HASH_ONLY + } else { + KEYS + }; + let s = select(&format!("SELECT * FROM \"t\" WHERE {condition}")); + let value_of = |op: &Operand| bound_value(op, &[]).cloned(); + let analysis = WhereAnalysis::new(s.where_clause.as_ref().unwrap(), keys, &value_of); + let checked = check_where( + &analysis, + keys, + &info.key_schema, + info, + &LimitsConfig::default(), + &[], + &value_of, + ) + .unwrap(); + let text = |v: &AttributeValue| match v { + AttributeValue::S(s) => s.clone(), + other => format!("{other:?}"), + }; + match ReadPlan::of( + Some(&checked), + keys, + on_table, + Direction::default(), + &value_of, + ) { + ReadPlan::Key(read) => format!("key +{}", read.filter.len()), + // A step that names one item shows its key; a Query shows its + // partition and a star. + ReadPlan::Steps(steps) => { + let shown: Vec = steps + .iter() + .map(|step| { + let partition = text(&value_of(step.read.partition).unwrap()); + let shown = match (&step.point, keys.sort, &step.read.sort) { + (None, _, _) => format!("{partition}*"), + ( + Some(_), + Some(_), + Some(SortCondition::Compare(CompareOp::Eq, sort)), + ) => format!("{partition}/{}", text(&value_of(sort).unwrap())), + (Some(_), _, _) => partition, + }; + // A step held back for its refusal shows a mark. + if step.held_back { + format!("!{shown}") + } else { + shown + } + }) + .collect(); + format!("steps {}", shown.join(" ")) + } + ReadPlan::Scan => "scan".to_owned(), + } + }; + let both = table(extenddb_core::types::ScalarAttributeType::S); + let hash = hash_table(); + for (condition, info, on_table, want) in [ + ("pk = 'a' AND sk = 'b'", &both, true, "steps a/b"), + ("pk = 'a' AND sk = 'b'", &both, false, "key +0"), + ("pk = 'a' AND sk = 'b' AND n = 1", &both, true, "steps a/b"), + ("pk = 'a' AND name = 'x'", &both, true, "key +1"), + ("pk = 'a' AND (x = 1 OR y = 2)", &both, true, "key +1"), + ("pk = 'a' AND sk > 'b'", &both, true, "key +0"), + ( + "pk = 'a' AND sk IN ['y', 'x']", + &both, + true, + "steps a/x a/y", + ), + // An index is read by one Query for each disjunct. + ("pk = 'a' AND sk IN ['x', 'y']", &both, false, "steps a* a*"), + ( + "pk = 'a' AND sk = 'x' OR 'a' = pk AND sk > 'y'", + &both, + true, + "steps a/x a*", + ), + ( + "pk = 'a' AND sk > 'y' OR 'a' = pk AND sk < 'x'", + &both, + true, + "steps a* a*", + ), + ( + "(pk = 'b' AND sk = 'x') OR (pk = 'a' AND sk = 'x')", + &both, + true, + "steps a/x b/x", + ), + ("pk IN ['b', 'a']", &both, true, "steps a* b*"), + ( + "pk = 'b' OR (pk = 'a' AND sk = 'x')", + &both, + true, + "steps a/x b*", + ), + ("pk IN ['a', 'b']", &both, false, "steps a* b*"), + ( + "(pk = 'a' AND sk = a) OR pk = 'b'", + &both, + false, + "steps !a* b*", + ), + ("pk = 'a' OR sk = 'b'", &both, true, "scan"), + ("pk = 'a' AND sk <> 'b'", &both, true, "scan"), + ("pk = \"a\"", &both, true, "scan"), + ("pk IN ['c', 'a', 'b']", &hash, true, "steps a b c"), + ( + "pk = 'c' OR pk = 'a' OR pk = 'zz'", + &hash, + true, + "steps a c zz", + ), + ("pk = 'a'", &hash, true, "steps a"), + // The disjuncts held back for a refusal take their places among + // the others and read nothing. + ( + "(pk = 'a' AND sk = a) OR pk = 'b'", + &both, + true, + "steps !a* b*", + ), + ( + "(pk = 'a' AND sk = a) OR (pk = 'b' AND sk = 'x')", + &both, + true, + "steps !a* b/x", + ), + ( + "(pk = 'c' AND sk = a) OR pk = 'b' OR pk = 'a'", + &both, + true, + "steps a* b* !c*", + ), + ] { + assert_eq!(plan(condition, info, on_table), want, "{condition}"); + } +} + +#[test] +fn disjuncts_are_read_in_key_order_or_its_reverse() { + let s = select("SELECT * FROM \"t\" WHERE pk IN [10, 9, 100]"); + let value_of = |op: &Operand| bound_value(op, &[]).cloned(); + let analysis = WhereAnalysis::new(s.where_clause.as_ref().unwrap(), HASH_ONLY, &value_of); + let reads: Vec<&KeyRead<'_>> = analysis.reads().iter().collect(); + let order = |partitions_descending| { + let direction = Direction { + partitions_descending, + items_descending: false, + }; + Step::all(&reads, &[], Some(HASH_ONLY), direction, &value_of) + .into_iter() + .map(|step| step.point.unwrap()) + .collect::>() + }; + let n = |v: &str| serde_json::json!({ "pk": { "N": v } }); + assert_eq!(order(false), vec![n("9"), n("10"), n("100")]); + assert_eq!(order(true), vec![n("100"), n("10"), n("9")]); +} + +#[test] +fn partitions_and_the_items_of_a_partition_take_their_own_directions() { + let s = select("SELECT * FROM \"t\" WHERE pk IN ['b', 'a'] AND sk IN ['y', 'x']"); + let value_of = |op: &Operand| bound_value(op, &[]).cloned(); + let analysis = WhereAnalysis::new(s.where_clause.as_ref().unwrap(), KEYS, &value_of); + let reads: Vec<&KeyRead<'_>> = analysis.reads().iter().collect(); + let order = |partitions_descending, items_descending| { + let direction = Direction { + partitions_descending, + items_descending, + }; + Step::all(&reads, &[], Some(KEYS), direction, &value_of) + .into_iter() + .map(|step| { + let key = step.point.unwrap(); + format!( + "{}/{}", + key["pk"]["S"].as_str().unwrap(), + key["sk"]["S"].as_str().unwrap() + ) + }) + .collect::>() + }; + assert_eq!(order(false, false), ["a/x", "a/y", "b/x", "b/y"]); + assert_eq!(order(true, false), ["b/x", "b/y", "a/x", "a/y"]); + assert_eq!(order(false, true), ["a/y", "a/x", "b/y", "b/x"]); + assert_eq!(order(true, true), ["b/y", "b/x", "a/y", "a/x"]); +} diff --git a/crates/engine/src/partiql/select/read_plan/tests/decomposed.rs b/crates/engine/src/partiql/select/read_plan/tests/decomposed.rs new file mode 100644 index 000000000..bed69eb54 --- /dev/null +++ b/crates/engine/src/partiql/select/read_plan/tests/decomposed.rs @@ -0,0 +1,166 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Unit tests for the refusal of a statement read by key that splits into +//! more than 50 disjuncts, and where it sits among the other refusals. + +use super::*; +use extenddb_core::types::ScalarAttributeType::S; + +const TOO_MANY: &str = "Too many decomposed read operations for a given query."; + +/// `000, 001, ...`, `n` string values. +fn values(prefix: &str, n: usize) -> String { + let values: Vec = (0..n).map(|i| format!("'{prefix}{i:03}'")).collect(); + values.join(", ") +} + +/// `n` equalities of the partition key ORed. +fn ors(n: usize) -> String { + let ors: Vec = (0..n).map(|i| format!("pk = 'v{i:03}'")).collect(); + ors.join(" OR ") +} + +/// `n` comparisons of other attributes ANDed. +fn comparisons(n: usize) -> String { + let terms: Vec = (1..=n).map(|i| format!("x{i:02} = 1")).collect(); + terms.join(" AND ") +} + +#[test] +fn a_statement_read_by_key_splits_into_at_most_50_disjuncts() { + let too_many = || Some(TOO_MANY.to_owned()); + let products = + |a: usize, b: usize| format!("pk IN [{}] AND sk IN [{}]", values("v", a), values("s", b)); + for (condition, want) in [ + (format!("pk IN [{}]", values("v", 50)), None), + (format!("pk IN [{}]", values("v", 51)), too_many()), + (ors(50), None), + (ors(51), too_many()), + (products(25, 2), None), + (products(5, 10), None), + (products(17, 3), too_many()), + (products(10, 10), too_many()), + (products(100, 100), too_many()), + ( + format!("pk IN [{}] AND (sk = 's' OR sk = 't')", values("v", 26)), + too_many(), + ), + ( + format!( + "(pk IN [{}] AND sk = 's') OR (pk IN [{}] AND sk = 't')", + values("v", 26), + values("w", 25) + ), + too_many(), + ), + (format!("pk = 'v000' AND sk IN [{}]", values("s", 50)), None), + ( + format!("pk = 'v000' AND sk IN [{}]", values("s", 51)), + too_many(), + ), + ( + format!("pk IN [{}] OR pk IN [{}]", values("v", 26), values("w", 25)), + too_many(), + ), + ( + format!("pk IN [{}] AND sk > 'a'", values("v", 51)), + too_many(), + ), + // The conditions of a filter are not counted. + ( + format!("pk IN [{}] AND {}", values("v", 50), comparisons(99)), + None, + ), + ( + format!("pk IN [{}] AND {}", values("v", 51), comparisons(1)), + too_many(), + ), + // A statement that is a scan is not counted. + (format!("pk IN [{}] OR x = 1", values("v", 51)), None), + (format!("x = 1 OR pk IN [{}]", values("v", 51)), None), + (format!("({}) OR x = 1", products(100, 100)), None), + ] { + let shown: String = condition.chars().take(60).collect(); + assert_eq!(answer(S, &condition, &[]), want, "{shown}"); + } + let params: Vec = (0..51) + .map(|i| AttributeValue::S(format!("v{i:03}"))) + .collect(); + let marks = |n: usize| vec!["?"; n].join(", "); + assert_eq!( + answer(S, &format!("pk IN [{}]", marks(51)), ¶ms), + too_many() + ); + assert_eq!( + answer(S, &format!("pk IN [{}]", marks(50)), ¶ms[..50]), + None + ); +} + +#[test] +fn the_disjuncts_held_back_for_a_refusal_are_counted_before_it() { + let big = "z".repeat(2049); + let hash = "One or more parameter values were invalid: Size of hashkey has exceeded the \ + maximum size limit of2048 bytes"; + let filter = "Filter Expression can only contain non-primary key attributes: Primary key \ + attribute: sk"; + for (condition, want) in [ + ( + format!("pk IN [{}, '{big}']", values("v", 50)), + Some(TOO_MANY), + ), + ( + format!("pk IN ['{big}', {}]", values("v", 50)), + Some(TOO_MANY), + ), + (format!("pk IN [{}, '{big}']", values("v", 49)), None), + ( + format!("pk IN [{}] AND sk = a", values("v", 51)), + Some(TOO_MANY), + ), + ( + format!("pk IN [{}] AND sk = a", values("v", 50)), + Some(filter), + ), + (format!("pk IN ['{big}']"), Some(hash)), + ] { + let shown: String = condition.chars().take(60).collect(); + assert_eq!(answer(S, &condition, &[]).as_deref(), want, "{shown}"); + } +} + +#[test] +fn key_values_and_the_overlap_refusal_come_before_the_count() { + let scalar = "Key attributes must be scalars; list random access '[]' and map lookup '.' \ + are not allowed: Key sk"; + let overlap = "Overlapping conditions with range keys are not supported in where clause"; + let repeated: Vec = (0..51).map(|i| format!("'v{:03}'", i % 26)).collect(); + for (condition, want) in [ + (format!("pk IN [{}, 1]", values("v", 50)), key_type("pk")), + (format!("pk IN [1, {}]", values("v", 50)), key_type("pk")), + ( + format!("pk IN [{}] AND sk = 1", values("v", 51)), + key_type("sk"), + ), + ( + format!("pk IN [{}] AND sk.x = 1", values("v", 51)), + Some(scalar.to_owned()), + ), + ( + format!("pk IN [{}]", repeated.join(", ")), + Some(overlap.to_owned()), + ), + ( + format!("pk IN [{}] AND (sk = 's' OR sk > 'a')", values("v", 26)), + Some(overlap.to_owned()), + ), + ( + format!("pk IN [{}] OR pk IN [{}]", values("v", 26), values("v", 26)), + Some(overlap.to_owned()), + ), + ] { + let shown: String = condition.chars().take(60).collect(); + assert_eq!(answer(S, &condition, &[]), want, "{shown}"); + } +} diff --git a/crates/engine/src/partiql/select/read_plan/tests/held_back.rs b/crates/engine/src/partiql/select/read_plan/tests/held_back.rs new file mode 100644 index 000000000..1404ca8b2 --- /dev/null +++ b/crates/engine/src/partiql/select/read_plan/tests/held_back.rs @@ -0,0 +1,113 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Unit tests for the disjuncts the caller may not read, held back with the +//! disjuncts that fail a check made when they are read. + +use super::*; +use extenddb_core::types::ScalarAttributeType::S; + +const HASH: &str = "One or more parameter values were invalid: Size of hashkey has exceeded \ + the maximum size limit of2048 bytes"; +const FILTER: &str = "Filter Expression can only contain non-primary key attributes: Primary \ + key attribute: sk"; + +/// How `check_reads` leaves a statement on a string-keyed table when the +/// caller may not read the partitions `denied`: the number of disjuncts read +/// and the refusal left for later, or the refusal the statement gets at +/// once. The refusal of a denied partition `v` is "denied v". +fn held_back(condition: &str, denied: &[&str]) -> Result<(usize, Option), String> { + let s = select(&format!("SELECT * FROM \"t\" WHERE {condition}")); + let info = table(S); + let value_of = |op: &Operand| bound_value(op, &[]).cloned(); + let analysis = WhereAnalysis::new(s.where_clause.as_ref().unwrap(), KEYS, &value_of); + let refusal = |read: &KeyRead<'_>| { + let Some(AttributeValue::S(partition)) = value_of(read.partition) else { + return None; + }; + denied + .contains(&partition.as_str()) + .then(|| DynamoDbError::AccessDeniedException(format!("denied {partition}"))) + }; + check_reads( + &analysis, + KEYS, + &info.key_schema, + &LimitsConfig::default(), + &[], + &value_of, + &refusal, + ) + .map(|c| (c.reads.len(), c.deferred.map(|e| e.message().to_owned()))) + .map_err(|e| e.message().to_owned()) +} + +#[test] +fn a_denied_disjunct_is_refused_for_authorization_before_its_checks() { + // Measured 2026-11-03 under ForAllValues LeadingKeys ["a"]: a disjunct + // the policy refuses that is past the key size limit gets AccessDenied + // on the page after the allowed disjuncts (D7, D8), and so does one with + // a filter on a key attribute (D15, under ["b"]). + let h = "z".repeat(2049); + let later = |reads: usize, message: String| Ok((reads, Some(message))); + assert_eq!( + held_back(&format!("pk IN ['a', 'b', '{h}']"), &["b", &h]), + later(1, format!("denied {h}")) + ); + assert_eq!( + held_back(&format!("pk IN ['{h}', 'a', 'b']"), &["b", &h]), + later(1, format!("denied {h}")) + ); + assert_eq!( + held_back("(pk = 'a' AND sk = a) OR pk = 'b' OR pk = 'c'", &["a", "c"]), + later(1, "denied c".to_owned()) + ); + assert_eq!( + held_back("(pk = 'b' AND sk = a) OR pk = 'a'", &["b"]), + later(1, "denied b".to_owned()) + ); +} + +#[test] +fn an_authorization_refusal_ranks_below_the_refusals_of_the_checks() { + // Measured 2026-11-03 under ForAllValues LeadingKeys ["a"]: both orders + // are refused at once with the filter refusal of the disjunct the policy + // allows, though b is read later (D13, D14). + for condition in [ + "(pk = 'a' AND sk = a) OR pk = 'b'", + "pk = 'b' OR (pk = 'a' AND sk = a)", + ] { + assert_eq!( + held_back(condition, &["b"]), + Err(FILTER.to_owned()), + "{condition}" + ); + } + // With a disjunct that is read, the refusal left for later is a + // check's, the last in read order of those. + let h = "z".repeat(2049); + assert_eq!( + held_back("pk = 'a' OR (pk = 'b' AND sk = a) OR pk = 'c'", &["c"]), + Ok((1, Some(FILTER.to_owned()))) + ); + assert_eq!( + held_back( + &format!("(pk = 'a' AND sk = a) OR pk = '{h}' OR pk = 'b' OR pk = 'c'"), + &["c"] + ), + Ok((1, Some(HASH.to_owned()))) + ); +} + +#[test] +fn a_statement_whose_every_disjunct_is_denied_gets_the_last_refusal() { + assert_eq!( + held_back("pk IN ['c', 'b']", &["b", "c"]), + Err("denied c".to_owned()) + ); + assert_eq!( + held_back("pk IN ['a', 'b']", &[]), + Ok((2, None)), + "nothing is denied" + ); +} diff --git a/crates/engine/src/partiql/select/read_plan/tests/key_order.rs b/crates/engine/src/partiql/select/read_plan/tests/key_order.rs new file mode 100644 index 000000000..44fcc15eb --- /dev/null +++ b/crates/engine/src/partiql/select/read_plan/tests/key_order.rs @@ -0,0 +1,116 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Unit tests for the order of the key checks and the ORDER BY refusals of +//! a SELECT on a table. + +use super::*; +use extenddb_core::types::ScalarAttributeType::S; + +const SCALAR: &str = "Key attributes must be scalars; list random access '[]' and map lookup \ + '.' are not allowed: Key pk"; +const TERM: &str = "Attribute n in ORDER BY clause must be part of the primary key"; +const HASH_KEY: &str = "Must have hash key in ORDER BY clause when more than one hash key \ + condition specified in WHERE clause."; +const OVERLAP: &str = "Overlapping conditions with range keys are not supported in where clause"; +const NO_WHERE: &str = "Must have WHERE clause in the statement when using ORDER BY clause."; +const TOO_MANY: &str = "Too many decomposed read operations for a given query."; + +/// The refusal `check_keys` gives `SELECT * FROM "t" ` on the pk/sk +/// table with string keys, or `None`. +fn refusal(tail: &str) -> Option { + let s = select(&format!("SELECT * FROM \"t\" {tail}")); + let info = table(S); + let value_of = |op: &Operand| bound_value(op, &[]).cloned(); + let analysis = s + .where_clause + .as_ref() + .map(|w| WhereAnalysis::new(w, KEYS, &value_of)); + check_keys( + analysis.as_ref(), + s.order_by.as_ref(), + KEYS, + &info, + true, + &[], + &value_of, + ) + .err() + .map(|e| e.message().to_owned()) +} + +fn values(n: usize) -> String { + let values: Vec = (0..n).map(|i| format!("'v{i:03}'")).collect(); + values.join(", ") +} + +#[test] +fn the_order_by_refusals_sit_among_the_key_checks() { + // Measured 2026-11-03 for a permitted caller, each WHERE with ORDER BY n + // and with ORDER BY sk: a path into a key, then the ORDER BY term, then + // the key values, then more than one partition key value without the + // partition key in the list, then overlapping disjuncts. + let key_type = |name: &str| key_type(name).unwrap(); + let fifty_one = format!("pk IN [{}]", values(51)); + for (condition, by_n, by_sk) in [ + ("pk.x = 1", SCALAR.to_owned(), SCALAR.to_owned()), + ("pk = 1", TERM.to_owned(), key_type("pk")), + ( + "(pk = 'a' OR pk = 'a')", + TERM.to_owned(), + OVERLAP.to_owned(), + ), + ( + "(pk = 'a' OR pk = 'b' OR pk = 'a')", + TERM.to_owned(), + HASH_KEY.to_owned(), + ), + (&fifty_one, TERM.to_owned(), HASH_KEY.to_owned()), + ("pk IN ['a', 1]", TERM.to_owned(), key_type("pk")), + ("pk = 'a' AND sk = 1", TERM.to_owned(), key_type("sk")), + ( + "pk = 'a' AND (sk = 'x' OR sk > 'a')", + TERM.to_owned(), + OVERLAP.to_owned(), + ), + ] { + let shown: String = condition.chars().take(40).collect(); + assert_eq!( + refusal(&format!("WHERE {condition} ORDER BY n")).as_deref(), + Some(by_n.as_str()), + "{shown} ORDER BY n" + ); + assert_eq!( + refusal(&format!("WHERE {condition} ORDER BY sk")).as_deref(), + Some(by_sk.as_str()), + "{shown} ORDER BY sk" + ); + } + assert_eq!(refusal("ORDER BY pk.x").as_deref(), Some(NO_WHERE)); + assert_eq!(refusal("WHERE n = 1 ORDER BY n").as_deref(), Some(TERM)); +} + +#[test] +fn the_key_checks_without_an_order_by_keep_their_order() { + // Measured 2026-11-03: a key value of the wrong type ahead of overlap + // and of the count, a path into a key ahead of a key value. + let key_type = |name: &str| key_type(name).unwrap(); + for (condition, want) in [ + ("pk = 1 OR pk = 1".to_owned(), key_type("pk")), + (format!("pk IN [{}, 1]", values(50)), key_type("pk")), + ("pk.x = 1 AND pk = 1".to_owned(), SCALAR.to_owned()), + (format!("pk IN [{}]", values(51)), TOO_MANY.to_owned()), + ( + format!("pk IN [{}] ORDER BY pk", values(51)), + TOO_MANY.to_owned(), + ), + ] { + let shown: String = condition.chars().take(40).collect(); + assert_eq!( + refusal(&format!("WHERE {condition}")).as_deref(), + Some(want.as_str()), + "{shown}" + ); + } + assert_eq!(refusal("WHERE pk IN ['a', 'b'] ORDER BY pk, sk"), None); +} diff --git a/crates/engine/src/partiql/select/request.rs b/crates/engine/src/partiql/select/request.rs new file mode 100644 index 000000000..37e4b291b --- /dev/null +++ b/crates/engine/src/partiql/select/request.rs @@ -0,0 +1,264 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The classic Query and Scan requests a SELECT is translated into. + +use serde_json::{Map, Value}; + +use extenddb_core::error::DynamoDbError; +use extenddb_core::partiql::ast::{Operand, SelectStatement, WhereExpr}; +use extenddb_core::partiql::plan::{KeyNames, SortCondition}; + +use crate::partiql::SingleRequest; +use crate::partiql::expr_bridge::{self, ExprBridge}; +use crate::{DispatchResult, OperationContext}; + +/// The `KeyConditionExpression` of a key read: the partition equality, and +/// the sort-key condition when there is one, written with the key on the left. +pub(super) fn key_condition( + bridge: &mut ExprBridge<'_>, + keys: KeyNames<'_>, + partition: &Operand, + sort: Option<&SortCondition<'_>>, +) -> Result { + let name = bridge.name_ref(keys.partition); + let value = bridge.operand(partition)?; + let mut kce = format!("{name} = {value}"); + if let (Some(sort_name), Some(condition)) = (keys.sort, sort) { + let name = bridge.name_ref(sort_name); + let condition = match *condition { + SortCondition::Compare(op, value) => { + format!( + "{name} {} {}", + expr_bridge::compare_symbol(op), + bridge.operand(value)? + ) + } + SortCondition::Between(low, high) => { + let low = bridge.operand(low)?; + let high = bridge.operand(high)?; + format!("{name} BETWEEN {low} AND {high}") + } + SortCondition::BeginsWith(prefix) => { + format!("begins_with({name}, {})", bridge.operand(prefix)?) + } + }; + kce.push_str(" AND "); + kce.push_str(&condition); + } + Ok(kce) +} + +/// Where a Query or Scan starts, how many items it may read, and whether a +/// Query reads the items of its partition in descending order (see +/// `order::direction`). +#[derive(Clone, Copy, Default)] +pub(super) struct Paging<'v> { + pub(super) start: Option<&'v Value>, + pub(super) limit: Option, + pub(super) descending: bool, +} + +/// Compose and dispatch a `Query` with a rendered key condition and filter. +pub(super) async fn query_request( + ctx: &OperationContext, + stmt: &SelectStatement, + req: &SingleRequest, + paging: Paging<'_>, + bridge: &ExprBridge<'_>, + kce: String, + filter: Option, +) -> Result { + let mut body = Map::new(); + body.insert( + "TableName".to_owned(), + Value::String(stmt.from.table.clone()), + ); + body.insert("KeyConditionExpression".to_owned(), Value::String(kce)); + if let Some(f) = filter { + body.insert("FilterExpression".to_owned(), Value::String(f)); + } + if let Some(index) = &stmt.from.index { + body.insert("IndexName".to_owned(), Value::String(index.clone())); + } + body.insert( + "ScanIndexForward".to_owned(), + Value::Bool(!paging.descending), + ); + insert_names_values(&mut body, bridge)?; + insert_common(&mut body, req, paging); + + crate::handle_query(Value::Object(body), ctx) + .await + .map_err(crate::partiql::bare_expression_error) +} + +/// Compose and dispatch a `Scan` for a Scan plan, with `filter` as its +/// filter. +pub(super) async fn scan_request( + ctx: &OperationContext, + stmt: &SelectStatement, + req: &SingleRequest, + paging: Paging<'_>, + filter: Option<&WhereExpr>, +) -> Result { + let mut bridge = ExprBridge::new(&req.parameters); + let filter = filter.map(|w| bridge.render_condition(w)).transpose()?; + + let mut body = Map::new(); + body.insert( + "TableName".to_owned(), + Value::String(stmt.from.table.clone()), + ); + if let Some(f) = filter { + body.insert("FilterExpression".to_owned(), Value::String(f)); + } + if let Some(index) = &stmt.from.index { + body.insert("IndexName".to_owned(), Value::String(index.clone())); + } + insert_names_values(&mut body, &bridge)?; + insert_common(&mut body, req, paging); + + crate::handle_scan(Value::Object(body), ctx) + .await + .map_err(crate::partiql::bare_expression_error) +} + +/// Join residual filter conjuncts into one `FilterExpression`, or `None`. +pub(super) fn render_filter( + bridge: &mut ExprBridge<'_>, + filter_terms: &[&WhereExpr], +) -> Result, DynamoDbError> { + if filter_terms.is_empty() { + return Ok(None); + } + let parts: Vec = filter_terms + .iter() + .map(|t| bridge.render_condition(t)) + .collect::>()?; + Ok(Some(parts.join(" AND "))) +} + +/// Attach the accumulated name/value maps to a composed request body. +fn insert_names_values( + body: &mut Map, + bridge: &ExprBridge<'_>, +) -> Result<(), DynamoDbError> { + if let Some(names) = bridge.names_value() { + body.insert("ExpressionAttributeNames".to_owned(), names); + } + if let Some(values) = bridge.values_value()? { + body.insert("ExpressionAttributeValues".to_owned(), values); + } + Ok(()) +} + +/// Attach the request parameters common to Query and Scan: Limit, an opaque +/// pagination cursor, ConsistentRead, and ReturnConsumedCapacity. +fn insert_common(body: &mut Map, req: &SingleRequest, paging: Paging<'_>) { + if let Some(limit) = paging.limit { + body.insert("Limit".to_owned(), Value::from(limit)); + } + if let Some(key) = paging.start { + body.insert("ExclusiveStartKey".to_owned(), key.clone()); + } + if req.consistent_read { + body.insert("ConsistentRead".to_owned(), Value::Bool(true)); + } + if let Some(rcc) = &req.return_consumed_capacity { + body.insert( + "ReturnConsumedCapacity".to_owned(), + Value::String(rcc.clone()), + ); + } +} + +#[cfg(test)] +mod tests { + use super::super::bound_value; + use super::super::read_plan::{ReadPlan, check_reads}; + use super::super::tests::{KEYS, select}; + use super::*; + use crate::partiql::TRANSLATED_EXPRESSION_LIMIT; + use extenddb_core::limits::LimitsConfig; + use extenddb_core::partiql::plan::{Direction, WhereAnalysis, keyed_condition}; + use extenddb_core::types::{KeySchemaElement, KeyType}; + + #[test] + fn the_key_condition_is_written_with_the_key_on_the_left() { + let s = select("SELECT * FROM \"t\" WHERE 'a' = pk AND 1 < sk AND x = 2"); + let value_of = |op: &Operand| bound_value(op, &[]).cloned(); + let analysis = WhereAnalysis::new(s.where_clause.as_ref().unwrap(), KEYS, &value_of); + let read = analysis.single().unwrap(); + let mut bridge = ExprBridge::new(&[]); + let kce = key_condition(&mut bridge, KEYS, read.partition, read.sort.as_ref()).unwrap(); + assert_eq!(kce, "#n0 = :v0 AND #n1 > :v1"); + assert_eq!(read.filter.len(), 1); + } + + #[test] + fn each_query_of_an_index_read_that_holds_a_disjunct_back_stays_within_the_bound() { + // The measured shape (2026-11-03): an index keyed on g read by 49 + // values and one past the key size limit, beside 99 comparisons. + let mut values: Vec = (0..47).map(|i| format!("'a{i:02}'")).collect(); + values.extend(["'a50'".to_owned(), "'a98'".to_owned()]); + let filter: Vec = (1..=99).map(|i| format!("x{i:02} = 1")).collect(); + let sql = format!( + "SELECT pk FROM \"t\".\"gix\" WHERE g IN [{}, '{}'] AND {}", + values.join(", "), + "z".repeat(2049), + filter.join(" AND ") + ); + let s = select(&sql); + let keys = KeyNames { + partition: "g", + sort: None, + }; + let schema = [KeySchemaElement { + attribute_name: "g".to_owned(), + key_type: KeyType::Hash, + }]; + let value_of = |op: &Operand| bound_value(op, &[]).cloned(); + let where_clause = s.where_clause.as_ref().unwrap(); + let analysis = WhereAnalysis::new(where_clause, keys, &value_of); + assert_eq!(analysis.disjunct_count(), 50); + let limits = LimitsConfig::default(); + let checked = + check_reads(&analysis, keys, &schema, &limits, &[], &value_of, &|_| None).unwrap(); + let ReadPlan::Steps(steps) = + ReadPlan::of(Some(&checked), keys, false, Direction::default(), &value_of) + else { + panic!("an index read of several disjuncts reads them one after another"); + }; + assert_eq!(steps.iter().filter(|step| step.held_back).count(), 1); + assert_eq!(steps.len(), 50); + // Each Query carries its own disjunct's filter: the 99 comparisons + // once. + for step in steps.iter().filter(|step| !step.held_back) { + let mut bridge = ExprBridge::new(&[]); + let kce = key_condition( + &mut bridge, + keys, + step.read.partition, + step.read.sort.as_ref(), + ) + .unwrap(); + let text = render_filter(&mut bridge, &step.read.filter) + .unwrap() + .unwrap(); + assert_eq!(text.matches(" AND ").count(), 98, "{text}"); + assert!(kce.len() + text.len() < TRANSLATED_EXPRESSION_LIMIT); + } + // A read that reaches back filters the base-table items on the WHERE + // once. + let joined = ExprBridge::new(&[]) + .render_condition(&keyed_condition(where_clause, keys)) + .unwrap(); + assert!( + joined.len() < 5 * sql.len(), + "{} for {}", + joined.len(), + sql.len() + ); + } +} diff --git a/crates/engine/src/partiql/select/response.rs b/crates/engine/src/partiql/select/response.rs new file mode 100644 index 000000000..3de3280fa --- /dev/null +++ b/crates/engine/src/partiql/select/response.rs @@ -0,0 +1,134 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The ExecuteStatement response of a SELECT and the projection of its items. + +use serde_json::{Map, Value}; + +use extenddb_core::partiql::ast::{Path, PathStep, Projection}; + +use crate::DispatchResult; +use crate::partiql::SingleRequest; +use crate::partiql::next_token::encode_next_token; + +/// Reshape a Query/Scan `DispatchResult` into the PartiQL `ExecuteStatement` +/// response: `Items` (projected), `ConsumedCapacity`, and, when the underlying +/// read was paged, both `LastEvaluatedKey` and an opaque `NextToken` bound to +/// the request's statement and parameter values. +pub(super) fn shape_response( + result: DispatchResult, + req: &SingleRequest, + projection: &Projection, +) -> DispatchResult { + let DispatchResult { body, metrics } = result; + let mut source = match body { + Value::Object(m) => m, + _ => Map::new(), + }; + + let raw_items = match source.remove("Items") { + Some(Value::Array(items)) => items, + _ => Vec::new(), + }; + let items: Vec = match projection { + Projection::All => raw_items, + Projection::Paths(paths) => raw_items + .iter() + .map(|item| project_item(item, paths)) + .collect(), + }; + + let mut out = Map::new(); + out.insert("Items".to_owned(), Value::Array(items)); + if let Some(cc) = source.remove("ConsumedCapacity") { + out.insert("ConsumedCapacity".to_owned(), cc); + } + if let Some(lek) = source.remove("LastEvaluatedKey") { + out.insert( + "NextToken".to_owned(), + Value::String(encode_next_token(&req.statement, &req.parameters, &lek)), + ); + out.insert("LastEvaluatedKey".to_owned(), lek); + } + + DispatchResult { + body: Value::Object(out), + metrics, + } +} + +/// Project one wire-form item (`{"attr": {typed value}}`) down to the selected +/// paths. A plain or nested attribute keys on its leaf name; any path with a +/// list-index step keys on the literal path text. A path that does not resolve +/// is omitted, as the service omits a missing projected attribute. +pub(crate) fn project_item(item: &Value, paths: &[Path]) -> Value { + let mut out = Map::new(); + for path in paths { + if let Some(value) = navigate(item, path) { + out.insert(output_key(path), value); + } + } + Value::Object(out) +} + +/// Walk a wire-form item along a document path, descending into `M` for an +/// attribute step and `L` for a list-index step. Returns the leaf value, or +/// `None` if any step is absent or the wrong shape. +fn navigate(item: &Value, path: &Path) -> Option { + let mut current = item.get(&path.root)?; + for step in &path.steps { + current = match step { + PathStep::Attribute(name) => current.get("M")?.get(name)?, + PathStep::Index(i) => current.get("L")?.get(*i)?, + }; + } + Some(current.clone()) +} + +/// The output map key for a projected path: the literal path text when it has a +/// list-index step, otherwise the leaf attribute name. +pub(super) fn output_key(path: &Path) -> String { + let has_index = path.steps.iter().any(|s| matches!(s, PathStep::Index(_))); + if has_index { + let mut text = path.root.clone(); + for step in &path.steps { + match step { + PathStep::Attribute(name) => { + text.push('.'); + text.push_str(name); + } + PathStep::Index(i) => text.push_str(&format!("[{i}]")), + } + } + text + } else { + // The leaf is the last attribute step, or the root when there are none. + match path.steps.last() { + Some(PathStep::Attribute(name)) => name.clone(), + _ => path.root.clone(), + } + } +} + +#[cfg(test)] +mod tests { + use super::super::tests::select; + use super::*; + + #[test] + fn projects_nested_path_on_leaf_and_index_on_literal_text() { + let item = serde_json::json!({ + "addr": {"M": {"city": {"S": "NYC"}}}, + "tags": {"L": [{"S": "a"}, {"S": "b"}]}, + "name": {"S": "n"} + }); + let s = select("SELECT addr.city, tags[0] FROM \"t\" WHERE pk='a' AND sk='b'"); + let Projection::Paths(paths) = &s.projection else { + panic!("expected paths"); + }; + let projected = project_item(&item, paths); + assert_eq!(projected["city"]["S"], "NYC"); + assert_eq!(projected["tags[0]"]["S"], "a"); + assert!(projected.get("addr").is_none()); + } +} diff --git a/crates/engine/src/partiql/select/steps.rs b/crates/engine/src/partiql/select/steps.rs new file mode 100644 index 000000000..94645d2f9 --- /dev/null +++ b/crates/engine/src/partiql/select/steps.rs @@ -0,0 +1,480 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The read of a SELECT's disjuncts one after another, a page at a time. + +use serde::Deserialize; +use serde_json::{Map, Value}; + +use extenddb_core::error::DynamoDbError; +use extenddb_core::partiql::ast::SelectStatement; +use extenddb_core::partiql::plan::{KeyNames, KeyRead}; +use extenddb_core::types::{Item, item_size_bytes}; + +use super::read_plan::Step; +use super::request::{Paging, key_condition, query_request, render_filter}; +use super::token_mismatch; +use super::visits::{Cursor, Ending, Read, Walk}; +use crate::partiql::SingleRequest; +use crate::partiql::expr_bridge::ExprBridge; +use crate::{DispatchMetrics, DispatchResult, OperationContext}; + +/// The size of a page of a read of disjuncts one after another: the 1 MB of +/// a Query's page. +const PAGE_BYTES: u64 = crate::read_helpers::PAGE_BYTE_BUDGET as u64; + +/// Read a page of the disjuncts `steps`, numbered in read order: all of them +/// for the first page, else those a NextToken says the last page left, in +/// the order the visit rules give (see the `visits` module). Measured +/// 2026-11-02: +/// +/// * a disjunct that names one item takes one unit of `Limit`, whether the +/// item exists or not (`pk IN ['a', 'zz']` with `Limit` 1 gives [a] and a +/// NextToken, then an empty page with none); +/// * any other disjunct is a Query with the units left; a Query that stops +/// at the limit ends the page, and a later page goes on inside it; a +/// Query that runs out leaves the units it did not evaluate to the next +/// disjunct in the same page (`(pk = 'b' AND sk = 'x') OR (pk = 'a' AND +/// sk > 'a')` with `Limit` 1 gives [a/x], [a/y], then [b/x]); +/// * once the last disjunct is read there is no NextToken, so a read of one +/// item never has one; +/// * a page stops before a whole item that would take it past 1 MB, and the +/// item is left for a later page (four items of about 380 KB, `pk IN +/// ['a', 'b', 'c', 'd']` without `Limit`, give [a, b] with a NextToken, +/// then [c, d]). +/// +/// The disjuncts of a page share one budget of [`PAGE_BYTES`], counted on +/// the items read as a Query's page counts them, before the projection and +/// the filter. A Query reads until the item that takes the page past it, +/// keeps that item, and ends the page there with that item's key as the +/// LastEvaluatedKey; the next page goes on inside that Query (measured +/// 2026-11-03: with partitions a and b of two items of about 380 KB, `pk IN +/// ['a', 'b']` gives [a/1, a/2, b/1] with the key b/1 and a NextToken, then +/// [b/2]; with partitions a, b and c of one such item, `pk IN ['a', 'b', +/// 'c']` gives all three in the first page). A Query that ends its own page +/// at 1 MB stops before that item (see `read_helpers::read_page`), so the +/// page reads that item with one more Query. +/// +/// A page visits at most 10 disjuncts (see `visits::PAGE_VISITS`), and the +/// rules above apply within that cap (measured 2026-11-03: `pk IN [...]` +/// with 25 values gives 10, 10 and 5 items with `Limit` 15 or 11, and 7, 7, +/// 7 and 4 with `Limit` 7; 11 partitions of 3 items give 30 items and then +/// 3). A disjunct held back for the refusal it gets when it is read takes a +/// visit and no unit of `Limit`, and is visited again by every later page +/// that reaches it. A request whose page visits only disjuncts held back +/// gets their refusal ([`StepTarget::refusal`]) instead of the page, also +/// when other disjuncts are left, so once only disjuncts held back are +/// left, the next request gets it (measured 2026-11-03: `pk IN [v000..v010, +/// <2049 bytes that sort after v005>]` gives v000 to v008, then v009 and +/// v010, then the refusal; `(pk IN [a00..a09] AND sk = a) OR pk IN +/// [b00..b10]` is refused on the first request; measured 2026-11-02: `(pk = +/// 'a' AND sk = a) OR pk = 'b' OR pk = 'c'` with `Limit` 1 gives [b/x], +/// then [c/x], then the refusal). +/// +/// A page that holds `Limit` items has a LastEvaluatedKey, the key of the +/// last item it returned in visit order (see [`StepTarget::last_key`]), +/// with or without a NextToken; a page that ends inside a Query at the byte +/// budget has the key of the item it ended on; any other page has none (see +/// [`page_key`]), so a page the cap ends with fewer items than `Limit` has +/// none (measured 2026-11-03: `pk IN [...]` with 11 values gives 10 items +/// with a NextToken and no LastEvaluatedKey; with 21 values and `Limit` 10 +/// the second page, which visits v016 to v020 and then v010 to v014, has +/// the key v014). +pub(super) async fn read_steps( + ctx: &OperationContext, + stmt: &SelectStatement, + req: &SingleRequest, + target: &StepTarget<'_>, + steps: &[Step<'_, '_>], + paging: Paging<'_>, +) -> Result { + let cursor = match paging.start { + None => Cursor::first(steps.len()), + Some(token) => Cursor::from_value(token, steps.len())?, + }; + let held_back: Vec = steps.iter().map(|step| step.held_back).collect(); + let limit = paging.limit.and_then(|limit| usize::try_from(limit).ok()); + let mut walk = Walk::new(cursor, &held_back, limit); + let reader = Reader { + ctx, + stmt, + req, + keys: target.keys, + descending: paging.descending, + filtered: target.filtered, + }; + let mut page = PageParts::default(); + let mut budget_key = None; + while let Some(visit) = walk.next_read() { + let current = &steps[visit.position]; + let read = if current.point.is_some() { + reader.read_item(current.read, &mut page).await? + } else { + let start = visit.start.as_ref(); + match reader + .read_query(current.read, start, walk.units(), &mut page) + .await? + { + QueryEnd::Ran(units) => Read::Done(units), + QueryEnd::RanPastBudget => Read::DoneAndEnd, + QueryEnd::AtLimit(key) => Read::Stopped(key), + QueryEnd::AtBudget(key) => { + budget_key = Some(key.clone()); + Read::Stopped(key) + } + } + }; + walk.record(visit.position, read); + } + let next = match walk.finish() { + Ending::Refused => { + return Err(target.refusal.cloned().unwrap_or_else(token_mismatch)); + } + Ending::Last => None, + Ending::More(cursor) => Some(cursor.into_value()), + }; + let last_key = budget_key.or_else(|| page_key(&page.items, &target.last_key, paging.limit)); + let mut body = Map::new(); + body.insert("Items".to_owned(), Value::Array(page.items)); + if let Some(capacity) = page.capacity { + body.insert("ConsumedCapacity".to_owned(), capacity); + } + Ok(StepsPage { + page: DispatchResult { + body: Value::Object(body), + metrics: page.metrics, + }, + next, + last_key, + }) +} + +/// How a Query of a disjunct stands against the page's byte budget. +enum BudgetEnd { + /// The page stays within its budget with every item the Query read. + Within(DispatchResult), + /// The page passes its budget inside the Query: the read up to and + /// including the item that takes it past. + Past(DispatchResult), +} + +/// How a page's Query of a disjunct ended. +enum QueryEnd { + /// The Query read the disjunct to its end, evaluating this many items. + Ran(usize), + /// The Query read the disjunct to its end and took the page past its + /// byte budget on its last item. + RanPastBudget, + /// The page ends inside the Query at `Limit`, on the item with this key. + AtLimit(Value), + /// The page ends inside the Query at the byte budget, on the item with + /// this key. + AtBudget(Value), +} + +/// What a read of disjuncts one after another reads besides the disjuncts. +pub(super) struct StepTarget<'t> { + /// The key names of the table or index read. + pub(super) keys: KeyNames<'t>, + /// The attributes of a page's LastEvaluatedKey: the key of what is read, + /// and for an index the table's key after it, as a Query of an index + /// names them. + pub(super) last_key: Vec<&'t str>, + /// Whether each Query carries its disjunct's filter. A read of an index + /// that reaches back filters the base-table items instead (see + /// `base_join`). + pub(super) filtered: bool, + /// The refusal of the disjuncts held back (see + /// `read_plan::Checked::deferred`), which a request whose page visits + /// only them gets. + pub(super) refusal: Option<&'t DynamoDbError>, +} + +/// What one read of a disjunct needs besides the disjunct and its paging. +struct Reader<'r> { + ctx: &'r OperationContext, + stmt: &'r SelectStatement, + req: &'r SingleRequest, + keys: KeyNames<'r>, + /// Whether a Query reads its items in descending order. + descending: bool, + /// Whether a Query carries its disjunct's filter (see + /// [`StepTarget::filtered`]). + filtered: bool, +} + +impl Reader<'_> { + /// Read the one item the disjunct `read` names into `page`, unless the + /// item would take the page past its byte budget. + async fn read_item( + &self, + read: &KeyRead<'_>, + page: &mut PageParts, + ) -> Result { + let result = self.query(read, Paging::default(), true).await?; + if page.bytes() > 0 + && page.bytes().saturating_add(result.metrics.returned_bytes) > PAGE_BYTES + { + return Ok(Read::Unread); + } + page.add(result, read); + Ok(Read::Done(1)) + } + + /// Read the Query of the disjunct `read` from `start` into `page`, with + /// `units` of `Limit` left. + async fn read_query( + &self, + read: &KeyRead<'_>, + start: Option<&Value>, + units: Option, + page: &mut PageParts, + ) -> Result { + let paging = Paging { + start, + limit: units.and_then(|u| i64::try_from(u).ok()), + ..Paging::default() + }; + let result = self.query(read, paging, true).await?; + let evaluated = scanned_count(&result); + match self.budget_end(read, result, start, page.bytes()).await? { + BudgetEnd::Within(result) => { + let limited = units.is_some_and(|u| evaluated >= u); + match page.add(result, read).0 { + None => Ok(QueryEnd::Ran(evaluated)), + Some(key) if limited => Ok(QueryEnd::AtLimit(key)), + Some(key) => { + // The Query ended its own page at 1 MB, before the + // item that takes this page past it: the page keeps + // that item. + let paging = Paging { + start: Some(&key), + limit: Some(1), + ..Paging::default() + }; + let result = self.query(read, paging, true).await?; + Ok(match page.add(result, read) { + (Some(key), _) => QueryEnd::AtBudget(key), + (None, more) => QueryEnd::Ran(evaluated + more), + }) + } + } + } + BudgetEnd::Past(result) => Ok(match page.add(result, read).0 { + Some(key) => QueryEnd::AtBudget(key), + None => QueryEnd::RanPastBudget, + }), + } + } + + /// Dispatch the Query of one disjunct, with its filter when `filtered` + /// and the read carries filters. + async fn query( + &self, + read: &KeyRead<'_>, + paging: Paging<'_>, + filtered: bool, + ) -> Result { + let mut bridge = ExprBridge::new(&self.req.parameters); + let kce = key_condition(&mut bridge, self.keys, read.partition, read.sort.as_ref())?; + let filter = if filtered && self.filtered { + render_filter(&mut bridge, &read.filter)? + } else { + None + }; + let paging = Paging { + descending: self.descending, + ..paging + }; + query_request(self.ctx, self.stmt, self.req, paging, &bridge, kce, filter).await + } + + /// Set `result`, the Query of `read` from `start`, against a page that + /// has read `spent` bytes. When the page passes its budget inside it, + /// the Query is read again with the number of items up to and including + /// the one that takes the page past as its `Limit`. + async fn budget_end( + &self, + read: &KeyRead<'_>, + result: DispatchResult, + start: Option<&Value>, + spent: u64, + ) -> Result { + if spent == 0 || spent.saturating_add(result.metrics.returned_bytes) <= PAGE_BYTES { + return Ok(BudgetEnd::Within(result)); + } + let evaluated = scanned_count(&result); + let sizes = self + .evaluated_sizes(read, &result, start, evaluated) + .await?; + let take = items_to_budget(spent, &sizes).max(1); + if take == evaluated && result.body.get("LastEvaluatedKey").is_some() { + return Ok(BudgetEnd::Past(result)); + } + let paging = Paging { + start, + limit: i64::try_from(take).ok(), + ..Paging::default() + }; + Ok(BudgetEnd::Past(self.query(read, paging, true).await?)) + } + + /// The sizes of the `evaluated` items a Query of `read` from `start` + /// read for `result`, in the order it read them. When the disjunct has a + /// filter, the items are read again without it, since the filter drops + /// some of them from `result`. + async fn evaluated_sizes( + &self, + read: &KeyRead<'_>, + result: &DispatchResult, + start: Option<&Value>, + evaluated: usize, + ) -> Result, DynamoDbError> { + let unfiltered; + let source = if read.filter.is_empty() || !self.filtered { + result + } else { + let paging = Paging { + start, + limit: i64::try_from(evaluated.max(1)).ok(), + ..Paging::default() + }; + unfiltered = self.query(read, paging, false).await?; + &unfiltered + }; + let items = source.body.get("Items").and_then(Value::as_array); + Ok(items.map_or_else(Vec::new, |items| items.iter().map(wire_item_size).collect())) + } +} + +/// The items, consumed capacity and metrics of the reads a page is made of. +#[derive(Default)] +struct PageParts { + items: Vec, + capacity: Option, + metrics: DispatchMetrics, +} + +impl PageParts { + /// The bytes the page has read, as a Query's page counts them. + fn bytes(&self) -> u64 { + self.metrics.returned_bytes + } + + /// Add one read of the disjunct `read` to the page, its items unless the + /// disjunct returns none of them (see `KeyRead::returns_nothing`). + /// Returns the read's LastEvaluatedKey and how many items it evaluated. + fn add(&mut self, result: DispatchResult, read: &KeyRead<'_>) -> (Option, usize) { + let evaluated = scanned_count(&result); + self.metrics.read_capacity_units += result.metrics.read_capacity_units; + self.metrics.returned_bytes += result.metrics.returned_bytes; + let Value::Object(mut body) = result.body else { + return (None, evaluated); + }; + if let Some(Value::Array(found)) = body.remove("Items") + && !read.returns_nothing() + { + self.metrics.returned_item_count += result.metrics.returned_item_count; + self.items.extend(found); + } + add_capacity(&mut self.capacity, body.remove("ConsumedCapacity")); + (body.remove("LastEvaluatedKey"), evaluated) + } +} + +/// How many items a read evaluated (its `ScannedCount`). +fn scanned_count(result: &DispatchResult) -> usize { + result + .body + .get("ScannedCount") + .and_then(Value::as_u64) + .and_then(|n| usize::try_from(n).ok()) + .unwrap_or(0) +} + +/// The size of a wire-form item, as a Query's page counts it. +fn wire_item_size(item: &Value) -> u64 { + Item::deserialize(item).map_or(0, |item| item_size_bytes(&item) as u64) +} + +/// How many of the items of `sizes` a page that has read `spent` bytes +/// takes: up to and including the one that takes it past [`PAGE_BYTES`], +/// or all of them. +fn items_to_budget(spent: u64, sizes: &[u64]) -> usize { + let mut total = spent; + for (index, size) in sizes.iter().enumerate() { + total = total.saturating_add(*size); + if total > PAGE_BYTES { + return index + 1; + } + } + sizes.len() +} + +/// A page of a read of disjuncts one after another. +pub(super) struct StepsPage { + pub(super) page: DispatchResult, + /// Where the next page goes on, when this one leaves disjuncts (see + /// [`Cursor`]). + pub(super) next: Option, + /// The page's LastEvaluatedKey. + pub(super) last_key: Option, +} + +/// The LastEvaluatedKey of a page of a read of disjuncts one after another +/// that does not end at the byte budget: the attributes `names` of the last +/// item when the page holds `limit` items, else none. Measured 2026-11-02: +/// with `Limit` 1, `pk IN ['a', 'b']` gives [a] with key a and a NextToken, +/// then [b] with key b and no NextToken; `pk IN ['a', 'zz']` gives [a] with +/// key a, then an empty page with none; `pk IN ['a', 'b', 'c']` with `Limit` +/// 2 gives [a, b] with key b, then [c] with none; a projection without the +/// key attributes has the key too; and a page read without `Limit` has +/// none, the one before a refused disjunct too. +fn page_key(items: &[Value], names: &[&str], limit: Option) -> Option { + let full = limit + .and_then(|limit| usize::try_from(limit).ok()) + .is_some_and(|limit| limit > 0 && items.len() == limit); + if !full { + return None; + } + let last = items.last()?; + let mut key = Map::new(); + for name in names { + key.insert((*name).to_owned(), last.get(*name)?.clone()); + } + Some(Value::Object(key)) +} + +/// Add one read's consumed capacity to a running total, number by number. +fn add_capacity(total: &mut Option, more: Option) { + let Some(more) = more else { + return; + }; + match total { + Some(sum) => add_numbers(sum, more), + None => *total = Some(more), + } +} + +fn add_numbers(sum: &mut Value, more: Value) { + match (sum, more) { + (Value::Object(sum), Value::Object(more)) => { + for (name, value) in more { + match sum.get_mut(&name) { + Some(existing) => add_numbers(existing, value), + None => { + sum.insert(name, value); + } + } + } + } + (sum, Value::Number(more)) if sum.is_number() => { + let total = sum.as_f64().unwrap_or(0.0) + more.as_f64().unwrap_or(0.0); + *sum = Value::from(total); + } + _ => {} + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/engine/src/partiql/select/steps/tests.rs b/crates/engine/src/partiql/select/steps/tests.rs new file mode 100644 index 000000000..de7ddc6cc --- /dev/null +++ b/crates/engine/src/partiql/select/steps/tests.rs @@ -0,0 +1,72 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Unit tests for the read of a SELECT's disjuncts one after another. + +use super::*; + +#[test] +fn a_full_page_names_the_key_of_its_last_item() { + let item = |pk: &str, sk: &str| serde_json::json!({ "pk": { "S": pk }, "sk": { "S": sk }, "n": { "N": "1" } }); + let key = |pk: &str, sk: &str| serde_json::json!({ "pk": { "S": pk }, "sk": { "S": sk } }); + let two = [item("a", "x"), item("b", "x")]; + let keys = ["pk", "sk"]; + assert_eq!(page_key(&two, &keys, Some(2)), Some(key("b", "x"))); + assert_eq!(page_key(&two[..1], &keys, Some(1)), Some(key("a", "x"))); + // A page with fewer items than the limit, an empty one, and one read + // without a limit have none. + assert_eq!(page_key(&two[..1], &keys, Some(2)), None); + assert_eq!(page_key(&[], &keys, Some(1)), None); + assert_eq!(page_key(&two, &keys, None), None); + // A table keyed on the partition key alone. + assert_eq!( + page_key(&two[..1], &["pk"], Some(1)), + Some(serde_json::json!({ "pk": { "S": "a" } })) + ); + // An index keyed on n, read with the table's key after its own. + assert_eq!( + page_key(&two[..1], &["n", "pk", "sk"], Some(1)), + Some(serde_json::json!({ "n": { "N": "1" }, "pk": { "S": "a" }, "sk": { "S": "x" } })) + ); +} + +#[test] +fn a_page_takes_items_up_to_the_one_that_passes_its_budget() { + let item = 380 * 1024; + // Two items read, the next one passes 1 MB and is kept. + assert_eq!(items_to_budget(2 * item, &[item, item]), 1); + // One item read, the second of the next two passes 1 MB. + assert_eq!(items_to_budget(item, &[item, item, item]), 2); + // Exactly 1 MB does not pass it. + assert_eq!(items_to_budget(PAGE_BYTES - 10, &[10, 1]), 2); + // A page that stays within its budget takes them all. + assert_eq!(items_to_budget(10, &[10, 10]), 2); + assert_eq!(items_to_budget(10, &[]), 0); +} + +#[test] +fn a_wire_item_is_sized_as_a_page_counts_it() { + let item = serde_json::json!({ + "pk": { "S": "a" }, "n": { "N": "12" }, "b": { "B": "AAEC" }, "t": { "BOOL": true } + }); + // pk 2 + 1, n 1 + 2, b 1 + 3 bytes, t 1 + 1. + assert_eq!(wire_item_size(&item), 12); + assert_eq!(wire_item_size(&serde_json::json!("not an item")), 0); +} + +#[test] +fn consumed_capacity_adds_up_number_by_number() { + let mut total = None; + let read = serde_json::json!({ + "TableName": "t", "CapacityUnits": 0.5, "Table": { "CapacityUnits": 0.5 } + }); + add_capacity(&mut total, Some(read.clone())); + add_capacity(&mut total, None); + add_capacity(&mut total, Some(read)); + assert_eq!( + total, + Some(serde_json::json!({ + "TableName": "t", "CapacityUnits": 1.0, "Table": { "CapacityUnits": 1.0 } + })) + ); +} diff --git a/crates/engine/src/partiql/select/tests.rs b/crates/engine/src/partiql/select/tests.rs new file mode 100644 index 000000000..889b68c83 --- /dev/null +++ b/crates/engine/src/partiql/select/tests.rs @@ -0,0 +1,66 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Unit tests for the SELECT executor, and the fixtures the tests of its modules share. + +use super::*; +use extenddb_core::partiql::ast::Statement; +use extenddb_core::partiql::parse; + +pub(super) fn select(sql: &str) -> SelectStatement { + match parse(sql).unwrap() { + Statement::Select(s) => s, + _ => panic!("not a select"), + } +} + +pub(super) const KEYS: KeyNames<'static> = KeyNames { + partition: "pk", + sort: Some("sk"), +}; + +pub(super) const HASH_ONLY: KeyNames<'static> = KeyNames { + partition: "pk", + sort: None, +}; + +fn projection_refusal(projection: &str) -> Option { + let sql = format!("SELECT {projection} FROM \"t\" WHERE pk = 'a' AND sk = 'b'"); + check_projection(&select(&sql)).err() +} + +#[test] +fn a_projection_names_each_identifier_once() { + for (projection, key) in [("a, a", "a"), ("m.k, k", "k"), ("a, \"a\"", "a")] { + assert_eq!( + projection_refusal(projection), + Some(format!("Duplicate identifiers in select clause: {key}")), + "{projection}" + ); + } +} + +#[test] +fn projected_paths_that_overlap_are_refused_later_path_first() { + for (projection, one, two) in [ + ("m.k, m", "[m]", "[m, k]"), + ("m, m.k", "[m, k]", "[m]"), + ("l[1], l", "[l]", "[l, [1]]"), + ] { + assert_eq!( + projection_refusal(projection), + Some(format!( + "Two document paths overlap with each other; must remove or rewrite one of \ + these paths; path one: {one}, path two: {two}" + )), + "{projection}" + ); + } +} + +#[test] +fn distinct_projected_paths_pass() { + for projection in ["*", "a, m.k", "m.k, m.j", "l[1], l[2]", "pk, sk"] { + assert_eq!(projection_refusal(projection), None, "{projection}"); + } +} diff --git a/crates/engine/src/partiql/select/visits.rs b/crates/engine/src/partiql/select/visits.rs new file mode 100644 index 000000000..8970d96bf --- /dev/null +++ b/crates/engine/src/partiql/select/visits.rs @@ -0,0 +1,298 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The order in which the pages of a read of disjuncts one after another +//! visit them, and the list each page leaves for the next (measured +//! 2026-11-03). +//! +//! The disjuncts are numbered 0 to N-1 in read order (key order, or the +//! order an ORDER BY gives), the ones held back for a refusal included (see +//! `Step::all`). The first page takes the list 0 to N-1. Each page visits +//! the list the page before it left in the order of [`visit_order`], visits +//! at most [`PAGE_VISITS`] disjuncts, and stops once its `Limit` is spent or +//! its byte budget is reached (see `read_steps`). The list it leaves holds +//! the disjuncts it visited and did not read to their end, in the order it +//! visited them, then the ones it did not visit, in their order: +//! +//! * a disjunct held back for a refusal takes a visit, no unit of `Limit` +//! and no read, and stays in the list, so it takes a visit again on every +//! later page that reaches it; +//! * a Query the page stops inside stays in the list, and goes on from +//! where it stopped when a later page reaches it, which need not be +//! first. +//! +//! When the list is empty there is no NextToken. A request whose page +//! visits only disjuncts held back gets their refusal instead of a page, +//! also when the page leaves other disjuncts unvisited, so once only +//! disjuncts held back are left, the next request gets it. Measured on +//! tables, an index, the sort key values of one partition and `ORDER BY pk +//! DESC`: 21 point reads give the items of positions 0 to 9, then 16 to 20 +//! and 10 to 14, then 15; with `Limit` 3, [0 1 2] [3 4 5] [6 7 8] [16 17 +//! 18] [19 20 9] [10 11 12] [13 14 15]; 32 point reads beside a value past +//! the key size limit at position 12 give [0..9], [32 10 11 13..18], +//! [19..27] and [28..31], each later page one item short for the visit the +//! held-back value takes, then the refusal; and 18 point reads beside a +//! Query of 21 items at position 10, with `Limit` 6, give on the page after +//! the Query stopped positions 16, 17 and 18 and then 3 items of the Query. +//! With the filter refusal held back for positions 0 to 9 of 21, the first +//! request is refused, with and without `Limit` 3; for 10 to 19 of 25, the +//! first page holds 0 to 9 and the second request is refused; for 0 to 8 +//! of 21, nine pages of one item and a page of three come before the +//! refusal. + +use std::collections::BTreeMap; + +use serde_json::{Map, Value}; + +use extenddb_core::error::DynamoDbError; + +use super::token_mismatch; + +/// The most disjuncts a page visits, whatever its `Limit` (measured +/// 2026-11-03: `pk IN [...]` with 11 values of items that exist gives 10 +/// items and then 1, with 25 values 10, 10 and 5, with `Limit` 100 or 15 +/// too). A Query the page goes on inside is one of them (measured +/// 2026-11-03: with partition a of 13 items before 20 partitions of one +/// item, `Limit` 12 gives 12 items, then the last item of a and 9 +/// partitions, then 10 partitions, then 1). +pub(super) const PAGE_VISITS: usize = 10; + +/// The order in which a page visits `left`, the positions the page before +/// it left, in the order it left them: by position modulo a bucket count, +/// and in list order within one bucket. The bucket count is 16, doubled +/// while the list holds more than three quarters of it, so it is the order +/// in which a hash set of the positions with that many buckets lists them +/// when they are added in list order. The count is larger than the list, +/// so the first page takes the disjuncts in read order, and so does every +/// page whose list holds no position at or above the count (measured +/// 2026-11-03: after a first page of 0 to 9 of 21 disjuncts, the list 10 to +/// 20 is visited 16 to 20, then 10 to 15; of 23 to 31 disjuncts, in read +/// order). +pub(super) fn visit_order(left: &[usize]) -> Vec { + let mut buckets = 16; + while left.len() > buckets * 3 / 4 { + buckets *= 2; + } + let mut order = left.to_vec(); + order.sort_by_key(|position| position % buckets); + order +} + +/// Where a read of disjuncts one after another goes on, as a NextToken +/// holds it. +#[derive(Debug, Clone, PartialEq)] +pub(super) struct Cursor { + /// The positions of the disjuncts left, in the order the last page left + /// them: at most the number of disjuncts, which is at most 50. + left: Vec, + /// The key from which the Query of each position a page stopped inside + /// goes on. + after: BTreeMap, +} + +impl Cursor { + /// The cursor of the first page of a read of `count` disjuncts. + pub(super) fn first(count: usize) -> Self { + Self { + left: (0..count).collect(), + after: BTreeMap::new(), + } + } + + /// The cursor as a NextToken holds it. + pub(super) fn into_value(self) -> Value { + let after: Vec = self + .after + .into_iter() + .map(|(position, key)| Value::Array(vec![Value::from(position), key])) + .collect(); + let mut out = Map::new(); + out.insert("left".to_owned(), Value::from(self.left)); + out.insert("after".to_owned(), Value::Array(after)); + Value::Object(out) + } + + /// The cursor a NextToken holds for a read of `count` disjuncts. + /// + /// # Errors + /// + /// The service's refusal of a NextToken this statement did not give, + /// for a value that is no list of distinct positions below `count`, or + /// that holds a key for a position not in the list. + pub(super) fn from_value(value: &Value, count: usize) -> Result { + let position = |value: &Value| { + value + .as_u64() + .and_then(|n| usize::try_from(n).ok()) + .filter(|position| *position < count) + }; + let left: Vec = value + .get("left") + .and_then(Value::as_array) + .and_then(|left| left.iter().map(position).collect()) + .ok_or_else(token_mismatch)?; + let mut listed = vec![false; count]; + for &at in &left { + if std::mem::replace(&mut listed[at], true) { + return Err(token_mismatch()); + } + } + let mut after = BTreeMap::new(); + for entry in value + .get("after") + .and_then(Value::as_array) + .ok_or_else(token_mismatch)? + { + let (Some(at), Some(key)) = (entry.get(0).and_then(position), entry.get(1)) else { + return Err(token_mismatch()); + }; + if !listed[at] || after.insert(at, key.clone()).is_some() { + return Err(token_mismatch()); + } + } + Ok(Self { left, after }) + } +} + +/// A disjunct a page reads, and the key its Query goes on from. +pub(super) struct Visit { + pub(super) position: usize, + pub(super) start: Option, +} + +/// How the read of a disjunct a page visits ended. +pub(super) enum Read { + /// Read to its end, with this many units of `Limit` spent. + Done(usize), + /// Read to its end, and the page ends after it. + DoneAndEnd, + /// The page ends inside the disjunct's Query, which goes on from this + /// key. + Stopped(Value), + /// The page ends before the disjunct, which is left as if it had not + /// been visited. + Unread, +} + +/// How a page ends. +pub(super) enum Ending { + /// Every visit of the page went to a disjunct held back for a refusal: + /// the request gets the refusal instead of the page. + Refused, + /// Every disjunct is read: the page has no NextToken. + Last, + /// The next page goes on from this cursor. + More(Cursor), +} + +/// The visits of one page. +pub(super) struct Walk<'h> { + /// Whether each position is held back for a refusal. + held_back: &'h [bool], + /// The order the page visits the positions left. + order: Vec, + /// The index in `order` of the next position to visit. + next: usize, + visits: usize, + /// Whether a visit of the page went to a disjunct that is not held back. + read: bool, + /// The units of `Limit` left, or `None` without a `Limit`. + units: Option, + /// The positions visited and left in the list, in the order visited. + kept: Vec, + after: BTreeMap, + ended: bool, +} + +impl<'h> Walk<'h> { + /// The visits of the page that goes on from `cursor`, with `limit` + /// units of `Limit`. `held_back` tells, by position, which disjuncts + /// are held back for a refusal. + pub(super) fn new(cursor: Cursor, held_back: &'h [bool], limit: Option) -> Self { + Self { + held_back, + order: visit_order(&cursor.left), + next: 0, + visits: 0, + read: false, + units: limit, + kept: Vec::new(), + after: cursor.after, + ended: false, + } + } + + /// The units of `Limit` the page has left, or `None` without a `Limit`. + pub(super) fn units(&self) -> Option { + self.units + } + + /// The next disjunct the page reads, or `None` once the page is over. A + /// disjunct held back for a refusal takes its visit here, without a + /// read. + pub(super) fn next_read(&mut self) -> Option { + while !self.ended && self.units != Some(0) && self.visits < PAGE_VISITS { + let position = *self.order.get(self.next)?; + self.next += 1; + self.visits += 1; + if self.held_back[position] { + self.kept.push(position); + continue; + } + self.read = true; + return Some(Visit { + position, + start: self.after.get(&position).cloned(), + }); + } + None + } + + /// Record how the read of `position`, the disjunct [`Walk::next_read`] + /// gave last, ended. + pub(super) fn record(&mut self, position: usize, read: Read) { + match read { + Read::Done(units) => { + self.after.remove(&position); + self.units = self.units.map(|left| left.saturating_sub(units)); + } + Read::DoneAndEnd => { + self.after.remove(&position); + self.ended = true; + } + Read::Stopped(key) => { + self.after.insert(position, key); + self.kept.push(position); + self.ended = true; + } + Read::Unread => { + self.next -= 1; + self.ended = true; + } + } + } + + /// How the page ends: refused when every one of its visits went to a + /// disjunct held back, else with what it leaves, the positions it kept, + /// in the order it visited them, then those it did not visit, in their + /// order. + pub(super) fn finish(self) -> Ending { + if self.visits > 0 && !self.read { + return Ending::Refused; + } + let mut left = self.kept; + left.extend_from_slice(&self.order[self.next..]); + if left.is_empty() { + return Ending::Last; + } + let after = self + .after + .into_iter() + .filter(|(position, _)| left.contains(position)) + .collect(); + Ending::More(Cursor { left, after }) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/engine/src/partiql/select/visits/tests.rs b/crates/engine/src/partiql/select/visits/tests.rs new file mode 100644 index 000000000..511c0d7b6 --- /dev/null +++ b/crates/engine/src/partiql/select/visits/tests.rs @@ -0,0 +1,371 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Unit tests for the visit order of a read of disjuncts one after another, +//! against the pages measured on the service (2026-11-03), written as the +//! positions of the items each page returns, in order. + +use super::*; +use serde_json::json; + +/// The pages of a read of `count` disjuncts with `limit`, the positions of +/// `held` held back for a refusal, and the Queries `queries` (position and +/// number of items); every other disjunct names one item that exists. Each +/// page is written as the positions of the items it returns, `p` for the +/// item a point read names and `pxn` for `n` items of the Query at `p`, and +/// the request the refusal ends is `R`. Each request takes the cursor +/// through its NextToken value. A Query reads from the key its last read +/// stopped at, and stops with a key when it reads `Limit` items, as the +/// classic Query does. +fn pages(count: usize, limit: Option, held: &[usize], queries: &[(usize, usize)]) -> String { + let held_back: Vec = (0..count).map(|p| held.contains(&p)).collect(); + let mut out = Vec::new(); + let mut cursor = Some(Cursor::first(count)); + while let Some(sent) = cursor.take() { + assert!(out.len() < 40, "more than 40 pages: {}", out.join(" | ")); + let token_value = sent.into_value(); + let mut walk = Walk::new( + Cursor::from_value(&token_value, count).unwrap(), + &held_back, + limit, + ); + let mut page = Vec::new(); + while let Some(visit) = walk.next_read() { + let position = visit.position; + let Some(&(_, items)) = queries.iter().find(|(q, _)| *q == position) else { + page.push(position.to_string()); + walk.record(position, Read::Done(1)); + continue; + }; + let from = visit + .start + .as_ref() + .and_then(Value::as_u64) + .map_or(0, |n| usize::try_from(n).unwrap()); + let units = walk.units(); + let take = units.map_or(items - from, |u| u.min(items - from)); + if take > 0 { + page.push(format!("{position}x{take}")); + } + let read = if take > 0 && units == Some(take) { + Read::Stopped(json!(from + take)) + } else { + Read::Done(take) + }; + walk.record(position, read); + } + match walk.finish() { + Ending::Refused => { + assert!(page.is_empty(), "a refused page read {page:?}"); + out.push("R".to_owned()); + } + Ending::Last => out.push(format!("[{}]", page.join(" "))), + Ending::More(next) => { + out.push(format!("[{}]", page.join(" "))); + cursor = Some(next); + } + } + } + out.join(" | ") +} + +/// The positions `from` to `to`, written as a page writes them. +fn run(from: usize, to: usize) -> String { + (from..=to) + .map(|p| p.to_string()) + .collect::>() + .join(" ") +} + +#[test] +fn a_list_is_visited_by_position_modulo_its_bucket_count() { + // 12 positions fit 16 buckets, 13 take 32. + let twelve: Vec = (10..22).collect(); + assert_eq!( + visit_order(&twelve), + [16, 17, 18, 19, 20, 21, 10, 11, 12, 13, 14, 15] + ); + let thirteen: Vec = (10..23).collect(); + assert_eq!(visit_order(&thirteen), thirteen); + // One bucket keeps the list's order. + assert_eq!(visit_order(&[18, 2, 34, 1]), [1, 18, 2, 34]); + assert_eq!(visit_order(&[2, 18, 1]), [1, 2, 18]); + // The first page of at most 50 disjuncts takes them in read order. + for count in [1, 12, 13, 24, 25, 48, 49, 50] { + let all: Vec = (0..count).collect(); + assert_eq!(visit_order(&all), all, "{count}"); + } +} + +#[test] +fn later_pages_of_point_reads_visit_in_the_measured_order() { + let first = format!("[{}]", run(0, 9)); + for (count, later) in [ + (17, "[16 10 11 12 13 14 15]"), + (18, "[16 17 10 11 12 13 14 15]"), + (20, "[16 17 18 19 10 11 12 13 14 15]"), + (21, "[16 17 18 19 20 10 11 12 13 14] | [15]"), + (22, "[16 17 18 19 20 21 10 11 12 13] | [14 15]"), + (23, "[10 11 12 13 14 15 16 17 18 19] | [20 21 22]"), + ( + 28, + "[10 11 12 13 14 15 16 17 18 19] | [20 21 22 23 24 25 26 27]", + ), + ( + 33, + "[32 10 11 12 13 14 15 16 17 18] | [19 20 21 22 23 24 25 26 27 28] | [29 30 31]", + ), + ( + 37, + "[10 11 12 13 14 15 16 17 18 19] | [32 33 34 35 36 20 21 22 23 24] \ + | [25 26 27 28 29 30 31]", + ), + ( + 41, + "[10 11 12 13 14 15 16 17 18 19] | [32 33 34 35 36 37 38 39 40 20] \ + | [21 22 23 24 25 26 27 28 29 30] | [31]", + ), + ( + 47, + "[10 11 12 13 14 15 16 17 18 19] | [20 21 22 23 24 25 26 27 28 29] \ + | [32 33 34 35 36 37 38 39 40 41] | [42 43 44 45 46 30 31]", + ), + ( + 49, + "[10 11 12 13 14 15 16 17 18 19] | [20 21 22 23 24 25 26 27 28 29] \ + | [32 33 34 35 36 37 38 39 40 41] | [48 42 43 44 45 46 30 47 31]", + ), + ( + 50, + "[10 11 12 13 14 15 16 17 18 19] | [20 21 22 23 24 25 26 27 28 29] \ + | [32 33 34 35 36 37 38 39 40 41] | [48 49 42 43 44 45 46 30 47 31]", + ), + ] { + let want = format!("{first} | {later}"); + assert_eq!(pages(count, None, &[], &[]), want, "{count}"); + } +} + +#[test] +fn pages_with_a_limit_visit_in_the_measured_order() { + for (count, limit, want) in [ + ( + 21, + 3, + "[0 1 2] | [3 4 5] | [6 7 8] | [16 17 18] | [19 20 9] | [10 11 12] | [13 14 15]", + ), + ( + 33, + 4, + "[0 1 2 3] | [4 5 6 7] | [8 9 10 11] | [32 12 13 14] | [15 16 17 18] \ + | [19 20 21 22] | [23 24 25 26] | [27 28 29 30] | [31]", + ), + ( + 21, + 10, + "[0 1 2 3 4 5 6 7 8 9] | [16 17 18 19 20 10 11 12 13 14] | [15]", + ), + ( + 33, + 7, + "[0 1 2 3 4 5 6] | [7 8 9 10 11 12 13] | [32 14 15 16 17 18 19] \ + | [20 21 22 23 24 25 26] | [27 28 29 30 31]", + ), + ] { + assert_eq!(pages(count, Some(limit), &[], &[]), want, "{count} {limit}"); + } +} + +#[test] +fn a_held_back_disjunct_takes_a_visit_on_every_page_that_reaches_it() { + for (held, want) in [ + ( + 0, + "[1 2 3 4 5 6 7 8 9] | [32 10 11 12 13 14 15 16 17] \ + | [18 19 20 21 22 23 24 25 26] | [27 28 29 30 31] | R", + ), + ( + 5, + "[0 1 2 3 4 6 7 8 9] | [32 10 11 12 13 14 15 16 17] \ + | [18 19 20 21 22 23 24 25 26] | [27 28 29 30 31] | R", + ), + ( + 12, + "[0 1 2 3 4 5 6 7 8 9] | [32 10 11 13 14 15 16 17 18] \ + | [19 20 21 22 23 24 25 26 27] | [28 29 30 31] | R", + ), + ( + 20, + "[0 1 2 3 4 5 6 7 8 9] | [32 10 11 12 13 14 15 16 17 18] \ + | [19 21 22 23 24 25 26 27 28] | [29 30 31] | R", + ), + ( + 32, + "[0 1 2 3 4 5 6 7 8 9] | [10 11 12 13 14 15 16 17 18] \ + | [19 20 21 22 23 24 25 26 27] | [28 29 30 31] | R", + ), + ] { + assert_eq!(pages(33, None, &[held], &[]), want, "{held}"); + } + for (count, limit, held, want) in [ + ( + 21, + None, + vec![20], + "[0 1 2 3 4 5 6 7 8 9] | [16 17 18 19 10 11 12 13 14] | [15] | R", + ), + ( + 21, + None, + vec![2], + "[0 1 3 4 5 6 7 8 9] | [16 17 18 19 20 10 11 12 13] | [14 15] | R", + ), + ( + 40, + None, + vec![39], + "[0 1 2 3 4 5 6 7 8 9] | [10 11 12 13 14 15 16 17 18 19] \ + | [32 33 34 35 36 37 38 20 21] | [22 23 24 25 26 27 28 29 30] | [31] | R", + ), + ( + 21, + Some(5), + vec![20], + "[0 1 2 3 4] | [5 6 7 8 9] | [16 17 18 19 10] | [11 12 13 14 15] | R", + ), + ( + 23, + Some(4), + vec![5, 17], + "[0 1 2 3] | [4 6 7 8] | [9 10 11 12] | [16 18 19 20] | [21 22 13 14] \ + | [15] | R", + ), + ] { + assert_eq!(pages(count, limit, &held, &[]), want, "{count} {held:?}"); + } +} + +#[test] +fn a_query_stopped_at_the_limit_goes_on_at_its_place() { + // Positions 0 to 18, a Query of 21 items at 10, `Limit` 6. + assert_eq!( + pages(19, Some(6), &[], &[(10, 21)]), + "[0 1 2 3 4 5] | [6 7 8 9 10x2] | [16 17 18 10x3] | [10x6] | [10x6] \ + | [10x4 11 12] | [13 14 15]" + ); + // A Query the page goes on inside is one of its 10 visits: a Query of + // 13 items at 0 and `Limit` 12 give 12, 10, 10 and 1 items. + assert_eq!( + pages(21, Some(12), &[], &[(0, 13)]), + "[0x12] | [0x1 1 2 3 4 5 6 7 8 9] | [16 17 18 19 20 10 11 12 13 14] | [15]" + ); +} + +#[test] +fn several_queries_stopped_inside_each_go_on_from_their_own_key() { + // A second Query, of 10 items at 16, that the page after the first + // stopped stops in too, with and without position 17 held back. + assert_eq!( + pages(19, Some(6), &[], &[(10, 21), (16, 10)]), + "[0 1 2 3 4 5] | [6 7 8 9 10x2] | [16x6] | [16x4 17 18] | [10x6] | [10x6] \ + | [10x6] | [10x1 11 12 13 14 15]" + ); + assert_eq!( + pages(19, Some(6), &[17], &[(10, 21), (16, 10)]), + "[0 1 2 3 4 5] | [6 7 8 9 10x2] | [16x6] | [16x4 18 10x1] | [10x6] | [10x6] \ + | [10x6] | [11 12 13 14 15] | R" + ); +} + +#[test] +fn a_request_whose_page_visits_only_held_back_disjuncts_is_refused() { + // Ten held back that sort first: the first request is refused, with and + // without `Limit` 3, though eleven disjuncts are left that read. + let first: Vec = (0..10).collect(); + assert_eq!(pages(21, None, &first, &[]), "R"); + assert_eq!(pages(21, Some(3), &first, &[]), "R"); + // Ten held back at 10 to 19 of 25: the second page's ten visits are + // theirs, so the second request is refused, though 20 to 24 are left. + let middle: Vec = (10..20).collect(); + assert_eq!(pages(25, None, &middle, &[]), "[0 1 2 3 4 5 6 7 8 9] | R"); + // Nine held back and one read make a page. + let nine: Vec = (0..9).collect(); + assert_eq!( + pages(21, None, &nine, &[]), + "[9] | [10] | [11] | [12] | [13] | [14] | [15] | [16] | [17] | [18 19 20] | R" + ); + // A page that leaves only disjuncts held back leaves them for the next + // request, which visits only them and is refused. + let held_back: Vec = (0..3).map(|p| p == 2).collect(); + let mut walk = Walk::new(Cursor::first(3), &held_back, None); + for position in [0, 1] { + assert_eq!(walk.next_read().unwrap().position, position); + walk.record(position, Read::Done(1)); + } + assert!(walk.next_read().is_none()); + let Ending::More(next) = walk.finish() else { + panic!("the page leaves position 2"); + }; + assert_eq!(next.left, [2]); + let mut walk = Walk::new(next, &held_back, None); + assert!(walk.next_read().is_none()); + assert!(matches!(walk.finish(), Ending::Refused)); +} + +#[test] +fn a_page_that_ends_before_an_item_leaves_it_first_of_those_not_visited() { + let held_back = vec![false; 4]; + let mut walk = Walk::new(Cursor::first(4), &held_back, None); + for position in [0, 1] { + assert_eq!(walk.next_read().unwrap().position, position); + walk.record(position, Read::Done(1)); + } + assert_eq!(walk.next_read().unwrap().position, 2); + walk.record(2, Read::Unread); + assert!(walk.next_read().is_none()); + let Ending::More(next) = walk.finish() else { + panic!("the page leaves 2 and 3"); + }; + assert_eq!(next.left, [2, 3]); +} + +#[test] +fn a_cursor_round_trips_through_its_token_value() { + let key = json!({ "pk": { "S": "a" }, "sk": { "S": "x" } }); + let cursor = Cursor { + left: vec![16, 3, 10, 4], + after: [(10, key.clone()), (16, json!({ "pk": { "S": "b" } }))] + .into_iter() + .collect(), + }; + let value = cursor.clone().into_value(); + assert_eq!( + value, + json!({ "left": [16, 3, 10, 4], "after": [[10, key], [16, { "pk": { "S": "b" } }]] }) + ); + assert_eq!(Cursor::from_value(&value, 17).unwrap(), cursor); + assert_eq!( + Cursor::from_value(&Cursor::first(3).into_value(), 3).unwrap(), + Cursor::first(3) + ); +} + +#[test] +fn a_value_that_is_no_cursor_of_the_statement_is_refused() { + for value in [ + json!({ "left": [0, 3], "after": [] }), + json!({ "left": [0, 0], "after": [] }), + json!({ "left": [0, -1], "after": [] }), + json!({ "left": [0, "1"], "after": [] }), + json!({ "left": [0, 1], "after": [[2, {}]] }), + json!({ "left": [0, 1], "after": [[1, {}], [1, {}]] }), + json!({ "left": [0, 1], "after": [[1]] }), + json!({ "left": [0, 1] }), + json!({ "after": [] }), + json!({ "step": 1 }), + json!({ "pk": { "S": "a" } }), + ] { + let err = Cursor::from_value(&value, 3).unwrap_err(); + assert_eq!(err.message(), "NextToken does not match request", "{value}"); + } +} diff --git a/crates/engine/src/partiql/snapshot.rs b/crates/engine/src/partiql/snapshot.rs new file mode 100644 index 000000000..73c265f67 --- /dev/null +++ b/crates/engine/src/partiql/snapshot.rs @@ -0,0 +1,192 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The catalog snapshot the members of a batch or a transaction run against. + +use std::future::Future; +use std::sync::Arc; + +use extenddb_core::error::DynamoDbError; +use extenddb_core::types::TableKeyInfo; +use extenddb_storage::error::StorageError; + +use crate::create_table::storage_err_to_dynamo; +use crate::{AuthorizedKeyInfo, OperationContext}; + +/// `ctx` with `authorized_key_info` as its catalog snapshot. +pub(super) fn context_with_key_info( + ctx: &OperationContext, + authorized_key_info: AuthorizedKeyInfo, +) -> OperationContext { + OperationContext { + storage: Arc::clone(&ctx.storage), + backup: Arc::clone(&ctx.backup), + limits: Arc::clone(&ctx.limits), + region: Arc::clone(&ctx.region), + account_id: Arc::clone(&ctx.account_id), + import_paths: Arc::clone(&ctx.import_paths), + export_paths: Arc::clone(&ctx.export_paths), + authorized_key_info, + auth_cache: ctx.auth_cache.clone(), + table_key_info_lookup: ctx.table_key_info_lookup.clone(), + item_authorizer: ctx.item_authorizer.clone(), + } +} + +/// The context an `ExecuteStatement` runs in: `ctx` with `table` read once, +/// as a single-table request reads it, and added to its catalog snapshot, +/// with what the read gave (`None` for a missing table). The statement is +/// authorized against that read, and every later lookup of the table in the +/// request resolves from it. +/// +/// # Errors +/// +/// Returns a failed catalog read other than a missing table, mapped as the +/// classic handlers map it. +pub(super) async fn pinned_context( + ctx: &OperationContext, + table: &str, +) -> Result<(OperationContext, Option), DynamoDbError> { + let info = match ctx.table_key_info(table).await { + Ok(info) => Some(info), + Err(StorageError::TableNotFound(_)) => None, + Err(e) => return Err(storage_err_to_dynamo(e)), + }; + let mut snapshot = ctx.authorized_key_info.clone(); + snapshot.insert(table.to_owned(), info.clone()); + Ok((context_with_key_info(ctx, snapshot), info)) +} + +/// The context the members of a batch or a transaction run in: `ctx` with +/// every table of `tables` read once and added to its catalog snapshot. +/// Every later lookup of those tables, in the PartiQL layer and in the +/// classic requests it builds, then resolves from that one read, as the +/// catalog rule on [`OperationContext`] asks. A table is read as the classic +/// batch and transaction handlers read it, with `table_key_info_uncached`, +/// and a missing one stays missing for the rest of the request. +/// +/// # Errors +/// +/// Returns a failed catalog read other than a missing table, mapped as the +/// classic handlers map it. +pub(super) async fn snapshot_context( + ctx: &OperationContext, + tables: &[&str], +) -> Result { + let snapshot = resolve_tables(&ctx.authorized_key_info, tables, |table| { + ctx.table_key_info_uncached(table) + }) + .await?; + Ok(context_with_key_info(ctx, snapshot)) +} + +/// `known` with every table of `tables` it does not hold read once with +/// `read`: its key info, or `None` when the table is missing. +async fn resolve_tables<'t, F, R>( + known: &AuthorizedKeyInfo, + tables: &[&'t str], + mut read: F, +) -> Result +where + F: FnMut(&'t str) -> R, + R: Future>, +{ + let mut snapshot = known.clone(); + for &table in tables { + if snapshot.contains_key(table) { + continue; + } + let entry = match read(table).await { + Ok(info) => Some(info), + Err(StorageError::TableNotFound(_)) => None, + Err(e) => return Err(storage_err_to_dynamo(e)), + }; + snapshot.insert(table.to_owned(), entry); + } + Ok(snapshot) +} + +#[cfg(test)] +mod tests { + use std::cell::RefCell; + + use super::*; + + fn block_on(future: F) -> F::Output { + tokio::runtime::Builder::new_current_thread() + .build() + .unwrap() + .block_on(future) + } + + fn info(table: &str) -> TableKeyInfo { + TableKeyInfo { + table_name: table.to_owned(), + ..TableKeyInfo::default() + } + } + + #[test] + fn each_table_is_read_once_and_a_missing_one_is_recorded() { + let reads = RefCell::new(Vec::new()); + let mut known = AuthorizedKeyInfo::new(); + known.insert("authorized".to_owned(), Some(info("authorized"))); + let tables = ["a", "b", "a", "missing", "authorized", "b", "missing"]; + let snapshot = block_on(resolve_tables(&known, &tables, |table| { + reads.borrow_mut().push(table.to_owned()); + async move { + match table { + "missing" => Err(StorageError::TableNotFound(table.to_owned())), + _ => Ok(info(table)), + } + } + })) + .unwrap(); + assert_eq!(*reads.borrow(), ["a", "b", "missing"]); + assert_eq!(snapshot.len(), 4); + assert_eq!(snapshot["a"].as_ref().unwrap().table_name, "a"); + assert_eq!( + snapshot["authorized"].as_ref().unwrap().table_name, + "authorized" + ); + assert!(snapshot["missing"].is_none()); + } + + #[test] + fn every_rebuilt_context_keeps_the_item_authorizer() { + use crate::ItemAuthorizer; + use crate::partiql::test_context::{self, Fixed}; + test_context::block_on(async { + let authorizer: Arc = Fixed::allowing(); + let ctx = test_context::context(Some(Arc::clone(&authorizer))).await; + let keeps = |rebuilt: &OperationContext| { + rebuilt + .item_authorizer + .as_ref() + .is_some_and(|kept| Arc::ptr_eq(kept, &authorizer)) + }; + assert!(keeps(&context_with_key_info( + &ctx, + AuthorizedKeyInfo::new() + ))); + // "t" is missing in the context's snapshot, so neither of these + // reads the catalog. + let (pinned, info) = pinned_context(&ctx, "t").await.unwrap(); + assert!(info.is_none()); + assert!(keeps(&pinned)); + assert!(keeps(&snapshot_context(&ctx, &["t"]).await.unwrap())); + assert!(keeps(&crate::partiql::translation_context(&ctx))); + }); + } + + #[test] + fn a_failed_read_fails_the_request() { + // A lost connection is a retryable fault, not a missing table. + let result = block_on(resolve_tables( + &AuthorizedKeyInfo::new(), + &["a"], + |_| async { Err(StorageError::Connection("connection lost".to_owned())) }, + )); + assert!(matches!(result, Err(DynamoDbError::ServiceUnavailable(_)))); + } +} diff --git a/crates/engine/src/partiql/test_context.rs b/crates/engine/src/partiql/test_context.rs new file mode 100644 index 000000000..443514af0 --- /dev/null +++ b/crates/engine/src/partiql/test_context.rs @@ -0,0 +1,100 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! An [`OperationContext`] for the unit tests that need one, and item +//! authorizers that answer every access the same way. The context's catalog +//! is an empty in-memory database, and its snapshot holds the table "t" as +//! missing, so that nothing the PartiQL layer does for "t" reads the +//! catalog. + +use std::future::Future; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use extenddb_core::error::DynamoDbError; +use extenddb_core::limits::LimitsConfig; +use extenddb_storage_sqlite::SqliteEngine; + +use crate::{ + AuthorizedKeyInfo, BackupRuntime, ItemAccess, ItemAuthorization, ItemAuthorizer, + OperationContext, +}; + +/// Run `future` to completion on a runtime of its own. +pub(super) fn block_on(future: F) -> F::Output { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(future) +} + +/// A context whose item authorizer is `authorizer`. +pub(super) async fn context(authorizer: Option>) -> OperationContext { + let storage = SqliteEngine::new(":memory:", 1, "us-east-1", 409_600) + .await + .expect("an in-memory database opens"); + let mut authorized_key_info = AuthorizedKeyInfo::new(); + authorized_key_info.insert("t".to_owned(), None); + OperationContext { + storage: Arc::new(storage), + backup: Arc::new(BackupRuntime::new(1)), + limits: Arc::new(LimitsConfig::default()), + region: Arc::from("us-east-1"), + account_id: Arc::from("000000000000"), + import_paths: Arc::from(Vec::new()), + export_paths: Arc::from(Vec::new()), + authorized_key_info, + auth_cache: extenddb_auth::AuthCacheRegistry::default(), + table_key_info_lookup: None, + item_authorizer: authorizer, + } +} + +/// Gives every access the same answer, and counts the accesses it is asked +/// about. +pub(super) struct Fixed { + answer: Result<(), DynamoDbError>, + asked: AtomicUsize, +} + +impl Fixed { + fn new(answer: Result<(), DynamoDbError>) -> Arc { + Arc::new(Self { + answer, + asked: AtomicUsize::new(0), + }) + } + + /// Allows every access. + pub(super) fn allowing() -> Arc { + Self::new(Ok(())) + } + + /// Refuses every access. + pub(super) fn refusing() -> Arc { + Self::new(Err(DynamoDbError::AccessDeniedException( + "refused".to_owned(), + ))) + } + + /// Fails every access with a fault. + pub(super) fn failing() -> Arc { + Self::new(Err(DynamoDbError::InternalServerError( + "Internal error during authorization".to_owned(), + ))) + } + + /// The number of accesses asked about. + pub(super) fn asked(&self) -> usize { + self.asked.load(Ordering::SeqCst) + } +} + +impl ItemAuthorizer for Fixed { + fn authorize<'a>(&'a self, _access: &'a ItemAccess) -> ItemAuthorization<'a> { + self.asked.fetch_add(1, Ordering::SeqCst); + let answer = self.answer.clone(); + Box::pin(async move { answer }) + } +} diff --git a/crates/engine/src/partiql/transaction.rs b/crates/engine/src/partiql/transaction.rs new file mode 100644 index 000000000..d4ae9e1e8 --- /dev/null +++ b/crates/engine/src/partiql/transaction.rs @@ -0,0 +1,634 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! PartiQL `ExecuteTransaction` executor. +//! +//! All-read or all-write, all-or-nothing. Reads behave like `TransactGetItems` +//! (`Responses[i] = {Item}`, no `TableName`); any write failure cancels with a +//! positional `CancellationReasons` list. `ClientRequestToken` gives +//! idempotency. 1..=100 statements, no two on the same item. Every behaviour is +//! justified by a measurement against the live service, cited where it is +//! implemented. +//! +//! The executor composes the engine's atomic multi-item handlers: +//! `handle_transact_get_items` for the read form and `handle_transact_write_items` +//! for the write form (which already enforces atomicity, conflicting-key +//! rejection, and `ClientRequestToken` idempotency). It translates each +//! PartiQL statement into the corresponding transact member, cancels the write +//! form with positional reasons when a member cannot be translated against its +//! table, and remaps two error strings the PartiQL surface words differently. +//! +//! The members are authorized after the checks of their form (see +//! `authorize_members`). In an all-read transaction, a member on a table +//! that exists and that reads an index, or does not name the full primary +//! key, fails the request before any member is authorized, as the form of +//! an EXISTS member, an EXISTS on an index and a `BatchExecuteStatement` +//! read without the full primary key do (measured 2026-11-03 for a caller +//! that may not run them). For these two refusals of a read transaction the +//! order against authorization is not measured. + +use serde_json::{Map, Value, json}; + +use extenddb_core::error::DynamoDbError; +use extenddb_core::expression::Projection as ItemProjection; +use extenddb_core::partiql::Statement; +use extenddb_core::partiql::ast::{Projection, Returning, SelectStatement}; +use extenddb_core::partiql::parser::parse_for_request; +use extenddb_core::types::{CancellationReason, Item}; + +use super::common; +use super::expr_bridge::ExprBridge; +use super::{SingleRequest, authorize, item_depth, write}; +use crate::expression_helpers::{build_expression_maps, parse_projection_expr}; +use crate::{ + DispatchResult, OperationContext, handle_transact_get_items, handle_transact_write_items, +}; + +use members::{build_write_member, check_exists_form, check_parameter_count, member_reason}; + +mod members; + +/// Execute an `ExecuteTransaction` request. +/// +/// # Errors +/// +/// Returns the service's request-level errors (read/write mix, conflicting +/// keys, size limit, idempotency mismatch) or `TransactionCanceledException` +/// with positional cancellation reasons. +pub(super) async fn run( + ctx: &OperationContext, + body: &Value, +) -> Result { + super::request_checks::check_execute_transaction(body)?; + let statements = body + .get("TransactStatements") + .and_then(Value::as_array) + .ok_or_else(|| { + DynamoDbError::ValidationException( + "TransactStatements must be a non-empty list".to_owned(), + ) + })?; + + // The service checks each member in order and stops at the first that + // fails, so a refusal of a member wins over any refusal of a later one + // (measured 2026-11-05). The members are read and parsed up to the first + // whose statement fails its checks; that refusal is reported once the + // members before it have passed the checks they get against their tables. + let mut parsed: Vec<(SingleRequest, Statement)> = Vec::with_capacity(statements.len()); + let mut refusal = None; + for (i, obj) in statements.iter().enumerate() { + match parse_member(i, obj) { + Ok(member) => parsed.push(member), + Err(error) => { + refusal = Some(error); + break; + } + } + } + + // Each table the statements name is read from the catalog once, and + // every member, and the TransactGetItems or TransactWriteItems request + // built from them, runs against that read. + let tables: Vec<&str> = parsed + .iter() + .map(|(_, stmt)| common::statement_table(stmt)) + .collect(); + let ctx = &super::snapshot::snapshot_context(ctx, &tables).await?; + + // A member's refusals against its table come ahead of the refusal of a + // read and a write in one request, which a member that differs from the + // members before it gets (measured 2026-11-05: an index read, a read + // without the full key, a key-only EXISTS and a RETURNING clause after a + // member of the other kind are refused for themselves). + let mut reads = None; + for (i, (req, stmt)) in parsed.iter().enumerate() { + check_member(ctx, i, stmt, req).await?; + if *reads.get_or_insert(stmt.is_read()) != stmt.is_read() { + return Err(DynamoDbError::ValidationException( + "ExecuteTransaction API does not support both read and write operations in the same request." + .to_owned(), + )); + } + } + if let Some(refusal) = refusal { + return Err(refusal); + } + + let rcc = body.get("ReturnConsumedCapacity").and_then(Value::as_str); + + if reads == Some(true) { + run_read(ctx, &parsed, rcc).await + } else { + run_write(ctx, body, &parsed, rcc).await + } +} + +/// Read and parse the member at position `i`, and make the checks its +/// statement gets before its table is read. +/// +/// A statement of only whitespace fails the whole request without the +/// statement's position, ahead of its parameters (measured 2026-11-05). A +/// malformed statement fails the whole request, its message prefixed +/// with the statement's position (measured 2026-11-02 on an empty WHERE). A +/// parameter that cannot be converted is reported without the statement's +/// position (measured 2026-11-02); the checks on a parsed statement name it. +/// A projected call (`COUNT(*)`) is among those checks, so it is refused +/// ahead of every table lookup (measured 2026-11-02), and so are a +/// projection list that names an identifier twice or two paths that +/// overlap, and a `?` without a parameter (measured 2026-11-05). +fn parse_member(i: usize, obj: &Value) -> Result<(SingleRequest, Statement), DynamoDbError> { + let positional = |message: &str| { + DynamoDbError::ValidationException(format!( + "Validation failed in TransactStatements[{i}]: {message}" + )) + }; + let req = SingleRequest::from_body(obj)?; + if super::request_checks::is_blank_statement(&req.statement) { + return Err(DynamoDbError::ValidationException( + super::request_checks::EMPTY_TRANSACTION_STATEMENT.to_owned(), + )); + } + super::check_parameters(obj.get("Parameters")).map_err(DynamoDbError::ValidationException)?; + let stmt = parse_for_request(&req.statement, req.parameters.len()) + .map_err(|e| positional(&e.message))?; + super::check_statement(&stmt, &req.parameters).map_err(|message| positional(&message))?; + if let Statement::Select(select) = &stmt { + super::select::check_projection(select).map_err(|message| positional(&message))?; + } + check_parameter_count(&stmt, req.parameters.len(), i)?; + Ok((req, stmt)) +} + +/// The refusals the member at position `i` gets against its table, before +/// any member is authorized. A member on a missing table gets none of them +/// but the refusal of an EXISTS through an index: the missing table cancels +/// the request once every member is checked. +/// +/// A read is refused when it reads an index, then when its WHERE does not +/// give the full primary key; a key value of the wrong type cancels the +/// request later (measured 2026-11-05). An EXISTS is refused when it reads +/// an index, ahead of a missing table, then when it is not a read of one +/// item with a further condition (see `members::check_exists_form`). A +/// `RETURNING` clause is refused (see [`reject_returning`]). +/// +/// # Errors +/// +/// The member's refusal. +async fn check_member( + ctx: &OperationContext, + i: usize, + stmt: &Statement, + req: &SingleRequest, +) -> Result<(), DynamoDbError> { + match stmt { + Statement::Select(select) => check_read_form(ctx, i, select, req).await, + Statement::Exists(_) => check_exists_form(ctx, i, stmt, req).await, + Statement::Insert(_) | Statement::Update(_) | Statement::Delete(_) => { + if let Some(table) = returning_table(stmt) + && ctx.table_key_info(table).await.is_ok() + { + reject_returning(stmt, i)?; + } + Ok(()) + } + } +} + +/// The refusals a read member at position `i` gets against its table: a +/// read of an index, existing or not, GSI or vector, then a read whose WHERE +/// is not of one item by its full primary key. +async fn check_read_form( + ctx: &OperationContext, + i: usize, + select: &SelectStatement, + req: &SingleRequest, +) -> Result<(), DynamoDbError> { + let Ok(key_info) = ctx.table_key_info(&select.from.table).await else { + return Ok(()); + }; + if select.from.index.is_some() { + return Err(DynamoDbError::ValidationException(format!( + "Validation failed in TransactStatements[{i}]: Reads on indices are not \ + supported within transactions." + ))); + } + // A multi-part base table key (this engine's preview extension) is + // refused once the members are authorized (see `run_read`). + if common::ensure_single_attribute_keys(&key_info.key_schema).is_err() { + return Ok(()); + } + let (pk, sk) = common::key_names(&key_info.key_schema); + let pk = pk.ok_or_else(|| { + DynamoDbError::InternalServerError("table has no partition key".to_owned()) + })?; + match common::point_read_key(select, pk, sk, &req.parameters)? { + Some(_) => Ok(()), + None => Err(DynamoDbError::ValidationException(format!( + "Validation failed in TransactStatements[{i}]: Select statements within \ + ExecuteTransaction must specify the primary key in the where clause." + ))), + } +} + +/// Execute an all-read transaction via `TransactGetItems`. +/// +/// The members are authorized once their form is checked: after the +/// refusals of a read of an index and of a read without the full primary +/// key, which fail the request (see [`check_member`]), and ahead of a +/// missing table and a key value of the wrong type, which cancel it (see +/// `authorize::gate`). +async fn run_read( + ctx: &OperationContext, + parsed: &[(SingleRequest, Statement)], + rcc: Option<&str>, +) -> Result { + // A member on a missing table reports ResourceNotFound and a member whose + // key value has the wrong type reports that the key does not match the + // schema, in one cancellation (measured 2026-11-05 with the two in either + // order); every other member reports `None`. + let mut transact_items: Vec = Vec::with_capacity(parsed.len()); + let mut projections: Vec> = Vec::with_capacity(parsed.len()); + let mut reasons: Vec = Vec::with_capacity(parsed.len()); + let mut multi_attribute = None; + for (req, stmt) in parsed { + let Statement::Select(select) = stmt else { + unreachable!("read transaction holds only SELECT statements"); + }; + let Ok(key_info) = ctx.table_key_info(&select.from.table).await else { + reasons.push(not_found_reason()); + continue; + }; + // A multi-part base table key (this engine's preview extension) is + // refused with the service's multi-attribute message. + if let Err(e) = common::ensure_single_attribute_keys(&key_info.key_schema) { + multi_attribute.get_or_insert(e); + reasons.push(CancellationReason::none()); + continue; + } + let (pk, sk) = common::key_names(&key_info.key_schema); + let pk = pk.ok_or_else(|| { + DynamoDbError::InternalServerError("table has no partition key".to_owned()) + })?; + let key = common::point_read_key(select, pk, sk, &req.parameters)?.ok_or_else(|| { + DynamoDbError::InternalServerError("a read member has its full key".to_owned()) + })?; + reasons.push(match common::check_key_types(&key, &key_info) { + Ok(()) => CancellationReason::none(), + Err(_) => CancellationReason::validation_error(common::KEY_SCHEMA_MISMATCH), + }); + projections.push(member_projection(ctx, select)?); + transact_items.push(json!({ + "Get": { "TableName": select.from.table, "Key": Value::Object(key) } + })); + } + authorize_members(ctx, parsed).await?; + if reasons + .iter() + .any(|reason| reason.code == "ResourceNotFound") + { + return Err(cancellation(reasons)); + } + if let Some(refusal) = multi_attribute { + return Err(refusal); + } + if reasons.iter().any(|reason| reason.code != "None") { + return Err(cancellation(reasons)); + } + + let mut tbody = json!({ "TransactItems": transact_items }); + if let Some(rcc) = rcc { + tbody["ReturnConsumedCapacity"] = json!(rcc); + } + + // The handler's output is the transaction read shape: Responses as a + // positional list of `{Item}` / `{}`, plus a ConsumedCapacity list. The + // items are read whole and projected here, since `TransactGetItems` drops + // an item that its projection leaves empty. + project_items(handle_transact_get_items(tbody, ctx).await?, &projections) +} + +/// The projection a read member's projection list makes, or `None` for +/// `SELECT *`. The item keeps the shape of the paths, as a +/// `ProjectionExpression` gives it: `SELECT a, m.k` returns `{a, m: {k}}` +/// (measured 2026-11-05). +/// +/// # Errors +/// +/// A refusal of the projection, which the statement checks make first. +fn member_projection( + ctx: &OperationContext, + select: &SelectStatement, +) -> Result, DynamoDbError> { + let Projection::Paths(paths) = &select.projection else { + return Ok(None); + }; + let mut bridge = ExprBridge::new(&[]); + let rendered: Vec = paths.iter().map(|path| bridge.path(path)).collect(); + let maps = build_expression_maps(Some(&bridge.names_map()), None); + let paths = parse_projection_expr(&rendered.join(", "), &ctx.limits)?; + ItemProjection::compile(&paths, &maps, true).map(Some) +} + +/// Give each item of a `TransactGetItems` answer its member's projection. +/// An item that has none of the projected paths is returned empty, and a +/// member whose item is missing returns no item (measured 2026-11-05). +fn project_items( + mut result: DispatchResult, + projections: &[Option], +) -> Result { + let Some(Value::Array(responses)) = result.body.get_mut("Responses") else { + return Ok(result); + }; + for (entry, projection) in responses.iter_mut().zip(projections) { + let (Some(projection), Some(item)) = (projection, entry.get("Item")) else { + continue; + }; + let projected = serde_json::from_value::(item.clone()) + .map(|item| projection.apply(&item)) + .and_then(serde_json::to_value) + .map_err(|e| { + tracing::error!(internal_error = %e, "failed to project a transaction read"); + DynamoDbError::InternalServerError("Internal server error".to_owned()) + })?; + entry["Item"] = projected; + } + Ok(result) +} + +/// Execute an all-write transaction via `TransactWriteItems`. +/// +/// Every member is translated against its table first. When one cannot be, +/// because its table is missing or its key does not match the table's schema, +/// the request is cancelled with one reason per member and nothing runs: no +/// condition is evaluated, no duplicate `INSERT` is found and two members on +/// one item are not refused (measured 2026-11-02). When a member would store +/// a value nested past the item limit, two members on one item are refused +/// next, then every member reports its own reason (see `item_depth`). +async fn run_write( + ctx: &OperationContext, + body: &Value, + parsed: &[(SingleRequest, Statement)], + rcc: Option<&str>, +) -> Result { + authorize_members(ctx, parsed).await?; + + let mut transact_items: Vec = Vec::with_capacity(parsed.len()); + // Positional record of which members are INSERTs, so a condition failure on + // one can be remapped to the DuplicateItem cancellation reason the service + // reports for a duplicate-key INSERT. + let mut is_insert: Vec = Vec::with_capacity(parsed.len()); + let mut reasons: Vec = Vec::with_capacity(parsed.len()); + let mut cancelled = false; + + for (i, (req, stmt)) in parsed.iter().enumerate() { + if let Some(reason) = member_reason(ctx, stmt, req).await? { + reasons.push(reason); + cancelled = true; + continue; + } + reasons.push(CancellationReason::none()); + // Built even after another member's reason, so a refusal of this + // statement itself fails the request rather than being hidden by it. + let (member, insert) = build_write_member(ctx, i, stmt, req).await?; + transact_items.push(member); + is_insert.push(insert); + } + if cancelled { + return Err(cancellation(reasons)); + } + if item_depth::has_member_past_the_item_limit(parsed) + && let Some(reasons) = + item_depth::transaction_reasons(ctx, parsed, &transact_items, &is_insert).await? + { + return Err(cancellation(reasons)); + } + + let mut tbody = json!({ "TransactItems": transact_items }); + if let Some(token) = body.get("ClientRequestToken").and_then(Value::as_str) { + tbody["ClientRequestToken"] = json!(token); + } + if let Some(rcc) = rcc { + tbody["ReturnConsumedCapacity"] = json!(rcc); + } + + match handle_transact_write_items(tbody, ctx).await { + Ok(result) => Ok(reshape_write(result)), + Err(e) => Err(remap_write_error(e, &is_insert)), + } +} + +/// Authorize every member, after the checks of the request and of the +/// members' form, and before a missing table, a key or a condition fails a +/// member. When the caller may not run one, the request is cancelled with +/// one reason per member, `AccessDenied` with the refusal for each refused +/// member and `None` for the others, and nothing is written (measured +/// 2026-11-03: a condition that fails beside a refused member has the +/// reason `None`). +async fn authorize_members( + ctx: &OperationContext, + parsed: &[(SingleRequest, Statement)], +) -> Result<(), DynamoDbError> { + let mut reasons = Vec::with_capacity(parsed.len()); + let mut refused = false; + for (req, stmt) in parsed { + let key_info = ctx.table_key_info(common::statement_table(stmt)).await.ok(); + let surface = authorize::Surface::Transaction; + match authorize::statement(ctx, stmt, key_info.as_ref(), req, surface).await { + Ok(_) => reasons.push(CancellationReason::none()), + Err(DynamoDbError::AccessDeniedException(message)) => { + refused = true; + reasons.push(CancellationReason { + code: "AccessDenied".to_owned(), + message: Some(message), + item: None, + }); + } + Err(e) => return Err(e), + } + } + if refused { + return Err(cancellation(reasons)); + } + Ok(()) +} + +/// The reason a member on a missing table reports. +fn not_found_reason() -> CancellationReason { + CancellationReason { + code: "ResourceNotFound".to_owned(), + message: Some("Requested resource not found".to_owned()), + item: None, + } +} + +/// A cancellation carrying one reason per member, its message listing their +/// codes in order. +fn cancellation(reasons: Vec) -> DynamoDbError { + let codes: Vec<&str> = reasons.iter().map(|r| r.code.as_str()).collect(); + DynamoDbError::TransactionCanceledException { + message: format!( + "Transaction cancelled, please refer cancellation reasons for specific reasons [{}]", + codes.join(", ") + ), + cancellation_reasons: reasons, + } +} + +/// Reshape a `TransactWriteItems` response into the PartiQL write shape: an +/// empty `Responses` list, carrying the `ConsumedCapacity` list when present. +fn reshape_write(result: DispatchResult) -> DispatchResult { + let mut out = Map::new(); + out.insert("Responses".to_owned(), Value::Array(Vec::new())); + if let Some(cc) = result.body.get("ConsumedCapacity") { + out.insert("ConsumedCapacity".to_owned(), cc.clone()); + } + DispatchResult { + body: Value::Object(out), + metrics: result.metrics, + } +} + +/// Remap the two write errors the PartiQL surface words differently from the +/// classic `TransactWriteItems` handler. +/// +/// A duplicate-key `INSERT` fails the underlying `Put`'s `attribute_not_exists` +/// condition, which the handler reports as `ConditionalCheckFailed`; the PartiQL +/// surface reports `DuplicateItem` for that position. A reused idempotency token +/// with a different body uses the PartiQL message. +fn remap_write_error(err: DynamoDbError, is_insert: &[bool]) -> DynamoDbError { + match err { + DynamoDbError::TransactionCanceledException { + cancellation_reasons, + .. + } => { + let mut reasons = cancellation_reasons; + for (i, reason) in reasons.iter_mut().enumerate() { + if is_insert.get(i) == Some(&true) && reason.code == "ConditionalCheckFailed" { + *reason = common::duplicate_item_reason(); + } + } + cancellation(reasons) + } + DynamoDbError::IdempotentParameterMismatchException(_) => { + DynamoDbError::IdempotentParameterMismatchException( + "Specified idempotent token was used with different request parameters within the idempotency window" + .to_owned(), + ) + } + other => other, + } +} + +/// The table of a member that carries a `RETURNING` clause, or `None`. +fn returning_table(stmt: &Statement) -> Option<&str> { + match stmt { + Statement::Delete(delete) if delete.returning.is_some() => Some(&delete.table), + Statement::Update(update) if update.returning.is_some() => Some(&update.table), + _ => None, + } +} + +/// Refuse a member that carries a `RETURNING` clause, naming its position. +/// +/// The service rejects the whole request with a `ValidationException`, so +/// nothing in the transaction applies, rather than running the member and +/// cancelling (measured 2026-11-02). A `DELETE` whose clause is not +/// `RETURNING ALL OLD *` reports that clause as invalid instead, the refusal a +/// single statement gives. +fn reject_returning(stmt: &Statement, index: usize) -> Result<(), DynamoDbError> { + let reason = match stmt { + Statement::Delete(delete) => match delete.returning { + None => return Ok(()), + Some(Returning::AllOld) => RETURNING_REFUSAL.to_owned(), + Some(other) => write::invalid_delete_returning(other).message().to_owned(), + }, + Statement::Update(update) if update.returning.is_some() => RETURNING_REFUSAL.to_owned(), + _ => return Ok(()), + }; + Err(DynamoDbError::ValidationException(format!( + "Validation failed in TransactStatements[{index}]: {reason}" + ))) +} + +/// The service's refusal of a `RETURNING` clause in a transaction member. +const RETURNING_REFUSAL: &str = "RETURNING clause is not supported in ExecuteTransaction."; + +#[cfg(test)] +mod tests { + use super::*; + use extenddb_core::partiql::parse; + + fn refusal(sql: &str, index: usize) -> Option { + reject_returning(&parse(sql).unwrap(), index) + .err() + .map(|e| e.message().to_owned()) + } + + #[test] + fn a_cancellation_lists_every_reason_code_in_order() { + let err = cancellation(vec![ + CancellationReason::none(), + not_found_reason(), + CancellationReason::validation_error(common::KEY_SCHEMA_MISMATCH), + ]); + assert_eq!( + err.message(), + "Transaction cancelled, please refer cancellation reasons for specific reasons \ + [None, ResourceNotFound, ValidationError]" + ); + } + + #[test] + fn every_update_returning_mode_is_refused_with_its_position() { + for mode in ["ALL OLD", "MODIFIED OLD", "ALL NEW", "MODIFIED NEW"] { + let sql = format!("UPDATE \"t\" SET a = 1 WHERE pk = 'k' RETURNING {mode} *"); + assert_eq!( + refusal(&sql, 3).as_deref(), + Some( + "Validation failed in TransactStatements[3]: RETURNING clause is not \ + supported in ExecuteTransaction." + ), + "{mode}" + ); + } + } + + #[test] + fn delete_returning_all_old_is_refused_like_an_update() { + assert_eq!( + refusal("DELETE FROM \"t\" WHERE pk = 'k' RETURNING ALL OLD *", 0).as_deref(), + Some( + "Validation failed in TransactStatements[0]: RETURNING clause is not \ + supported in ExecuteTransaction." + ) + ); + } + + #[test] + fn delete_with_another_returning_clause_reports_the_invalid_clause() { + assert_eq!( + refusal( + "DELETE FROM \"t\" WHERE pk = 'k' RETURNING MODIFIED OLD *", + 1 + ) + .as_deref(), + Some( + "Validation failed in TransactStatements[1]: Invalid returning clause: \ + RETURNING MODIFIED OLD *. Only RETURNING ALL OLD * is allowed in DELETE \ + statements." + ) + ); + } + + #[test] + fn members_without_returning_pass() { + for sql in [ + "UPDATE \"t\" SET a = 1 WHERE pk = 'k'", + "DELETE FROM \"t\" WHERE pk = 'k'", + "INSERT INTO \"t\" VALUE {'pk': 'k'}", + "SELECT * FROM \"t\" WHERE pk = 'k'", + ] { + assert_eq!(refusal(sql, 0), None, "{sql}"); + } + } +} diff --git a/crates/engine/src/partiql/transaction/members.rs b/crates/engine/src/partiql/transaction/members.rs new file mode 100644 index 000000000..b7064d8be --- /dev/null +++ b/crates/engine/src/partiql/transaction/members.rs @@ -0,0 +1,508 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! A write transaction's members: their TransactWriteItems form and their cancellation reasons. + +use serde_json::{Value, json}; + +use extenddb_core::error::DynamoDbError; +use extenddb_core::partiql::Statement; +use extenddb_core::partiql::ast::{InsertStatement, SelectStatement, UpdateStatement, WhereExpr}; +use extenddb_core::types::{AttributeValue, CancellationReason, Item, TableKeyInfo}; +use extenddb_core::validation::validate_item_keys; + +use super::not_found_reason; +use crate::OperationContext; +use crate::create_table::storage_err_to_dynamo; +use crate::partiql::{SingleRequest, common, write}; + +/// Translate one PartiQL write statement (at position `index`) into its +/// `TransactWriteItems` member, returning the member and whether it is an +/// `INSERT`. +pub(super) async fn build_write_member( + ctx: &OperationContext, + index: usize, + stmt: &Statement, + req: &SingleRequest, +) -> Result<(Value, bool), DynamoDbError> { + match stmt { + Statement::Exists(select) => Ok((exists_member(ctx, index, select, req).await?, false)), + Statement::Insert(insert) => { + let parts = write::build_insert_parts(ctx, insert, &req.parameters).await?; + let mut put = json!({ + "TableName": parts.table, + "Item": parts.item, + "ConditionExpression": parts.condition, + "ExpressionAttributeNames": Value::Object(parts.names), + }); + attach_rvoccf(&mut put, req); + Ok((json!({ "Put": put }), true)) + } + Statement::Update(update) => { + let parts = write::build_update_parts(ctx, update, &req.parameters).await?; + let mut upd = json!({ + "TableName": parts.table, + "Key": Value::Object(parts.key), + "UpdateExpression": parts.update_expression, + "ConditionExpression": parts.condition, + }); + if !parts.names.is_empty() { + upd["ExpressionAttributeNames"] = Value::Object(parts.names); + } + if !parts.values.is_empty() { + upd["ExpressionAttributeValues"] = Value::Object(parts.values); + } + attach_rvoccf(&mut upd, req); + Ok((json!({ "Update": upd }), false)) + } + Statement::Delete(delete) => { + let parts = write::build_delete_parts(ctx, delete, &req.parameters).await?; + let mut del = json!({ + "TableName": parts.table, + "Key": Value::Object(parts.key), + }); + if let Some(condition) = parts.condition { + del["ConditionExpression"] = json!(condition); + } + if !parts.names.is_empty() { + del["ExpressionAttributeNames"] = Value::Object(parts.names); + } + if !parts.values.is_empty() { + del["ExpressionAttributeValues"] = Value::Object(parts.values); + } + attach_rvoccf(&mut del, req); + Ok((json!({ "Delete": del }), false)) + } + Statement::Select(_) => Err(DynamoDbError::InternalServerError( + "SELECT routed to the transaction write path".to_owned(), + )), + } +} + +/// The `ConditionCheck` member of an `EXISTS` statement at position +/// `index`. +/// +/// # Errors +/// +/// The refusal of a check through an index, and of one that is not a read +/// of one item with a further condition, naming the position. +async fn exists_member( + ctx: &OperationContext, + index: usize, + select: &SelectStatement, + req: &SingleRequest, +) -> Result { + if select.from.index.is_some() { + return Err(exists_index_refusal(index)); + } + let parts = write::build_exists_parts(ctx, index, select, &req.parameters) + .await? + .ok_or_else(|| { + DynamoDbError::ValidationException(format!( + "Validation failed in TransactStatements[{index}]: EXISTS() must \ + contain a single item read with additional condition" + )) + })?; + let mut check = json!({ + "TableName": parts.table, + "Key": Value::Object(parts.key), + "ConditionExpression": parts.condition, + "ExpressionAttributeNames": Value::Object(parts.names), + }); + if !parts.values.is_empty() { + check["ExpressionAttributeValues"] = Value::Object(parts.values); + } + // A failed check returns no item for `ALL_OLD` (measured + // 2026-11-02 with one key value and with two: the reason holds + // no item though the item exists). + Ok(json!({ "ConditionCheck": check })) +} + +/// The refusal of a condition check through an index, at position `index` +/// (measured 2026-11-02 on an index the table does not have). +fn exists_index_refusal(index: usize) -> DynamoDbError { + DynamoDbError::ValidationException(format!( + "Validation failed in TransactStatements[{index}]: Operations on indices are \ + not supported in transactions" + )) +} + +/// Refuse an `EXISTS` member at position `index` whose form the service +/// refuses (see [`exists_member`]), before the members are authorized: a +/// caller that may not read the table gets the refusal of the form +/// (measured 2026-11-03). A check through an index is refused ahead of a +/// missing table (measured 2026-11-05 with a table and an index that do not +/// exist); any other member on a missing table is left to the checks made +/// after authorization. +/// +/// # Errors +/// +/// The refusal of the member's form. +pub(super) async fn check_exists_form( + ctx: &OperationContext, + index: usize, + stmt: &Statement, + req: &SingleRequest, +) -> Result<(), DynamoDbError> { + let Statement::Exists(select) = stmt else { + return Ok(()); + }; + if select.from.index.is_some() { + return Err(exists_index_refusal(index)); + } + if ctx.table_key_info(&select.from.table).await.is_err() { + return Ok(()); + } + exists_member(ctx, index, select, req).await.map(|_| ()) +} + +/// Carry a statement's per-statement `ReturnValuesOnConditionCheckFailure` onto +/// its transact member, so a conditional cancel returns the pre-image item. +fn attach_rvoccf(member: &mut Value, req: &SingleRequest) { + if let Some(rvoccf) = &req.return_values_on_condition_check_failure { + member["ReturnValuesOnConditionCheckFailure"] = json!(rvoccf); + } +} + +/// Refuse a member whose statement has more `?` placeholders than the request +/// gives it parameters, naming its position (measured 2026-11-02). Extra +/// parameters are ignored, as for a single statement. +pub(super) fn check_parameter_count( + stmt: &Statement, + supplied: usize, + index: usize, +) -> Result<(), DynamoDbError> { + if supplied < stmt.parameter_count() { + return Err(DynamoDbError::ValidationException(format!( + "Validation failed in TransactStatements[{index}]: Number of parameters in request \ + and statement don't match." + ))); + } + Ok(()) +} + +/// The reason a write member cancels the request with when it cannot be +/// translated against its table, or `None` when it can. +/// +/// A missing table is `ResourceNotFound`. A WHERE that does not give the full +/// key by equality, or an `UPDATE` key value of the wrong type, is a +/// `ValidationError` saying the key does not match the schema, and so is a +/// WHERE that tests a path into a key attribute against values (measured +/// 2026-11-05 on UPDATE and DELETE); a path into a key attribute among the +/// other conditions of an `UPDATE`, a `DELETE` or an `EXISTS` is a +/// `ValidationError` with its refusal (see [`condition_key_reason`]); a `SET` of a +/// key attribute and an `INSERT` without one are `ValidationError`s naming +/// the attribute (measured 2026-11-02); a secondary index key given a value +/// of the wrong type is a `ValidationError` with the `UpdateItem` or +/// `PutItem` refusal (see [`index_key_reason`]). A `DELETE` key value of the +/// wrong type is treated like an `UPDATE` one, a `REMOVE` of a key attribute +/// like a `SET`, and an `INSERT` key value of the wrong type gets the message +/// a `TransactWriteItems` `Put` gives for it. +/// +/// # Errors +/// +/// Returns the refusal of a multi-attribute key, which fails the request. +pub(super) async fn member_reason( + ctx: &OperationContext, + stmt: &Statement, + req: &SingleRequest, +) -> Result, DynamoDbError> { + let key_info = match ctx.table_key_info(common::statement_table(stmt)).await { + Ok(info) => info, + Err(e) => { + return match storage_err_to_dynamo(e) { + DynamoDbError::ResourceNotFoundException(_) => Ok(Some(not_found_reason())), + other => Err(other), + }; + } + }; + common::ensure_single_attribute_keys(&key_info.key_schema)?; + let params = &req.parameters; + let condition_key = + |where_clause: Option<&WhereExpr>| condition_key_reason(where_clause, &key_info, params); + match stmt { + Statement::Insert(insert) => match insert_key_reason(insert, &key_info, params)? { + Some(reason) => Ok(Some(reason)), + None => Ok(index_key_reason(ctx, stmt, &key_info, params).await), + }, + Statement::Update(update) => { + if let Some(reason) = key_reason(stmt, &key_info, params)? + .or_else(|| key_update_reason(update, &key_info)) + { + return Ok(Some(reason)); + } + Ok(match index_key_reason(ctx, stmt, &key_info, params).await { + Some(reason) => Some(reason), + None => condition_key(update.where_clause.as_ref()), + }) + } + Statement::Delete(delete) => Ok(key_reason(stmt, &key_info, params)? + .or_else(|| condition_key(Some(&delete.where_clause)))), + Statement::Exists(select) => Ok(condition_key(select.where_clause.as_ref())), + Statement::Select(_) => Ok(None), + } +} + +/// The reason an `INSERT` or an `UPDATE` member reports when its request +/// gives a secondary index key a value of the wrong type: the `INSERT`'s +/// item, or a value the `UPDATE` sets. Each member that does so has the +/// reason beside the reasons of the other members, and an `UPDATE` has it +/// after a SET of a key attribute and ahead of a path into a key attribute +/// among its other conditions (measured 2026-11-05). A member whose request +/// cannot be built has none here: its refusal comes when it is built. +async fn index_key_reason( + ctx: &OperationContext, + stmt: &Statement, + key_info: &TableKeyInfo, + params: &[AttributeValue], +) -> Option { + let refusal = match stmt { + Statement::Insert(insert) => { + let parts = write::build_insert_parts(ctx, insert, params).await.ok()?; + write::insert_refusal(&parts, key_info).ok()? + } + Statement::Update(update) => { + let parts = write::build_update_parts(ctx, update, params).await.ok()?; + write::update_index_key_refusal(&ctx.limits, &parts, key_info).ok()? + } + Statement::Delete(_) | Statement::Exists(_) | Statement::Select(_) => None, + }; + refusal.map(|error| CancellationReason::validation_error(error.message())) +} + +/// The reason a member reports when its WHERE pins its item and reads a +/// path into a key attribute among its other conditions (see +/// `common::condition_key_refusal`), after the reasons of its key and of a +/// SET of a key attribute (measured 2026-11-05 on UPDATE, DELETE and +/// EXISTS). +fn condition_key_reason( + where_clause: Option<&WhereExpr>, + key_info: &TableKeyInfo, + params: &[AttributeValue], +) -> Option { + let (Some(pk), sk) = common::key_names(&key_info.key_schema) else { + return None; + }; + common::condition_key_refusal(where_clause?, pk, sk, params) + .map(CancellationReason::validation_error) +} + +/// The reason an `UPDATE` or `DELETE` reports when its WHERE does not give the +/// full primary key by equality, or gives a key value whose type is not the +/// one the table defines. +fn key_reason( + stmt: &Statement, + key_info: &TableKeyInfo, + params: &[AttributeValue], +) -> Result, DynamoDbError> { + let (Some(pk), sk) = common::key_names(&key_info.key_schema) else { + return Err(DynamoDbError::InternalServerError( + "table has no partition key".to_owned(), + )); + }; + let matches = common::write_key(stmt, pk, sk, params)?.is_some_and(|key| { + key.iter() + .all(|(name, value)| common::wire_key_value_matches(key_info, name, value)) + }); + if matches { + Ok(None) + } else { + Ok(Some(CancellationReason::validation_error( + common::KEY_SCHEMA_MISMATCH, + ))) + } +} + +/// The reason an `UPDATE` reports when it sets or removes a key attribute, +/// naming the first one in statement order. +fn key_update_reason( + update: &UpdateStatement, + key_info: &TableKeyInfo, +) -> Option { + update + .sets + .iter() + .map(|set| &set.path) + .chain(&update.removes) + .find_map(|path| { + key_info + .key_schema + .iter() + .find(|key| key.attribute_name == path.root) + }) + .map(|key| { + CancellationReason::validation_error(format!( + "One or more parameter values were invalid: Cannot update attribute {}. \ + This attribute is part of the key", + key.attribute_name + )) + }) +} + +/// The reason an `INSERT` reports when its item leaves out a key attribute or +/// gives one a value of the wrong type, worded as a `TransactWriteItems` `Put` +/// of that item words it rather than as `ExecuteStatement` does. +fn insert_key_reason( + insert: &InsertStatement, + key_info: &TableKeyInfo, + params: &[AttributeValue], +) -> Result, DynamoDbError> { + let mut keys = Item::new(); + for (name, operand) in common::insert_values(insert) { + if key_info + .key_schema + .iter() + .any(|key| key.attribute_name == name) + { + keys.insert(name.to_owned(), write::resolve_av(operand, params)?); + } + } + Ok( + validate_item_keys(&keys, &key_info.key_schema, &key_info.attribute_definitions) + .err() + .map(|e| CancellationReason::validation_error(e.message())), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use extenddb_core::partiql::parse; + use extenddb_core::types::{ + AttributeDefinition, KeySchemaElement, KeyType, ScalarAttributeType, + }; + + /// A pk(S)/sk(S) table. + fn composite_table() -> TableKeyInfo { + let key = |name: &str, key_type| KeySchemaElement { + attribute_name: name.to_owned(), + key_type, + }; + let string = |name: &str| AttributeDefinition { + attribute_name: name.to_owned(), + attribute_type: ScalarAttributeType::S, + }; + TableKeyInfo { + table_name: "t".to_owned(), + key_schema: vec![key("pk", KeyType::Hash), key("sk", KeyType::Range)], + attribute_definitions: vec![string("pk"), string("sk")], + ..TableKeyInfo::default() + } + } + + /// The reason message a write member reports on the composite table. + fn member_message(sql: &str) -> Option { + let table = composite_table(); + let stmt = parse(sql).unwrap(); + let reason = match &stmt { + Statement::Insert(insert) => insert_key_reason(insert, &table, &[]).unwrap(), + Statement::Update(update) => key_reason(&stmt, &table, &[]) + .unwrap() + .or_else(|| key_update_reason(update, &table)), + Statement::Delete(_) => key_reason(&stmt, &table, &[]).unwrap(), + Statement::Select(_) | Statement::Exists(_) => unreachable!("not a write"), + }; + reason.map(|r| { + assert_eq!(r.code, "ValidationError", "{sql}"); + r.message.unwrap() + }) + } + + #[test] + fn a_member_without_the_full_key_does_not_match_the_schema() { + for sql in [ + "UPDATE \"t\" SET a = 1 WHERE pk = 'k'", + "UPDATE \"t\" SET a = 1 WHERE a = 2", + "DELETE FROM \"t\" WHERE pk = 'k'", + "UPDATE \"t\" SET a = 1 WHERE pk = 'k' AND sk = 1", + "DELETE FROM \"t\" WHERE pk = 'k' AND sk = 1", + ] { + assert_eq!( + member_message(sql).as_deref(), + Some("The provided key element does not match the schema"), + "{sql}" + ); + } + } + + #[test] + fn a_member_with_the_full_key_translates() { + for sql in [ + "UPDATE \"t\" SET a = 1 WHERE pk = 'k' AND sk = 's'", + "UPDATE \"t\" SET a = 1 WHERE pk = 'k' AND sk = 's' AND a = 2", + "DELETE FROM \"t\" WHERE sk = 's' AND pk = 'k'", + "INSERT INTO \"t\" VALUE {'pk': 'k', 'sk': 's', 'a': 1}", + ] { + assert_eq!(member_message(sql), None, "{sql}"); + } + } + + #[test] + fn an_update_of_a_key_attribute_names_it() { + for (sql, name) in [ + ( + "UPDATE \"t\" SET pk = 'z' WHERE pk = 'k' AND sk = 's'", + "pk", + ), + ("UPDATE \"t\" REMOVE sk WHERE pk = 'k' AND sk = 's'", "sk"), + ( + "UPDATE \"t\" SET a = 1, sk = 'z' WHERE pk = 'k' AND sk = 's'", + "sk", + ), + ] { + assert_eq!( + member_message(sql), + Some(format!( + "One or more parameter values were invalid: Cannot update attribute \ + {name}. This attribute is part of the key" + )), + "{sql}" + ); + } + } + + #[test] + fn an_insert_key_problem_takes_the_put_message() { + for (sql, message) in [ + ( + "INSERT INTO \"t\" VALUE {'pk': 'k'}", + "Missing the key sk in the item", + ), + ( + "INSERT INTO \"t\" VALUE {'sk': 's'}", + "Missing the key pk in the item", + ), + ( + "INSERT INTO \"t\" VALUE {'pk': 'k', 'sk': 1}", + "Type mismatch for key sk expected: S actual: N", + ), + ] { + assert_eq!( + member_message(sql), + Some(format!( + "One or more parameter values were invalid: {message}" + )), + "{sql}" + ); + } + } + + #[test] + fn too_few_parameters_name_the_member() { + let stmt = parse("UPDATE \"t\" SET a = ? WHERE pk = ? AND sk = 's'").unwrap(); + assert_eq!( + check_parameter_count(&stmt, 1, 2) + .err() + .map(|e| e.message().to_owned()) + .as_deref(), + Some( + "Validation failed in TransactStatements[2]: Number of parameters in request \ + and statement don't match." + ) + ); + assert!(check_parameter_count(&stmt, 2, 2).is_ok()); + assert!(check_parameter_count(&stmt, 3, 2).is_ok()); + let quoted = parse("UPDATE \"t\" SET a = '?' WHERE pk = 'k'").unwrap(); + assert!(check_parameter_count("ed, 0, 0).is_ok()); + } +} diff --git a/crates/engine/src/partiql/write.rs b/crates/engine/src/partiql/write.rs new file mode 100644 index 000000000..904096317 --- /dev/null +++ b/crates/engine/src/partiql/write.rs @@ -0,0 +1,410 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! PartiQL write executor: `INSERT`, `UPDATE`, `DELETE`. +//! +//! Drives the engine's put/update/delete handlers with the PartiQL-specific +//! semantics on top: INSERT is create-only (`DuplicateItemException`), UPDATE +//! carries an implicit existence condition (not an upsert), both require the +//! full key, extra WHERE terms become a condition expression, and `RETURNING` +//! shapes the `Items` payload. Every behaviour is justified by a measurement +//! against the live service, cited where it is implemented. +//! +//! The statement is translated into the classic request shapes +//! (`PutItem`/`UpdateItem`/`DeleteItem` bodies) and handed to the existing +//! handlers, so the measured classic semantics (condition evaluation, capacity, +//! item sizing, error mapping) are reused rather than re-implemented. + +use std::collections::BTreeMap; + +use serde_json::{Map, Value, json}; + +use extenddb_core::error::DynamoDbError; +use extenddb_core::partiql::Statement; +use extenddb_core::partiql::ast::{Operand, Returning, WhereExpr}; +use extenddb_core::partiql::plan::SCALAR_KEY_REFUSAL; +use extenddb_core::types::{AttributeValue, ScalarAttributeType, TableKeyInfo}; +use extenddb_core::validation::IndexKeyRef; + +use super::authorize::Gate; +use super::{SingleRequest, bind_parameter, common}; +use crate::{DispatchResult, OperationContext}; + +use delete::run_delete; +use expr_builder::ExprBuilder; +use insert::run_insert; +use update::run_update; + +pub(super) use delete::build_delete_parts; +pub(super) use exists::build_exists_parts; +pub(super) use insert::{build_insert_parts, insert_refusal}; +pub(super) use update::{ + UpdateParts, build_update_parts, check_overlapping_paths, first_overlap, + update_index_key_refusal, update_refusal, +}; + +mod delete; +mod exists; +mod expr_builder; +mod insert; +mod update; + +/// Execute a parsed write statement (`INSERT`/`UPDATE`/`DELETE`), passing +/// `gate` once its checks made before authorization are done and before it +/// writes (see `authorize::gate`). +/// +/// # Errors +/// +/// Returns the service's error for the failure (duplicate key, failed +/// condition, missing full key, invalid returning clause), as measured, and +/// the statement's refusal. +pub(super) async fn run( + ctx: &OperationContext, + stmt: &Statement, + req: &SingleRequest, + gate: &mut Gate<'_>, +) -> Result { + match stmt { + Statement::Insert(insert) => run_insert(ctx, insert, req, gate).await, + Statement::Update(update) => run_update(ctx, update, req, gate).await, + Statement::Delete(delete) => run_delete(ctx, delete, req, gate).await, + Statement::Exists(_) => Err(exists_outside_transaction()), + // SELECT is dispatched to the read executor by the caller; reaching + // here is a routing bug, not a client error. + Statement::Select(_) => Err(DynamoDbError::InternalServerError( + "SELECT routed to the write executor".to_owned(), + )), + } +} + +/// Translate a parsed write statement into the request it runs as, without +/// running it: the checks [`run`] makes before it authorizes the statement. +/// +/// # Errors +/// +/// The refusal [`run`] answers for a statement that fails one of them. +pub(super) async fn check_translation( + ctx: &OperationContext, + stmt: &Statement, + params: &[AttributeValue], +) -> Result<(), DynamoDbError> { + match stmt { + Statement::Insert(insert) => build_insert_parts(ctx, insert, params).await.map(drop), + Statement::Update(update) => build_update_parts(ctx, update, params).await.map(drop), + Statement::Delete(delete) => { + if let Some(returning) = delete.returning + && returning != Returning::AllOld + { + return Err(invalid_delete_returning(returning)); + } + build_delete_parts(ctx, delete, params).await.map(drop) + } + Statement::Exists(_) => Err(exists_outside_transaction()), + Statement::Select(_) => Err(DynamoDbError::InternalServerError( + "SELECT routed to the write executor".to_owned(), + )), + } +} + +/// `EXISTS(SELECT ...)` is a condition check for an `ExecuteTransaction` +/// member. `ExecuteStatement` refuses it, and a `BatchExecuteStatement` +/// member reports it as its error (measured 2026-11-02). +pub(super) fn exists_outside_transaction() -> DynamoDbError { + DynamoDbError::ValidationException( + "EXISTS can only be used in ExecuteTransaction write requests.".to_owned(), + ) +} + +// --- response shaping ----------------------------------------------------- + +/// Reshape a classic put/update/delete response into the PartiQL write shape: +/// a `RETURNING` item becomes the single element of `Items`, an absent one +/// leaves `Items` empty, and `ConsumedCapacity` passes through when present. +fn reshape(result: DispatchResult) -> DispatchResult { + let mut items: Vec = Vec::new(); + if let Some(attributes) = result.body.get("Attributes") { + items.push(attributes.clone()); + } + let mut body = json!({ "Items": items }); + if let Some(cc) = result.body.get("ConsumedCapacity") { + body["ConsumedCapacity"] = cc.clone(); + } + DispatchResult { + body, + metrics: result.metrics, + } +} + +// --- key / condition translation ----------------------------------------- + +/// The partition and (optional) sort key attribute names of the target table. +/// +/// A multi-part base table key (this engine's preview extension) is refused +/// with the service's multi-attribute message before any write is attempted. +fn key_names(key_info: &TableKeyInfo) -> Result<(&str, Option<&str>), DynamoDbError> { + super::common::ensure_single_attribute_keys(&key_info.key_schema)?; + let (partition, sort) = common::key_names(&key_info.key_schema); + let partition = partition.ok_or_else(|| { + DynamoDbError::InternalServerError("table has no partition key".to_owned()) + })?; + Ok((partition, sort)) +} + +/// Split a write's WHERE into the full primary key (returned in wire form) and +/// the residual condition terms. A key equality may have the value on either +/// side (measured 2026-11-02: `UPDATE ... WHERE 'a' = pk` updates the item). +/// +/// # Errors +/// +/// The service's refusal of a key attribute with two different values +/// (measured 2026-11-02: `pk = 'a' AND 1 = pk`) and of a path into a key +/// attribute tested against values (see `common::where_key`), and its +/// full-key message when the key is not pinned by equality. +fn split_key_where<'a>( + where_clause: &'a WhereExpr, + pk_name: &str, + sk_name: Option<&str>, + params: &[AttributeValue], +) -> Result<(Map, Vec<&'a WhereExpr>), DynamoDbError> { + match split_full_key(where_clause, pk_name, sk_name, params)? { + FullKey::Pinned(key, residual) => Ok((key, residual)), + FullKey::Conflict(name) => Err(DynamoDbError::ValidationException(format!( + "Multiple conditions on same key {name}. Only single item Update/Insert/Delete are \ + supported" + ))), + FullKey::IntoKey(name) => Err(DynamoDbError::ValidationException(format!( + "{SCALAR_KEY_REFUSAL}{name}" + ))), + FullKey::Partial => Err(full_key_error()), + } +} + +/// The service's refusal of a write whose WHERE does not pin the whole key. +fn full_key_error() -> DynamoDbError { + DynamoDbError::ValidationException( + "Where clause does not contain a mandatory equality on all key attributes".to_owned(), + ) +} + +/// How a write's WHERE pins its item (see `common::where_key`), with the key +/// in wire form. +enum FullKey<'a> { + /// The full primary key, and the residual WHERE terms. + Pinned(Map, Vec<&'a WhereExpr>), + /// A key attribute without an equality. + Partial, + /// A key attribute with two different values, named. + Conflict(String), + /// A path into a key attribute tested against values, named. + IntoKey(String), +} + +/// Read a write's WHERE for its full primary key. +fn split_full_key<'a>( + where_clause: &'a WhereExpr, + pk_name: &str, + sk_name: Option<&str>, + params: &[AttributeValue], +) -> Result, DynamoDbError> { + let (partition, sort, residual) = + match common::where_key(where_clause, pk_name, sk_name, params) { + common::WhereKey::Pinned { + partition, + sort, + residual, + } => (partition, sort, residual), + common::WhereKey::Partial => return Ok(FullKey::Partial), + common::WhereKey::Conflict { name, .. } => return Ok(FullKey::Conflict(name)), + common::WhereKey::IntoKey { name } => return Ok(FullKey::IntoKey(name)), + }; + let key = wire_key(pk_name, sk_name, partition, sort, params)?; + Ok(FullKey::Pinned(key, residual)) +} + +/// The wire-form primary key of the given key values. +fn wire_key( + pk_name: &str, + sk_name: Option<&str>, + partition: &Operand, + sort: Option<&Operand>, + params: &[AttributeValue], +) -> Result, DynamoDbError> { + let mut key = Map::new(); + key.insert( + pk_name.to_owned(), + to_json_value(&resolve_av(partition, params)?)?, + ); + if let (Some(sort_name), Some(value)) = (sk_name, sort) { + key.insert( + sort_name.to_owned(), + to_json_value(&resolve_av(value, params)?)?, + ); + } + Ok(key) +} + +/// Build a condition expression from residual WHERE terms, or `None` when there +/// are none. Each term is a self-contained predicate, so multiple terms are +/// ANDed with each wrapped in parentheses. +fn build_condition( + builder: &mut ExprBuilder, + residual: &[&WhereExpr], + params: &[AttributeValue], +) -> Result, DynamoDbError> { + if residual.is_empty() { + return Ok(None); + } + if residual.len() == 1 { + return Ok(Some(builder.condition(residual[0], params)?)); + } + let mut parts = Vec::new(); + for term in residual { + parts.push(format!("({})", builder.condition(term, params)?)); + } + Ok(Some(parts.join(" AND "))) +} + +// --- helpers -------------------------------------------------------------- + +/// Resolve an operand that must be a literal or a positional parameter to its +/// AttributeValue (used for INSERT item values and WHERE key equalities). +pub(super) fn resolve_av( + operand: &Operand, + params: &[AttributeValue], +) -> Result { + match operand { + Operand::Literal(av) => Ok(av.clone()), + Operand::Parameter(index) => bind_parameter(params, *index), + _ => Err(DynamoDbError::ValidationException( + "Unsupported value expression".to_owned(), + )), + } +} + +/// Validate that every key attribute is present in an INSERT item and typed to +/// match the table schema, with the PartiQL-specific messages. +fn validate_insert_keys( + item: &BTreeMap, + key_info: &TableKeyInfo, +) -> Result<(), DynamoDbError> { + for element in &key_info.key_schema { + let name = &element.attribute_name; + let Some(value) = item.get(name) else { + return Err(DynamoDbError::ValidationException(format!( + "Key attribute should be present in the item: Key {name}" + ))); + }; + if let Some(definition) = key_info + .attribute_definitions + .iter() + .find(|d| &d.attribute_name == name) + { + let matches = match definition.attribute_type { + ScalarAttributeType::S => matches!(value, AttributeValue::S(_)), + ScalarAttributeType::N => matches!(value, AttributeValue::N(_)), + ScalarAttributeType::B => matches!(value, AttributeValue::B(_)), + }; + if !matches { + return Err(DynamoDbError::ValidationException(format!( + "Key attribute's data type should match its data type in table's schema: Key {name}" + ))); + } + } + } + Ok(()) +} + +/// Map a `RETURNING` mode to the classic `ReturnValues` string, or `None` when +/// the statement has no `RETURNING` clause. A `MODIFIED` mode asks for the +/// whole item on its side; `returning::project_modified` then keeps only the +/// statement's paths. +fn returning_to_return_values(returning: Option) -> Option<&'static str> { + returning.map(|mode| match mode { + Returning::AllOld | Returning::ModifiedOld => "ALL_OLD", + Returning::AllNew | Returning::ModifiedNew => "ALL_NEW", + }) +} + +/// The start of the refusal of a `DELETE` returning clause. +pub(super) const INVALID_RETURNING: &str = "Invalid returning clause: RETURNING "; + +/// The refusal of any `DELETE` clause other than `RETURNING ALL OLD *`, with +/// the clause named verbatim. It comes before the statement is authorized +/// (measured 2026-11-03). +pub(super) fn invalid_delete_returning(returning: Returning) -> DynamoDbError { + DynamoDbError::ValidationException(format!( + "{INVALID_RETURNING}{}. Only RETURNING ALL OLD * is allowed in DELETE statements.", + returning_clause_text(returning) + )) +} + +/// The clause text as it appears in the DELETE invalid-returning message. +fn returning_clause_text(returning: Returning) -> &'static str { + match returning { + Returning::AllOld => "ALL OLD *", + Returning::ModifiedOld => "MODIFIED OLD *", + Returning::AllNew => "ALL NEW *", + Returning::ModifiedNew => "MODIFIED NEW *", + } +} + +/// The key schema of each secondary index of the table. +fn index_keys(key_info: &TableKeyInfo) -> Vec> { + key_info + .global_secondary_indexes + .iter() + .chain(&key_info.local_secondary_indexes) + .map(|index| IndexKeyRef { + index_name: &index.index_name, + key_schema: &index.key_schema, + }) + .collect() +} + +/// Read back a map the translation built in wire form. +fn from_wire(map: &Map) -> Result { + serde_json::from_value(Value::Object(map.clone())).map_err(|e| { + tracing::error!(internal_error = %e, "failed to read a translated PartiQL map"); + internal() + }) +} + +fn internal() -> DynamoDbError { + DynamoDbError::InternalServerError("Internal server error".to_owned()) +} + +/// Serialize an item map to the DynamoDB JSON wire form. +fn to_json(item: &BTreeMap) -> Result { + serde_json::to_value(item).map_err(|e| { + tracing::error!(internal_error = %e, "failed to serialize PartiQL item"); + DynamoDbError::InternalServerError("Internal server error".to_owned()) + }) +} + +/// Serialize a single AttributeValue to the DynamoDB JSON wire form. +pub(super) fn to_json_value(av: &AttributeValue) -> Result { + serde_json::to_value(av).map_err(|e| { + tracing::error!(internal_error = %e, "failed to serialize PartiQL value"); + DynamoDbError::InternalServerError("Internal server error".to_owned()) + }) +} + +/// The DynamoDB type label of an attribute value, for the operand-type errors +/// ("operand type: S"). +pub(super) fn av_type_label(av: &AttributeValue) -> &'static str { + match av { + AttributeValue::S(_) => "S", + AttributeValue::N(_) => "N", + AttributeValue::B(_) => "B", + AttributeValue::SS(_) => "SS", + AttributeValue::NS(_) => "NS", + AttributeValue::BS(_) => "BS", + AttributeValue::Bool(_) => "BOOL", + AttributeValue::Null => "NULL", + AttributeValue::L(_) => "L", + AttributeValue::M(_) => "M", + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/engine/src/partiql/write/delete.rs b/crates/engine/src/partiql/write/delete.rs new file mode 100644 index 000000000..b3cd65ed0 --- /dev/null +++ b/crates/engine/src/partiql/write/delete.rs @@ -0,0 +1,144 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The DELETE statement: a DeleteItem of one item. + +use serde_json::{Map, Value, json}; + +use extenddb_core::error::DynamoDbError; +use extenddb_core::partiql::ast::{DeleteStatement, Returning, WhereExpr}; +use extenddb_core::types::AttributeValue; + +use super::expr_builder::ExprBuilder; +use super::{build_condition, invalid_delete_returning, key_names, reshape, split_key_where}; +use crate::create_table::storage_err_to_dynamo; +use crate::partiql::authorize::Gate; +use crate::partiql::{SingleRequest, common}; +use crate::{DispatchResult, OperationContext, handle_delete_item}; + +// --- DELETE --------------------------------------------------------------- + +pub(super) async fn run_delete( + ctx: &OperationContext, + delete: &DeleteStatement, + req: &SingleRequest, + gate: &mut Gate<'_>, +) -> Result { + // DELETE allows only `RETURNING ALL OLD *`; any other form is rejected with + // the clause named verbatim. + if let Some(returning) = delete.returning + && returning != Returning::AllOld + { + return Err(invalid_delete_returning(returning)); + } + + let parts = build_delete_parts(ctx, delete, &req.parameters).await?; + gate.pass().await?; + if let Some(refusal) = parts.condition_key_refusal { + return Err(DynamoDbError::ValidationException(refusal)); + } + + let mut body = json!({ + "TableName": parts.table, + "Key": Value::Object(parts.key), + }); + if let Some(condition) = parts.condition { + body["ConditionExpression"] = json!(condition); + } + if !parts.names.is_empty() { + body["ExpressionAttributeNames"] = Value::Object(parts.names); + } + if !parts.values.is_empty() { + body["ExpressionAttributeValues"] = Value::Object(parts.values); + } + if matches!(delete.returning, Some(Returning::AllOld)) { + body["ReturnValues"] = json!("ALL_OLD"); + } + if let Some(rcc) = &req.return_consumed_capacity { + body["ReturnConsumedCapacity"] = json!(rcc); + } + if let Some(rvoccf) = &req.return_values_on_condition_check_failure { + body["ReturnValuesOnConditionCheckFailure"] = json!(rvoccf); + } + + Ok(reshape( + handle_delete_item(body, ctx) + .await + .map_err(crate::partiql::bare_expression_error)?, + )) +} + +/// The translated classic `DeleteItem` pieces for a `DELETE`: the key, an +/// optional residual condition, and the names/values maps. Shared by the +/// single-statement and transaction executors. +pub(crate) struct DeleteParts { + pub table: String, + pub key: Map, + pub condition: Option, + pub names: Map, + pub values: Map, + /// The refusal of a path into a key attribute among the WHERE's other + /// conditions (see `common::condition_key_refusal`), which the + /// statement gets once it is authorized, whether or not the item exists. + pub condition_key_refusal: Option, +} + +/// Translate a `DELETE` into its `DeleteItem` pieces. Does not validate the +/// `RETURNING` clause (the single-statement path does that) and does not +/// dispatch. +/// +/// # Errors +/// +/// Returns the service's full-key message when the WHERE does not pin the whole +/// primary key, or `ResourceNotFoundException` for a missing table. +pub(crate) async fn build_delete_parts( + ctx: &OperationContext, + delete: &DeleteStatement, + params: &[AttributeValue], +) -> Result { + let key_info = ctx + .table_key_info(&delete.table) + .await + .map_err(storage_err_to_dynamo)?; + let (pk_name, sk_name) = key_names(&key_info)?; + + let (key, residual) = split_key_where(&delete.where_clause, pk_name, sk_name, params)?; + common::check_key_types(&key, &key_info)?; + let condition_key_refusal = + common::condition_key_refusal(&delete.where_clause, pk_name, sk_name, params); + + let mut builder = ExprBuilder::default(); + let condition = delete_condition(&mut builder, pk_name, &residual, params)?; + + Ok(DeleteParts { + table: delete.table.clone(), + key, + condition, + names: builder.names, + values: builder.values, + condition_key_refusal, + }) +} + +/// The condition a `DELETE` carries, or `None` when its WHERE is the key alone. +/// +/// DELETE has no implicit existence condition: deleting a missing item is a +/// silent success. A residual WHERE term is checked only against an item that +/// exists, so a missing key with a non-key predicate is a silent no-op too, +/// not a failed condition (measured 2026-11-02). `attribute_not_exists` on the +/// partition key is true exactly when the key is missing, which lets the +/// classic handler express that in one condition. +fn delete_condition( + builder: &mut ExprBuilder, + pk_name: &str, + residual: &[&WhereExpr], + params: &[AttributeValue], +) -> Result, DynamoDbError> { + let Some(residual_expr) = build_condition(builder, residual, params)? else { + return Ok(None); + }; + let pk_ref = builder.name(pk_name); + Ok(Some(format!( + "attribute_not_exists({pk_ref}) OR ({residual_expr})" + ))) +} diff --git a/crates/engine/src/partiql/write/exists.rs b/crates/engine/src/partiql/write/exists.rs new file mode 100644 index 000000000..655767107 --- /dev/null +++ b/crates/engine/src/partiql/write/exists.rs @@ -0,0 +1,103 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The EXISTS check of a transaction member: a ConditionCheck of one item. + +use serde_json::{Map, Value}; + +use extenddb_core::error::DynamoDbError; +use extenddb_core::partiql::ast::SelectStatement; +use extenddb_core::partiql::plan::SCALAR_KEY_REFUSAL; +use extenddb_core::types::AttributeValue; + +use super::expr_builder::ExprBuilder; +use super::{build_condition, key_names, wire_key}; +use crate::OperationContext; +use crate::create_table::storage_err_to_dynamo; +use crate::partiql::common; + +// --- EXISTS --------------------------------------------------------------- + +/// The translated `TransactWriteItems` `ConditionCheck` pieces for an +/// `EXISTS(SELECT ...)` transaction member. +pub(crate) struct ConditionCheckParts { + pub table: String, + pub key: Map, + pub condition: String, + pub names: Map, + pub values: Map, +} + +/// Translate an `EXISTS(SELECT ...)` transaction member into its +/// `ConditionCheck` pieces. `None` when the SELECT is not a read of one item +/// (the full primary key by equality, on the table) with a further condition: +/// the service refuses a read with no further condition (measured +/// 2026-11-02). The check is the further condition alone: a missing item +/// whose condition holds, such as `w IS MISSING`, lets the transaction write +/// (measured 2026-11-02). +/// +/// A key attribute given two values names no item, and such a check fails, +/// with or without a further condition: it reads the item the first values +/// name, with the whole WHERE as its condition (measured 2026-11-02: `pk = +/// 'a' AND pk = 'b' AND a = 1` and `pk = 'a' AND pk = 'b'` cancel the +/// transaction with `ConditionalCheckFailed` for the member). +/// +/// A condition that tests a path into a key attribute against values (see +/// `common::where_key`) fails the request, naming the member at position +/// `index` (measured 2026-11-05: `EXISTS(SELECT * FROM t WHERE pk = 'k' AND +/// sk = 'b' AND pk.c = 1)`). +/// +/// # Errors +/// +/// Returns `ResourceNotFoundException` for a missing table, the +/// multi-attribute refusal for a multi-part key, and the refusal of a path +/// into a key attribute tested against values. +pub(crate) async fn build_exists_parts( + ctx: &OperationContext, + index: usize, + select: &SelectStatement, + params: &[AttributeValue], +) -> Result, DynamoDbError> { + let key_info = ctx + .table_key_info(&select.from.table) + .await + .map_err(storage_err_to_dynamo)?; + let (pk_name, sk_name) = key_names(&key_info)?; + let (None, Some(where_clause)) = (&select.from.index, &select.where_clause) else { + return Ok(None); + }; + let (key, condition) = match common::where_key(where_clause, pk_name, sk_name, params) { + common::WhereKey::Pinned { + partition, + sort, + residual, + } => ( + wire_key(pk_name, sk_name, partition, sort, params)?, + residual, + ), + common::WhereKey::Conflict { + first: Some((partition, sort)), + .. + } => ( + wire_key(pk_name, sk_name, partition, sort, params)?, + vec![where_clause], + ), + common::WhereKey::IntoKey { name } => { + return Err(DynamoDbError::ValidationException(format!( + "Validation failed in TransactStatements[{index}]: {SCALAR_KEY_REFUSAL}{name}" + ))); + } + common::WhereKey::Partial | common::WhereKey::Conflict { .. } => return Ok(None), + }; + let mut builder = ExprBuilder::default(); + let Some(further) = build_condition(&mut builder, &condition, params)? else { + return Ok(None); + }; + Ok(Some(ConditionCheckParts { + table: select.from.table.clone(), + key, + condition: further, + names: builder.names, + values: builder.values, + })) +} diff --git a/crates/engine/src/partiql/write/expr_builder.rs b/crates/engine/src/partiql/write/expr_builder.rs new file mode 100644 index 000000000..3d9c76534 --- /dev/null +++ b/crates/engine/src/partiql/write/expr_builder.rs @@ -0,0 +1,209 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The builder of the classic expressions and placeholder maps a write is translated into. + +use serde_json::{Map, Value}; + +use extenddb_core::error::DynamoDbError; +use extenddb_core::partiql::ast::{ArithOp, Operand, Path, PathStep, TypeName, WhereExpr}; +use extenddb_core::types::AttributeValue; + +use super::{av_type_label, to_json_value}; +use crate::partiql::bind_parameter; +use crate::partiql::expr_bridge::compare_symbol; + +// --- expression builder --------------------------------------------------- + +/// Allocates `#name` / `:value` placeholders and the maps the classic handlers +/// expect, shared across a statement's update and condition expressions. +#[derive(Default)] +pub(super) struct ExprBuilder { + pub(super) names: Map, + pub(super) values: Map, + next_name: usize, + next_value: usize, +} + +impl ExprBuilder { + /// Allocate a name placeholder for a real attribute name. + pub(super) fn name(&mut self, real: &str) -> String { + let placeholder = format!("#n{}", self.next_name); + self.next_name += 1; + self.names + .insert(placeholder.clone(), Value::String(real.to_owned())); + placeholder + } + + /// Allocate a value placeholder for a typed AttributeValue. + pub(super) fn value(&mut self, av: &AttributeValue) -> Result { + let placeholder = format!(":v{}", self.next_value); + self.next_value += 1; + self.values.insert(placeholder.clone(), to_json_value(av)?); + Ok(placeholder) + } + + /// Translate a document path to placeholder form (`#n0.#n1[2]`). + pub(super) fn path(&mut self, path: &Path) -> String { + let mut rendered = self.name(&path.root); + for step in &path.steps { + match step { + PathStep::Attribute(attr) => { + rendered.push('.'); + rendered.push_str(&self.name(attr)); + } + PathStep::Index(index) => { + rendered.push_str(&format!("[{index}]")); + } + } + } + rendered + } + + /// Translate an operand to expression text. + pub(super) fn operand( + &mut self, + operand: &Operand, + params: &[AttributeValue], + ) -> Result { + Ok(match operand { + Operand::Literal(av) => self.value(av)?, + Operand::Parameter(index) => { + let av = bind_parameter(params, *index)?; + self.value(&av)? + } + Operand::Path(path) => self.path(path), + Operand::Arith(left, op, right) => { + let symbol = match op { + ArithOp::Add => "+", + ArithOp::Sub => "-", + }; + // The service type-checks an inline literal operand of an + // arithmetic expression up front: a non-number literal is an + // operand-type error naming the operator and the type, distinct + // from the generic runtime message a string ATTRIBUTE operand + // produces in the classic evaluator. + for side in [left.as_ref(), right.as_ref()] { + if let Operand::Literal(av) = side + && !matches!(av, AttributeValue::N(_)) + { + return Err(DynamoDbError::ValidationException(format!( + "Incorrect operand type for operator or function; operator or function: {symbol}, operand type: {}", + av_type_label(av) + ))); + } + } + let left = self.operand(left, params)?; + let right = self.operand(right, params)?; + format!("{left} {symbol} {right}") + } + Operand::Function(name, args) => { + let mut rendered = Vec::new(); + for arg in args { + rendered.push(self.operand(arg, params)?); + } + format!("{}({})", name, rendered.join(", ")) + } + }) + } + + /// Translate a boolean WHERE/condition expression to expression text, + /// written by `expr_bridge::condition_text`. An OR at the top is not in + /// parentheses: the callers put each term in parentheses themselves. + pub(super) fn condition( + &mut self, + expr: &WhereExpr, + params: &[AttributeValue], + ) -> Result { + crate::partiql::expr_bridge::condition_text(expr, false, |predicate, negated| { + self.predicate(predicate, negated, params) + }) + } + + /// Translate one predicate of a condition, negated when `negated` is set. + fn predicate( + &mut self, + expr: &WhereExpr, + negated: bool, + params: &[AttributeValue], + ) -> Result { + let text = match expr { + WhereExpr::Compare(left, op, right) => { + let left = self.operand(left, params)?; + let right = self.operand(right, params)?; + format!("{left} {} {right}", compare_symbol(*op)) + } + WhereExpr::Between(value, low, high) => { + let value = self.operand(value, params)?; + let low = self.operand(low, params)?; + let high = self.operand(high, params)?; + format!("{value} BETWEEN {low} AND {high}") + } + WhereExpr::In(value, items) => { + let value = self.operand(value, params)?; + let mut rendered = Vec::new(); + for item in items { + rendered.push(self.operand(item, params)?); + } + format!("{value} IN ({})", rendered.join(", ")) + } + WhereExpr::BeginsWith(value, prefix) => { + let value = self.operand(value, params)?; + let prefix = self.operand(prefix, params)?; + format!("begins_with({value}, {prefix})") + } + WhereExpr::Contains(value, operand) => { + let value = self.operand(value, params)?; + let operand = self.operand(operand, params)?; + format!("contains({value}, {operand})") + } + WhereExpr::AttributeType(value, kind) => { + let value = self.operand(value, params)?; + let kind = self.operand(kind, params)?; + format!("attribute_type({value}, {kind})") + } + WhereExpr::Function(name, args) => { + let mut rendered = Vec::new(); + for arg in args { + rendered.push(self.operand(arg, params)?); + } + format!("{}({})", name, rendered.join(", ")) + } + WhereExpr::IsMissing(path, missing) => { + let rendered = self.path(path); + return Ok(if *missing == negated { + format!("attribute_exists({rendered})") + } else { + format!("attribute_not_exists({rendered})") + }); + } + // As on a read: `IS NULL` is an equality with NULL, `IS ` + // an attribute_type test. + WhereExpr::IsType(operand, ty, is_not) => { + let value = self.operand(operand, params)?; + let test = if *ty == TypeName::Null { + let null = self.value(&AttributeValue::Null)?; + format!("{value} = {null}") + } else { + let code = ty.attribute_type().ok_or_else(|| { + DynamoDbError::ValidationException(ty.unsupported_filter_message()) + })?; + let code = self.value(&AttributeValue::S(code.to_owned()))?; + format!("attribute_type({value}, {code})") + }; + return Ok(if *is_not == negated { + test + } else { + format!("NOT {test}") + }); + } + WhereExpr::And(..) + | WhereExpr::Or(..) + | WhereExpr::Not(_) + | WhereExpr::ComparedWithBoolean(..) => { + unreachable!("condition_text writes AND, OR and NOT itself") + } + }; + Ok(if negated { format!("NOT {text}") } else { text }) + } +} diff --git a/crates/engine/src/partiql/write/insert.rs b/crates/engine/src/partiql/write/insert.rs new file mode 100644 index 000000000..d40138996 --- /dev/null +++ b/crates/engine/src/partiql/write/insert.rs @@ -0,0 +1,143 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The INSERT statement: a create-only PutItem. + +use std::collections::BTreeMap; + +use serde_json::{Map, Value, json}; + +use extenddb_core::error::DynamoDbError; +use extenddb_core::partiql::ast::InsertStatement; +use extenddb_core::types::{AttributeValue, Item, TableKeyInfo}; +use extenddb_core::validation::validate_index_key_types; + +use super::expr_builder::ExprBuilder; +use super::{ + from_wire, index_keys, internal, key_names, reshape, resolve_av, to_json, validate_insert_keys, +}; +use crate::create_table::storage_err_to_dynamo; +use crate::partiql::authorize::Gate; +use crate::partiql::{SingleRequest, common, item_depth}; +use crate::{DispatchResult, OperationContext, handle_put_item}; + +// --- INSERT --------------------------------------------------------------- + +pub(super) async fn run_insert( + ctx: &OperationContext, + insert: &InsertStatement, + req: &SingleRequest, + gate: &mut Gate<'_>, +) -> Result { + let parts = build_insert_parts(ctx, insert, &req.parameters).await?; + gate.pass().await?; + let mut body = json!({ + "TableName": parts.table, + "Item": parts.item, + "ConditionExpression": parts.condition, + "ExpressionAttributeNames": Value::Object(parts.names), + }); + if let Some(rcc) = &req.return_consumed_capacity { + body["ReturnConsumedCapacity"] = json!(rcc); + } + + let result = handle_put_item(body, ctx).await.map_err(|e| { + crate::partiql::bare_expression_error(item_depth::item_nesting_error(map_insert_error(e))) + }); + Ok(reshape(result?)) +} + +/// The translated classic `PutItem` pieces for an `INSERT`: the item, the +/// create-only condition, and the names map it references. Shared by the +/// single-statement executor and the transaction executor (which wraps them in +/// a `TransactWriteItems` `Put` member). +pub(crate) struct PutParts { + pub table: String, + pub item: Value, + pub condition: String, + pub names: Map, +} + +/// Translate an `INSERT` into its `PutItem` pieces (create-only condition, +/// item, names), resolving `?` parameters. Does not dispatch. +/// +/// # Errors +/// +/// Returns the service's key-validation message when a key attribute is missing +/// or mistyped, or `ResourceNotFoundException` for a missing table. +pub(crate) async fn build_insert_parts( + ctx: &OperationContext, + insert: &InsertStatement, + params: &[AttributeValue], +) -> Result { + let key_info = ctx + .table_key_info(&insert.table) + .await + .map_err(storage_err_to_dynamo)?; + + // Build the item from the VALUE map, resolving `?` parameters to their typed + // AttributeValues. The values are the ones authorization read the key + // from (see `common::insert_values`). + let mut item: BTreeMap = BTreeMap::new(); + for (name, operand) in common::insert_values(insert) { + item.insert(name.to_owned(), resolve_av(operand, params)?); + } + + // PartiQL names the missing or mistyped key attribute with its own message, + // which differs from the classic PutItem key-validation message, so the + // check runs here before the put. + validate_insert_keys(&item, &key_info)?; + + // Create-only: a duplicate full key must fail. `attribute_not_exists` on the + // partition key is the create-only condition; the target slot is addressed + // by its full key, so an item with the same partition key but a different + // sort key does not trip it. + let (pk_name, _sk_name) = key_names(&key_info)?; + let mut builder = ExprBuilder::default(); + let pk_ref = builder.name(pk_name); + let condition = format!("attribute_not_exists({pk_ref})"); + + Ok(PutParts { + table: insert.table.clone(), + item: to_json(&item)?, + condition, + names: builder.names, + }) +} + +/// The refusal of the `PutItem` request of `parts` on the table `key_info` +/// when its item gives a secondary index key a value of the wrong type. +/// +/// # Errors +/// +/// A fault of reading back the item. +pub(crate) fn insert_refusal( + parts: &PutParts, + key_info: &TableKeyInfo, +) -> Result, DynamoDbError> { + let Value::Object(item) = &parts.item else { + return Err(internal()); + }; + let item: Item = from_wire(item)?; + Ok(validate_index_key_types( + &item, + &index_keys(key_info), + &key_info.attribute_definitions, + ) + .err()) +} + +/// Map a create-only condition failure to the PartiQL duplicate-key error. +/// +/// The only condition the INSERT path sets is `attribute_not_exists(pk)`, so a +/// `ConditionalCheckFailedException` from it means the full key already exists. +fn map_insert_error(e: DynamoDbError) -> DynamoDbError { + match e { + DynamoDbError::ConditionalCheckFailedException(..) => { + DynamoDbError::DuplicateItemException( + "Duplicate primary key exists in table".to_owned(), + ) + } + other => other, + } +} diff --git a/crates/engine/src/partiql/write/tests.rs b/crates/engine/src/partiql/write/tests.rs new file mode 100644 index 000000000..f209b83b6 --- /dev/null +++ b/crates/engine/src/partiql/write/tests.rs @@ -0,0 +1,225 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Unit tests for the write executor. + +use super::*; +use extenddb_core::types::{AttributeDefinition, KeySchemaElement, KeyType}; + +/// A pk(N)/sk(S) table. +fn number_string_table() -> TableKeyInfo { + let key = |name: &str, key_type| KeySchemaElement { + attribute_name: name.to_owned(), + key_type, + }; + let definition = |name: &str, attribute_type| AttributeDefinition { + attribute_name: name.to_owned(), + attribute_type, + }; + TableKeyInfo { + table_name: "t".to_owned(), + key_schema: vec![key("pk", KeyType::Hash), key("sk", KeyType::Range)], + attribute_definitions: vec![ + definition("pk", ScalarAttributeType::N), + definition("sk", ScalarAttributeType::S), + ], + ..TableKeyInfo::default() + } +} + +fn key_type_answer(key: &Value) -> Option { + common::check_key_types(key.as_object().unwrap(), &number_string_table()) + .err() + .map(|e| e.message().to_owned()) +} + +fn overlap_answer(sql: &str) -> Option { + match extenddb_core::partiql::parse(sql).unwrap() { + Statement::Update(update) => check_overlapping_paths(&update).err(), + other => panic!("expected an UPDATE, got {other:?}"), + } +} + +#[test] +fn a_deep_write_condition_is_written_for_the_classic_parser() { + let condition_of = |sql: &str| match extenddb_core::partiql::parse(sql).unwrap() { + Statement::Update(update) => update.where_clause.unwrap(), + other => panic!("expected an UPDATE, got {other:?}"), + }; + // 99 levels of NOT and AND hold 298 operators, within the service's + // 300; written as nested as the statement, the text passed the + // classic parser's 150 levels. + let deep = "NOT (a = 1 AND ".repeat(99) + "a = 1" + &")".repeat(99); + let condition = condition_of(&format!("UPDATE \"t\" SET q = 1 WHERE {deep}")); + let text = std::thread::Builder::new() + .stack_size(128 << 10) + .spawn(move || ExprBuilder::default().condition(&condition, &[]).unwrap()) + .unwrap() + .join() + .unwrap(); + let wrapped = format!("attribute_exists(#pk) AND ({text})"); + let limits = extenddb_core::limits::LimitsConfig::default(); + assert!(crate::expression_helpers::parse_condition_expr(&wrapped, &limits).is_ok()); + // An OR at the top is left for the caller to put in parentheses. + let or = condition_of("UPDATE \"t\" SET q = 1 WHERE a = 1 OR NOT (b = 1 OR c = 2)"); + assert_eq!( + ExprBuilder::default().condition(&or, &[]).unwrap(), + "#n0 = :v0 OR NOT #n1 = :v1 AND NOT #n2 = :v2" + ); +} + +#[test] +fn update_paths_that_overlap_are_refused_naming_the_later_one_first() { + for (assignments, one, two) in [ + ("SET q = 1, q = 2", "[q]", "[q]"), + ("SET q = 1 REMOVE q", "[q]", "[q]"), + ("SET m = {'a': 1}, m.a = 2", "[m, a]", "[m]"), + ( + "SET ss = set_add(ss, <<'b'>>), ss = set_add(ss, <<'c'>>)", + "[ss]", + "[ss]", + ), + ( + "SET ss = 'plain', ss = set_add(ss, <<'b'>>)", + "[ss]", + "[ss]", + ), + ("SET ss = set_delete(ss, <<'a'>>) REMOVE ss", "[ss]", "[ss]"), + ] { + let sql = format!("UPDATE \"t\" {assignments} WHERE pk = 'a'"); + assert_eq!( + overlap_answer(&sql), + Some(format!( + "Two document paths overlap with each other; must remove or rewrite one \ + of these paths; path one: {one}, path two: {two}" + )), + "{sql}" + ); + } + assert!(overlap_answer("UPDATE \"t\" SET l[0] = 1, l = [2] WHERE pk = 'a'").is_some()); +} + +#[test] +fn update_paths_that_name_one_element_two_ways_conflict() { + for (assignments, one, two) in [ + ("SET a.b = 1, a[0] = 2", "[a, [0]]", "[a, b]"), + ("SET a[0] = 2, a.b = 1", "[a, b]", "[a, [0]]"), + ("SET m.x.b = 1 REMOVE m.x[3]", "[m, x, [3]]", "[m, x, b]"), + ("SET a[1].b = 1, a.c = 2", "[a, c]", "[a, [1], b]"), + ] { + let sql = format!("UPDATE \"t\" {assignments} WHERE pk = 'a'"); + assert_eq!( + overlap_answer(&sql), + Some(format!( + "Two document paths conflict with each other; must remove or rewrite one \ + of these paths; path one: {one}, path two: {two}" + )), + "{sql}" + ); + } + // Paths that part at two members, or at two elements, do not. + assert_eq!( + overlap_answer("UPDATE \"t\" SET a.b = 1, a.c[0] = 2 WHERE pk = 'a'"), + None + ); + assert_eq!( + overlap_answer("UPDATE \"t\" SET a[0].b = 1, a[1] = 2 WHERE pk = 'a'"), + None + ); +} + +#[test] +fn update_paths_that_do_not_overlap_pass() { + for assignments in [ + "SET m.a = 1, m.b = 2", + "SET l[0] = 1, l[1] = 2", + "SET a = 1 REMOVE b", + "SET ab = 1, a = 2", + "SET ss = set_add(ss, <<'b'>>), ns = set_delete(ns, <<1>>)", + ] { + let sql = format!("UPDATE \"t\" {assignments} WHERE pk = 'a'"); + assert_eq!(overlap_answer(&sql), None, "{sql}"); + } +} + +#[test] +fn a_write_key_of_the_wrong_type_names_the_sort_key_first() { + let message = |name: &str| { + Some(format!( + "Key attribute's data type should match its data type in table's schema: Key {name}" + )) + }; + assert_eq!( + key_type_answer(&json!({"pk": {"N": "1"}, "sk": {"S": "a"}})), + None + ); + assert_eq!( + key_type_answer(&json!({"pk": {"S": "a"}, "sk": {"S": "a"}})), + message("pk") + ); + assert_eq!( + key_type_answer(&json!({"pk": {"N": "1"}, "sk": {"N": "1"}})), + message("sk") + ); + assert_eq!( + key_type_answer(&json!({"sk": {"N": "1"}, "pk": {"S": "a"}})), + message("sk") + ); + assert_eq!( + key_type_answer(&json!({"pk": {"S": "a"}, "sk": {"N": "1"}})), + message("sk") + ); +} + +/// The key and residual count a write's WHERE gives on the pk/sk table, +/// or its refusal. +fn write_key_answer(condition: &str) -> Result<(Value, usize), String> { + let sql = format!("DELETE FROM \"t\" WHERE {condition}"); + let Statement::Delete(delete) = extenddb_core::partiql::parse(&sql).unwrap() else { + panic!("expected a DELETE"); + }; + split_key_where(&delete.where_clause, "pk", Some("sk"), &[]) + .map(|(key, residual)| (Value::Object(key), residual.len())) + .map_err(|e| e.message().to_owned()) +} + +#[test] +fn a_write_key_equality_may_have_the_value_on_either_side() { + let key = json!({"pk": {"N": "1"}, "sk": {"S": "x"}}); + for (condition, residual) in [ + ("pk = 1 AND sk = 'x'", 0), + ("1 = pk AND 'x' = sk", 0), + ("pk = 1 AND pk = 1 AND sk = 'x'", 0), + ("pk = 1 AND sk = 'x' AND 'x' = sk", 0), + ("pk = 1 AND sk = 'x' AND sk <> 'y'", 1), + ("pk = 1 AND sk = 'x' AND sk = a", 1), + ] { + assert_eq!( + write_key_answer(condition), + Ok((key.clone(), residual)), + "{condition}" + ); + } +} + +#[test] +fn a_write_key_with_two_values_is_refused_naming_the_key() { + let multiple = |name: &str| { + Err(format!( + "Multiple conditions on same key {name}. Only single item Update/Insert/Delete \ + are supported" + )) + }; + assert_eq!( + write_key_answer("pk = 'a' AND 1 = pk AND sk = 'x'"), + multiple("pk") + ); + assert_eq!( + write_key_answer("pk = 'a' AND sk = 'x' AND sk = 'y'"), + multiple("sk") + ); + assert_eq!( + write_key_answer("pk = 'a' AND sk > 'x'"), + Err("Where clause does not contain a mandatory equality on all key attributes".to_owned()) + ); +} diff --git a/crates/engine/src/partiql/write/update.rs b/crates/engine/src/partiql/write/update.rs new file mode 100644 index 000000000..6f1ba14e9 --- /dev/null +++ b/crates/engine/src/partiql/write/update.rs @@ -0,0 +1,361 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The UPDATE statement: an UpdateItem of an item that exists, and the checks of its paths. + +use std::collections::HashMap; + +use serde_json::{Map, Value, json}; + +use extenddb_core::error::DynamoDbError; +use extenddb_core::expression::{ExpressionMaps, UpdateAction}; +use extenddb_core::limits::LimitsConfig; +use extenddb_core::partiql::ast::{ + Path, PathStep, SetMembersAction, SetMembersOp, UpdateStatement, +}; +use extenddb_core::types::{AttributeValue, TableKeyInfo}; +use extenddb_core::validation::validate_index_key_literal_operands; + +use super::expr_builder::ExprBuilder; +use super::{ + build_condition, from_wire, full_key_error, index_keys, key_names, reshape, resolve_av, + returning_to_return_values, split_key_where, +}; +use crate::create_table::storage_err_to_dynamo; +use crate::expression_helpers::{build_expression_maps, parse_update_expr}; +use crate::partiql::authorize::Gate; +use crate::partiql::{SingleRequest, common, item_depth}; +use crate::transact_write_helpers::validate_no_key_updates; +use crate::update_item::handle_update_item_with; +use crate::{DispatchResult, OperationContext}; + +// --- UPDATE --------------------------------------------------------------- + +pub(super) async fn run_update( + ctx: &OperationContext, + update: &UpdateStatement, + req: &SingleRequest, + gate: &mut Gate<'_>, +) -> Result { + let parts = build_update_parts(ctx, update, &req.parameters).await?; + // The item check below reads the item. + gate.pass().await?; + // A path into a key attribute among the WHERE's other conditions is + // refused after the refusals of the `UpdateItem` request, which the + // update below gives: a SET or REMOVE of a key attribute, then a + // secondary index key given a value of the wrong type (measured + // 2026-11-05). + if let Some(refusal) = &parts.condition_key_refusal { + let key_info = ctx + .table_key_info(&update.table) + .await + .map_err(storage_err_to_dynamo)?; + if update_refusal(&ctx.limits, &parts, &key_info)?.is_none() { + return Err(DynamoDbError::ValidationException(refusal.clone())); + } + } + if item_depth::sets_a_value_past_the_item_limit(update, &req.parameters) + && let Some(err) = item_depth::unwritable_update_error(ctx, &parts, req).await + { + return Err(crate::partiql::bare_expression_error(err)); + } + + let mut body = json!({ + "TableName": parts.table, + "Key": Value::Object(parts.key), + "UpdateExpression": parts.update_expression, + "ConditionExpression": parts.condition, + }); + if !parts.names.is_empty() { + body["ExpressionAttributeNames"] = Value::Object(parts.names); + } + if !parts.values.is_empty() { + body["ExpressionAttributeValues"] = Value::Object(parts.values); + } + if let Some(rv) = returning_to_return_values(update.returning) { + body["ReturnValues"] = json!(rv); + } + if let Some(rcc) = &req.return_consumed_capacity { + body["ReturnConsumedCapacity"] = json!(rcc); + } + if let Some(rvoccf) = &req.return_values_on_condition_check_failure { + body["ReturnValuesOnConditionCheckFailure"] = json!(rvoccf); + } + + // The service sizes an UPDATE by its finished item alone, not by the + // statement as it sizes UpdateItem (measured 2026-11-04). + let finished_item_sizing = true; + let mut result = handle_update_item_with(body, ctx, finished_item_sizing) + .await + .map_err(crate::partiql::bare_expression_error)?; + crate::partiql::returning::project_modified(&mut result.body, update)?; + Ok(reshape(result)) +} + +/// The translated classic `UpdateItem` pieces for an `UPDATE`: the key, the +/// update expression, the implicit-existence-plus-residual condition, and the +/// names/values maps. Shared by the single-statement and transaction executors. +pub(crate) struct UpdateParts { + pub table: String, + pub key: Map, + pub update_expression: String, + pub condition: String, + pub names: Map, + pub values: Map, + /// The refusal of a path into a key attribute among the WHERE's other + /// conditions (see `common::condition_key_refusal`), which the + /// statement gets once it is authorized. + pub condition_key_refusal: Option, +} + +/// The refusal `UpdateItem` gives the request of `parts` on the table +/// `key_info` before it reads the item, in its words and in its order: a +/// SET or REMOVE of a key attribute, then a secondary index key given a +/// value of the wrong type. A statement is refused for a path into a key +/// attribute among its other conditions only after these (measured +/// 2026-11-05 on `ExecuteStatement` and `BatchExecuteStatement`). +/// +/// # Errors +/// +/// A fault of reading back the request. +pub(crate) fn update_refusal( + limits: &LimitsConfig, + parts: &UpdateParts, + key_info: &TableKeyInfo, +) -> Result, DynamoDbError> { + let (actions, maps) = request_actions(limits, parts)?; + Ok(validate_no_key_updates(&actions, key_info, &maps) + .and_then(|()| check_index_keys(&actions, key_info, &maps)) + .err()) +} + +/// The second refusal of [`update_refusal`]: the request of `parts` gives a +/// secondary index key a value of the wrong type. +/// +/// # Errors +/// +/// A fault of reading back the request. +pub(crate) fn update_index_key_refusal( + limits: &LimitsConfig, + parts: &UpdateParts, + key_info: &TableKeyInfo, +) -> Result, DynamoDbError> { + let (actions, maps) = request_actions(limits, parts)?; + Ok(check_index_keys(&actions, key_info, &maps).err()) +} + +/// The actions and the expression maps of the request of `parts`, read as +/// `UpdateItem` reads them. +fn request_actions( + limits: &LimitsConfig, + parts: &UpdateParts, +) -> Result<(Vec, ExpressionMaps), DynamoDbError> { + let names: HashMap = from_wire(&parts.names)?; + let values: HashMap = from_wire(&parts.values)?; + let maps = build_expression_maps(Some(&names), Some(&values)); + Ok((parse_update_expr(&parts.update_expression, limits)?, maps)) +} + +/// Refuse an action that gives a secondary index key of the table a value +/// of the wrong type as a literal operand, as `UpdateItem` does. +fn check_index_keys( + actions: &[UpdateAction], + key_info: &TableKeyInfo, + maps: &ExpressionMaps, +) -> Result<(), DynamoDbError> { + validate_index_key_literal_operands( + actions, + &index_keys(key_info), + &key_info.attribute_definitions, + maps, + ) +} + +/// Translate an `UPDATE` into its `UpdateItem` pieces. Does not dispatch. +/// +/// # Errors +/// +/// Returns the service's full-key message when the WHERE does not pin the whole +/// primary key, or `ResourceNotFoundException` for a missing table. +pub(crate) async fn build_update_parts( + ctx: &OperationContext, + update: &UpdateStatement, + params: &[AttributeValue], +) -> Result { + let key_info = ctx + .table_key_info(&update.table) + .await + .map_err(storage_err_to_dynamo)?; + // Only the FROM-first form can omit WHERE. It is refused like an UPDATE + // without the full key, after a missing table (measured 2026-11-02). + let where_clause = update.where_clause.as_ref().ok_or_else(full_key_error)?; + let (pk_name, sk_name) = key_names(&key_info)?; + + let (key, residual) = split_key_where(where_clause, pk_name, sk_name, params)?; + common::check_key_types(&key, &key_info)?; + let condition_key_refusal = + common::condition_key_refusal(where_clause, pk_name, sk_name, params); + + let mut builder = ExprBuilder::default(); + + let mut set_parts = Vec::new(); + for action in &update.sets { + let path = builder.path(&action.path); + let value = builder.operand(&action.value, params)?; + set_parts.push(format!("{path} = {value}")); + } + let mut remove_parts = Vec::new(); + for path in &update.removes { + remove_parts.push(builder.path(path)); + } + let (add_parts, delete_parts) = set_member_parts(&mut builder, &update.set_members, params)?; + let update_expression = [ + ("SET", set_parts), + ("REMOVE", remove_parts), + ("ADD", add_parts), + ("DELETE", delete_parts), + ] + .into_iter() + .filter(|(_, parts)| !parts.is_empty()) + .map(|(clause, parts)| format!("{clause} {}", parts.join(", "))) + .collect::>() + .join(" "); + + // UPDATE is not an upsert: an implicit existence condition on the partition + // key makes a missing item fail with ConditionalCheckFailedException. Any + // residual WHERE term is ANDed on as the caller's condition. + let pk_ref = builder.name(pk_name); + let condition = match build_condition(&mut builder, &residual, params)? { + Some(residual_expr) => format!("attribute_exists({pk_ref}) AND ({residual_expr})"), + None => format!("attribute_exists({pk_ref})"), + }; + + Ok(UpdateParts { + table: update.table.clone(), + key, + update_expression, + condition, + names: builder.names, + values: builder.values, + condition_key_refusal, + }) +} + +/// Refuse an `UPDATE` whose targets name one document path twice, a path and +/// a path inside it, or one element both as a map member and as a list +/// element. The targets are taken in this order: the `SET` targets, the +/// attributes `set_add` and `set_delete` assign, then the `REMOVE` paths, and +/// each is compared with the ones before it. Nothing is read or written +/// (measured 2026-11-02: `SET q = 1, q = 2`, `SET q = 1 REMOVE q`, `set_add` +/// twice on one set, a `SET` beside a `set_add` of the same attribute, and +/// `SET m = {...}, m.a = 2`, with and without `RETURNING`; measured +/// 2026-11-03: `SET a.b = 1, a[0] = 2` on an item where a is a map, a list or +/// absent, and on a missing item). +/// +/// The message names the later of the two paths first, which gives the +/// measured `path one: [m, a], path two: [m]` for `SET m = {...}, m.a = 2` +/// and `path one: [a, [0]], path two: [a, b]` for `SET a.b = 1, a[0] = 2`; +/// the order with the child path first was not measured. +/// +/// # Errors +/// +/// Returns the service's message for the first such pair. +pub(crate) fn check_overlapping_paths(update: &UpdateStatement) -> Result<(), String> { + let paths: Vec<&Path> = update + .sets + .iter() + .map(|action| &action.path) + .chain(update.set_members.iter().map(|action| &action.path)) + .chain(&update.removes) + .collect(); + first_overlap(&paths) +} + +/// Refuse the first of `paths` that names one document path with a path +/// before it, a path inside it or around it, or one element both as a map +/// member and as a list element, naming the later path first (see +/// [`check_overlapping_paths`]). A SELECT's projected paths get this refusal +/// too (measured 2026-11-05: `m.k, m` in `ExecuteStatement`, +/// `BatchExecuteStatement` and `ExecuteTransaction`, `m, m.k` and `l[1], l` +/// in `ExecuteTransaction`). +/// +/// # Errors +/// +/// Returns the service's message for the first such pair. +pub(crate) fn first_overlap(paths: &[&Path]) -> Result<(), String> { + for (i, later) in paths.iter().enumerate() { + if let Some((earlier, meeting)) = paths[..i] + .iter() + .find_map(|earlier| meeting(earlier, later).map(|meeting| (earlier, meeting))) + { + return Err(format!( + "Two document paths {meeting} with each other; must remove or rewrite one of \ + these paths; path one: {}, path two: {}", + path_list(later), + path_list(earlier) + )); + } + } + Ok(()) +} + +/// How two paths meet, as the refusal words it: `overlap` when one is the +/// other or leads into it, `conflict` when they part at one element that one +/// names as a map member and the other as a list element. +fn meeting(a: &Path, b: &Path) -> Option<&'static str> { + if a.root != b.root { + return None; + } + for (x, y) in a.steps.iter().zip(&b.steps) { + match (x, y) { + _ if x == y => {} + (PathStep::Attribute(_), PathStep::Index(_)) + | (PathStep::Index(_), PathStep::Attribute(_)) => return Some("conflict"), + _ => return None, + } + } + Some("overlap") +} + +/// A path as the overlap message lists it: `[m, a]`, with an index as `[0]`. +fn path_list(path: &Path) -> String { + let mut parts = vec![path.root.clone()]; + for step in &path.steps { + parts.push(match step { + PathStep::Attribute(name) => name.clone(), + PathStep::Index(index) => format!("[{index}]"), + }); + } + format!("[{}]", parts.join(", ")) +} + +/// The classic `ADD` and `DELETE` actions for `set_add` and `set_delete` +/// assignments: the service adds the members to the set, or removes them, the +/// way those actions do (measured 2026-11-02). A `?` parameter has to bind to +/// a set; any other value gets the refusal a literal that is not a set gets. +fn set_member_parts( + builder: &mut ExprBuilder, + actions: &[SetMembersAction], + params: &[AttributeValue], +) -> Result<(Vec, Vec), DynamoDbError> { + let mut add_parts = Vec::new(); + let mut delete_parts = Vec::new(); + for action in actions { + let members = resolve_av(&action.members, params)?; + if !matches!( + members, + AttributeValue::SS(_) | AttributeValue::NS(_) | AttributeValue::BS(_) + ) { + return Err(DynamoDbError::ValidationException( + action.op.not_a_set_message(&action.first_argument_at), + )); + } + let path = builder.path(&action.path); + let value = builder.value(&members)?; + match action.op { + SetMembersOp::Add => add_parts.push(format!("{path} {value}")), + SetMembersOp::Delete => delete_parts.push(format!("{path} {value}")), + } + } + Ok((add_parts, delete_parts)) +} diff --git a/crates/engine/src/update_item.rs b/crates/engine/src/update_item.rs index ee9658117..c54a45f1b 100755 --- a/crates/engine/src/update_item.rs +++ b/crates/engine/src/update_item.rs @@ -36,6 +36,17 @@ use crate::{DispatchMetrics, DispatchResult}; pub async fn handle_update_item( body: Value, ctx: &OperationContext, +) -> Result { + handle_update_item_with(body, ctx, false).await +} + +/// [`handle_update_item`], sizing the update by its finished item alone when +/// `finished_item_sizing` is set, as the service sizes a PartiQL `UPDATE` +/// (see `validate_update_item_size`). +pub(crate) async fn handle_update_item_with( + body: Value, + ctx: &OperationContext, + finished_item_sizing: bool, ) -> Result { // Sequential rather than aggregated, deliberately: UpdateItem stops at the // FIRST invalid enum and reports one error, where PutItem and DeleteItem @@ -161,6 +172,7 @@ pub async fn handle_update_item( // The storage layer sizes a desugared legacy update against the item limit // the way the service sizes `AttributeUpdates`, not an expression. maps.legacy_attribute_updates = input.attribute_updates.is_some(); + maps.finished_item_sizing = finished_item_sizing; // No UpdateExpression and no AttributeUpdates: no-op upsert. // Some("") still errors via parse_update_expr. diff --git a/crates/server/src/handler.rs b/crates/server/src/handler.rs index aeef4ccd1..923bd7ff8 100755 --- a/crates/server/src/handler.rs +++ b/crates/server/src/handler.rs @@ -108,6 +108,7 @@ pub(crate) async fn handle_request( // --- Authz segment --- let authz_start = std::time::Instant::now(); let authorized_key_info; + let item_authorizer; { if state.catalog_store.is_none() { tracing::error!("Authorization required but catalog_store is not configured"); @@ -140,6 +141,9 @@ pub(crate) async fn handle_request( Ok(ki) => authorized_key_info = ki, Err(e) => return error_response(&e, &request_id), } + // The engine authorizes each PartiQL statement through it. + item_authorizer = + crate::item_authz::for_request(&state, identity, &account_id, source_ip, user_agent); } #[allow(clippy::cast_precision_loss)] let authz_us = authz_start.elapsed().as_micros() as f64; @@ -158,6 +162,7 @@ pub(crate) async fn handle_request( table_key_info_lookup: Some( state.table_key_info_cache.clone() as Arc ), + item_authorizer: Some(item_authorizer), }; let table_name = extract_table_name(&input); diff --git a/crates/server/src/item_authz.rs b/crates/server/src/item_authz.rs new file mode 100644 index 000000000..2dce334c1 --- /dev/null +++ b/crates/server/src/item_authz.rs @@ -0,0 +1,231 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The item authorizer the handler gives the engine for one request. +//! +//! The PartiQL operations name their tables inside their statements, so +//! `authorize_request` does not authorize them; the engine authorizes each +//! statement through this authorizer before it reads or writes (see +//! `extenddb_engine::ItemAuthorizer`). It evaluates the caller's policies +//! with `check_authorization`, against this deployment's ARN of the table or +//! index, with the request-level condition keys every other operation +//! carries and the item-level keys of the statement. + +use std::sync::Arc; + +use extenddb_auth::AuthIdentity; +use extenddb_auth::policy::context::RequestParams; +use extenddb_core::types::AttributeValue; +use extenddb_engine::{ItemAccess, ItemAuthorization, ItemAuthorizer}; + +use crate::AppState; +use crate::authorization; +use crate::authz_resource::{self, AuthzTarget}; +use crate::leading_keys::{NumberForm, extract_typed_value}; + +/// The item authorizer of a request signed by `identity`, from `source_ip` +/// with `user_agent`. In developer mode it allows every access, as +/// `authorize_request` skips the IAM decision there. +pub(crate) fn for_request( + state: &Arc, + identity: AuthIdentity, + account_id: &Arc, + source_ip: String, + user_agent: Option, +) -> Arc { + if state.dev_mode { + return Arc::new(AllowEveryAccess); + } + Arc::new(PolicyAuthorizer { + state: Arc::clone(state), + identity, + account_id: Arc::clone(account_id), + source_ip, + user_agent, + }) +} + +/// Developer mode: the request is authenticated, and no policy is evaluated. +struct AllowEveryAccess; + +impl ItemAuthorizer for AllowEveryAccess { + fn authorize<'a>(&'a self, _access: &'a ItemAccess) -> ItemAuthorization<'a> { + Box::pin(async { Ok(()) }) + } +} + +/// Evaluates the caller's policies for each access. +struct PolicyAuthorizer { + state: Arc, + identity: AuthIdentity, + account_id: Arc, + source_ip: String, + user_agent: Option, +} + +impl ItemAuthorizer for PolicyAuthorizer { + fn authorize<'a>(&'a self, access: &'a ItemAccess) -> ItemAuthorization<'a> { + Box::pin(async move { + // A statement names its table by name, so the denial names the + // canonical ARN. + let target = AuthzTarget::new( + access.action, + resource_arn(access, &self.state.region, &self.account_id), + ); + let params = RequestParams { + region: Some(self.state.region.to_string()), + secure_transport: self.state.tls_enabled, + source_ip: Some(self.source_ip.clone()), + user_agent: self.user_agent.clone(), + ..item_params(access) + }; + Box::pin(authorization::check_authorization( + self.state.authz_cache.as_ref(), + &self.identity, + &target, + false, + params, + )) + .await + }) + } +} + +/// The resource an access is authorized against: this deployment's ARN of +/// its table, or of its index for a read of an index, which a read of the +/// index is authorized against alone (measured 2026-11-03). The table name +/// has passed the table name rule, so it holds no `/` or `:`. +fn resource_arn(access: &ItemAccess, region: &str, account_id: &str) -> String { + let table = authz_resource::table_resource_arn(&access.table, region, account_id); + match &access.index { + Some(index) => format!("{table}/index/{index}"), + None => table, + } +} + +/// The item-level condition keys of an access. +fn item_params(access: &ItemAccess) -> RequestParams { + RequestParams { + leading_keys: access.partition_keys.as_deref().and_then(leading_keys), + attributes: access.attributes.clone(), + select: access.select.map(ToOwned::to_owned), + return_values: access.return_values.map(ToOwned::to_owned), + enclosing_operation: access.enclosing_operation.map(ToOwned::to_owned), + full_table_scan: access.full_table_scan, + ..RequestParams::default() + } +} + +/// The `dynamodb:LeadingKeys` values of partition key values, or `None` +/// when one cannot be read, which leaves the key absent as it does for the +/// classic operations. A number is written as Java's `BigDecimal` prints the +/// text the statement or the parameter gives it (`NumberForm::Decimal`): +/// under `ForAllValues` `["10"]`, `pk = 10` and `pk = 010` run and `pk = +/// 10.0`, `pk = 1E1` and the parameter `{"N": "10.0"}` are refused, in all +/// three operations (measured 2026-11-03). +fn leading_keys(values: &[AttributeValue]) -> Option> { + values + .iter() + .map(|value| { + let wire = serde_json::to_value(value).ok()?; + extract_typed_value(&wire, NumberForm::Decimal) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn access(table: &str, index: Option<&str>) -> ItemAccess { + ItemAccess { + action: "PartiQLSelect", + table: table.to_owned(), + index: index.map(ToOwned::to_owned), + ..ItemAccess::default() + } + } + + #[test] + fn a_table_read_is_authorized_against_the_table_arn() { + assert_eq!( + resource_arn(&access("orders", None), "us-east-1", "123456789012"), + "arn:aws:dynamodb:us-east-1:123456789012:table/orders" + ); + } + + #[test] + fn an_index_read_is_authorized_against_the_index_arn() { + assert_eq!( + resource_arn(&access("orders", Some("gix")), "eu-west-1", "123456789012"), + "arn:aws:dynamodb:eu-west-1:123456789012:table/orders/index/gix" + ); + } + + #[test] + fn a_number_partition_value_keeps_the_scale_it_is_written_with() { + // Measured 2026-11-03 under ForAllValues ["10"] and ["10.0"]. + for (text, written) in [ + ("10", "10"), + ("010", "10"), + ("10.0", "10.0"), + ("1E1", "1E+1"), + ("010.0", "10.0"), + ("-10", "-10"), + ] { + assert_eq!( + leading_keys(&[AttributeValue::N(text.to_owned())]), + Some(vec![written.to_owned()]), + "{text}" + ); + } + } + + #[test] + fn string_and_binary_partition_values_are_written_as_sent() { + let values = [ + AttributeValue::S("alice".to_owned()), + AttributeValue::B(vec![1, 2]), + ]; + assert_eq!( + leading_keys(&values), + Some(vec!["alice".to_owned(), "AQI=".to_owned()]) + ); + } + + #[test] + fn a_number_that_cannot_be_read_leaves_the_key_absent() { + let values = [ + AttributeValue::S("a".to_owned()), + AttributeValue::N("not a number".to_owned()), + ]; + assert_eq!(leading_keys(&values), None); + } + + #[test] + fn the_item_keys_of_an_access_reach_the_policy_request() { + let access = ItemAccess { + action: "PartiQLUpdate", + table: "t".to_owned(), + partition_keys: Some(vec![AttributeValue::S("a".to_owned())]), + attributes: Some(vec!["pk".to_owned(), "q".to_owned()]), + return_values: Some("ALL_NEW"), + enclosing_operation: Some("ExecuteTransaction"), + full_table_scan: Some(false), + ..ItemAccess::default() + }; + let params = item_params(&access); + assert_eq!(params.leading_keys, Some(vec!["a".to_owned()])); + assert_eq!( + params.attributes, + Some(vec!["pk".to_owned(), "q".to_owned()]) + ); + assert_eq!(params.return_values.as_deref(), Some("ALL_NEW")); + assert_eq!( + params.enclosing_operation.as_deref(), + Some("ExecuteTransaction") + ); + assert_eq!(params.full_table_scan, Some(false)); + assert_eq!(params.select, None); + } +} diff --git a/crates/server/src/leading_keys.rs b/crates/server/src/leading_keys.rs index 0f2577985..3a6c733b7 100644 --- a/crates/server/src/leading_keys.rs +++ b/crates/server/src/leading_keys.rs @@ -117,7 +117,7 @@ pub(crate) enum NumberForm { /// spelling `10.0` matches on `GetItem` and does not match on /// `TransactGetItems`. An N value the engine would reject as malformed yields /// `None`, so the engine returns `ValidationException` before any data flows. -fn extract_typed_value(type_val: &Value, form: NumberForm) -> Option { +pub(crate) fn extract_typed_value(type_val: &Value, form: NumberForm) -> Option { let (type_tag, inner) = type_val.as_object()?.iter().next()?; let raw = inner.as_str().unwrap_or_default(); if type_tag == "N" { diff --git a/crates/server/src/lib.rs b/crates/server/src/lib.rs index 73e1a1fe9..a51d7320d 100755 --- a/crates/server/src/lib.rs +++ b/crates/server/src/lib.rs @@ -15,6 +15,7 @@ pub mod console; pub mod dev_credentials; mod handler; mod import_export_paths; +mod item_authz; pub mod key_info_cache; mod leading_keys; pub mod management; diff --git a/crates/server/src/request_helpers.rs b/crates/server/src/request_helpers.rs index 8c0d6e39c..7bd4e91ee 100755 --- a/crates/server/src/request_helpers.rs +++ b/crates/server/src/request_helpers.rs @@ -152,6 +152,20 @@ pub(crate) async fn authorize_request( return Ok(authorized_key_info); } + // A PartiQL request names its tables inside its statements. The service + // authorizes each statement with the action of its type + // (`dynamodb:PartiQLSelect` and the others) against the table or index it + // reads or writes, and needs no action for the operation itself (measured + // 2026-11-03). The engine authorizes each statement through the item + // authorizer the handler gives it, and refuses every statement without + // one (see `item_authz`). + if matches!( + operation, + "ExecuteStatement" | "BatchExecuteStatement" | "ExecuteTransaction" + ) { + return Ok(authorized_key_info); + } + // Request-level condition keys every evaluation below carries: the // deployment region, aws:SecureTransport, aws:SourceIp and aws:UserAgent, // and the tag condition keys the operation populates (aws:TagKeys and diff --git a/docs/differences-from-dynamodb.md b/docs/differences-from-dynamodb.md index 13bb1c8d5..5ddd9e631 100755 --- a/docs/differences-from-dynamodb.md +++ b/docs/differences-from-dynamodb.md @@ -14,7 +14,7 @@ from a request sent to that service or backend. | Storage backend | Proprietary distributed storage | PostgreSQL (default), SQLite, MongoDB, or Cassandra, selected by mutually exclusive Cargo features at build time; one backend per binary | | Global Tables | CreateGlobalTable, replication | Not implemented (returns UnknownOperationException) | | DAX (Accelerator) | In-memory caching layer | Not applicable | -| PartiQL | ExecuteStatement, BatchExecuteStatement | Not implemented (returns UnknownOperationException) | +| PartiQL | ExecuteStatement, BatchExecuteStatement, ExecuteTransaction | Supported for all four statement types (see the PartiQL section below). Built behind a default-on `partiql` build feature. | | Numeric precision on partition/sort keys (MongoDB backend only) | 38 significant digits | 34 significant digits (BSON Decimal128). Values that exceed this precision are rejected at write and query time with a ValidationException rather than silently downcast. PostgreSQL backend supports the full 38 digits. | | Inverted numeric `BETWEEN` on a sort key (MongoDB backend only) | ValidationException ("The BETWEEN operator requires upper bound to be greater than or equal to lower bound") | Same error in all practical cases. The inversion guard compares bounds via `f64`, so a `KeyConditionExpression` `BETWEEN` whose bounds are inverted only beyond f64's ~15–17 significant digits (e.g. `BETWEEN 10000000000000002 AND 10000000000000001`) is not rejected and returns an empty result set instead. Valid ranges are never wrongly rejected. | | `TransactGetItems` overlapping an in-flight `TransactWriteItems` on the same item | May cancel the read with `TransactionCanceledException` and reason `TransactionConflict` | Never cancels for this reason on PostgreSQL: the read runs in one `REPEATABLE READ` snapshot taken at its first read, so it sees either all of the write or none of it and returns the pre-write state while the write is in flight. That is a legal serial order (the read ordered before the write) and the all-or-nothing guarantee holds, but a client that retries on `TransactionConflict` to observe the newest state will not get that signal here. SQLite behaves the same (one WAL snapshot); MongoDB with the default `snapshot` read concern likewise, and the row above covers the other read concerns. | @@ -174,9 +174,14 @@ remains. The key and untouched attributes are left out. Both the statement figur and the finished item are checked against the limit, and a breach refuses with `Item size to update has exceeded the maximum allowed size`. +A transacted `Update` and a PartiQL `UPDATE` are sized by the finished item +alone, with the same wording. For PartiQL this holds in `ExecuteStatement`, +`BatchExecuteStatement` and `ExecuteTransaction` (measured in us-east-1 on +2026-11-04). + | Area | DynamoDB | ExtendDB | |------|----------|------| -| `UpdateItem` item-size accounting | In 31 of 33 commercial regions a standalone `UpdateItem` is sized by the statement (the rule above). At a one-byte key the largest finished item it can produce is 409,581 bytes. Two regions (`eu-south-2`, `me-central-1`) size an `UpdateItem` on the finished item, like a `PutItem`. | PostgreSQL and SQLite follow the 31-region statement rule. A transacted `Update` is sized on the finished item on both, as on the service. Cassandra sizes every update on the finished item against a fixed 409,600 bytes; MongoDB applies no item-size limit on updates. | +| `UpdateItem` item-size accounting | In 31 of 33 commercial regions a standalone `UpdateItem` is sized by the statement (the rule above). At a one-byte key the largest finished item it can produce is 409,581 bytes. Two regions (`eu-south-2`, `me-central-1`) size an `UpdateItem` on the finished item, like a `PutItem`. | PostgreSQL and SQLite follow the 31-region statement rule. A transacted `Update` and a PartiQL `UPDATE` are sized on the finished item on both, as on the service. Cassandra sizes every update on the finished item against a fixed 409,600 bytes; MongoDB applies no item-size limit on updates. | ## Capacity and Throttling @@ -186,13 +191,30 @@ and the finished item are checked against the limit, and a breach refuses with | On-demand capacity | Automatic scaling | Fixed initial burst capacity (4000 WCU / 12000 RCU), no auto-scaling | | Throttling | Always on; throttles requests that exceed provisioned/burst capacity. No setting to disable | Configurable via `throttling_enabled` runtime setting (default: `true`) | +## PartiQL + +ExtendDB implements `ExecuteStatement`, `BatchExecuteStatement`, and +`ExecuteTransaction` as a translation over the existing data plane: a `SELECT` +runs as a point read, a Query, or a Scan by the shape of its `WHERE`; `INSERT`, +`UPDATE`, and `DELETE` run as the matching item writes. Behaviour was measured +against the service in us-east-1 on 2026-11-02. The build feature `partiql` is on +by default; with it off the three operations return `UnknownOperationException`. + +| Area | DynamoDB | ExtendDB | +|------|----------|------| +| Request-rate throttling of PartiQL operations | A PartiQL request is throttled against the table's provisioned or burst capacity like the classic operation it performs | Not throttled by table. The server's throttle is keyed on a request's top-level `TableName`, which a PartiQL request does not carry (the table is named inside the statement), so a PartiQL request is not counted against a table's token bucket. Consumed capacity is still reported. Parsing the statement in the throttle path to recover the table would duplicate the PartiQL parser outside the feature boundary, so it is deliberately not done. | +| PartiQL on a multi-attribute key schema | A `SELECT`, `INSERT`, `UPDATE`, or `DELETE` against a table or index whose key schema has more than one HASH or more than one RANGE attribute is refused with `ValidationException` "PartiQL statements cannot be executed on tables with multi-attribute HASH keys or multi-attribute RANGE keys." (A multi-attribute key is reachable on the service only through the few preview APIs that create one.) | Same. The base table's own key is single or composite (one HASH, optional one RANGE) in both the standard and the multi-part-key preview; the refusal applies when the statement's target (base table or named index) carries a multi-attribute key, and a write to a single-key base table is unaffected by a multi-attribute GSI on the same table. | +| `BatchExecuteStatement` SELECT that names an index | Refused per statement with "Select statements within BatchExecuteStatement must specify the primary key in the where clause." (a batch read is a base-table point read; no index read is possible) | Same, whatever the `WHERE` names. A missing index is the per-statement "The table does not have the specified index". | +| `ExecuteTransaction` SELECT that names an index | Refused with `ValidationException` "Validation failed in TransactStatements[i]: Reads on indices are not supported within transactions." | Same, for an existing GSI, a vector index, or a missing index. | +| `ExecuteTransaction` read naming an unknown table | Cancels the whole transaction: `TransactionCanceledException` with a positional `ResourceNotFound` cancellation reason | Same. | +| `ConsistentRead=true` on an index SELECT | A GSI or vector index is refused with "Strongly consistent read is not supported on Global Secondary Indexes", ahead of the vector refusal | Same ordering. | + ## Operations Not Implemented The following operations return `UnknownOperationException`: - CreateGlobalTable, DescribeGlobalTable, ListGlobalTables, UpdateGlobalTable - DescribeGlobalTableSettings, UpdateGlobalTableSettings -- ExecuteStatement, BatchExecuteStatement, ExecuteTransaction - DescribeContributorInsights, UpdateContributorInsights - DescribeKinesisStreamingDestination, EnableKinesisStreamingDestination, DisableKinesisStreamingDestination - DescribeTableReplicaAutoScaling, UpdateTableReplicaAutoScaling diff --git a/tests/partiql_authz_helpers.py b/tests/partiql_authz_helpers.py new file mode 100644 index 000000000..7cf79c16b --- /dev/null +++ b/tests/partiql_authz_helpers.py @@ -0,0 +1,371 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""Principals, policies and checks shared by the PartiQL authorization tests. + +The tests give an IAM user the permissions the measured session had, in its +identity policy, so a refusal names the user and an identity-based policy. +A test class derives from :class:`PartiqlAuthzCase` and imports the module +fixtures it uses (``auth_env``, ``mgmt``, ``region``, ``account_id``). +""" + +from __future__ import annotations + +import os +import time +import uuid +from types import SimpleNamespace +from typing import Any + +import boto3 +import pytest +from botocore.config import Config as BotoConfig +from botocore.exceptions import ClientError + +from conftest import wait_for_active +from management_helpers import ManagementClient + +PARTIQL = [ + "dynamodb:PartiQLSelect", + "dynamodb:PartiQLInsert", + "dynamodb:PartiQLUpdate", + "dynamodb:PartiQLDelete", +] +CLASSIC = [ + "dynamodb:GetItem", + "dynamodb:Query", + "dynamodb:Scan", + "dynamodb:PutItem", + "dynamodb:UpdateItem", + "dynamodb:DeleteItem", + "dynamodb:BatchGetItem", + "dynamodb:BatchWriteItem", + "dynamodb:ConditionCheckItem", +] +ALLOW_ALL = { + "Version": "2012-10-17", + "Statement": [{"Effect": "Allow", "Action": "dynamodb:*", "Resource": "*"}], +} +AX = "pk = 'a' AND sk = 'x'" +BX = "pk = 'b' AND sk = 'x'" +CANCELLED = "Transaction cancelled, please refer cancellation reasons for specific reasons " + + +def _require_auth_env() -> tuple[str, str, str]: + endpoint = os.environ.get("EXTENDDB_TEST_ENDPOINT", "").strip() + admin_user = os.environ.get("EXTENDDB_ADMIN_USER", "").strip() + admin_pass = os.environ.get("EXTENDDB_ADMIN_PASSWORD", "").strip() + if not endpoint or not admin_user or not admin_pass: + pytest.fail( + "MISCONFIGURED: PartiQL authorization tests require EXTENDDB_TEST_ENDPOINT, " + "EXTENDDB_ADMIN_USER, and EXTENDDB_ADMIN_PASSWORD. " + "These must be set by devtools/run-tests before test execution." + ) + return endpoint, admin_user, admin_pass + + +def _make_client(endpoint_url: str, access_key: str, secret_key: str, region: str, + session_token: str | None = None, validate: bool = True) -> Any: + kwargs: dict = dict( + service_name="dynamodb", + endpoint_url=endpoint_url, + aws_access_key_id=access_key, + aws_secret_access_key=secret_key, + region_name=region, + config=BotoConfig(retries={"max_attempts": 0}, parameter_validation=validate), + ) + if session_token is not None: + kwargs["aws_session_token"] = session_token + if endpoint_url.startswith("https://"): + kwargs["verify"] = False + return boto3.client(**kwargs) + + +def select(table: str, where: str | None = None, what: str = "*", + index: str | None = None) -> str: + target = f'"{table}"' if index is None else f'"{table}"."{index}"' + return f"SELECT {what} FROM {target}" + (f" WHERE {where}" if where else "") + + +def keys_of(items: list[dict]) -> list[str]: + return sorted("/".join(i[n]["S"] for n in ("pk", "sk") if n in i) for i in items) + + +def statement_error(client: Any, statement: str, **kwargs) -> dict: + with pytest.raises(ClientError) as exc: + client.execute_statement(Statement=statement, **kwargs) + return exc.value.response["Error"] + + +def batch(client: Any, statements: list[str]) -> list[dict]: + resp = client.batch_execute_statement( + Statements=[{"Statement": s} for s in statements]) + return resp["Responses"] + + +def transaction(client: Any, statements: list[str]) -> dict: + return client.execute_transaction( + TransactStatements=[{"Statement": s} for s in statements]) + + +def cancelled(client: Any, statements: list[str]) -> dict: + with pytest.raises(ClientError) as exc: + transaction(client, statements) + return exc.value.response + + +def transaction_error(client: Any, statements: list[str]) -> dict: + return cancelled(client, statements)["Error"] + + +@pytest.fixture(scope="module") +def auth_env(): + return _require_auth_env() + + +@pytest.fixture(scope="module") +def mgmt(auth_env) -> ManagementClient: + endpoint, admin_user, admin_pass = auth_env + return ManagementClient(endpoint, admin_user, admin_pass) + + +@pytest.fixture(scope="module") +def region() -> str: + return os.environ.get("AWS_DEFAULT_REGION", "us-east-1") + + +@pytest.fixture(scope="module") +def account_id(mgmt) -> str: + acct_id = f"{uuid.uuid4().int % 10**12:012d}" + resp = mgmt.create_account(acct_id, f"pq-authz-{acct_id}") + assert resp.status_code == 201, resp.text + yield acct_id + mgmt.delete_account(acct_id) + + +class PartiqlAuthzCase: + """Principals and policies in an account of their own, and the checks of + the answers a refused statement gets. A table fixture sets ``self._run``, + the run id in the names of the tables.""" + + _run: str + + @pytest.fixture(autouse=True) + def setup(self, auth_env, mgmt, account_id, region): + self.endpoint = auth_env[0] + self.mgmt = mgmt + self.account_id = account_id + self.region = region + self._users: list[str] = [] + self._roles: list[str] = [] + yield + for u in self._users: + try: + self.mgmt.delete_user(self.account_id, u) + except Exception: + pass + for r in self._roles: + try: + self.mgmt.delete_role(self.account_id, r) + except Exception: + pass + + # -- principals and policies --------------------------------------------- + + def _user(self, policy: dict, validate: bool = True) -> tuple[Any, str]: + """A user with `policy` as its identity policy: its client and ARN.""" + name = f"pqz-{uuid.uuid4().hex[:8]}" + resp = self.mgmt.create_user(self.account_id, name, "TestPass123!") + assert resp.status_code == 201, resp.text + self._users.append(name) + resp = self.mgmt.create_access_key(self.account_id, name) + assert resp.status_code == 201, resp.text + creds = resp.json() + resp = self.mgmt.put_user_policy(self.account_id, name, "p", policy) + assert resp.status_code == 204, resp.text + client = _make_client(self.endpoint, creds["access_key_id"], + creds["secret_access_key"], self.region, validate=validate) + return client, f"arn:aws:iam::{self.account_id}:user/{name}" + + def _client(self, *statements: dict, validate: bool = True) -> tuple[Any, str]: + return self._user({"Version": "2012-10-17", "Statement": list(statements)}, + validate=validate) + + def _session(self, session_policy: dict) -> Any: + """A role session: the role is allowed dynamodb:*, the session is + narrowed by `session_policy`.""" + _, caller_arn = self._client( + {"Effect": "Allow", "Action": "sts:AssumeRole", "Resource": "*"}) + role = f"pqr-{uuid.uuid4().hex[:8]}" + trust = { + "Version": "2012-10-17", + "Statement": [{"Effect": "Allow", "Principal": {"AWS": caller_arn}, + "Action": "sts:AssumeRole"}], + } + assert self.mgmt.create_role(self.account_id, role, trust).status_code == 201 + self._roles.append(role) + assert self.mgmt.put_role_policy( + self.account_id, role, "full", ALLOW_ALL).status_code == 204 + resp = self.mgmt.assume_role(self.account_id, role, caller_arn, "pq-authz", + session_policy=session_policy) + assert resp.status_code == 201, resp.text + creds = resp.json() + return _make_client(self.endpoint, creds["access_key_id"], + creds["secret_access_key"], self.region, + session_token=creds["session_token"]) + + def _arn(self, table: str, index: str | None = None) -> str: + arn = f"arn:aws:dynamodb:{self.region}:{self.account_id}:table/{table}" + return arn if index is None else f"{arn}/index/{index}" + + def _tables_pattern(self) -> list[str]: + """The tables named pqa-* and their indexes, as the measured identity + policy names them.""" + base = f"arn:aws:dynamodb:{self.region}:{self.account_id}:table/pqa-*" + return [base, base + "/index/*"] + + def _allow(self, actions: list[str], condition: dict | None = None, + resources: list[str] | None = None) -> dict: + st: dict = {"Effect": "Allow", "Action": actions, + "Resource": resources or self._tables_pattern()} + if condition: + st["Condition"] = condition + return st + + def _deny(self, actions: list[str], condition: dict | None = None, + resources: list[str] | None = None) -> dict: + st = self._allow(actions, condition, resources) + st["Effect"] = "Deny" + return st + + def _identity(self, validate: bool = True) -> tuple[Any, str]: + """The measured identity policy: dynamodb:* on pqa-* and their + indexes, PartiQLDelete denied on pqa-deny-*.""" + return self._client( + self._allow(["dynamodb:*"]), + self._deny(["dynamodb:PartiQLDelete"], + resources=[self._arn(f"pqa-deny-{self._run}")]), + validate=validate, + ) + + def _fenced(self, operator: str, values: list[str] | str) -> tuple[Any, str]: + return self._client(self._allow( + PARTIQL, {operator: {"dynamodb:LeadingKeys": values}})) + + def _with(self, actions: list[str], condition: dict | None = None, + resources: list[str] | None = None) -> tuple[Any, str]: + """A user allowed `actions` on the tables named pqa-* and their + indexes, under `condition`.""" + return self._client(self._allow(actions, condition, resources)) + + # -- tables ---------------------------------------------------------------- + + @pytest.fixture() + def r(self): + """The tables of the second measurement: C (pk, sk, GSI gix on g) + holding a/x (g ga), a/y, b/x (g gb) and c/x, each with a = '1'; K + (pk, sk, KEYS_ONLY GSI kix on g) holding a/x (g ga); N keyed on a + number pk holding 10; L (pk, sk, KEYS_ONLY LSI lix on l) holding a/x + with l and an attribute u the index does not hold; B outside the + pqa-* pattern, keyed on pk with GSI gix on g, holding a (g ga).""" + self._run = uuid.uuid4().hex[:6] + owner, _ = self._user(ALLOW_ALL) + names = SimpleNamespace( + c=f"pqa-{self._run}-c", k=f"pqa-{self._run}-k", n=f"pqa-{self._run}-n", + l=f"pqa-{self._run}-l", b=f"pqb-{self._run}", + missing_a=f"pqa-{self._run}-missing", missing_b=f"pqb-{self._run}-missing", + owner=owner) + + def create(name: str, attrs: list[str | tuple[str, str]], + keys: list[tuple[str, str]], **extra: Any) -> None: + typed = [a if isinstance(a, tuple) else (a, "S") for a in attrs] + owner.create_table( + TableName=name, + AttributeDefinitions=[{"AttributeName": a, "AttributeType": t} + for a, t in typed], + KeySchema=[{"AttributeName": a, "KeyType": k} for a, k in keys], + BillingMode="PAY_PER_REQUEST", **extra) + wait_for_active(owner, name) + + def on_g(index: str, projection: str) -> dict: + return {"GlobalSecondaryIndexes": [{ + "IndexName": index, + "KeySchema": [{"AttributeName": "g", "KeyType": "HASH"}], + "Projection": {"ProjectionType": projection}}]} + + def put(name: str, **attrs: str) -> None: + owner.put_item(TableName=name, Item={k: {"S": v} for k, v in attrs.items()}) + + pk_sk = [("pk", "HASH"), ("sk", "RANGE")] + create(names.c, ["pk", "sk", "g"], pk_sk, **on_g("gix", "ALL")) + create(names.k, ["pk", "sk", "g"], pk_sk, **on_g("kix", "KEYS_ONLY")) + create(names.b, ["pk", "g"], [("pk", "HASH")], **on_g("gix", "ALL")) + create(names.n, [("pk", "N")], [("pk", "HASH")]) + create(names.l, ["pk", "sk", "l"], pk_sk, LocalSecondaryIndexes=[{ + "IndexName": "lix", + "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "l", "KeyType": "RANGE"}], + "Projection": {"ProjectionType": "KEYS_ONLY"}}]) + owner.put_item(TableName=names.n, Item={"pk": {"N": "10"}, "a": {"S": "1"}}) + put(names.l, pk="a", sk="x", l="la", u="hidden") + put(names.c, pk="a", sk="x", g="ga", a="1") + put(names.c, pk="a", sk="y", a="1") + put(names.c, pk="b", sk="x", g="gb", a="1") + put(names.c, pk="c", sk="x", a="1") + put(names.k, pk="a", sk="x", g="ga", a="1") + put(names.b, pk="a", g="ga") + for name, index, count in ((names.c, "gix", 2), (names.k, "kix", 1), + (names.b, "gix", 1)): + deadline = time.monotonic() + 15 + while owner.scan(TableName=name, IndexName=index, + Select="COUNT")["Count"] < count: + assert time.monotonic() < deadline, f"index {index} of {name} not filled" + time.sleep(0.05) + yield names + for name in (names.c, names.k, names.n, names.l, names.b): + try: + owner.delete_table(TableName=name) + except Exception: + pass + + # -- the answers to a refused statement ------------------------------------ + + @staticmethod + def _refusal(principal: str, action: str, resource: str, + explicit: bool = False) -> str: + head = (f"User: {principal} is not authorized to perform: dynamodb:{action} " + f"on resource: {resource}") + if explicit: + return head + " with an explicit deny in an identity-based policy" + return head + f" because no identity-based policy allows the dynamodb:{action} action" + + def _denied(self, error: dict, principal: str, action: str, resource: str, + explicit: bool = False) -> None: + assert error["Code"] == "AccessDeniedException", error + assert error["Message"] == self._refusal(principal, action, resource, explicit) + + def _denied_entry(self, entry: dict, principal: str, action: str, table: str, + explicit: bool = False) -> None: + assert entry == { + "Error": {"Code": "AccessDenied", + "Message": self._refusal(principal, action, self._arn(table), + explicit)}, + "TableName": table, + } + + def _cancelled_by(self, resp: dict, reasons: list[tuple[str, str] | None], + principal: str) -> None: + """A transaction cancelled with AccessDenied, as (action, table), for + each refused member and None for the others.""" + codes = ["None" if r is None else "AccessDenied" for r in reasons] + assert resp["Error"]["Code"] == "TransactionCanceledException", resp + assert resp["Error"]["Message"] == CANCELLED + f"[{', '.join(codes)}]" + want = [] + for r in reasons: + if r is None: + want.append({"Code": "None"}) + else: + action, table, *explicit = r + want.append({"Code": "AccessDenied", "Message": self._refusal( + principal, action, self._arn(table), bool(explicit))}) + assert resp["CancellationReasons"] == want diff --git a/tests/partiql_gsi_table.py b/tests/partiql_gsi_table.py new file mode 100644 index 000000000..3a9097bef --- /dev/null +++ b/tests/partiql_gsi_table.py @@ -0,0 +1,107 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""A table with a global secondary index for the PartiQL batch and +transaction tests, and the items those tests write in it. + +The table has the key pk(S)/sk(S) and the index `gsi` on g(S)/gs(S) with +projection ALL, on demand. EXTENDDB_TEST_GSI_TABLE names a table to use in +place of a new one: it is created when it is missing and kept afterwards. +Without it the table is new and is deleted at the end. Every key a test +writes carries the run's suffix (see `key`), so a kept table holds no item a +test reads from an earlier run. +""" + +from __future__ import annotations + +import contextlib +import os +import uuid + +from conftest import wait_for_active + +INDEX = "gsi" + +RUN = uuid.uuid4().hex[:8] + + +def key(name: str) -> str: + """The partition key `name` takes in this run.""" + return f"{name}-{RUN}" + + +def item(pk: str, sk: str = "b", gs: str = "1", **extra) -> dict: + """The item the tests write: `{pk, sk, a: 1, b: 2, x: 0, m: {k: 1, 'my + key': 1}, g: pk, gs}`, with `extra` attributes added.""" + value = { + "pk": {"S": pk}, + "sk": {"S": sk}, + "a": {"N": "1"}, + "b": {"N": "2"}, + "x": {"N": "0"}, + "m": {"M": {"k": {"N": "1"}, "my key": {"N": "1"}}}, + "g": {"S": pk}, + "gs": {"S": gs}, + } + value.update(extra) + return value + + +def put(client, table: str, name: str, **extra) -> str: + """Write the test item under the key `name` takes in this run, and + return that key.""" + pk = key(name) + client.put_item(TableName=table, Item=item(pk, **extra)) + return pk + + +def get(client, table: str, pk: str): + """The item stored under `pk`, read consistently, or None.""" + return client.get_item( + TableName=table, Key={"pk": {"S": pk}, "sk": {"S": "b"}}, ConsistentRead=True + ).get("Item") + + +@contextlib.contextmanager +def gsi_table(client, prefix: str): + """The name of the table, created for the test module unless + EXTENDDB_TEST_GSI_TABLE names one.""" + kept = os.environ.get("EXTENDDB_TEST_GSI_TABLE", "").strip() + name = kept or f"{prefix}-{uuid.uuid4().hex[:12]}" + if not kept or not _exists(client, name): + client.create_table( + TableName=name, + BillingMode="PAY_PER_REQUEST", + AttributeDefinitions=[ + {"AttributeName": n, "AttributeType": "S"} for n in ("pk", "sk", "g", "gs") + ], + KeySchema=[ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"}, + ], + GlobalSecondaryIndexes=[ + { + "IndexName": INDEX, + "KeySchema": [ + {"AttributeName": "g", "KeyType": "HASH"}, + {"AttributeName": "gs", "KeyType": "RANGE"}, + ], + "Projection": {"ProjectionType": "ALL"}, + } + ], + ) + wait_for_active(client, name) + try: + yield name + finally: + if not kept: + with contextlib.suppress(client.exceptions.ResourceNotFoundException): + client.delete_table(TableName=name) + + +def _exists(client, name: str) -> bool: + try: + client.describe_table(TableName=name) + except client.exceptions.ResourceNotFoundException: + return False + return True diff --git a/tests/test_partiql_authorization.py b/tests/test_partiql_authorization.py new file mode 100644 index 000000000..6e2ef4622 --- /dev/null +++ b/tests/test_partiql_authorization.py @@ -0,0 +1,677 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""Per-statement authorization of ExecuteStatement, BatchExecuteStatement and +ExecuteTransaction. + +A PartiQL request names its tables inside its statements. The service +authorizes each statement with the action of its type (dynamodb:PartiQLSelect, +dynamodb:PartiQLInsert, dynamodb:PartiQLUpdate, dynamodb:PartiQLDelete) against +the table it names, or for a read of an index against the index, and needs no +action for the operation itself. + +Every expected answer here was measured against live DynamoDB and IAM on +2026-11-03, in a role session whose identity policy allowed dynamodb:* on the +tables named pqa-* and denied dynamodb:PartiQLDelete on pqa-deny-*, narrowed by +a session policy for most cases. These tests give an IAM user the same +permissions in its identity policy, so a refusal names the user and an +identity-based policy; the session policy cases use a role session. + +Measured model: + - The action is the statement's type. The classic item actions allow no + statement, and a statement needs no other action. + - The resource is the table ARN, or for a read of an index the index ARN + alone. + - A refused ExecuteStatement is AccessDeniedException. In a + BatchExecuteStatement the refused statement's entry is + {"Error": {"Code": "AccessDenied", "Message": ...}, "TableName": ...} and + the others run. A refused ExecuteTransaction member cancels the + transaction with AccessDenied for it and None for the others, and nothing + is written. + - Syntax, the EXISTS form and the table name rule are checked before + authorization; a missing table, a missing index and the key checks after. + test_partiql_authorization_order.py has the other checks on each side. + - dynamodb:LeadingKeys holds the partition key values a statement reads or + writes, the index's for a read of an index, and is absent for a scan. Each + disjunct of a SELECT read by key is authorized by itself: a refused one is + held back while the items of the others come back with a NextToken. + - dynamodb:FullTableScan is "true" for an ExecuteStatement scan, "false" for + an ExecuteStatement read by key, and absent in the other two operations. + - dynamodb:EnclosingOperation is BatchExecuteStatement or ExecuteTransaction + in those operations and absent in ExecuteStatement. + - dynamodb:Attributes holds the table's key attributes and the top-level + names a statement names, the WHERE included; dynamodb:Select is + SPECIFIC_ATTRIBUTES for a projection list, ALL_ATTRIBUTES for `*` on a + table and ALL_PROJECTED_ATTRIBUTES for `*` on an index; + dynamodb:ReturnValues comes from RETURNING, and is NONE for a write + without it. + +Prerequisites: + - extenddb running with ``auth.provider = "builtin"`` on EXTENDDB_TEST_ENDPOINT + - Admin credentials in EXTENDDB_ADMIN_USER / EXTENDDB_ADMIN_PASSWORD env vars +""" + +from __future__ import annotations + +import time +import uuid +from types import SimpleNamespace + +import pytest +from botocore.exceptions import ClientError + +from conftest import wait_for_active +from partiql_authz_helpers import ( # noqa: F401 (the fixtures are used by name) + ALLOW_ALL, + AX, + BX, + CANCELLED, + CLASSIC, + PARTIQL, + PartiqlAuthzCase, + account_id, + auth_env, + batch, + cancelled, + keys_of, + mgmt, + region, + select, + statement_error, + transaction, +) + + +class TestPartiqlAuthorization(PartiqlAuthzCase): + """Each statement of a PartiQL request is authorized by itself.""" + + # -- tables ---------------------------------------------------------------- + + @pytest.fixture() + def t(self): + """The measured tables: C (pk, sk, GSI gix on g) holding a/x (g ga), + a/y and b/x; H and D (pqa-deny-*) keyed on pk; B outside the pqa-* + pattern, keyed on pk with GSI gix on g.""" + self._run = uuid.uuid4().hex[:6] + owner, _ = self._user(ALLOW_ALL) + names = SimpleNamespace( + c=f"pqa-{self._run}-c", h=f"pqa-{self._run}-h", d=f"pqa-deny-{self._run}", + b=f"pqb-{self._run}", missing_a=f"pqa-{self._run}-missing", + missing_b=f"pqb-{self._run}-missing", owner=owner) + + def create(name: str, attrs: list[str], keys: list[tuple[str, str]], + gsi: bool = False) -> None: + extra: dict = {} + if gsi: + extra["GlobalSecondaryIndexes"] = [{ + "IndexName": "gix", + "KeySchema": [{"AttributeName": "g", "KeyType": "HASH"}], + "Projection": {"ProjectionType": "ALL"}}] + owner.create_table( + TableName=name, + AttributeDefinitions=[{"AttributeName": a, "AttributeType": "S"} + for a in attrs], + KeySchema=[{"AttributeName": a, "KeyType": k} for a, k in keys], + BillingMode="PAY_PER_REQUEST", **extra) + wait_for_active(owner, name) + + def put(name: str, **attrs: str) -> None: + owner.put_item(TableName=name, Item={k: {"S": v} for k, v in attrs.items()}) + + create(names.c, ["pk", "sk", "g"], [("pk", "HASH"), ("sk", "RANGE")], gsi=True) + create(names.h, ["pk"], [("pk", "HASH")]) + create(names.d, ["pk"], [("pk", "HASH")]) + create(names.b, ["pk", "g"], [("pk", "HASH")], gsi=True) + put(names.c, pk="a", sk="x", g="ga", a="1") + put(names.c, pk="a", sk="y", a="1") + put(names.c, pk="b", sk="x", g="gb", a="1") + put(names.h, pk="a") + put(names.h, pk="b") + put(names.d, pk="a") + put(names.b, pk="a", g="ga") + for name, count in ((names.c, 2), (names.b, 1)): + deadline = time.monotonic() + 15 + while owner.scan(TableName=name, IndexName="gix", + Select="COUNT")["Count"] < count: + assert time.monotonic() < deadline, f"index of {name} not filled" + time.sleep(0.05) + yield names + for name in (names.c, names.h, names.d, names.b): + try: + owner.delete_table(TableName=name) + except Exception: + pass + + def _items(self, t, table: str) -> list[str]: + return keys_of(t.owner.scan(TableName=table, ConsistentRead=True)["Items"]) + + # -- 1. actions and resources --------------------------------------------- + + def test_each_statement_type_is_authorized_with_its_action(self, t): + # Measured 2026-11-03: on a table outside the policy each statement + # is refused naming the action of its type and the table's ARN; on a + # table inside it a SELECT runs. + c, me = self._identity() + assert len(c.execute_statement(Statement=select(t.c, "pk = 'a'"))["Items"]) == 2 + for statement, action in [ + (select(t.b, "pk = 'a'"), "PartiQLSelect"), + (f"INSERT INTO \"{t.b}\" VALUE {{'pk': 'z'}}", "PartiQLInsert"), + (f"UPDATE \"{t.b}\" SET q = 1 WHERE pk = 'a'", "PartiQLUpdate"), + (f"DELETE FROM \"{t.b}\" WHERE pk = 'zz'", "PartiQLDelete"), + ]: + self._denied(statement_error(c, statement), me, action, self._arn(t.b)) + assert self._items(t, t.b) == ["a"] + + def test_a_partiql_action_allows_every_operation_its_statement_runs_in(self, t): + # Measured 2026-11-03: with dynamodb:PartiQLSelect alone a SELECT runs + # in each of the three operations, and an INSERT or UPDATE is refused + # naming its own action. + c, me = self._client(self._allow(["dynamodb:PartiQLSelect"])) + assert len(c.execute_statement(Statement=select(t.c, "pk = 'a'"))["Items"]) == 2 + assert len(c.execute_statement( + Statement=select(t.c, "g = 'ga'", index="gix"))["Items"]) == 1 + assert len(c.execute_statement(Statement=select(t.c, index="gix"))["Items"]) == 2 + assert "Item" in batch(c, [select(t.c, AX)])[0] + assert len(transaction(c, [select(t.c, AX)])["Responses"]) == 1 + self._denied(statement_error( + c, f"INSERT INTO \"{t.c}\" VALUE {{'pk': 'n', 'sk': 'n'}}"), + me, "PartiQLInsert", self._arn(t.c)) + self._denied(statement_error(c, f"UPDATE \"{t.c}\" SET q = 1 WHERE {AX}"), + me, "PartiQLUpdate", self._arn(t.c)) + + def test_the_classic_item_actions_allow_no_statement(self, t): + # Measured 2026-11-03. + c, me = self._client(self._allow(CLASSIC)) + self._denied(statement_error(c, select(t.c, "pk = 'a'")), + me, "PartiQLSelect", self._arn(t.c)) + self._denied(statement_error( + c, f"INSERT INTO \"{t.c}\" VALUE {{'pk': 'n3', 'sk': 's'}}"), + me, "PartiQLInsert", self._arn(t.c)) + self._denied_entry(batch(c, [select(t.c, AX)])[0], me, "PartiQLSelect", t.c) + self._cancelled_by(cancelled(c, [select(t.c, AX)]), + [("PartiQLSelect", t.c)], me) + + def test_an_index_read_is_authorized_against_the_index_alone(self, t): + # Measured 2026-11-03: allowed on the indexes alone, a read of the + # index runs and a read of the table is refused; a read of an index of + # a table outside the policy is refused naming the index ARN, also for + # an index the table does not have. + idx = [f"arn:aws:dynamodb:{self.region}:{self.account_id}:table/pqa-*/index/*"] + c, me = self._client(self._allow(["dynamodb:PartiQLSelect"], resources=idx)) + assert len(c.execute_statement( + Statement=select(t.c, "g = 'ga'", index="gix"))["Items"]) == 1 + self._denied(statement_error(c, select(t.c, "pk = 'a'")), + me, "PartiQLSelect", self._arn(t.c)) + c, me = self._identity() + self._denied(statement_error(c, select(t.b, "g = 'ga'", index="gix")), + me, "PartiQLSelect", self._arn(t.b, "gix")) + self._denied(statement_error(c, select(t.b, "g = 'ga'", index="nix")), + me, "PartiQLSelect", self._arn(t.b, "nix")) + + def test_returning_needs_no_other_action(self, t): + # Measured 2026-11-03. + c, _ = self._client(self._allow(["dynamodb:PartiQLUpdate"])) + resp = c.execute_statement( + Statement=f"UPDATE \"{t.c}\" SET q = 2 WHERE {AX} RETURNING ALL NEW *") + assert len(resp["Items"]) == 1 + resp = c.execute_statement( + Statement=f"UPDATE \"{t.c}\" SET q = 3 WHERE {AX} RETURNING MODIFIED OLD *") + assert len(resp["Items"]) == 1 + c, _ = self._client(self._allow(["dynamodb:PartiQLDelete"])) + resp = c.execute_statement(Statement=f"DELETE FROM \"{t.c}\" WHERE " + f"pk = 'zz' AND sk = 'zz' RETURNING ALL OLD *") + assert resp["Items"] == [] + + def test_an_explicit_deny_on_one_action_leaves_the_others(self, t): + # Measured 2026-11-03: with PartiQLDelete denied on D, a DELETE is + # refused with the explicit deny wording and a SELECT and an UPDATE + # run. + c, me = self._identity() + self._denied(statement_error(c, f"DELETE FROM \"{t.d}\" WHERE pk = 'zz'"), + me, "PartiQLDelete", self._arn(t.d), explicit=True) + assert len(c.execute_statement(Statement=select(t.d, "pk = 'a'"))["Items"]) == 1 + assert c.execute_statement( + Statement=f"UPDATE \"{t.d}\" SET q = 1 WHERE pk = 'a'")["Items"] == [] + + # -- 2. the answers to a refused statement ---------------------------------- + + def test_a_refused_batch_statement_is_its_entry_and_the_others_run(self, t): + # Measured 2026-11-03. + c, me = self._identity() + out = batch(c, [select(t.c, AX), select(t.b, "pk = 'a'")]) + assert "Item" in out[0] + self._denied_entry(out[1], me, "PartiQLSelect", t.b) + out = batch(c, [select(t.b, "pk = 'a'"), select(t.c, AX)]) + self._denied_entry(out[0], me, "PartiQLSelect", t.b) + assert "Item" in out[1] + out = batch(c, [f"UPDATE \"{t.c}\" SET q = 1 WHERE {AX}", + f"UPDATE \"{t.b}\" SET q = 1 WHERE pk = 'a'"]) + assert out[0] == {"TableName": t.c} + self._denied_entry(out[1], me, "PartiQLUpdate", t.b) + out = batch(c, [f"DELETE FROM \"{t.d}\" WHERE pk = 'zz'", + f"UPDATE \"{t.c}\" SET q = 2 WHERE {AX}"]) + self._denied_entry(out[0], me, "PartiQLDelete", t.d, explicit=True) + assert out[1] == {"TableName": t.c} + item = t.owner.get_item(TableName=t.c, Key={"pk": {"S": "a"}, "sk": {"S": "x"}}) + assert item["Item"]["q"] == {"N": "2"}, "the allowed statements ran" + + def test_a_missing_table_entry_beside_a_refusal_has_no_table_name(self, t): + # Measured 2026-11-03. + c, me = self._identity() + out = batch(c, [select(t.b, "pk = 'a'"), select(t.missing_a, "pk = 'a'")]) + self._denied_entry(out[0], me, "PartiQLSelect", t.b) + assert out[1] == {"Error": {"Code": "ResourceNotFound", + "Message": "Requested resource not found"}} + + def test_a_refused_member_cancels_the_transaction_and_nothing_is_written(self, t): + # Measured 2026-11-03. + c, me = self._identity() + resp = cancelled(c, [f"UPDATE \"{t.c}\" SET q = 3 WHERE {AX}", + f"UPDATE \"{t.b}\" SET q = 3 WHERE pk = 'a'"]) + self._cancelled_by(resp, [None, ("PartiQLUpdate", t.b)], me) + resp = cancelled(c, [select(t.c, AX), select(t.b, "pk = 'a'")]) + self._cancelled_by(resp, [None, ("PartiQLSelect", t.b)], me) + resp = cancelled(c, [f"DELETE FROM \"{t.d}\" WHERE pk = 'zz'", + f"UPDATE \"{t.c}\" SET q = 5 WHERE {AX}"]) + self._cancelled_by(resp, [("PartiQLDelete", t.d, "explicit"), None], me) + item = t.owner.get_item(TableName=t.c, Key={"pk": {"S": "a"}, "sk": {"S": "x"}}) + assert "q" not in item["Item"], "a cancelled transaction writes nothing" + + # -- 3. where authorization sits among the checks --------------------------- + + def test_the_form_of_a_statement_is_checked_before_authorization(self, t): + # Measured 2026-11-03: a syntax error, an EXISTS without a condition + # beyond the key and a table name that breaks the name rule are + # refused whatever the caller may do. + c, _ = self._identity() + error = statement_error(c, f"SELEC * FROM \"{t.b}\"") + assert error == {"Code": "ValidationException", "Message": + "Statement wasn't well formed, can't be processed: " + "Expected data manipulation"} + with pytest.raises(ClientError) as exc: + transaction(c, [f"EXISTS(SELECT * FROM \"{t.b}\" WHERE pk = 'a')", + f"UPDATE \"{t.c}\" SET q = 4 WHERE {AX}"]) + assert exc.value.response["Error"] == { + "Code": "ValidationException", + "Message": "Validation failed in TransactStatements[0]: EXISTS() must " + "contain a single item read with additional condition"} + error = statement_error(c, select(self._arn(t.c), "pk = 'a'")) + assert error == {"Code": "ValidationException", "Message": + "Table name should be within the length [3, 255] and only " + "contain 'a-z', 'A-Z', '0-9', '-', '_', '.'."} + + def test_the_table_the_index_and_the_keys_are_checked_after_authorization(self, t): + # Measured 2026-11-03: a caller that may not read a table gets + # AccessDenied for a missing table, a missing index, an UPDATE without + # the full key and a key value of the wrong type; one that may gets + # ResourceNotFoundException for a missing table. + c, me = self._identity() + self._denied(statement_error(c, select(t.missing_b, "pk = 'a'")), + me, "PartiQLSelect", self._arn(t.missing_b)) + error = statement_error(c, select(t.missing_a, "pk = 'a'")) + assert error == {"Code": "ResourceNotFoundException", + "Message": "Requested resource not found"} + self._denied(statement_error(c, f"UPDATE \"{t.b}\" SET q = 1 WHERE q = 2"), + me, "PartiQLUpdate", self._arn(t.b)) + self._denied(statement_error(c, select(t.b, "pk = 1")), + me, "PartiQLSelect", self._arn(t.b)) + self._denied(statement_error(c, select(t.b)), + me, "PartiQLSelect", self._arn(t.b)) + self._denied(statement_error(c, select(t.b, "pk = ?"), + Parameters=[{"S": "a"}]), + me, "PartiQLSelect", self._arn(t.b)) + resp = cancelled(c, [f"UPDATE \"{t.b}\" SET q = 1 WHERE q = 2", + f"UPDATE \"{t.c}\" SET q = 6 WHERE {AX}"]) + self._cancelled_by(resp, [("PartiQLUpdate", t.b), None], me) + + def test_an_allowed_write_reaches_the_item_checks(self, t): + # Measured 2026-11-03. + c, _ = self._client(self._allow(["dynamodb:PartiQLUpdate"])) + error = statement_error( + c, f"UPDATE \"{t.c}\" SET q = 1 WHERE pk = 'zz' AND sk = 'zz'") + assert error["Code"] == "ConditionalCheckFailedException" + c, _ = self._client(self._allow(["dynamodb:PartiQLInsert"])) + error = statement_error(c, f"INSERT INTO \"{t.c}\" VALUE {{'pk': 'a', 'sk': 'x'}}") + assert error == {"Code": "DuplicateItemException", + "Message": "Duplicate primary key exists in table"} + + # -- 4. dynamodb:LeadingKeys ------------------------------------------------- + + def test_a_read_by_key_is_fenced_on_its_partition_key(self, t): + # Measured 2026-11-03 under ForAllValues:StringEquals ["a"]. + c, me = self._fenced("ForAllValues:StringEquals", ["a"]) + assert len(c.execute_statement(Statement=select(t.c, "pk = 'a'"))["Items"]) == 2 + assert len(c.execute_statement(Statement=select(t.c, AX))["Items"]) == 1 + assert len(c.execute_statement(Statement=select(t.c, "pk IN ['a']"))["Items"]) == 2 + resp = c.execute_statement(Statement=select(t.c, "pk = ?"), + Parameters=[{"S": "a"}]) + assert len(resp["Items"]) == 2 + self._denied(statement_error(c, select(t.c, "pk = 'b'")), + me, "PartiQLSelect", self._arn(t.c)) + + def test_each_disjunct_of_a_read_by_key_is_authorized_by_itself(self, t): + # Measured 2026-11-03: the refused partition b is held back, and the + # items of a come back with a NextToken. + for operator in ("ForAllValues:StringEquals", "ForAnyValue:StringEquals"): + c, _ = self._fenced(operator, ["a"]) + for where in ("pk IN ['a', 'b']", "pk = 'a' OR pk = 'b'"): + resp = c.execute_statement(Statement=select(t.c, where)) + assert keys_of(resp["Items"]) == ["a/x", "a/y"], (operator, where) + assert "NextToken" in resp, (operator, where) + + def test_an_index_read_is_fenced_on_the_index_partition_key(self, t): + # Measured 2026-11-03. + c, me = self._fenced("ForAllValues:StringEquals", ["a"]) + assert c.execute_statement( + Statement=select(t.c, "g = 'a'", index="gix"))["Items"] == [] + self._denied(statement_error(c, select(t.c, "g = 'ga'", index="gix")), + me, "PartiQLSelect", self._arn(t.c, "gix")) + + def test_a_scan_carries_no_leading_keys(self, t): + # Measured 2026-11-03: allowed under ForAllValues, refused under + # ForAnyValue and under a bare StringEquals. + c, _ = self._fenced("ForAllValues:StringEquals", ["a"]) + assert len(c.execute_statement(Statement=select(t.c))["Items"]) == 3 + assert len(c.execute_statement(Statement=select(t.c, "sk = 'x'"))["Items"]) == 2 + for operator, values in (("ForAnyValue:StringEquals", ["a"]), + ("StringEquals", "a")): + c, me = self._fenced(operator, values) + self._denied(statement_error(c, select(t.c)), + me, "PartiQLSelect", self._arn(t.c)) + + def test_a_bare_string_equals_never_matches_leading_keys(self, t): + # Measured 2026-11-03. + c, me = self._fenced("StringEquals", "a") + for where in ("pk = 'a'", "pk IN ['a', 'b']"): + self._denied(statement_error(c, select(t.c, where)), + me, "PartiQLSelect", self._arn(t.c)) + + def test_a_write_is_fenced_on_its_item_partition_key(self, t): + # Measured 2026-11-03. + c, me = self._fenced("ForAllValues:StringEquals", ["a"]) + c.execute_statement(Statement=f"INSERT INTO \"{t.c}\" VALUE {{'pk': 'a', 'sk': 'n4'}}") + self._denied(statement_error( + c, f"INSERT INTO \"{t.c}\" VALUE {{'pk': 'b', 'sk': 'n4'}}"), + me, "PartiQLInsert", self._arn(t.c)) + c.execute_statement(Statement=f"UPDATE \"{t.c}\" SET q = 1 WHERE {AX}") + self._denied(statement_error(c, f"UPDATE \"{t.c}\" SET q = 1 WHERE {BX}"), + me, "PartiQLUpdate", self._arn(t.c)) + self._denied(statement_error( + c, f"DELETE FROM \"{t.c}\" WHERE pk = 'b' AND sk = 'zz'"), + me, "PartiQLDelete", self._arn(t.c)) + c.execute_statement(Statement=f"DELETE FROM \"{t.c}\" WHERE pk = 'a' AND sk = 'zz'") + assert self._items(t, t.c) == ["a/n4", "a/x", "a/y", "b/x"] + + def test_batch_and_transaction_statements_carry_their_own_keys(self, t): + # Measured 2026-11-03. + c, me = self._fenced("ForAllValues:StringEquals", ["a"]) + out = batch(c, [select(t.c, AX), select(t.c, BX)]) + assert "Item" in out[0] + self._denied_entry(out[1], me, "PartiQLSelect", t.c) + out = batch(c, [select(t.c, BX), select(t.c, AX)]) + self._denied_entry(out[0], me, "PartiQLSelect", t.c) + assert "Item" in out[1] + out = batch(c, [f"UPDATE \"{t.c}\" SET q = 2 WHERE {AX}", + f"UPDATE \"{t.c}\" SET q = 2 WHERE {BX}"]) + assert out[0] == {"TableName": t.c} + self._denied_entry(out[1], me, "PartiQLUpdate", t.c) + self._cancelled_by(cancelled(c, [select(t.c, AX), select(t.c, BX)]), + [None, ("PartiQLSelect", t.c)], me) + self._cancelled_by(cancelled(c, [f"UPDATE \"{t.c}\" SET q = 3 WHERE {AX}", + f"UPDATE \"{t.c}\" SET q = 3 WHERE {BX}"]), + [None, ("PartiQLUpdate", t.c)], me) + + # -- 5. the other condition keys --------------------------------------------- + + def test_full_table_scan_is_true_for_a_scan_and_false_for_a_read_by_key(self, t): + # Measured 2026-11-03 with dynamodb:PartiQLSelect denied when + # dynamodb:FullTableScan is true. + c, me = self._client( + self._allow(PARTIQL), + self._deny(["dynamodb:PartiQLSelect"], + {"Bool": {"dynamodb:FullTableScan": "true"}})) + for where in (None, "a = '1'", "pk = 'a' OR a = '1'"): + self._denied(statement_error(c, select(t.c, where)), + me, "PartiQLSelect", self._arn(t.c), explicit=True) + self._denied(statement_error(c, select(t.c, index="gix")), + me, "PartiQLSelect", self._arn(t.c, "gix"), explicit=True) + assert len(c.execute_statement(Statement=select(t.c, "pk = 'a'"))["Items"]) == 2 + assert len(c.execute_statement(Statement=select(t.c, AX))["Items"]) == 1 + assert len(c.execute_statement( + Statement=select(t.c, "g = 'ga'", index="gix"))["Items"]) == 1 + assert len(c.execute_statement( + Statement=select(t.c, "pk IN ['a', 'b']"))["Items"]) == 3 + assert "Item" in batch(c, [select(t.c, AX)])[0] + assert len(transaction(c, [select(t.c, AX)])["Responses"]) == 1 + c.execute_statement(Statement=f"UPDATE \"{t.c}\" SET q = 1 WHERE {AX}") + + def test_full_table_scan_false_allows_reads_by_key_only(self, t): + # Measured 2026-11-03 with dynamodb:PartiQLSelect allowed only when + # dynamodb:FullTableScan is false; the key is absent in a batch and a + # transaction. + c, me = self._client(self._allow( + ["dynamodb:PartiQLSelect"], {"Bool": {"dynamodb:FullTableScan": "false"}})) + assert len(c.execute_statement(Statement=select(t.c, "pk = 'a'"))["Items"]) == 2 + assert len(c.execute_statement(Statement=select(t.c, AX))["Items"]) == 1 + assert len(c.execute_statement( + Statement=select(t.c, "g = 'ga'", index="gix"))["Items"]) == 1 + self._denied(statement_error(c, select(t.c)), me, "PartiQLSelect", self._arn(t.c)) + self._denied_entry(batch(c, [select(t.c, AX)])[0], me, "PartiQLSelect", t.c) + self._cancelled_by(cancelled(c, [select(t.c, AX)]), + [("PartiQLSelect", t.c)], me) + + def test_enclosing_operation_names_the_batch_or_the_transaction(self, t): + # Measured 2026-11-03: absent in ExecuteStatement, so a condition on + # "ExecuteStatement" matches nothing. + update = f"UPDATE \"{t.c}\" SET q = 1 WHERE {AX}" + allowed = { + "ExecuteTransaction": {"et"}, + "BatchExecuteStatement": {"bes"}, + "ExecuteStatement": set(), + } + for value, runs in allowed.items(): + c, me = self._client(self._allow( + PARTIQL, {"StringEquals": {"dynamodb:EnclosingOperation": value}})) + if "es" in runs: + c.execute_statement(Statement=select(t.c, AX)) + else: + self._denied(statement_error(c, select(t.c, AX)), + me, "PartiQLSelect", self._arn(t.c)) + for statement, action in ((select(t.c, AX), "PartiQLSelect"), + (update, "PartiQLUpdate")): + if "bes" in runs: + assert "Error" not in batch(c, [statement])[0], value + else: + self._denied_entry(batch(c, [statement])[0], me, action, t.c) + if "et" in runs: + transaction(c, [statement]) + else: + self._cancelled_by(cancelled(c, [statement]), [(action, t.c)], me) + + def test_enclosing_operation_is_absent_in_execute_statement(self, t): + # Measured 2026-11-03. + c, me = self._client( + self._allow(PARTIQL), + self._deny(PARTIQL, {"StringEquals": { + "dynamodb:EnclosingOperation": "ExecuteTransaction"}})) + assert len(c.execute_statement(Statement=select(t.c, AX))["Items"]) == 1 + assert "Item" in batch(c, [select(t.c, AX)])[0] + self._cancelled_by(cancelled(c, [select(t.c, AX)]), + [("PartiQLSelect", t.c, "explicit")], me) + c, me = self._client(self._allow( + PARTIQL, {"Null": {"dynamodb:EnclosingOperation": "true"}})) + assert len(c.execute_statement(Statement=select(t.c, AX))["Items"]) == 1 + self._denied_entry(batch(c, [select(t.c, AX)])[0], me, "PartiQLSelect", t.c) + self._cancelled_by(cancelled(c, [select(t.c, AX)]), + [("PartiQLSelect", t.c)], me) + + def test_attributes_are_the_names_a_statement_names(self, t): + # Measured 2026-11-03 under ForAllValues:StringEquals + # dynamodb:Attributes [pk, sk, a, q]: a name in the WHERE counts as + # one in the projection or a SET does. + c, me = self._client(self._allow(PARTIQL, { + "ForAllValues:StringEquals": {"dynamodb:Attributes": ["pk", "sk", "a", "q"]}})) + assert len(c.execute_statement( + Statement=select(t.c, "pk = 'a'", what="pk, a"))["Items"]) == 2 + assert len(c.execute_statement(Statement=select(t.c, "pk = 'a'"))["Items"]) == 2 + for statement in (select(t.c, "pk = 'a'", what="g"), + select(t.c, "pk = 'a' AND g = 'ga'", what="pk")): + self._denied(statement_error(c, statement), me, "PartiQLSelect", self._arn(t.c)) + c.execute_statement( + Statement=f"INSERT INTO \"{t.c}\" VALUE {{'pk': 'n5', 'sk': 's', 'a': '1'}}") + self._denied(statement_error( + c, f"INSERT INTO \"{t.c}\" VALUE {{'pk': 'n6', 'sk': 's', 'g': 'x'}}"), + me, "PartiQLInsert", self._arn(t.c)) + c.execute_statement(Statement=f"UPDATE \"{t.c}\" SET q = 1 WHERE {AX}") + for statement in (f"UPDATE \"{t.c}\" SET g = 'ga' WHERE {AX}", + f"UPDATE \"{t.c}\" SET q = 2 WHERE {AX} AND g = 'ga'"): + self._denied(statement_error(c, statement), me, "PartiQLUpdate", self._arn(t.c)) + c.execute_statement(Statement=f"DELETE FROM \"{t.c}\" WHERE pk = 'zz' AND sk = 'zz'") + + def test_select_is_how_a_statement_projects(self, t): + # Measured 2026-11-03. + for value, runs, refused in ( + ("SPECIFIC_ATTRIBUTES", "pk", "*"), + ("ALL_ATTRIBUTES", "*", "pk")): + c, me = self._client(self._allow( + ["dynamodb:PartiQLSelect"], {"StringEquals": {"dynamodb:Select": value}})) + resp = c.execute_statement(Statement=select(t.c, "pk = 'a'", what=runs)) + assert len(resp["Items"]) == 2, value + self._denied(statement_error(c, select(t.c, "pk = 'a'", what=refused)), + me, "PartiQLSelect", self._arn(t.c)) + + def test_return_values_come_from_returning(self, t): + # Measured 2026-11-03. + c, me = self._client(self._allow( + ["dynamodb:PartiQLUpdate", "dynamodb:PartiQLDelete"], + {"StringEqualsIfExists": {"dynamodb:ReturnValues": "NONE"}})) + c.execute_statement(Statement=f"UPDATE \"{t.c}\" SET q = 3 WHERE {AX}") + self._denied(statement_error( + c, f"UPDATE \"{t.c}\" SET q = 4 WHERE {AX} RETURNING ALL NEW *"), + me, "PartiQLUpdate", self._arn(t.c)) + self._denied(statement_error( + c, f"DELETE FROM \"{t.c}\" WHERE pk = 'zz' AND sk = 'zz' RETURNING ALL OLD *"), + me, "PartiQLDelete", self._arn(t.c)) + c, me = self._client(self._allow( + ["dynamodb:PartiQLUpdate"], + {"StringEquals": {"dynamodb:ReturnValues": "ALL_NEW"}})) + resp = c.execute_statement( + Statement=f"UPDATE \"{t.c}\" SET q = 5 WHERE {AX} RETURNING ALL NEW *") + assert len(resp["Items"]) == 1 + self._denied(statement_error(c, f"UPDATE \"{t.c}\" SET q = 6 WHERE {AX}"), + me, "PartiQLUpdate", self._arn(t.c)) + + # -- session policies ---------------------------------------------------------- + + def test_a_session_policy_narrows_each_statement(self, t): + # Measured 2026-11-03: a session allowed PartiQLSelect alone is + # refused an INSERT; one whose session policy denies H is refused H in + # each operation while C runs. + s = self._session({"Version": "2012-10-17", "Statement": [ + self._allow(["dynamodb:PartiQLSelect"], resources=[self._tables_pattern()[0]])]}) + assert len(s.execute_statement(Statement=select(t.c, "pk = 'a'"))["Items"]) == 2 + error = statement_error(s, f"INSERT INTO \"{t.c}\" VALUE {{'pk': 'n', 'sk': 'n'}}") + assert error["Code"] == "AccessDeniedException" + assert (f" is not authorized to perform: dynamodb:PartiQLInsert on resource: " + f"{self._arn(t.c)} ") in error["Message"] + s = self._session({"Version": "2012-10-17", "Statement": [ + self._allow(["dynamodb:PartiQLSelect"]), + self._deny(["dynamodb:PartiQLSelect"], resources=[self._arn(t.h)])]}) + error = statement_error(s, select(t.h, "pk = 'a'")) + assert error["Code"] == "AccessDeniedException" + out = batch(s, [select(t.c, AX), select(t.h, "pk = 'a'")]) + assert "Item" in out[0] + assert out[1]["Error"]["Code"] == "AccessDenied" + assert out[1]["TableName"] == t.h + resp = cancelled(s, [select(t.c, AX), select(t.h, "pk = 'a'")]) + assert resp["Error"]["Message"] == CANCELLED + "[None, AccessDenied]" + assert [r["Code"] for r in resp["CancellationReasons"]] == ["None", "AccessDenied"] + + @pytest.mark.xfail( + reason="the shared authorization message names a role session " + "arn:aws:iam:::assumed-role/... and an identity-based policy for " + "every refusal; the service names arn:aws:sts:::assumed-role/... " + "and the session policy", + raises=AssertionError, + strict=True, + ) + def test_a_session_policy_refusal_names_the_session_and_its_policy(self, t): + # Measured 2026-11-03. + s = self._session({"Version": "2012-10-17", "Statement": [ + self._allow(["dynamodb:PartiQLSelect"]), + self._deny(["dynamodb:PartiQLSelect"], resources=[self._arn(t.h)])]}) + roles = [r for r in self._roles] + principal = f"arn:aws:sts::{self.account_id}:assumed-role/{roles[-1]}/pq-authz" + head = f"User: {principal} is not authorized to perform: " + error = statement_error(s, f"INSERT INTO \"{t.c}\" VALUE {{'pk': 'n', 'sk': 'n'}}") + assert error["Message"] == ( + f"{head}dynamodb:PartiQLInsert on resource: {self._arn(t.c)} because no " + f"session policy allows the dynamodb:PartiQLInsert action") + error = statement_error(s, select(t.h, "pk = 'a'")) + assert error["Message"] == ( + f"{head}dynamodb:PartiQLSelect on resource: {self._arn(t.h)} with an " + f"explicit deny in a session policy") + + # -- every path asks for authorization ------------------------------------------ + + def test_every_statement_path_is_authorized(self, t): + # Each way a statement reads or writes items is refused for a caller + # the policy gives no PartiQL action, and nothing is written. + c, me = self._client(self._allow(CLASSIC)) + before = self._items(t, t.c) + arn = self._arn(t.c) + for statement in ( + select(t.c), + select(t.c, "a = '1'"), + select(t.c, AX), + select(t.c, "pk = 'a'"), + select(t.c, "pk IN ['a', 'b']"), + select(t.c, "pk = 'a' AND sk IN ['x', 'y']"), + select(t.c, "pk IN ['a', 'b'] ORDER BY pk DESC"), + ): + self._denied(statement_error(c, statement), me, "PartiQLSelect", arn) + for where in ("g = 'ga'", "g IN ['ga', 'gb']", None): + self._denied(statement_error(c, select(t.c, where, index="gix")), + me, "PartiQLSelect", self._arn(t.c, "gix")) + for statement, action in ( + (f"INSERT INTO \"{t.c}\" VALUE {{'pk': 'n', 'sk': 'n'}}", "PartiQLInsert"), + (f"UPDATE \"{t.c}\" SET q = 1 WHERE {AX}", "PartiQLUpdate"), + (f"UPDATE \"{t.c}\" SET q = 1 WHERE {AX} RETURNING ALL NEW *", "PartiQLUpdate"), + (f"DELETE FROM \"{t.c}\" WHERE {AX}", "PartiQLDelete"), + (f"DELETE FROM \"{t.c}\" WHERE {AX} RETURNING ALL OLD *", "PartiQLDelete"), + ): + self._denied(statement_error(c, statement), me, action, arn) + out = batch(c, [select(t.c, AX), select(t.c, BX)]) + for entry in out: + self._denied_entry(entry, me, "PartiQLSelect", t.c) + out = batch(c, [f"INSERT INTO \"{t.c}\" VALUE {{'pk': 'n', 'sk': 'n'}}", + f"UPDATE \"{t.c}\" SET q = 1 WHERE {AX}", + f"DELETE FROM \"{t.c}\" WHERE {BX}"]) + for entry, action in zip(out, ("PartiQLInsert", "PartiQLUpdate", "PartiQLDelete")): + self._denied_entry(entry, me, action, t.c) + self._cancelled_by(cancelled(c, [select(t.c, AX), select(t.c, BX)]), + [("PartiQLSelect", t.c), ("PartiQLSelect", t.c)], me) + self._cancelled_by( + cancelled(c, [f"EXISTS(SELECT * FROM \"{t.c}\" WHERE {AX} AND a = '1')", + f"INSERT INTO \"{t.c}\" VALUE {{'pk': 'n', 'sk': 'n'}}", + f"UPDATE \"{t.c}\" SET q = 1 WHERE {BX}", + f"DELETE FROM \"{t.c}\" WHERE pk = 'a' AND sk = 'y'"]), + [("PartiQLSelect", t.c), ("PartiQLInsert", t.c), ("PartiQLUpdate", t.c), + ("PartiQLDelete", t.c)], me) + assert self._items(t, t.c) == before + + def test_every_page_is_authorized(self, t): + # A NextToken carries no permission: the request that presents it is + # authorized as the first one was. + owner = t.owner + c, me = self._client(self._allow(CLASSIC)) + for statement in (select(t.c), select(t.c, "pk IN ['a', 'b']")): + resp = owner.execute_statement(Statement=statement, Limit=1) + assert "NextToken" in resp, statement + self._denied(statement_error(c, statement, NextToken=resp["NextToken"]), + me, "PartiQLSelect", self._arn(t.c)) + fenced, fenced_arn = self._fenced("ForAllValues:StringEquals", ["b"]) + statement = select(t.c, "pk = 'a'") + resp = owner.execute_statement(Statement=statement, Limit=1) + self._denied(statement_error(fenced, statement, NextToken=resp["NextToken"]), + fenced_arn, "PartiQLSelect", self._arn(t.c)) diff --git a/tests/test_partiql_authorization_keys.py b/tests/test_partiql_authorization_keys.py new file mode 100644 index 000000000..e63a3dcad --- /dev/null +++ b/tests/test_partiql_authorization_keys.py @@ -0,0 +1,297 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""The condition keys of a PartiQL statement, measured with the tables of +the second measurement. + +Every expected answer here was measured against live DynamoDB and IAM on +2026-11-03, with the principals of test_partiql_authorization.py. + +Measured model: + - An EXISTS member of ExecuteTransaction is authorized as a read: + dynamodb:PartiQLSelect on the table, with the LeadingKeys, Attributes and + EnclosingOperation of its SELECT. + - dynamodb:ReturnValues is "NONE" for an INSERT, UPDATE or DELETE without + RETURNING, in all three operations; MODIFIED OLD is "UPDATED_OLD", + MODIFIED NEW "UPDATED_NEW", ALL OLD "ALL_OLD". + - dynamodb:Attributes always holds the table's key attribute names, with + the top-level names the statement uses; `*` adds none. + - dynamodb:Select is "ALL_PROJECTED_ATTRIBUTES" for `*` on an index, + "ALL_ATTRIBUTES" for `*` on a table and "SPECIFIC_ATTRIBUTES" for a + projection list, in all three operations. + - Only the PartiQL actions allow a statement. + - A number partition key is compared as Java's BigDecimal prints the text + the statement or the parameter gives it: `10` and `010` are "10", `10.0` + is "10.0", `1E1` is "1E+1". + - A write carries no dynamodb:FullTableScan. + - A read of an index that returns attributes the index does not hold is + authorized against the index ARN alone. + +Prerequisites: + - extenddb running with ``auth.provider = "builtin"`` on EXTENDDB_TEST_ENDPOINT + - Admin credentials in EXTENDDB_ADMIN_USER / EXTENDDB_ADMIN_PASSWORD env vars +""" + +from __future__ import annotations + +from partiql_authz_helpers import ( # noqa: F401 (the fixtures are used by name) + AX, + BX, + PARTIQL, + PartiqlAuthzCase, + account_id, + auth_env, + batch, + cancelled, + mgmt, + region, + select, + statement_error, + transaction, +) + +WRITES = ["dynamodb:PartiQLUpdate", "dynamodb:PartiQLDelete", "dynamodb:PartiQLInsert"] +SELECT_ONLY = ["dynamodb:PartiQLSelect"] + + +def attributes(*names: str) -> dict: + return {"ForAllValues:StringEquals": {"dynamodb:Attributes": list(names)}} + + +def absent(key: str) -> dict: + return {"Null": {f"dynamodb:{key}": "true"}} + + +class TestPartiqlAuthorizationKeys(PartiqlAuthzCase): + """The condition keys each statement is authorized with.""" + + # -- EXISTS ------------------------------------------------------------------ + + def test_an_exists_member_is_authorized_as_a_read(self, r): + # Measured 2026-11-03 (X1 to X7). + exists_ax = f"EXISTS(SELECT * FROM \"{r.c}\" WHERE {AX} AND a = '1')" + exists_bx = f"EXISTS(SELECT * FROM \"{r.c}\" WHERE {BX} AND a = '1')" + update_ax = f"UPDATE \"{r.c}\" SET q = 1 WHERE {AX}" + update_bx = f"UPDATE \"{r.c}\" SET q = 1 WHERE {BX}" + c, _ = self._identity() + assert transaction(c, [exists_ax, update_bx])["Responses"] == [] + c, me = self._with(SELECT_ONLY) + self._cancelled_by(cancelled(c, [exists_ax, update_bx]), + [None, ("PartiQLUpdate", r.c)], me) + c, me = self._with(["dynamodb:PartiQLUpdate"]) + self._cancelled_by(cancelled(c, [exists_ax, update_bx]), + [("PartiQLSelect", r.c), None], me) + c, me = self._identity() + exists_b = f"EXISTS(SELECT * FROM \"{r.b}\" WHERE pk = 'a' AND g = 'ga')" + self._cancelled_by(cancelled(c, [exists_b, update_bx]), + [("PartiQLSelect", r.b), None], me) + c, me = self._fenced("ForAllValues:StringEquals", ["a"]) + self._cancelled_by(cancelled(c, [exists_bx, update_ax]), + [("PartiQLSelect", r.c), None], me) + c, me = self._with(PARTIQL, attributes("pk", "sk", "q")) + self._cancelled_by(cancelled(c, [exists_ax, update_bx]), + [("PartiQLSelect", r.c), None], me) + c, _ = self._with(PARTIQL, {"StringEquals": { + "dynamodb:EnclosingOperation": "ExecuteTransaction"}}) + assert transaction(c, [exists_ax, update_bx])["Responses"] == [] + + # -- dynamodb:ReturnValues ----------------------------------------------------- + + def test_return_values_is_none_for_a_write_without_returning(self, r): + # Measured 2026-11-03: an Allow on StringEquals "NONE" runs them (R1 + # to R5) and an Allow on Null "true" refuses them (R6 to R10). + update = f"UPDATE \"{r.c}\" SET q = 1 WHERE {AX}" + delete = f"DELETE FROM \"{r.c}\" WHERE pk = 'zz' AND sk = 'zz'" + c, _ = self._with(WRITES, {"StringEquals": {"dynamodb:ReturnValues": "NONE"}}) + assert c.execute_statement(Statement=update)["Items"] == [] + assert c.execute_statement(Statement=delete)["Items"] == [] + c.execute_statement(Statement=f"INSERT INTO \"{r.c}\" VALUE {{'pk': 'r1', 'sk': 's'}}") + assert batch(c, [update]) == [{"TableName": r.c}] + assert transaction(c, [update])["Responses"] == [] + c, me = self._with(WRITES, absent("ReturnValues")) + arn = self._arn(r.c) + self._denied(statement_error(c, update), me, "PartiQLUpdate", arn) + self._denied(statement_error(c, delete), me, "PartiQLDelete", arn) + self._denied(statement_error( + c, f"INSERT INTO \"{r.c}\" VALUE {{'pk': 'r2', 'sk': 's'}}"), + me, "PartiQLInsert", arn) + self._denied_entry(batch(c, [update])[0], me, "PartiQLUpdate", r.c) + self._cancelled_by(cancelled(c, [update]), [("PartiQLUpdate", r.c)], me) + + def test_return_values_names_each_returning_mode(self, r): + # Measured 2026-11-03 (R11 to R14), on an item that holds q. + r.owner.update_item(TableName=r.c, Key={"pk": {"S": "a"}, "sk": {"S": "x"}}, + UpdateExpression="SET q = :v", + ExpressionAttributeValues={":v": {"N": "1"}}) + for value, statement, items in ( + ("UPDATED_OLD", f"UPDATE \"{r.c}\" SET q = 7 WHERE {AX} " + "RETURNING MODIFIED OLD *", 1), + ("UPDATED_NEW", f"UPDATE \"{r.c}\" SET q = 8 WHERE {AX} " + "RETURNING MODIFIED NEW *", 1), + ("ALL_OLD", f"UPDATE \"{r.c}\" SET q = 9 WHERE {AX} RETURNING ALL OLD *", 1), + ("ALL_OLD", f"DELETE FROM \"{r.c}\" WHERE pk = 'zz' AND sk = 'zz' " + "RETURNING ALL OLD *", 0), + ): + c, _ = self._with(WRITES, {"StringEquals": {"dynamodb:ReturnValues": value}}) + assert len(c.execute_statement(Statement=statement)["Items"]) == items, statement + + # -- dynamodb:Attributes ------------------------------------------------------- + + def test_attributes_always_hold_the_table_key(self, r): + # Measured 2026-11-03: under Null "true" every statement is refused, + # and a statement that does not name sk is refused under a set + # without it (A3 to A5, A9, A10, A14 to A16, A18, N21, N22). + arn = self._arn(r.c) + c, me = self._with(PARTIQL, absent("Attributes")) + for statement, action in ( + (select(r.c, "pk = 'a'"), "PartiQLSelect"), + (select(r.c), "PartiQLSelect"), + (select(r.c, "pk = 'a'", what="pk"), "PartiQLSelect"), + (f"DELETE FROM \"{r.c}\" WHERE pk = 'zz' AND sk = 'zz'", "PartiQLDelete"), + (f"UPDATE \"{r.c}\" SET q = 7 WHERE {AX}", "PartiQLUpdate"), + ): + self._denied(statement_error(c, statement), me, action, arn) + self._denied_entry(batch(c, [select(r.c, AX)])[0], me, "PartiQLSelect", r.c) + for names, statement in ( + (("pk", "a"), select(r.c, "a = '1'", what="pk")), + (("pk",), select(r.c, "a = '1'", what="pk")), + (("q",), select(r.c, "pk = 'a'", what="pk")), + (("pk", "q"), select(r.c, "pk = 'a'", what="pk")), + (("pk", "q"), select(r.c, "pk = 'a' ORDER BY sk", what="pk")), + ): + c, me = self._with(PARTIQL, attributes(*names)) + self._denied(statement_error(c, statement), me, "PartiQLSelect", arn) + + def test_attributes_hold_the_names_a_statement_uses(self, r): + # Measured 2026-11-03 (A1, A2, A6 to A8, A11, A17, N20). + arn = self._arn(r.c) + c, me = self._with(PARTIQL, attributes("q")) + self._denied(statement_error(c, f"UPDATE \"{r.c}\" SET q = 1 WHERE {AX}"), + me, "PartiQLUpdate", arn) + c, me = self._with(PARTIQL, attributes("a")) + self._denied(statement_error(c, select(r.c, "pk = 'a'", what="a")), + me, "PartiQLSelect", arn) + # A nested path counts by its top-level name. + c, _ = self._with(PARTIQL, attributes("pk", "sk", "m")) + assert len(c.execute_statement( + Statement=select(r.c, "pk = 'a'", what="m.x"))["Items"]) == 2 + c.execute_statement( + Statement=f"INSERT INTO \"{r.c}\" VALUE {{'pk': 'n7', 'sk': 's', 'm': {{'x': 1}}}}") + # A condition, a REMOVE target and a name in a SET value count. + c, me = self._with(PARTIQL, attributes("pk", "sk")) + self._denied(statement_error( + c, f"DELETE FROM \"{r.c}\" WHERE pk = 'zz' AND sk = 'zz' AND g = 'x'"), + me, "PartiQLDelete", arn) + c, me = self._with(PARTIQL, attributes("pk", "sk", "a")) + self._denied(statement_error(c, f"UPDATE \"{r.c}\" REMOVE q WHERE {AX}"), + me, "PartiQLUpdate", arn) + # `*` adds no name, though the items hold g. + assert len(c.execute_statement(Statement=select(r.c, "pk = 'a'"))["Items"]) == 2 + c, me = self._with(PARTIQL, attributes("pk", "sk", "q")) + self._denied(statement_error(c, f"UPDATE \"{r.c}\" SET q = g WHERE {AX}"), + me, "PartiQLUpdate", arn) + + def test_attributes_are_carried_by_each_statement(self, r): + # Measured 2026-11-03 (A12, A13). + c, me = self._with(PARTIQL, attributes("pk", "sk", "a", "q")) + out = batch(c, [f"UPDATE \"{r.c}\" SET q = 5 WHERE {AX}", + f"UPDATE \"{r.c}\" SET g = 'gz' WHERE {BX}"]) + assert out[0] == {"TableName": r.c} + self._denied_entry(out[1], me, "PartiQLUpdate", r.c) + self._cancelled_by(cancelled(c, [f"UPDATE \"{r.c}\" SET q = 6 WHERE {AX}", + f"UPDATE \"{r.c}\" SET g = 'gz' WHERE {BX}"]), + [None, ("PartiQLUpdate", r.c)], me) + + # -- dynamodb:Select ------------------------------------------------------------- + + def test_select_names_how_a_statement_projects(self, r): + # Measured 2026-11-03 (S1 to S13). + def under(value: str) -> tuple: + return self._with(SELECT_ONLY, {"StringEquals": {"dynamodb:Select": value}}) + + on_gix = select(r.c, "g = 'ga'", index="gix") + on_kix = select(r.k, "g = 'ga'", index="kix") + c, me = under("ALL_PROJECTED_ATTRIBUTES") + assert len(c.execute_statement(Statement=on_gix)["Items"]) == 1 + assert len(c.execute_statement(Statement=on_kix)["Items"]) == 1 + self._denied(statement_error(c, select(r.c, "pk = 'a'")), + me, "PartiQLSelect", self._arn(r.c)) + c, me = under("ALL_ATTRIBUTES") + self._denied(statement_error(c, on_gix), me, "PartiQLSelect", self._arn(r.c, "gix")) + self._denied(statement_error(c, on_kix), me, "PartiQLSelect", self._arn(r.k, "kix")) + assert "Item" in batch(c, [select(r.c, AX)])[0] + assert len(transaction(c, [select(r.c, AX)])["Responses"]) == 1 + c, _ = under("SPECIFIC_ATTRIBUTES") + assert len(c.execute_statement( + Statement=select(r.c, "g = 'ga'", what="pk", index="gix"))["Items"]) == 1 + assert "Item" in batch(c, [select(r.c, AX, what="pk")])[0] + c, me = self._with(SELECT_ONLY, absent("Select")) + self._denied_entry(batch(c, [select(r.c, AX)])[0], me, "PartiQLSelect", r.c) + self._cancelled_by(cancelled(c, [select(r.c, AX)]), [("PartiQLSelect", r.c)], me) + for statement in (select(r.c, "pk = 'a'"), select(r.c)): + self._denied(statement_error(c, statement), me, "PartiQLSelect", self._arn(r.c)) + + # -- the actions, number keys, writes and index reads ---------------------------- + + def test_only_the_partiql_actions_allow_a_statement(self, r): + # Measured 2026-11-03: a caller allowed the three operations' own + # actions on every resource is refused every statement (N1 to N3). + c, me = self._client({"Effect": "Allow", "Resource": "*", "Action": [ + "dynamodb:ExecuteStatement", "dynamodb:BatchExecuteStatement", + "dynamodb:ExecuteTransaction"]}) + self._denied(statement_error(c, select(r.c, "pk = 'a'")), + me, "PartiQLSelect", self._arn(r.c)) + self._denied_entry(batch(c, [select(r.c, AX)])[0], me, "PartiQLSelect", r.c) + self._cancelled_by(cancelled(c, [select(r.c, AX)]), [("PartiQLSelect", r.c)], me) + + def test_a_number_partition_value_keeps_the_scale_it_is_written_with(self, r): + # Measured 2026-11-03 under ForAllValues LeadingKeys ["10"] and + # ["10.0"] (N4 to N11). + arn = self._arn(r.n) + c, me = self._fenced("ForAllValues:StringEquals", ["10"]) + for literal in ("10", "010"): + resp = c.execute_statement(Statement=select(r.n, f"pk = {literal}")) + assert len(resp["Items"]) == 1, literal + for literal in ("10.0", "1E1"): + self._denied(statement_error(c, select(r.n, f"pk = {literal}")), + me, "PartiQLSelect", arn) + self._denied(statement_error(c, select(r.n, "pk = ?"), Parameters=[{"N": "10.0"}]), + me, "PartiQLSelect", arn) + for literal in ("10.0", "1E1"): + self._denied(statement_error( + c, f"UPDATE \"{r.n}\" SET q = 1 WHERE pk = {literal}"), + me, "PartiQLUpdate", arn) + self._denied_entry(batch(c, [select(r.n, "pk = 1E1")])[0], me, "PartiQLSelect", r.n) + self._cancelled_by(cancelled(c, [select(r.n, "pk = 1E1")]), + [("PartiQLSelect", r.n)], me) + c, me = self._fenced("ForAllValues:StringEquals", ["10.0"]) + assert len(c.execute_statement(Statement=select(r.n, "pk = 10.0"))["Items"]) == 1 + self._denied(statement_error(c, select(r.n, "pk = 10")), me, "PartiQLSelect", arn) + + def test_a_write_carries_no_full_table_scan(self, r): + # Measured 2026-11-03: an Allow on Bool "false" refuses every write + # (N12 to N14). + arn = self._arn(r.c) + c, me = self._with(PARTIQL, {"Bool": {"dynamodb:FullTableScan": "false"}}) + for statement, action in ( + (f"INSERT INTO \"{r.c}\" VALUE {{'pk': 'f1', 'sk': 's'}}", "PartiQLInsert"), + (f"UPDATE \"{r.c}\" SET q = 1 WHERE {AX}", "PartiQLUpdate"), + (f"DELETE FROM \"{r.c}\" WHERE pk = 'zz' AND sk = 'zz'", "PartiQLDelete"), + ): + self._denied(statement_error(c, statement), me, action, arn) + + def test_an_index_read_that_reads_the_table_is_authorized_against_the_index(self, r): + # Measured 2026-11-03: a read of the KEYS_ONLY LSI that returns u, + # which the index does not hold, runs for a caller allowed the + # indexes alone, and is refused naming the index ARN for one allowed + # the table alone (N15 to N19). + on_lix = select(r.l, "pk = 'a'", what="u", index="lix") + indexes = [self._tables_pattern()[1]] + c, _ = self._with(SELECT_ONLY, resources=indexes) + for what in ("u", "*", "l"): + resp = c.execute_statement(Statement=select(r.l, "pk = 'a'", what=what, index="lix")) + assert len(resp["Items"]) == 1, what + c, me = self._with(SELECT_ONLY, resources=[self._arn(r.l)]) + self._denied(statement_error(c, on_lix), me, "PartiQLSelect", self._arn(r.l, "lix")) + c, _ = self._identity() + assert c.execute_statement(Statement=on_lix)["Items"] == [{"u": {"S": "hidden"}}] diff --git a/tests/test_partiql_authorization_order.py b/tests/test_partiql_authorization_order.py new file mode 100644 index 000000000..e6fae2e17 --- /dev/null +++ b/tests/test_partiql_authorization_order.py @@ -0,0 +1,367 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""Where the authorization of a PartiQL statement sits among its checks, and +the pages of a read by key whose refused disjuncts are held back. + +Every expected answer here was measured against live DynamoDB and IAM on +2026-11-03, with the principals of test_partiql_authorization.py. B (pqb-*) +is outside the caller's policy, so there a check the service makes after +authorization is hidden by the refusal, and one it makes before is answered. + +Measured model: + - Before authorization: the parse of the statement, a VALUE that names an + attribute twice among its refusals; the checks of the request (Limit, + the number of statements, reads mixed with writes); fewer parameters than + placeholders; ConsistentRead on a global secondary index; more than 100 + IN operands; more than 50 disjuncts; overlapping disjuncts; a path into a + key attribute; a NextToken that does not decode; a DELETE RETURNING + clause other than ALL OLD *; a BatchExecuteStatement read without the + full primary key; SELECT COUNT(*); and in ExecuteTransaction the EXISTS + form and an EXISTS on an index, on a table that exists. + - After it: a parameter the statement does not use, ORDER BY on another + attribute, a key value of the wrong type, an INSERT without the key, a + key value over the size limit, SET of a key attribute, a missing index, a + missing table. A transaction is cancelled for authorization before any + condition is evaluated. + - A refused disjunct of a read by key is held back: the request whose page + visits only held-back disjuncts gets AccessDeniedException. Within a + disjunct, authorization comes before the key size and key-attribute + filter checks; across disjuncts, an authorization refusal ranks below the + other refusals, and is the answer only when every held-back disjunct is + refused for authorization. + +Prerequisites: + - extenddb running with ``auth.provider = "builtin"`` on EXTENDDB_TEST_ENDPOINT + - Admin credentials in EXTENDDB_ADMIN_USER / EXTENDDB_ADMIN_PASSWORD env vars +""" + +from __future__ import annotations + +from typing import Any + +import pytest +from botocore.exceptions import ClientError + +from partiql_authz_helpers import ( # noqa: F401 (the fixtures are used by name) + AX, + BX, + PARTIQL, + PartiqlAuthzCase, + account_id, + auth_env, + batch, + cancelled, + keys_of, + mgmt, + region, + select, + statement_error, + transaction_error, +) + +FILTER = ("Filter Expression can only contain non-primary key attributes: Primary key " + "attribute: sk") +LK = "ForAllValues:StringEquals" + + +def pages(client: Any, statement: str, **kwargs: Any) -> tuple[list[list[str]], dict | None]: + """The keys of the items of each page of `statement`, and the error that + ends the pages, if any.""" + out: list[list[str]] = [] + token = None + for _ in range(8): + args = dict(kwargs) + if token: + args["NextToken"] = token + try: + resp = client.execute_statement(Statement=statement, **args) + except ClientError as exc: + return out, exc.response["Error"] + out.append(keys_of(resp["Items"])) + token = resp.get("NextToken") + if not token: + return out, None + raise AssertionError(f"more than 8 pages: {out}") + + +def values(n: int) -> str: + return ", ".join(f"'v{i:03}'" for i in range(n)) + + +def validation(message: str) -> dict: + return {"Code": "ValidationException", "Message": message} + + +class TestPartiqlAuthorizationOrder(PartiqlAuthzCase): + """Each statement is authorized at its place among its checks.""" + + # -- the pages of a read with a refused disjunct --------------------------------- + + def test_the_request_whose_page_visits_only_refused_disjuncts_is_refused(self, r): + # Measured 2026-11-03 (D1, D2, D4 to D6, D10 to D12). + arn = self._arn(r.c) + c, me = self._fenced(LK, ["a"]) + for where, first in ( + ("pk IN ['a', 'b']", ["a/x", "a/y"]), + ("pk IN ['a', 'b', 'c']", ["a/x", "a/y"]), + (f"({AX}) OR ({BX})", ["a/x"]), + ("pk IN ['a', 'b'] ORDER BY pk DESC", ["a/x", "a/y"]), + ("pk IN ['a', 'b'] AND a = '1'", ["a/x", "a/y"]), + ): + got, error = pages(c, select(r.c, where)) + assert got == [first], where + self._denied(error, me, "PartiQLSelect", arn) + got, error = pages(c, select(r.c, "pk IN ['b']")) + assert got == [] + self._denied(error, me, "PartiQLSelect", arn) + c, me = self._fenced(LK, ["b"]) + got, error = pages(c, select(r.c, "pk IN ['a', 'b']")) + assert got == [["b/x"]] + self._denied(error, me, "PartiQLSelect", arn) + c, me = self._fenced(LK, ["a", "c"]) + got, error = pages(c, select(r.c, "pk IN ['a', 'b', 'c']")) + assert got == [["a/x", "a/y", "c/x"]] + self._denied(error, me, "PartiQLSelect", arn) + + def test_a_limit_pages_the_allowed_disjuncts_before_the_refusal(self, r): + # Measured 2026-11-03 (D3): [a/x] | [a/y] | [] | AccessDenied. The + # empty page is the end of a partition's answer, not asserted here. + c, me = self._fenced(LK, ["a"]) + got, error = pages(c, select(r.c, "pk IN ['a', 'b']"), Limit=1) + assert got[:2] == [["a/x"], ["a/y"]] + assert all(page == [] for page in got[2:]), got + self._denied(error, me, "PartiQLSelect", self._arn(r.c)) + + def test_a_refused_disjunct_is_refused_for_authorization_before_its_checks(self, r): + # Measured 2026-11-03: under ["a"] a refused disjunct past the key + # size limit gets AccessDenied once the allowed one is read, last in + # read order or first (D7, D8); under ["b"] so does one with a filter + # on a key attribute (D15). + arn = self._arn(r.c) + c, me = self._fenced(LK, ["a"]) + big, big_first = "z" * 2049, "0" + "z" * 2048 + for where in (f"pk IN ['a', 'b', '{big}']", f"pk IN ['{big_first}', 'a', 'b']"): + got, error = pages(c, select(r.c, where)) + assert got == [["a/x", "a/y"]] + self._denied(error, me, "PartiQLSelect", arn) + c, me = self._fenced(LK, ["b"]) + got, error = pages(c, select(r.c, "(pk = 'a' AND sk = a) OR pk = 'b' OR pk = 'c'")) + assert got == [["b/x"]] + self._denied(error, me, "PartiQLSelect", arn) + + def test_an_authorization_refusal_ranks_below_the_refusals_of_the_checks(self, r): + # Measured 2026-11-03 under ["a"] (D13, D14): refused at once with the + # filter refusal of the disjunct the policy allows, though b is read + # later. + c, _ = self._fenced(LK, ["a"]) + for where in ("(pk = 'a' AND sk = a) OR pk = 'b'", + "pk = 'b' OR (pk = 'a' AND sk = a)"): + assert statement_error(c, select(r.c, where)) == validation(FILTER), where + + def test_a_read_of_an_index_holds_back_its_refused_disjuncts(self, r): + # Measured 2026-11-03 under ["ga"] (D9): the refusal names the index. + c, me = self._fenced(LK, ["ga"]) + got, error = pages(c, select(r.c, "g IN ['ga', 'gb']", index="gix")) + assert got == [["a/x"]] + self._denied(error, me, "PartiQLSelect", self._arn(r.c, "gix")) + + # -- the checks made before authorization ---------------------------------------- + + def test_the_checks_made_before_authorization_answer_first(self, r): + # Measured 2026-11-03 on B (O1 to O4, O6, O7, O15, O17, V11). + c, _ = self._identity() + b = r.b + for statement, kwargs, message in ( + (select(b, "pk = ?"), {}, + "Number of parameters in request and statement don't match."), + (select(b, "g = 'ga'", index="gix"), {"ConsistentRead": True}, + "Strongly consistent read is not supported on Global Secondary Indexes"), + (select(b, f"pk IN [{values(101)}]"), {}, + "The IN operator is provided with too many operands; number of operands: 101"), + (select(b, f"pk IN [{values(51)}]"), {}, + "Too many decomposed read operations for a given query."), + (select(b, "pk = 'a' OR pk = 'a'"), {}, + "Overlapping conditions with range keys are not supported in where clause"), + (select(b, "pk.x = 1"), {}, + "Key attributes must be scalars; list random access '[]' and map lookup '.' " + "are not allowed: Key pk"), + (select(b, "pk = 'a'"), {"NextToken": "garbage"}, "Invalid NextToken"), + (f"DELETE FROM \"{b}\" WHERE pk = 'a' RETURNING ALL NEW *", {}, + "Invalid returning clause: RETURNING ALL NEW *. Only RETURNING ALL OLD * is " + "allowed in DELETE statements."), + (select(b, "pk = 'a'", what="COUNT(*)"), {}, + "Unexpected path component at 1:8:5"), + ): + assert statement_error(c, statement, **kwargs) == validation(message), statement + + def test_the_checks_of_the_request_are_made_before_authorization(self, r): + # Measured 2026-11-03 on B (O8, O14, O19, O20). + c, _ = self._identity(validate=False) + b = r.b + error = statement_error(c, select(b, "pk = 'a'"), Limit=0) + assert error == validation( + "1 validation error detected: Value '0' at 'limit' failed to satisfy constraint: " + "Member must have value greater than or equal to 1") + statements = [select(b, f"pk = 'v{i:03}'") for i in range(26)] + with pytest.raises(ClientError) as exc: + batch(c, statements) + echo = ", ".join( + f"BatchStatementRequest(statement={s}, parameters=null, consistentRead=null, " + "returnValuesOnConditionCheckFailure=null)" for s in statements) + assert exc.value.response["Error"] == validation( + f"1 validation error detected: Value '[{echo}]' at 'statements' failed to satisfy " + "constraint: Member must have length less than or equal to 25") + mixed = [select(b, "pk = 'a'"), f"UPDATE \"{r.c}\" SET q = 1 WHERE {AX}"] + assert transaction_error(c, mixed) == validation( + "ExecuteTransaction API does not support both read and write operations in the " + "same request.") + with pytest.raises(ClientError) as exc: + batch(c, mixed) + assert exc.value.response["Error"] == validation( + "Read and write requests together in the same batch is not supported.") + + def test_a_batch_read_is_refused_its_form_before_authorization(self, r): + # Measured 2026-11-03: a read without the full primary key and a + # COUNT(*), on B and on a missing table outside the policy, are the + # statement's ValidationError (O23, V12, V13). + c, _ = self._identity() + out = batch(c, [select(r.b, "g = 'ga'"), + select(r.b, "pk = 'a'", what="COUNT(*)"), + select(r.missing_b, "pk = 'a'", what="COUNT(*)")]) + assert out == [ + {"Error": {"Code": "ValidationError", "Message": + "Select statements within BatchExecuteStatement must specify the " + "primary key in the where clause."}}, + {"Error": {"Code": "ValidationError", + "Message": "Unexpected path component at 1:8:5"}}, + {"Error": {"Code": "ValidationError", + "Message": "Unexpected path component at 1:8:5"}}, + ] + + def test_a_transaction_member_is_refused_its_form_before_authorization(self, r): + # Measured 2026-11-03 on B and a missing table outside the policy: an + # EXISTS without a condition beyond the key, an EXISTS on an index and + # a COUNT(*) (V7, V10, V14, V15). + c, _ = self._identity() + head = "Validation failed in TransactStatements[0]: " + for statement, message in ( + (f"EXISTS(SELECT * FROM \"{r.b}\" WHERE pk = 'a')", + "EXISTS() must contain a single item read with additional condition"), + (f"EXISTS(SELECT * FROM \"{r.b}\".\"gix\" WHERE g = 'ga' AND pk = 'a')", + "Operations on indices are not supported in transactions"), + (select(r.b, "pk = 'a'", what="COUNT(*)"), "Unexpected path component at 1:8:5"), + (select(r.missing_b, "pk = 'a'", what="COUNT(*)"), + "Unexpected path component at 1:8:5"), + ): + assert transaction_error(c, [statement]) == validation(head + message), statement + + # -- the checks made after authorization ----------------------------------------- + + def test_the_checks_made_after_authorization_are_hidden_by_its_refusal(self, r): + # Measured 2026-11-03 on B (O5, O9 to O11, O16, O18, O21). + c, me = self._identity() + b = r.b + for statement, kwargs, action, resource in ( + (select(b, "pk = 'a' ORDER BY n"), {}, "PartiQLSelect", self._arn(b)), + (f"INSERT INTO \"{b}\" VALUE {{'pk': 1}}", {}, "PartiQLInsert", self._arn(b)), + (f"INSERT INTO \"{b}\" VALUE {{'x': 'y'}}", {}, "PartiQLInsert", self._arn(b)), + (select(b, f"pk = '{'z' * 2049}'"), {}, "PartiQLSelect", self._arn(b)), + (f"UPDATE \"{b}\" SET pk = 'z' WHERE pk = 'a'", {}, "PartiQLUpdate", + self._arn(b)), + (select(b, "pk = 'a'"), {"Parameters": [{"S": "x"}]}, "PartiQLSelect", + self._arn(b)), + (select(b, "g = 'ga'", index="nix"), {"ConsistentRead": True}, + "PartiQLSelect", self._arn(b, "nix")), + ): + self._denied(statement_error(c, statement, **kwargs), me, action, resource) + + def test_a_transaction_is_cancelled_for_authorization_before_its_conditions(self, r): + # Measured 2026-11-03: a condition that fails beside a refused member + # has the reason None, and each refused member has its own refusal + # (O12, O13, O22). + c, me = self._identity() + refused = f"UPDATE \"{r.b}\" SET q = 1 WHERE pk = 'a'" + failing = f"UPDATE \"{r.c}\" SET q = 1 WHERE {AX} AND a = 'nope'" + self._cancelled_by(cancelled(c, [refused, failing]), + [("PartiQLUpdate", r.b), None], me) + self._cancelled_by(cancelled(c, [failing, refused]), + [None, ("PartiQLUpdate", r.b)], me) + self._cancelled_by( + cancelled(c, [refused, f"UPDATE \"{r.b}\" SET q = 2 WHERE pk = 'b'"]), + [("PartiQLUpdate", r.b), ("PartiQLUpdate", r.b)], me) + + def test_a_transaction_member_on_a_missing_table_is_authorized_first(self, r): + # Measured 2026-11-03: the EXISTS form is not checked on a missing + # table; the member is refused, or for a caller that may read the + # table cancelled with ResourceNotFound (V8, V9). + c, me = self._identity() + bare = "EXISTS(SELECT * FROM \"{}\" WHERE pk = 'a')" + self._cancelled_by(cancelled(c, [bare.format(r.missing_b)]), + [("PartiQLSelect", r.missing_b)], me) + resp = cancelled(c, [bare.format(r.missing_a)]) + assert resp["Error"]["Message"].endswith("[ResourceNotFound]") + assert resp["CancellationReasons"] == [ + {"Code": "ResourceNotFound", "Message": "Requested resource not found"}] + + def test_a_read_of_an_index_of_a_missing_table_is_authorized_against_the_table(self, r): + # Measured 2026-11-03 (V16); an existing table's missing index is + # authorized against the index (O21). + c, me = self._identity() + self._denied(statement_error(c, select(r.missing_b, "g = 'ga'", index="gix")), + me, "PartiQLSelect", self._arn(r.missing_b)) + + def test_a_missing_table_or_index_carries_no_full_table_scan(self, r): + # Measured 2026-11-03: a Deny of PartiQLSelect on FullTableScan "true" + # does not refuse these, which get the answer of the missing table or + # index (N23 to N25). + c, _ = self._client( + self._allow(PARTIQL), + self._deny(["dynamodb:PartiQLSelect"], {"Bool": {"dynamodb:FullTableScan": "true"}})) + not_found = {"Code": "ResourceNotFoundException", + "Message": "Requested resource not found"} + assert statement_error(c, select(r.missing_a)) == not_found + assert statement_error(c, select(r.missing_a, "pk = 'a'")) == not_found + assert statement_error(c, select(r.c, index="nix")) == validation( + "The table does not have the specified index") + + # -- a VALUE that names an attribute twice ---------------------------------------- + + def test_a_value_that_names_an_attribute_twice_is_refused_before_authorization(self, r): + # Measured 2026-11-03: refused at the second name, for any caller, + # and nothing is written (V1 to V5). The table names are 12 + # characters long, as the measured ones were. + c, _ = self._identity() + lk_a, _ = self._fenced(LK, ["a"]) + table = f"\"{r.c}\"" + for client, value, refusal, keys in ( + (c, "{'pk': 'd1', 'pk': 'd2', 'sk': 's'}", "pk under root at 1:47:4", + [("d1", "s"), ("d2", "s")]), + (lk_a, "{'pk': 'a', 'pk': 'b', 'sk': 'dup'}", "pk under root at 1:46:4", + [("a", "dup"), ("b", "dup")]), + (lk_a, "{'pk': 'b', 'pk': 'a', 'sk': 'dup2'}", "pk under root at 1:46:4", + [("a", "dup2"), ("b", "dup2")]), + (c, "{'pk': 'd3', 'sk': 's', 'v': '1', 'v': '2'}", "v under root at 1:68:3", + [("d3", "s")]), + (c, "{'pk': 'd4', 'sk': 's1', 'sk': 's2'}", "sk under root at 1:59:4", + [("d4", "s1"), ("d4", "s2")]), + ): + error = statement_error(client, f"INSERT INTO {table} VALUE {value}") + assert error == validation(f"Duplicate keys in tuple. Key: {refusal}"), value + for pk, sk in keys: + item = r.owner.get_item(TableName=r.c, ConsistentRead=True, + Key={"pk": {"S": pk}, "sk": {"S": sk}}) + assert "Item" not in item, (value, pk, sk) + + def test_a_name_given_twice_in_a_batch_or_a_transaction(self, r): + # Measured 2026-11-03: in BatchExecuteStatement the statement's + # ValidationError (V6); in ExecuteTransaction the refusal names the + # statement, as a parse refusal does (with a permitted caller, in + # probe_dups). + c, _ = self._identity() + insert = f"INSERT INTO \"{r.c}\" VALUE {{'pk': 'd5', 'pk': 'd6', 'sk': 's'}}" + refusal = "Duplicate keys in tuple. Key: pk under root at 1:47:4" + assert batch(c, [insert]) == [ + {"Error": {"Code": "ValidationError", "Message": refusal}}] + assert transaction_error(c, [insert]) == validation( + f"Validation failed in TransactStatements[0]: {refusal}") diff --git a/tests/test_partiql_batch.py b/tests/test_partiql_batch.py new file mode 100644 index 000000000..dfb0b4fab --- /dev/null +++ b/tests/test_partiql_batch.py @@ -0,0 +1,1034 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""PartiQL BatchExecuteStatement over the wire. + +Dual-target: identical assertions run against real DynamoDB and against an +ExtendDB fix server (selected by EXTENDDB_TEST_ENDPOINT, see conftest). Every +expected response and error Code+Message was measured on the live service; the +records are in conf-run/partiql/char-batch-txn.md (Part 1). + +A batch is all reads or all writes. Reads behave like BatchGetItem +(Responses[i] = {TableName, Item}); writes carry per-statement Error entries and +real partial success. The one shared composite pk(S)/sk(S) table is reused with +a distinct partition key per test for isolation. +""" + +from __future__ import annotations + +import os +import uuid + +import boto3 +import pytest +from botocore.config import Config +from botocore.exceptions import ClientError + +from conftest import wait_for_active +from partiql_gsi_table import get as _gget +from partiql_gsi_table import gsi_table, item, key, put + + +@pytest.fixture(scope="session") +def dynamodb_client(endpoint_url): + """DynamoDB client honoring EXTENDDB_TEST_PROFILE for the live-service run.""" + region = os.environ.get("AWS_DEFAULT_REGION", "us-east-1") + profile = os.environ.get("EXTENDDB_TEST_PROFILE") or None + session = boto3.Session(profile_name=profile, region_name=region) + kwargs: dict = {"service_name": "dynamodb"} + if endpoint_url: + kwargs["endpoint_url"] = endpoint_url + if endpoint_url.startswith("https://"): + kwargs["verify"] = False + kwargs["config"] = Config(parameter_validation=True) + return session.client(**kwargs) + + +@pytest.fixture(scope="module") +def btable(dynamodb_client): + """A composite pk(S)/sk(S) table, PAY_PER_REQUEST.""" + name = f"extenddb-measure-2c-batch-{uuid.uuid4().hex[:12]}" + dynamodb_client.create_table( + TableName=name, + AttributeDefinitions=[ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "sk", "AttributeType": "S"}, + ], + KeySchema=[ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"}, + ], + BillingMode="PAY_PER_REQUEST", + ) + wait_for_active(dynamodb_client, name) + try: + yield name + finally: + try: + dynamodb_client.delete_table(TableName=name) + except dynamodb_client.exceptions.ResourceNotFoundException: + pass + + +def _batch(client, statements, **kwargs): + return client.batch_execute_statement(Statements=statements, **kwargs) + + +def _error(excinfo): + err = excinfo.value.response["Error"] + return err["Code"], err["Message"] + + +def _seed(client, table, pk, sk="s1", a=1): + client.put_item( + TableName=table, + Item={"pk": {"S": pk}, "sk": {"S": sk}, "a": {"N": str(a)}}, + ) + + +def _get(client, table, pk, sk="s1"): + return client.get_item( + TableName=table, Key={"pk": {"S": pk}, "sk": {"S": sk}} + ).get("Item") + + +# --- B1 / B2: reads -------------------------------------------------------- + + +def test_all_reads_success_shape(dynamodb_client, btable): + _seed(dynamodb_client, btable, "p1", a=1) + _seed(dynamodb_client, btable, "p2", a=2) + resp = _batch( + dynamodb_client, + [ + {"Statement": f"SELECT * FROM \"{btable}\" WHERE pk='p1' AND sk='s1'"}, + {"Statement": f"SELECT * FROM \"{btable}\" WHERE pk='p2' AND sk='s1'"}, + ], + ) + assert resp["Responses"] == [ + {"TableName": btable, "Item": {"pk": {"S": "p1"}, "sk": {"S": "s1"}, "a": {"N": "1"}}}, + {"TableName": btable, "Item": {"pk": {"S": "p2"}, "sk": {"S": "s1"}, "a": {"N": "2"}}}, + ] + + +def test_read_absent_key_omits_item(dynamodb_client, btable): + resp = _batch( + dynamodb_client, + [{"Statement": f"SELECT * FROM \"{btable}\" WHERE pk='nope' AND sk='nope'"}], + ) + assert resp["Responses"] == [{"TableName": btable}] + + +def test_read_partial_key_is_per_statement_validation_error(dynamodb_client, btable): + _seed(dynamodb_client, btable, "pp") + resp = _batch( + dynamodb_client, + [{"Statement": f"SELECT * FROM \"{btable}\" WHERE pk='pp'"}], + ) + assert resp["Responses"] == [ + { + "Error": { + "Code": "ValidationError", + "Message": "Select statements within BatchExecuteStatement must specify the primary key in the where clause.", + } + } + ] + + +# --- B4 / B6: writes ------------------------------------------------------- + + +def test_all_writes_insert_success_shape(dynamodb_client, btable): + resp = _batch( + dynamodb_client, + [ + {"Statement": f"INSERT INTO \"{btable}\" VALUE {{'pk':'w1','sk':'s1','a':10}}"}, + {"Statement": f"INSERT INTO \"{btable}\" VALUE {{'pk':'w2','sk':'s1','a':20}}"}, + ], + ) + assert resp["Responses"] == [{"TableName": btable}, {"TableName": btable}] + assert _get(dynamodb_client, btable, "w1")["a"] == {"N": "10"} + assert _get(dynamodb_client, btable, "w2")["a"] == {"N": "20"} + + +def test_partial_success_good_insert_and_duplicate(dynamodb_client, btable): + _seed(dynamodb_client, btable, "exist", a=1) + resp = _batch( + dynamodb_client, + [ + {"Statement": f"INSERT INTO \"{btable}\" VALUE {{'pk':'fresh','sk':'s1','a':1}}"}, + {"Statement": f"INSERT INTO \"{btable}\" VALUE {{'pk':'exist','sk':'s1','a':999}}"}, + ], + ) + assert resp["Responses"] == [ + {"TableName": btable}, + { + "Error": {"Code": "DuplicateItem", "Message": "Duplicate primary key exists in table"}, + "TableName": btable, + }, + ] + # The first INSERT committed; the duplicate left the existing item untouched. + assert _get(dynamodb_client, btable, "fresh")["a"] == {"N": "1"} + assert _get(dynamodb_client, btable, "exist")["a"] == {"N": "1"} + + +# --- B5: read/write mix (request-level) ----------------------------------- + + +def test_mixed_read_and_write_rejects_whole_request(dynamodb_client, btable): + with pytest.raises(ClientError) as excinfo: + _batch( + dynamodb_client, + [ + {"Statement": f"SELECT * FROM \"{btable}\" WHERE pk='p1' AND sk='s1'"}, + {"Statement": f"INSERT INTO \"{btable}\" VALUE {{'pk':'x','sk':'s1'}}"}, + ], + ) + code, message = _error(excinfo) + assert code == "ValidationException" + assert message == "Read and write requests together in the same batch is not supported." + + +# --- B7: duplicate key in one batch (request-level) ----------------------- + + +def test_duplicate_key_two_updates_rejects(dynamodb_client, btable): + _seed(dynamodb_client, btable, "dupu") + with pytest.raises(ClientError) as excinfo: + _batch( + dynamodb_client, + [ + {"Statement": f"UPDATE \"{btable}\" SET a=1 WHERE pk='dupu' AND sk='s1'"}, + {"Statement": f"UPDATE \"{btable}\" SET a=2 WHERE pk='dupu' AND sk='s1'"}, + ], + ) + code, message = _error(excinfo) + assert code == "ValidationException" + assert message == "Provided list of item keys contains duplicates" + + +def test_duplicate_key_insert_and_delete_rejects(dynamodb_client, btable): + with pytest.raises(ClientError) as excinfo: + _batch( + dynamodb_client, + [ + {"Statement": f"INSERT INTO \"{btable}\" VALUE {{'pk':'dupd','sk':'s1'}}"}, + {"Statement": f"DELETE FROM \"{btable}\" WHERE pk='dupd' AND sk='s1'"}, + ], + ) + code, message = _error(excinfo) + assert code == "ValidationException" + assert message == "Provided list of item keys contains duplicates" + + +# --- B8: ReturnConsumedCapacity (a list) ---------------------------------- + + +def test_consumed_capacity_reads_indexes_is_a_list(dynamodb_client, btable): + _seed(dynamodb_client, btable, "c1") + _seed(dynamodb_client, btable, "c2") + resp = _batch( + dynamodb_client, + [ + {"Statement": f"SELECT * FROM \"{btable}\" WHERE pk='c1' AND sk='s1'"}, + {"Statement": f"SELECT * FROM \"{btable}\" WHERE pk='c2' AND sk='s1'"}, + ], + ReturnConsumedCapacity="INDEXES", + ) + assert resp["ConsumedCapacity"] == [ + {"TableName": btable, "CapacityUnits": 1.0, "Table": {"CapacityUnits": 1.0}} + ] + + +def test_consumed_capacity_write_total_is_a_list(dynamodb_client, btable): + _seed(dynamodb_client, btable, "cw") + resp = _batch( + dynamodb_client, + [{"Statement": f"UPDATE \"{btable}\" SET a=2 WHERE pk='cw' AND sk='s1'"}], + ReturnConsumedCapacity="TOTAL", + ) + assert resp["ConsumedCapacity"] == [{"TableName": btable, "CapacityUnits": 1.0}] + + +# --- B9: statement-count limit -------------------------------------------- + + +def test_twenty_five_statements_ok(dynamodb_client, btable): + statements = [ + {"Statement": f"SELECT * FROM \"{btable}\" WHERE pk='n{i}' AND sk='s1'"} + for i in range(25) + ] + resp = _batch(dynamodb_client, statements) + assert len(resp["Responses"]) == 25 + + +def test_twenty_six_statements_rejected(dynamodb_client, btable): + statements = [ + {"Statement": f"SELECT * FROM \"{btable}\" WHERE pk='n{i}' AND sk='s1'"} + for i in range(26) + ] + with pytest.raises(ClientError) as excinfo: + _batch(dynamodb_client, statements) + code, message = _error(excinfo) + assert code == "ValidationException" + assert message.endswith("Member must have length less than or equal to 25") + + +# --- B10: per-statement syntax / resource errors -------------------------- + + +def test_syntax_error_is_per_statement(dynamodb_client, btable): + _seed(dynamodb_client, btable, "ok1") + resp = _batch( + dynamodb_client, + [ + {"Statement": f"SELECT * FROM \"{btable}\" WHERE pk='ok1' AND sk='s1'"}, + {"Statement": f"SELCT * FROM \"{btable}\""}, + ], + ) + assert resp["Responses"][0]["Item"]["pk"] == {"S": "ok1"} + assert resp["Responses"][1] == { + "Error": { + "Code": "ValidationError", + "Message": "Statement wasn't well formed, can't be processed: Expected data manipulation", + } + } + + +def test_unknown_table_is_per_statement_resource_not_found(dynamodb_client, btable): + resp = _batch( + dynamodb_client, + [ + { + "Statement": "INSERT INTO \"extenddb-measure-2c-no-such-table\" VALUE {'pk':'x','sk':'s1'}" + } + ], + ) + assert resp["Responses"] == [ + {"Error": {"Code": "ResourceNotFound", "Message": "Requested resource not found"}} + ] + + +# --- RETURNING on a write member ------------------------------------------ +# A member honours RETURNING and carries the row as Responses[i].Item, with +# the projection ExecuteStatement puts in Items. An empty projection leaves +# Item out (measured 2026-11-02). + + +def test_member_delete_returning_all_old_carries_the_deleted_item(dynamodb_client, btable): + _seed(dynamodb_client, btable, "ret-del", a=7) + resp = _batch( + dynamodb_client, + [ + { + "Statement": f"DELETE FROM \"{btable}\" WHERE pk='ret-del' AND sk='s1' RETURNING ALL OLD *" + } + ], + ) + assert resp["Responses"] == [ + { + "TableName": btable, + "Item": {"pk": {"S": "ret-del"}, "sk": {"S": "s1"}, "a": {"N": "7"}}, + } + ] + assert _get(dynamodb_client, btable, "ret-del") is None + + +def test_member_update_returning_all_new_carries_the_whole_item(dynamodb_client, btable): + _seed(dynamodb_client, btable, "ret-allnew", a=1) + resp = _batch( + dynamodb_client, + [ + { + "Statement": f"UPDATE \"{btable}\" SET a=2 WHERE pk='ret-allnew' AND sk='s1' RETURNING ALL NEW *" + } + ], + ) + assert resp["Responses"] == [ + { + "TableName": btable, + "Item": {"pk": {"S": "ret-allnew"}, "sk": {"S": "s1"}, "a": {"N": "2"}}, + } + ] + + +def test_member_update_returning_modified_new_carries_the_changed_attribute( + dynamodb_client, btable +): + _seed(dynamodb_client, btable, "ret-modnew", a=1) + resp = _batch( + dynamodb_client, + [ + { + "Statement": f"UPDATE \"{btable}\" SET a=2 WHERE pk='ret-modnew' AND sk='s1' RETURNING MODIFIED NEW *" + } + ], + ) + assert resp["Responses"] == [{"TableName": btable, "Item": {"a": {"N": "2"}}}] + + +def test_member_update_with_an_empty_modified_projection_omits_item(dynamodb_client, btable): + _seed(dynamodb_client, btable, "ret-modempty", a=1) + resp = _batch( + dynamodb_client, + [ + { + "Statement": f"UPDATE \"{btable}\" REMOVE a WHERE pk='ret-modempty' AND sk='s1' RETURNING MODIFIED NEW *" + } + ], + ) + assert resp["Responses"] == [{"TableName": btable}] + assert "a" not in _get(dynamodb_client, btable, "ret-modempty") + + +# --- ConsistentRead per member -------------------------------------------- +# ConsistentRead is rated per member: a consistent read of a small item costs +# a full unit, an eventual one half, and a mixed batch sums the two (measured +# 2026-11-02). + + +def _read_units(resp): + return sum(entry["CapacityUnits"] for entry in resp["ConsumedCapacity"]) + + +def test_consistent_read_is_rated_per_member(dynamodb_client, btable): + _seed(dynamodb_client, btable, "cr1") + _seed(dynamodb_client, btable, "cr2") + + def member(pk, **extra): + return { + "Statement": f"SELECT * FROM \"{btable}\" WHERE pk='{pk}' AND sk='s1'", + **extra, + } + + def units(members): + return _read_units(_batch(dynamodb_client, members, ReturnConsumedCapacity="TOTAL")) + + assert units([member("cr1")]) == 0.5 + assert units([member("cr1", ConsistentRead=False)]) == 0.5 + assert units([member("cr1", ConsistentRead=True)]) == 1.0 + assert units([member("cr1", ConsistentRead=True), member("cr2")]) == 1.5 + + +# --- ReturnValuesOnConditionCheckFailure on a member ------------------------ +# With ALL_OLD, a member whose condition fails carries the existing item on its +# Error, never on the entry; NONE and an absent option carry none (measured +# 2026-11-02). + + +@pytest.mark.parametrize("setting", ["ALL_OLD", "NONE", None]) +def test_condition_failure_item_rides_on_the_member_error(dynamodb_client, btable, setting): + pk = f"rvccf-{(setting or 'absent').lower()}" + _seed(dynamodb_client, btable, pk, a=1) + member = {"Statement": f"UPDATE \"{btable}\" SET a=2 WHERE pk='{pk}' AND sk='s1' AND a=99"} + if setting is not None: + member["ReturnValuesOnConditionCheckFailure"] = setting + resp = _batch(dynamodb_client, [member]) + error = {"Code": "ConditionalCheckFailed", "Message": "The conditional request failed"} + if setting == "ALL_OLD": + error["Item"] = {"pk": {"S": pk}, "sk": {"S": "s1"}, "a": {"N": "1"}} + assert resp["Responses"] == [{"Error": error, "TableName": btable}] + assert _get(dynamodb_client, btable, pk)["a"] == {"N": "1"} + + +# --- empty Statements (request-level) ------------------------------------- + + +def test_empty_statements_list_rejected(dynamodb_client_no_validation): + with pytest.raises(ClientError) as excinfo: + dynamodb_client_no_validation.batch_execute_statement(Statements=[]) + code, message = _error(excinfo) + assert code == "ValidationException" + assert message == ( + "1 validation error detected: Value '[]' at 'statements' failed to satisfy " + "constraint: Member must have length greater than or equal to 1" + ) + + +# --- DELETE on a missing key with a non-key predicate ----------------------- +# The predicate is a condition on an item that exists, so a member naming a +# missing key is a silent no-op, as the statement is on its own. + + +def test_member_delete_missing_key_with_predicate_is_a_no_op(dynamodb_client, btable): + resp = _batch( + dynamodb_client, + [ + { + "Statement": f"DELETE FROM \"{btable}\" WHERE pk='ghost-pred' AND sk='s1' AND a=1" + } + ], + ) + assert resp["Responses"] == [{"TableName": btable}] + + +# --- statements that fail before they run ----------------------------------- +# The service translates each statement into the read or the write it runs +# as. A statement whose translation fails gets its own error and counts as +# neither a read nor a write, nor in the check for two statements on one item. +# A statement that translates and fails the check of its request counts as a +# read or a write, and not in the check for two statements on one item. The +# errors of the item operation a statement runs as name its table (measured +# 2026-11-05 against a table with a global secondary index). + +MIXED = "Read and write requests together in the same batch is not supported." +DUPLICATES = "Provided list of item keys contains duplicates" +PARAMETERS = "Number of parameters in request and statement don't match." +NOT_FOUND = {"Error": {"Code": "ResourceNotFound", "Message": "Requested resource not found"}} +KEY_TYPE = "Key attribute's data type should match its data type in table's schema: Key sk" +INDEX_KEY_TYPE = ( + "One or more parameter values were invalid: Type mismatch for Index Key g Expected: S " + "Actual: N IndexName: gsi" +) + + +def _key_update(name): + return ( + f"One or more parameter values were invalid: Cannot update attribute {name}. " + "This attribute is part of the key" + ) + + +def _validation(message, table=None): + entry = {"Error": {"Code": "ValidationError", "Message": message}} + if table is not None: + entry["TableName"] = table + return entry + + +@pytest.fixture(scope="module") +def gtable(dynamodb_client): + """The table with the index `gsi`, holding the item of the reads.""" + with gsi_table(dynamodb_client, "extenddb-measure-2c-batchg") as name: + put(dynamodb_client, name, "seed") + yield name + + +def _sql(template, table, **names): + """`template` with `{t}` as the quoted table and each other field as the + key that name takes in this run.""" + return template.format(t=f'"{table}"', **{n: key(v) for n, v in names.items()}) + + +def _seed_read(table): + return _sql("SELECT * FROM {t} WHERE pk = '{s}' AND sk = 'b'", table, s="seed") + + +def _seed_answer(table): + return {"TableName": table, "Item": item(key("seed"))} + + +@pytest.mark.parametrize( + "statement, error", + [ + ("SELECT * FROM nosuch_tbl WHERE pk = 'a' AND sk = 'b'", NOT_FOUND), + ( + "SELECT * FROM {t}.\"nosuch\" WHERE g = 'a' AND gs = '1'", + _validation("The table does not have the specified index"), + ), + ("SELECT * FROM {t} WHERE pk = '{s}' AND sk = 1", _validation(KEY_TYPE)), + ( + "SELECT a, a FROM {t} WHERE pk = '{s}' AND sk = 'b'", + _validation("Duplicate identifiers in select clause: a"), + ), + ("SELECT * FROM {t} WHERE pk = ? AND sk = 'b'", _validation(PARAMETERS)), + ], + ids=["missing-table", "missing-index", "key-type", "repeated-identifier", "parameter"], +) +def test_a_read_that_does_not_translate_is_no_read(dynamodb_client, gtable, statement, error): + """A read that fails its translation leaves the batch to its writes.""" + inserted = f"r-untranslated-{uuid.uuid4().hex[:6]}" + resp = _batch( + dynamodb_client, + [ + {"Statement": _sql(statement, gtable, s="seed")}, + {"Statement": _sql("INSERT INTO {t} VALUE {{'pk':'{k}','sk':'b','a':1}}", gtable, + k=inserted)}, + ], + ) + assert resp["Responses"] == [error, {"TableName": gtable}] + assert _gget(dynamodb_client, gtable, key(inserted)) is not None + + +@pytest.mark.parametrize( + "statement, error", + [ + ( + "INSERT INTO {t} VALUE {{'pk':'{k}'}}", + _validation("Key attribute should be present in the item: Key sk"), + ), + ("INSERT INTO {t} VALUE {{'pk':'{k}','sk':1}}", _validation(KEY_TYPE)), + ( + "UPDATE {t} SET a = 9 WHERE pk = '{k}'", + _validation("Where clause does not contain a mandatory equality on all key attributes"), + ), + ("INSERT INTO {t} VALUE {{'pk':'{k}','sk':'b','x':?}}", _validation(PARAMETERS)), + ( + "DELETE FROM {t} WHERE pk = '{k}' AND sk = 'b' RETURNING ALL NEW *", + _validation( + "Invalid returning clause: RETURNING ALL NEW *. Only RETURNING ALL OLD * is " + "allowed in DELETE statements." + ), + ), + ("UPDATE nosuch_tbl SET a = 9 WHERE pk = 'a' AND sk = 'b'", NOT_FOUND), + ( + "UPDATE {t} SET a = 1, a = 2 WHERE pk = '{k}' AND sk = 'b'", + _validation( + "Two document paths overlap with each other; must remove or rewrite one of " + "these paths; path one: [a], path two: [a]" + ), + ), + ], + ids=[ + "no-sort-key", + "key-type", + "partial-key", + "parameter", + "returning", + "missing-table", + "overlapping-paths", + ], +) +def test_a_write_that_does_not_translate_is_no_write(dynamodb_client, gtable, statement, error): + """A write that fails its translation leaves the batch to its reads, and + writes nothing.""" + pk = put(dynamodb_client, gtable, "w-untranslated") + resp = _batch( + dynamodb_client, + [ + {"Statement": _sql(statement, gtable, k="w-untranslated")}, + {"Statement": _seed_read(gtable)}, + ], + ) + assert resp["Responses"] == [error, _seed_answer(gtable)] + assert _gget(dynamodb_client, gtable, pk) == item(pk) + + +def test_a_failed_read_beside_a_write_lets_the_write_run(dynamodb_client, gtable): + resp = _batch( + dynamodb_client, + [ + {"Statement": "SELECT * FROM nosuch_tbl WHERE pk = 'a' AND sk = 'b'"}, + {"Statement": _sql("INSERT INTO {t} VALUE {{'pk':'{k}','sk':'b','a':1}}", gtable, + k="cb1")}, + ], + ) + assert resp["Responses"] == [NOT_FOUND, {"TableName": gtable}] + pk = key("cb1") + assert _gget(dynamodb_client, gtable, pk) == { + "pk": {"S": pk}, + "sk": {"S": "b"}, + "a": {"N": "1"}, + } + + +@pytest.mark.parametrize( + "statements", + [ + ["SELECT * FROM {t} WHERE pk = '{s}' AND sk = 'b'", + "UPDATE {t} SET a = 9 WHERE pk = '{k}' AND sk = 'b'"], + ["SELECT * FROM {t} WHERE pk = '{s}'", + "INSERT INTO {t} VALUE {{'pk':'{n}','sk':'b','a':1}}"], + ["SELECT * FROM {t}.\"gsi\" WHERE g = '{s}' AND gs = '1'", + "INSERT INTO {t} VALUE {{'pk':'{n}','sk':'b','a':1}}"], + ["SELECT * FROM {t} WHERE pk = '{s}' AND sk = 'b'", + "UPDATE {t} SET pk = 'z' WHERE pk = '{k}' AND sk = 'b'"], + ["SELECT * FROM {t} WHERE pk = '{s}' AND sk = 'b'", + "UPDATE {t} SET a = 9 WHERE pk = '{n}' AND sk = 'b'"], + ["SELECT * FROM {t} WHERE pk = '{s}' AND sk = 'b'", + "UPDATE {t} SET g = 1 WHERE pk = '{k}' AND sk = 'b'"], + ["SELECT * FROM {t} WHERE pk = '{s}' AND sk = 'b'", + "UPDATE {t} SET a.b = 1 WHERE pk = '{k}' AND sk = 'b'"], + ], + ids=[ + "update", + "partial-key-read", + "index-read", + "key-update", + "missing-item", + "index-key-type", + "document-path", + ], +) +def test_a_statement_that_translates_counts_toward_the_mix(dynamodb_client, gtable, statements): + """A statement that translates is a read or a write, whatever its + request then answers.""" + pk = put(dynamodb_client, gtable, "w-counted") + with pytest.raises(ClientError) as excinfo: + _batch( + dynamodb_client, + [{"Statement": _sql(s, gtable, s="seed", k="w-counted", n="w-counted-new")} + for s in statements], + ) + assert _error(excinfo) == ("ValidationException", MIXED) + assert _gget(dynamodb_client, gtable, pk) == item(pk) + assert _gget(dynamodb_client, gtable, key("w-counted-new")) is None + + +@pytest.mark.parametrize( + "first, error", + [ + ("INSERT INTO {t} VALUE {{'pk':'{k}','sk':'b','x':?}}", _validation(PARAMETERS)), + ("UPDATE {t} SET a = ? WHERE pk = '{k}' AND sk = 'b'", _validation(PARAMETERS)), + ("UPDATE {t} SET a = 9 WHERE pk = '{k}' AND sk = 'b' AND a = ?", _validation(PARAMETERS)), + ("UPDATE {t} SET pk = 'z' WHERE pk = '{k}' AND sk = 'b'", "key-update:pk"), + ("UPDATE {t} REMOVE sk WHERE pk = '{k}' AND sk = 'b'", "key-update:sk"), + ("UPDATE {t} SET g = 1 WHERE pk = '{k}' AND sk = 'b'", "index-key-type"), + ("INSERT INTO {t} VALUE {{'pk':'{k}','sk':'b','g':1}}", "index-key-type"), + ( + "UPDATE {t} SET a = 1, a = 2 WHERE pk = '{k}' AND sk = 'b'", + _validation( + "Two document paths overlap with each other; must remove or rewrite one of " + "these paths; path one: [a], path two: [a]" + ), + ), + ], + ids=[ + "insert-parameter", + "set-parameter", + "condition-parameter", + "set-partition-key", + "remove-sort-key", + "set-index-key-type", + "insert-index-key-type", + "overlapping-paths", + ], +) +def test_a_write_refused_before_its_item_is_not_a_duplicate( + dynamodb_client, gtable, first, error +): + """A write refused before it reaches its item leaves the item to the + other statement on it.""" + pk = put(dynamodb_client, gtable, "dup-refused") + resp = _batch( + dynamodb_client, + [ + {"Statement": _sql(first, gtable, k="dup-refused")}, + {"Statement": _sql("DELETE FROM {t} WHERE pk = '{k}' AND sk = 'b'", gtable, + k="dup-refused")}, + ], + ) + if error == "index-key-type": + error = _validation(INDEX_KEY_TYPE, gtable) + elif isinstance(error, str): + error = _validation(_key_update(error.split(":")[1]), gtable) + assert resp["Responses"] == [error, {"TableName": gtable}] + assert _gget(dynamodb_client, gtable, pk) is None + + +def test_a_delete_with_another_returning_clause_is_not_a_duplicate(dynamodb_client, gtable): + pk = put(dynamodb_client, gtable, "dup-returning") + resp = _batch( + dynamodb_client, + [ + {"Statement": _sql( + "DELETE FROM {t} WHERE pk = '{k}' AND sk = 'b' RETURNING ALL NEW *", gtable, + k="dup-returning")}, + {"Statement": _sql("UPDATE {t} SET a = 9 WHERE pk = '{k}' AND sk = 'b'", gtable, + k="dup-returning")}, + ], + ) + assert resp["Responses"] == [ + _validation( + "Invalid returning clause: RETURNING ALL NEW *. Only RETURNING ALL OLD * is " + "allowed in DELETE statements." + ), + {"TableName": gtable}, + ] + assert _gget(dynamodb_client, gtable, pk)["a"] == {"N": "9"} + + +@pytest.mark.parametrize( + "first, error", + [ + ( + "SELECT a, a FROM {t} WHERE pk = '{s}' AND sk = 'b'", + _validation("Duplicate identifiers in select clause: a"), + ), + ( + "SELECT * FROM {t}.\"nosuch\" WHERE pk = '{s}' AND sk = 'b'", + _validation("The table does not have the specified index"), + ), + ( + "SELECT * FROM {t}.\"gsi\" WHERE pk = '{s}' AND sk = 'b'", + _validation( + "Select statements within BatchExecuteStatement must specify the primary key " + "in the where clause." + ), + ), + ], + ids=["repeated-identifier", "missing-index", "index-read"], +) +def test_a_read_refused_before_its_item_is_not_a_duplicate( + dynamodb_client, gtable, first, error +): + resp = _batch( + dynamodb_client, + [{"Statement": _sql(first, gtable, s="seed")}, {"Statement": _seed_read(gtable)}], + ) + assert resp["Responses"] == [error, _seed_answer(gtable)] + + +@pytest.mark.parametrize( + "first", + [ + "UPDATE {t} SET a = 9 WHERE pk = '{k}' AND sk = 'b'", + "UPDATE {t} SET a = 9 WHERE pk = '{k}' AND sk = 'b' AND a = 99", + "UPDATE {t} SET b = m + 1 WHERE pk = '{k}' AND sk = 'b'", + "UPDATE {t} SET a.b = 1 WHERE pk = '{k}' AND sk = 'b'", + ], + ids=["update", "failed-condition", "operand-type", "document-path"], +) +def test_two_writes_that_reach_one_item_are_duplicates(dynamodb_client, gtable, first): + pk = put(dynamodb_client, gtable, "dup-reached") + with pytest.raises(ClientError) as excinfo: + _batch( + dynamodb_client, + [ + {"Statement": _sql(first, gtable, k="dup-reached")}, + {"Statement": _sql("DELETE FROM {t} WHERE pk = '{k}' AND sk = 'b'", gtable, + k="dup-reached")}, + ], + ) + assert _error(excinfo) == ("ValidationException", DUPLICATES) + assert _gget(dynamodb_client, gtable, pk) == item(pk) + + +@pytest.mark.parametrize( + "statement, message", + [ + ("UPDATE {t} SET pk = 'z' WHERE pk = '{k}' AND sk = 'b'", _key_update("pk")), + ("UPDATE {t} REMOVE sk WHERE pk = '{k}' AND sk = 'b'", _key_update("sk")), + ( + "UPDATE {t} SET b = m + 1 WHERE pk = '{k}' AND sk = 'b'", + "An operand in the update expression has an incorrect data type", + ), + ( + "UPDATE {t} SET a = set_add(a, <<'x'>>) WHERE pk = '{k}' AND sk = 'b'", + "An operand in the update expression has an incorrect data type", + ), + ( + "UPDATE {t} SET a.b = 1 WHERE pk = '{k}' AND sk = 'b'", + "The document path provided in the update expression is invalid for update", + ), + ( + "UPDATE {t} SET m[0] = 1 WHERE pk = '{k}' AND sk = 'b'", + "The document path provided in the update expression is invalid for update", + ), + ("UPDATE {t} SET g = 1 WHERE pk = '{k}' AND sk = 'b'", INDEX_KEY_TYPE), + ("INSERT INTO {t} VALUE {{'pk':'{n}','sk':'b','g':1}}", INDEX_KEY_TYPE), + # Both come ahead of a path into a key attribute among the other + # conditions (measured 2026-11-05). + ( + "UPDATE {t} SET pk = 'z' WHERE pk = '{k}' AND sk = 'b' AND NOT pk.c = 1", + _key_update("pk"), + ), + ("UPDATE {t} SET g = 1 WHERE pk = '{k}' AND sk = 'b' AND NOT pk.c = 1", INDEX_KEY_TYPE), + ( + "UPDATE {t} SET gs = 1 WHERE pk = '{k}' AND sk = 'b' AND NOT pk.c = 1", + INDEX_KEY_TYPE.replace("Index Key g ", "Index Key gs "), + ), + ( + "UPDATE {t} SET g = 1, pk = 'z' WHERE pk = '{k}' AND sk = 'b' AND NOT pk.c = 1", + _key_update("pk"), + ), + # An index key given the value of another attribute is checked on + # the written item, after the path into a key attribute. + ( + "UPDATE {t} SET g = a WHERE pk = '{k}' AND sk = 'b' AND NOT pk.c = 1", + "Key attributes must be scalars; list random access '[]' and map lookup '.' are not " + "allowed: Key: pk", + ), + ], + ids=[ + "set-partition-key", + "remove-sort-key", + "operand-type", + "set-add-to-a-number", + "path-into-a-number", + "index-into-a-map", + "set-index-key-type", + "insert-index-key-type", + "set-partition-key-ahead-of-a-path-into-a-key", + "set-index-key-type-ahead-of-a-path-into-a-key", + "set-index-sort-key-type-ahead-of-a-path-into-a-key", + "set-partition-key-ahead-of-an-index-key-type", + "index-key-from-another-attribute-after-a-path-into-a-key", + ], +) +def test_an_error_of_the_item_operation_names_the_table( + dynamodb_client, gtable, statement, message +): + pk = put(dynamodb_client, gtable, "tn") + resp = _batch( + dynamodb_client, [{"Statement": _sql(statement, gtable, k="tn", n="tn-new")}] + ) + assert resp["Responses"] == [_validation(message, gtable)] + assert _gget(dynamodb_client, gtable, pk) == item(pk) + + +def test_an_error_of_reading_the_statement_names_no_table(dynamodb_client, gtable): + resp = _batch( + dynamodb_client, + [ + {"Statement": _seed_read(gtable)}, + {"Statement": _sql("SELECT * FROM {t} WHERE pk = ? AND sk = 'b'", gtable)}, + ], + ) + assert resp["Responses"] == [_seed_answer(gtable), _validation(PARAMETERS)] + + +# --- projections ------------------------------------------------------------ +# A batch read flattens a projected path to its leaf, and refuses an +# identifier named twice and two paths that overlap as the statement's own +# error (measured 2026-11-05). + + +def test_a_projected_read_keys_each_path_on_its_leaf(dynamodb_client, gtable): + resp = _batch( + dynamodb_client, + [{"Statement": _sql("SELECT a, m.k FROM {t} WHERE pk = '{s}' AND sk = 'b'", gtable, + s="seed")}], + ) + assert resp["Responses"] == [ + {"TableName": gtable, "Item": {"a": {"N": "1"}, "k": {"N": "1"}}} + ] + + +@pytest.mark.parametrize( + "projection, message", + [ + ("a, a", "Duplicate identifiers in select clause: a"), + ("m.k, k", "Duplicate identifiers in select clause: k"), + ( + "m.k, m", + "Two document paths overlap with each other; must remove or rewrite one of these " + "paths; path one: [m], path two: [m, k]", + ), + ], + ids=["repeated", "repeated-leaf", "overlapping"], +) +def test_a_projection_refusal_is_the_statement_error( + dynamodb_client, gtable, projection, message +): + resp = _batch( + dynamodb_client, + [{"Statement": _sql(f"SELECT {projection} FROM {{t}} WHERE pk = '{{s}}' AND sk = 'b'", + gtable, s="seed")}], + ) + assert resp["Responses"] == [_validation(message)] + + +# --- ConsumedCapacity with an index and with failed writes -------------------- +# A batch sums the units of its statements per table, with the units of each +# index at INDEXES. A write that fails at its item is charged the units of the +# item stored under its key (one when there is none), counted in its table's +# CapacityUnits alone; a table on which every write failed reports nothing +# (measured 2026-11-05). + +BIG = {"big": {"S": "x" * 3000}} + + +def _capacity(client, table, statements, rcc, **names): + resp = _batch( + client, + [{"Statement": _sql(s, table, **names)} for s in statements], + ReturnConsumedCapacity=rcc, + ) + return resp.get("ConsumedCapacity") + + +def _arm(table, units, table_units=None, gsi_units=None): + arm = {"TableName": table, "CapacityUnits": units} + if table_units is not None: + arm["Table"] = {"CapacityUnits": table_units} + if gsi_units is not None: + arm["GlobalSecondaryIndexes"] = {"gsi": {"CapacityUnits": gsi_units}} + return arm + + +UPDATE_A = "UPDATE {t} SET a = 9 WHERE pk = '{ok}' AND sk = 'b'" +UPDATE_MISSING = "UPDATE {t} SET a = 9 WHERE pk = '{missing}' AND sk = 'b'" + + +def test_an_index_key_change_reports_the_index_units(dynamodb_client, gtable): + put(dynamodb_client, gtable, "cb17") + assert _capacity( + dynamodb_client, gtable, ["UPDATE {t} SET g = 'x' WHERE pk = '{k}' AND sk = 'b'"], + "INDEXES", k="cb17", + ) == [_arm(gtable, 3.0, 1.0, 2.0)] + + +@pytest.mark.parametrize( + "statements, rcc, expected", + [ + ([UPDATE_A, UPDATE_MISSING], "TOTAL", (3.0, None, None)), + ([UPDATE_A, UPDATE_MISSING], "INDEXES", (3.0, 1.0, 1.0)), + ([UPDATE_MISSING, UPDATE_A], "INDEXES", (3.0, 1.0, 1.0)), + ([UPDATE_A, UPDATE_MISSING, + "UPDATE {t} SET a = 9 WHERE pk = '{missing2}' AND sk = 'b'"], "TOTAL", (4.0, None, None)), + ([UPDATE_A, "UPDATE {t} SET a = 9 WHERE pk = '{ok2}' AND sk = 'b'"], "TOTAL", + (4.0, None, None)), + ([UPDATE_A, "UPDATE {t} SET a = 9 WHERE pk = '{big}' AND sk = 'b' AND a = 99"], "INDEXES", + (5.0, 1.0, 1.0)), + ([UPDATE_A, "INSERT INTO {t} VALUE {{'pk':'{big}','sk':'b','a':1}}"], "TOTAL", + (5.0, None, None)), + ([UPDATE_A, "DELETE FROM {t} WHERE pk = '{ok2}' AND sk = 'b' AND a = 99"], "TOTAL", + (3.0, None, None)), + ([UPDATE_A, "UPDATE {t} SET b = m + 1 WHERE pk = '{ok2}' AND sk = 'b'"], "TOTAL", + (3.0, None, None)), + ([UPDATE_A, "UPDATE {t} SET a.b = 1 WHERE pk = '{ok2}' AND sk = 'b'"], "TOTAL", + (3.0, None, None)), + ([UPDATE_A, "UPDATE {t} SET pk = 'z' WHERE pk = '{ok2}' AND sk = 'b'"], "TOTAL", + (2.0, None, None)), + ([UPDATE_A, "UPDATE {t} SET g = 1 WHERE pk = '{ok2}' AND sk = 'b'"], "TOTAL", + (2.0, None, None)), + (["DELETE FROM {t} WHERE pk = '{ok}' AND sk = 'b'", + "DELETE FROM {t} WHERE pk = '{missing}' AND sk = 'b'"], "INDEXES", (3.0, 2.0, 1.0)), + (["INSERT INTO {t} VALUE {{'pk':'{missing}','sk':'b','a':1,'g':'q','gs':'1'}}", + "INSERT INTO {t} VALUE {{'pk':'{missing2}','sk':'b','a':1}}"], "INDEXES", + (3.0, 2.0, 1.0)), + ], + ids=[ + "failed-condition-total", + "failed-condition-indexes", + "failure-first", + "two-failures", + "two-successes", + "failed-condition-on-a-3kb-item", + "duplicate-over-a-3kb-item", + "failed-delete-condition", + "operand-type", + "document-path", + "key-update-is-not-charged", + "index-key-type-is-not-charged", + "deletes", + "inserts", + ], +) +def test_a_failed_write_is_charged_on_its_table(dynamodb_client, gtable, statements, rcc, expected): + put(dynamodb_client, gtable, "cap-ok") + put(dynamodb_client, gtable, "cap-ok2") + put(dynamodb_client, gtable, "cap-big", **BIG) + names = { + "ok": "cap-ok", + "ok2": "cap-ok2", + "big": "cap-big", + "missing": f"cap-missing-{uuid.uuid4().hex[:6]}", + "missing2": f"cap-missing-{uuid.uuid4().hex[:6]}", + } + units, table_units, gsi_units = expected + assert _capacity(dynamodb_client, gtable, statements, rcc, **names) == [ + _arm(gtable, units, table_units, gsi_units) + ] + + +def test_failed_writes_alone_report_no_capacity(dynamodb_client, gtable): + assert _capacity( + dynamodb_client, + gtable, + [UPDATE_MISSING, "UPDATE {t} SET a = 9 WHERE pk = '{missing2}' AND sk = 'b'"], + "INDEXES", + missing=f"cap-missing-{uuid.uuid4().hex[:6]}", + missing2=f"cap-missing-{uuid.uuid4().hex[:6]}", + ) is None diff --git a/tests/test_partiql_boolean_keys.py b/tests/test_partiql_boolean_keys.py new file mode 100644 index 000000000..476a6a23e --- /dev/null +++ b/tests/test_partiql_boolean_keys.py @@ -0,0 +1,141 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""Dual-target wire tests for a key condition compared with a boolean in a +PartiQL SELECT. + +When a key condition (an equality of the partition key with a value, a key +condition on the sort key, or an AND of them) is compared with a boolean +literal, the service reads by that key condition and the literal alone +decides: compared with true (`= true` or `<> true`) every item read is +returned, compared with false (`= false` or `<> false`) none. Any other +condition compared with a boolean keeps its meaning as a condition. + +Dual-target: EXTENDDB_TEST_ENDPOINT selects an extenddb server; unset, the +tests run against the live service with the default credential chain. The +assertions are identical on both. Every expected value was measured on the +service (2026-11-03). +""" + +from __future__ import annotations + +import pytest + +from conftest import scoped_table + + +def _filled(client, keys): + """A table keyed on string pk and sk holding the items `keys`, each with + n = 1.""" + with scoped_table( + client, + attribute_definitions=[ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "sk", "AttributeType": "S"}, + ], + key_schema=[ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"}, + ], + ) as name: + for pk, sk in keys: + client.put_item( + TableName=name, Item={"pk": {"S": pk}, "sk": {"S": sk}, "n": {"N": "1"}} + ) + yield name + + +@pytest.fixture(scope="module") +def four(dynamodb_client): + """Items a/x, a/y, b/x and c/x.""" + yield from _filled(dynamodb_client, [("a", "x"), ("a", "y"), ("b", "x"), ("c", "x")]) + + +@pytest.fixture(scope="module") +def five(dynamodb_client): + """Items a/x, a/y, a/z, b/x and b/y.""" + yield from _filled( + dynamodb_client, [("a", "x"), ("a", "y"), ("a", "z"), ("b", "x"), ("b", "y")] + ) + + +def _keys(client, table, condition): + statement = f'SELECT * FROM "{table}" WHERE {condition}' + items, token = [], None + for _ in range(8): + args = {"NextToken": token} if token else {} + response = client.execute_statement(Statement=statement, **args) + items += response["Items"] + token = response.get("NextToken") + if not token: + break + return sorted(f"{item['pk']['S']}/{item['sk']['S']}" for item in items) + + +@pytest.mark.parametrize( + ("condition", "keys"), + [ + ("pk = 'a' <> true", ["a/x", "a/y", "a/z"]), + ("pk = 'b' <> true", ["b/x", "b/y"]), + ("pk = 'a' = true", ["a/x", "a/y", "a/z"]), + ("pk = 'a' AND sk = 'y' <> true", ["a/y"]), + ("pk = 'a' AND sk = 'x' = true", ["a/x"]), + ("pk = 'a' AND sk > 'x' <> true", ["a/y", "a/z"]), + ("(pk = 'a' AND sk = 'x') <> true", ["a/x"]), + ("pk = 'a' <> true AND n = 1", ["a/x", "a/y", "a/z"]), + ("pk = 'a' <> true OR pk = 'b'", ["a/x", "a/y", "a/z", "b/x", "b/y"]), + ], +) +def test_a_key_condition_compared_with_true_returns_what_it_reads( + dynamodb_client, five, condition, keys +): + assert _keys(dynamodb_client, five, condition) == keys + + +@pytest.mark.parametrize( + "condition", + [ + "pk = 'a' <> false", + "pk = 'a' AND sk > 'x' = false", + "(pk = 'a' AND sk = 'x') = false", + ], +) +def test_a_key_condition_compared_with_false_returns_nothing(dynamodb_client, five, condition): + assert _keys(dynamodb_client, five, condition) == [] + + +@pytest.mark.parametrize( + ("condition", "keys"), + [ + ("pk = 'a' = false", []), + ("(pk = 'a') = false", []), + ("pk = 'a' = false AND n = 1", []), + ("pk = 'a' AND sk = 'x' = false", []), + ("pk = 'a' AND (sk = 'x') = false", []), + ("pk = 'a' AND sk = 'x' <> true", ["a/x"]), + ("pk = 'a' <> true", ["a/x", "a/y"]), + ("pk = 'a' = false OR pk = 'b'", ["b/x"]), + ("pk = 'b' OR pk = 'a' = false", ["b/x"]), + ], +) +def test_the_literal_alone_decides_beside_other_disjuncts(dynamodb_client, four, condition, keys): + assert _keys(dynamodb_client, four, condition) == keys + + +@pytest.mark.parametrize( + ("condition", "keys"), + [ + ("pk = 'a' AND n = 1 <> true", []), + ("pk = 'a' AND n = 1 = false", []), + ("pk IN ['a'] <> true", ["b/x", "b/y"]), + ("pk IN ['a'] = false", ["b/x", "b/y"]), + ], +) +def test_another_condition_compared_with_a_boolean_keeps_its_meaning( + dynamodb_client, five, condition, keys +): + assert _keys(dynamodb_client, five, condition) == keys + + +def test_a_not_over_a_key_condition_is_a_scan(dynamodb_client, four): + assert _keys(dynamodb_client, four, "NOT pk = 'a'") == ["b/x", "c/x"] diff --git a/tests/test_partiql_check_order.py b/tests/test_partiql_check_order.py new file mode 100644 index 000000000..ee8529c52 --- /dev/null +++ b/tests/test_partiql_check_order.py @@ -0,0 +1,317 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""Dual-target wire tests for the order of the checks on a PartiQL statement +and its WHERE condition. + +The service walks the condition as a tree, within the operator limit and past +it: an operator before its operands, the operands in the order they are +written, and the arguments of a call before its name. It reports the first +number literal that breaks an item number rule, `?` with no parameter left in +the request, or refusal it meets on that walk, ahead of the operator count. +An alias on the table and a table name outside the name rule come before all +of those, then an alias in the select list, an UPDATE of an index and a LIMIT +clause. In ExecuteStatement an EXISTS statement comes before the name rule. +The SET clause of an UPDATE, and parameters beyond the ones the statement +reads, come after the operator count; in the clause a number comes before a +unary plus. ExecuteStatement, BatchExecuteStatement and ExecuteTransaction +keep the same order. + +Dual-target: EXTENDDB_TEST_ENDPOINT selects an extenddb server; unset, the +tests run against the live service with the default credential chain. The +assertions are identical on both. Every expected value was measured on the +service (2026-11-02). +""" + +from __future__ import annotations + +import pytest +from botocore.exceptions import ClientError + +from conftest import scoped_table + +TOO_MANY_OPERATORS = "The expression contains too many operators; operator count: 301" +PARAMETERS = "Number of parameters in request and statement don't match." +OVERFLOW = ( + "Number overflow. Attempting to store a number with magnitude larger than supported " + "range under " +) +UNDERFLOW = ( + "Number underflow. Attempting to store a number with magnitude smaller than supported " + "range under " +) +PLUS = "Unsupported operator in Condition Expression. Operator: +" +LIKE = "Unsupported operator in Condition Expression. Operator: like" +MISSING = "MISSING must be compared using IS operator" +IS_LEFT_SIDE = "The left hand side argument to IS must be a Variable Reference, Literal, or Path" +NAME_RULE = ( + "Table name should be within the length [3, 255] and only contain 'a-z', 'A-Z', '0-9', " + "'-', '_', '.'." +) +NOTS = "NOT " * 300 + "a = 1" +DEEP = "[" * 33 + "1" + "]" * 33 +NESTED = "Nesting Levels have exceeded supported limits under root" + "[0]" * 32 +IN_OF_101 = "a IN [" + ", ".join(str(i) for i in range(101)) + "]" +PROJECTED_CALL = "Unexpected path component at 1:8:5" + + +@pytest.fixture(scope="module") +def table(dynamodb_client): + """A table keyed on a string pk.""" + with scoped_table(dynamodb_client) as name: + yield name + + +def _message(client, statement: str, **kwargs) -> str: + with pytest.raises(ClientError) as e: + client.execute_statement(Statement=statement, **kwargs) + err = e.value.response["Error"] + assert err["Code"] == "ValidationException", err + return err["Message"] + + +@pytest.mark.parametrize( + "condition, message", + [ + ("b = 1e126 AND c = ?", OVERFLOW + "root"), + ("c = ? AND b = 1e126", PARAMETERS), + ("b = 1 + 1 AND c = 1e126", PLUS), + ("c = 1e126 AND b = 1 + 1", OVERFLOW + "root"), + ("c = 1e126 AND foo(b)", OVERFLOW + "root"), + ("c = 1e126 AND b = MISSING", OVERFLOW + "root"), + ("c = 1e126 AND d = 123456789012345678901234567890123456789", OVERFLOW + "root"), + ("b = [1e126]", OVERFLOW + "root[0]"), + ("b = {'k': 1e-131}", UNDERFLOW + "root.k"), + ], +) +def test_the_first_problem_in_the_condition_comes_before_the_operator_count( + dynamodb_client, table, condition, message +): + statement = f'SELECT * FROM "{table}" WHERE {condition} AND {NOTS}' + assert _message(dynamodb_client, statement) == message + + +def test_a_parameter_after_the_operators_comes_before_their_count(dynamodb_client, table): + statement = f'SELECT * FROM "{table}" WHERE {NOTS} AND c = ?' + assert _message(dynamodb_client, statement) == PARAMETERS + + +def test_an_alias_comes_before_the_condition(dynamodb_client, table): + statement = f'SELECT * FROM "{table}" AS x WHERE c = 1e126 AND {NOTS}' + assert _message(dynamodb_client, statement) == "Aliasing is not supported" + + +@pytest.mark.parametrize( + "condition, given, message", + [ + ("c = ?", [{"N": "1"}, {"N": "2"}], TOO_MANY_OPERATORS), + ("c = ? AND d = ?", [{"N": "1"}], PARAMETERS), + ], +) +def test_the_parameters_the_condition_reads_are_counted_before_the_operators( + dynamodb_client, table, condition, given, message +): + statement = f'SELECT * FROM "{table}" WHERE {condition} AND {NOTS}' + assert _message(dynamodb_client, statement, Parameters=given) == message + + +@pytest.mark.parametrize("value", ["1e126", "[1, 1e126]", "?", "+b", "b IS NULL", "[?]"]) +def test_a_set_value_comes_after_the_operator_count(dynamodb_client, table, value): + statement = f"UPDATE \"{table}\" SET q = {value} WHERE pk = 'a' AND {NOTS}" + assert _message(dynamodb_client, statement) == TOO_MANY_OPERATORS + + +def test_a_set_clause_after_the_condition_comes_after_the_operator_count( + dynamodb_client, table +): + statement = f"FROM \"{table}\" WHERE pk = 'a' AND {NOTS} SET q = +b" + assert _message(dynamodb_client, statement) == TOO_MANY_OPERATORS + + +@pytest.mark.parametrize( + "condition, message", + [ + ("b = 1e126 + 1", PLUS), + ("b = 1 + 1e126", PLUS), + ("? IS NULL AND c = 1e126", IS_LEFT_SIDE), + ("foo(1e126)", OVERFLOW + "root"), + ], +) +def test_the_walk_meets_an_operator_before_its_operands_and_a_name_after_its_arguments( + dynamodb_client, table, condition, message +): + statement = f'SELECT * FROM "{table}" WHERE {condition} AND {NOTS}' + assert _message(dynamodb_client, statement) == message + + +@pytest.mark.parametrize( + "condition, message", + [ + ("c = 1e126 AND b = 1 + 1", OVERFLOW + "root"), + ("b = 1 + 1 AND c = 1e126", PLUS), + ("c = 1e126 AND foo(b)", OVERFLOW + "root"), + ("c = 1e126 AND b = MISSING", OVERFLOW + "root"), + ("b = MISSING AND c = 1e126", MISSING), + ("c = 1e126 AND b = 'x' || 'y'", OVERFLOW + "root"), + ("c = 1e126 AND a LIKE 'x'", OVERFLOW + "root"), + ("a LIKE 'x' AND c = 1e126", LIKE), + ("c = ? AND b = 1 + 1", PARAMETERS), + ("b = 1 + 1 AND c = ?", PLUS), + ("c = 1e126 AND a IS INT4", OVERFLOW + "root"), + (f"c = 1e126 AND b = {DEEP}", OVERFLOW + "root"), + (f"foo({DEEP})", NESTED), + ("foo(1 + 1)", PLUS), + ], +) +def test_the_walk_keeps_its_order_within_the_operator_limit( + dynamodb_client, table, condition, message +): + statement = f'SELECT * FROM "{table}" WHERE {condition}' + assert _message(dynamodb_client, statement) == message + + +def test_an_unknown_function_before_a_number_is_met_first(dynamodb_client, table): + statement = f'SELECT * FROM "{table}" WHERE foo(b) AND c = 1e126' + column = statement.index("foo(") + 1 + assert _message(dynamodb_client, statement) == f"Unrecognized function: foo at 1:{column}:3" + + +def test_a_parameter_inside_the_argument_of_an_unknown_function_is_met_first( + dynamodb_client, table +): + statement = f'SELECT * FROM "{table}" WHERE foo([?])' + column = statement.index("?") + 1 + assert _message(dynamodb_client, statement) == ( + f"Unsupported data type: Parameter under key root[0] at 1:{column}:1" + ) + + +def test_an_operand_with_a_plus_as_the_condition_is_refused_for_it(dynamodb_client, table): + assert _message(dynamodb_client, f'SELECT * FROM "{table}" WHERE a + 1') == PLUS + + +@pytest.mark.parametrize( + "condition", + [ + "b = 1 + 1", + "c = 1e126", + NOTS, + f"c = 1e126 AND {NOTS}", + "a IN [" + ", ".join(str(i) for i in range(101)) + "]", + ], + ids=["plus", "overflow", "301-operators", "overflow-then-301-operators", "in-of-101"], +) +def test_the_table_name_rule_comes_first(dynamodb_client, condition): + statement = f'SELECT * FROM "tt" WHERE {condition}' + assert _message(dynamodb_client, statement) == NAME_RULE + + +@pytest.mark.parametrize( + "condition, message", + [("c = ?", PARAMETERS), ("c = 1e126", OVERFLOW + "root")], +) +def test_a_batch_keeps_the_order(dynamodb_client, table, condition, message): + statement = f'SELECT * FROM "{table}" WHERE {condition} AND {NOTS}' + response = dynamodb_client.batch_execute_statement(Statements=[{"Statement": statement}]) + assert response["Responses"] == [{"Error": {"Code": "ValidationError", "Message": message}}] + + +@pytest.mark.parametrize( + "condition, message", + [("c = ?", PARAMETERS), ("c = 1e126", OVERFLOW + "root")], +) +def test_a_transaction_keeps_the_order(dynamodb_client, table, condition, message): + statement = f'SELECT * FROM "{table}" WHERE {condition} AND {NOTS}' + with pytest.raises(ClientError) as e: + dynamodb_client.execute_transaction(TransactStatements=[{"Statement": statement}]) + err = e.value.response["Error"] + assert err["Code"] == "ValidationException", err + assert err["Message"] == f"Validation failed in TransactStatements[0]: {message}" + + +@pytest.mark.parametrize( + "statement", + [ + "SELECT a AS b FROM \"tt\" WHERE pk = 'a'", + "UPDATE \"tt\".\"idx\" SET q = 1 WHERE pk = 'a'", + ], + ids=["select-list-alias", "update-of-an-index"], +) +def test_the_table_name_rule_comes_before_a_select_list_alias_and_an_update_of_an_index( + dynamodb_client, statement +): + assert _message(dynamodb_client, statement) == NAME_RULE + + +def test_exists_in_execute_statement_comes_before_the_table_name_rule(dynamodb_client): + statement = "EXISTS(SELECT * FROM \"tt\" WHERE pk = 'a' AND b = 1 + 1)" + assert _message(dynamodb_client, statement) == ( + "EXISTS can only be used in ExecuteTransaction write requests." + ) + + +def test_a_select_list_call_comes_before_an_in_with_no_list(dynamodb_client, table): + statement = f'SELECT COUNT(*) FROM "{table}" WHERE a IN 1' + assert _message(dynamodb_client, statement) == PROJECTED_CALL + response = dynamodb_client.batch_execute_statement(Statements=[{"Statement": statement}]) + assert response["Responses"] == [ + {"Error": {"Code": "ValidationError", "Message": PROJECTED_CALL}} + ] + with pytest.raises(ClientError) as e: + dynamodb_client.execute_transaction(TransactStatements=[{"Statement": statement}]) + err = e.value.response["Error"] + assert err["Code"] == "ValidationException", err + assert err["Message"] == f"Validation failed in TransactStatements[0]: {PROJECTED_CALL}" + + +@pytest.mark.parametrize("sets", ["q = 1e126, r = +b", "r = +b, q = 1e126"]) +def test_a_number_in_a_set_clause_comes_before_a_unary_plus(dynamodb_client, table, sets): + statement = f"UPDATE \"{table}\" SET {sets} WHERE pk = 'a'" + assert _message(dynamodb_client, statement) == OVERFLOW + "root" + + +@pytest.mark.parametrize( + "condition", + ["c = 1e126", "c = ?", "b = 1 + 1", NOTS, IN_OF_101], + ids=["overflow", "no-parameter", "plus", "301-operators", "in-of-101"], +) +def test_a_limit_clause_comes_before_the_condition(dynamodb_client, table, condition): + statement = f'SELECT * FROM "{table}" WHERE {condition} LIMIT 2' + # The token after LIMIT is named. + column = statement.index("LIMIT 2") + len("LIMIT ") + 1 + assert _message(dynamodb_client, statement) == f"Unsupported clause: LIMIT at 1:{column}:1" + + +def test_a_delete_returning_all_with_nothing_after_it_comes_before_the_condition( + dynamodb_client, table +): + statement = f"DELETE FROM \"{table}\" WHERE pk = 'a' AND c = 1e126 RETURNING ALL" + assert _message(dynamodb_client, statement) == ( + "Statement wasn't well formed, can't be processed: " + "Expected ( MODIFIED | ALL ) ( NEW | OLD ) in each returning element." + ) + + +@pytest.mark.parametrize("tail", ["", f" AND {NOTS}"], ids=["alone", "then-301-operators"]) +def test_the_refusal_of_is_int4_comes_after_its_left_side(dynamodb_client, table, tail): + statement = f'SELECT * FROM "{table}" WHERE 1e126 IS INT4{tail}' + assert _message(dynamodb_client, statement) == OVERFLOW + "root" + + +def test_a_set_parameter_comes_after_the_operator_count_in_a_batch(dynamodb_client, table): + statement = f"UPDATE \"{table}\" SET q = ? WHERE pk = 'a' AND {NOTS}" + response = dynamodb_client.batch_execute_statement(Statements=[{"Statement": statement}]) + assert response["Responses"] == [ + {"Error": {"Code": "ValidationError", "Message": TOO_MANY_OPERATORS}} + ] + + +def test_a_set_parameter_comes_after_the_operator_count_in_a_transaction( + dynamodb_client, table +): + statement = f"UPDATE \"{table}\" SET q = ? WHERE pk = 'a' AND {NOTS}" + with pytest.raises(ClientError) as e: + dynamodb_client.execute_transaction(TransactStatements=[{"Statement": statement}]) + assert e.value.response["Error"]["Message"] == ( + f"Validation failed in TransactStatements[0]: {TOO_MANY_OPERATORS}" + ) diff --git a/tests/test_partiql_decomposed_reads.py b/tests/test_partiql_decomposed_reads.py new file mode 100644 index 000000000..220fc0056 --- /dev/null +++ b/tests/test_partiql_decomposed_reads.py @@ -0,0 +1,373 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""Dual-target wire tests for the number of disjuncts a PartiQL SELECT read +by key may split into. + +A SELECT whose every disjunct reads by key splits into at most 50 of them: +each value of an IN on a key attribute and each side of an OR of key +conditions is one, an AND multiplies them, and a disjunct held back for a +refusal made when it is read counts too. A 51st is refused with "Too many +decomposed read operations for a given query." on the first request, on a +table and on an index. The conditions of a filter are not counted, and a +statement with a disjunct that is a scan runs as a scan. + +The refusal comes after the operator count, the missing table, the scalar +key refusal, the key type check, the ORDER BY refusals and the overlap +refusal, and before the key size check and the refusal of a key attribute +in a filter. + +Dual-target: EXTENDDB_TEST_ENDPOINT selects an extenddb server; unset, the +tests run against the live service with the default credential chain. The +assertions are identical on both. Every expected value was measured on the +service (2026-11-03). +""" + +from __future__ import annotations + +import time +import uuid + +import pytest +from botocore.exceptions import ClientError + +from conftest import scoped_table + +TOO_MANY = "Too many decomposed read operations for a given query." +TOO_MANY_OPERATORS = "The expression contains too many operators; operator count: 301" +KEY_TYPE = "Key attribute's data type should match its data type in table's schema: Key {}" +SCALAR = ( + "Key attributes must be scalars; list random access '[]' and map lookup '.' are not " + "allowed: Key sk" +) +OVERLAP = "Overlapping conditions with range keys are not supported in where clause" +ORDER_BY_HASH = ( + "Must have hash key in ORDER BY clause when more than one hash key condition specified in " + "WHERE clause." +) +ORDER_BY_KEY = "Attribute n in ORDER BY clause must be part of the primary key" +BIG_HASH = "z" * 2049 +FILTER = " AND ".join(f"x{i:02} = 1" for i in range(1, 100)) + + +def _values(n, prefix="v", start=0): + """`n` string values `000`, `001`, ... from `start`.""" + return ", ".join(f"'{prefix}{i:03}'" for i in range(start, start + n)) + + +def _ors(values): + """An OR of equalities of pk with each of `values`.""" + return " OR ".join(f"pk = 'v{i:03}'" for i in values) + + +def _wait_for_index(client, table, count): + """Wait until the index gix shows every item written (it fills + asynchronously on the service).""" + for _ in range(240): + scan = client.scan(TableName=table, IndexName="gix", Select="COUNT") + if scan["Count"] == count: + return + time.sleep(0.5) + raise TimeoutError(f"gix of {table} did not reach {count} items") + + +@pytest.fixture(scope="module") +def htable(dynamodb_client): + """A table keyed on a string pk holding v000 to v029, each with x = 2.""" + with scoped_table(dynamodb_client) as name: + for i in range(30): + dynamodb_client.put_item( + TableName=name, Item={"pk": {"S": f"v{i:03}"}, "x": {"N": "2"}} + ) + yield name + + +@pytest.fixture(scope="module") +def ctable(dynamodb_client): + """A table keyed on string pk and sk holding v000/s to v029/s, each with + n = 1.""" + with scoped_table( + dynamodb_client, + attribute_definitions=[ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "sk", "AttributeType": "S"}, + ], + key_schema=[ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"}, + ], + ) as name: + for i in range(30): + dynamodb_client.put_item( + TableName=name, + Item={"pk": {"S": f"v{i:03}"}, "sk": {"S": "s"}, "n": {"N": "1"}}, + ) + yield name + + +@pytest.fixture(scope="module") +def gtable(dynamodb_client): + """A table keyed on pk with a GSI gix keyed on g, holding p000 to p029 + with g v000 to v029.""" + with scoped_table( + dynamodb_client, + attribute_definitions=[ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "g", "AttributeType": "S"}, + ], + GlobalSecondaryIndexes=[ + { + "IndexName": "gix", + "KeySchema": [{"AttributeName": "g", "KeyType": "HASH"}], + "Projection": {"ProjectionType": "ALL"}, + } + ], + ) as name: + for i in range(30): + dynamodb_client.put_item( + TableName=name, Item={"pk": {"S": f"p{i:03}"}, "g": {"S": f"v{i:03}"}} + ) + _wait_for_index(dynamodb_client, name, 30) + yield name + + +@pytest.fixture(scope="module") +def g2table(dynamodb_client): + """A table keyed on pk with a GSI gix keyed on g and gs, holding p000 to + p014 in index partition a with gs w000 to w014.""" + with scoped_table( + dynamodb_client, + attribute_definitions=[ + {"AttributeName": name, "AttributeType": "S"} for name in ("pk", "g", "gs") + ], + GlobalSecondaryIndexes=[ + { + "IndexName": "gix", + "KeySchema": [ + {"AttributeName": "g", "KeyType": "HASH"}, + {"AttributeName": "gs", "KeyType": "RANGE"}, + ], + "Projection": {"ProjectionType": "ALL"}, + } + ], + ) as name: + for i in range(15): + dynamodb_client.put_item( + TableName=name, + Item={"pk": {"S": f"p{i:03}"}, "g": {"S": "a"}, "gs": {"S": f"w{i:03}"}}, + ) + _wait_for_index(dynamodb_client, name, 15) + yield name + + +def _count(client, statement, **kwargs): + """How many items `statement` returns over all its pages, following + NextToken, or the message of the ValidationException that refuses it.""" + count, token = 0, None + for _ in range(40): + args = dict(kwargs) + if token: + args["NextToken"] = token + try: + response = client.execute_statement(Statement=statement, **args) + except ClientError as e: + assert e.response["Error"]["Code"] == "ValidationException", e.response + return e.response["Error"]["Message"] + count += len(response["Items"]) + token = response.get("NextToken") + if not token: + return count + raise AssertionError(f"more than 40 pages: {statement[:80]}") + + +# ---- what counts ------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("condition", "result"), + [ + (f"pk IN [{_values(50)}]", 30), + (f"pk IN [{_values(51)}]", TOO_MANY), + (f"pk IN [{_values(100)}]", TOO_MANY), + (_ors(range(50)), 30), + (_ors(range(51)), TOO_MANY), + (_ors(range(150)), TOO_MANY), + (f"pk IN [{_values(26)}] OR pk IN [{_values(25, start=26)}]", TOO_MANY), + # The conditions of a filter are not counted. + (f"pk IN [{_values(50)}] AND {FILTER}", 0), + (f"pk IN [{_values(99)}] AND x01 = 1", TOO_MANY), + # A statement with a disjunct that is a scan runs as a scan. + (f"pk IN [{_values(51)}] OR x = 1", 30), + (f"x = 1 OR pk IN [{_values(51)}]", 30), + ], + ids=[ + "in-50", + "in-51", + "in-100", + "or-50", + "or-51", + "or-150", + "two-in-lists", + "in-50-and-a-long-filter", + "in-99-and-a-short-filter", + "in-51-or-a-scan", + "a-scan-or-in-51", + ], +) +def test_a_read_by_partition_key_splits_into_at_most_50_disjuncts( + dynamodb_client, htable, condition, result +): + assert _count(dynamodb_client, f'SELECT * FROM "{htable}" WHERE {condition}') == result + + +@pytest.mark.parametrize( + ("condition", "result"), + [ + (f"pk IN [{_values(25)}] AND sk IN [{_values(2, 's')}]", 0), + (f"pk IN [{_values(10)}] AND sk IN [{_values(5, 's')}]", 0), + (f"pk IN [{_values(5)}] AND sk IN [{_values(10, 's')}]", 0), + (f"pk IN [{_values(17)}] AND sk IN [{_values(3, 's')}]", TOO_MANY), + (f"pk IN [{_values(10)}] AND sk IN [{_values(10, 's')}]", TOO_MANY), + (f"pk IN [{_values(50)}] AND sk = 's'", 30), + (f"pk IN [{_values(51)}] AND sk = 's'", TOO_MANY), + (f"pk IN [{_values(51)}] AND sk > 'a'", TOO_MANY), + (f"pk IN [{_values(26)}] AND (sk = 's' OR sk = 't')", TOO_MANY), + ( + f"(pk IN [{_values(26)}] AND sk = 's') OR " + f"(pk IN [{_values(25, start=26)}] AND sk = 't')", + TOO_MANY, + ), + (f"pk = 'v000' AND sk IN [{_values(50, 's')}]", 0), + (f"pk = 'v000' AND sk IN [{_values(51, 's')}]", TOO_MANY), + # Not every disjunct reads by key: a scan. + (f"pk IN [{_values(51)}] AND sk = 's' AND pk = 'v000'", 1), + ], + ids=[ + "25-by-2", + "10-by-5", + "5-by-10", + "17-by-3", + "10-by-10", + "items-50", + "items-51", + "ranges-51", + "in-26-by-two-sort-keys", + "26-or-25", + "sort-key-in-50", + "sort-key-in-51", + "a-second-partition-value", + ], +) +def test_a_read_by_both_keys_counts_every_disjunct_of_the_product( + dynamodb_client, ctable, condition, result +): + assert _count(dynamodb_client, f'SELECT * FROM "{ctable}" WHERE {condition}') == result + + +def test_parameters_are_counted(dynamodb_client, htable): + parameters = [{"S": f"v{i:03}"} for i in range(51)] + + def statement(n): + return f"SELECT * FROM \"{htable}\" WHERE pk IN [{', '.join(['?'] * n)}]" + + assert _count(dynamodb_client, statement(51), Parameters=parameters) == TOO_MANY + assert _count(dynamodb_client, statement(50), Parameters=parameters[:50]) == 30 + + +@pytest.mark.parametrize( + ("condition", "result"), + [ + (f"g IN [{_values(50)}]", 30), + (f"g IN [{_values(51)}]", TOO_MANY), + (f"g IN [{_values(50)}] AND {FILTER}", 0), + (f"g IN [{_values(99)}] AND x01 = 1", TOO_MANY), + ], + ids=["in-50", "in-51", "in-50-and-a-long-filter", "in-99-and-a-short-filter"], +) +def test_an_index_read_splits_into_at_most_50_disjuncts( + dynamodb_client, gtable, condition, result +): + statement = f'SELECT * FROM "{gtable}"."gix" WHERE {condition}' + assert _count(dynamodb_client, statement) == result + + +@pytest.mark.parametrize(("n", "result"), [(50, 15), (51, TOO_MANY)]) +def test_sort_key_values_of_one_index_partition_are_counted(dynamodb_client, g2table, n, result): + statement = f"SELECT * FROM \"{g2table}\".\"gix\" WHERE g = 'a' AND gs IN [{_values(n, 'w')}]" + assert _count(dynamodb_client, statement) == result + + +# ---- where the refusal comes ------------------------------------------------- + + +@pytest.mark.parametrize( + ("table", "condition", "result"), + [ + ("h", _ors(range(200)), TOO_MANY_OPERATORS), + ("c", f"pk IN [{_values(51)}] AND sk.x = 1", SCALAR), + ("h", f"pk IN [{_values(50)}, 1]", KEY_TYPE.format("pk")), + ("h", f"pk IN [1, {_values(50)}]", KEY_TYPE.format("pk")), + ("c", f"pk IN [{_values(51)}] AND sk = 1", KEY_TYPE.format("sk")), + ("c", f"pk IN [{_values(51)}] ORDER BY sk DESC", ORDER_BY_HASH), + ("c", f"pk IN [{_values(51)}] ORDER BY n", ORDER_BY_KEY), + ("h", "pk IN [" + ", ".join(f"'v{i % 26:03}'" for i in range(51)) + "]", OVERLAP), + ("h", "pk IN [" + ", ".join(f"'v{i % 51:03}'" for i in range(60)) + "]", OVERLAP), + ("h", "pk IN [" + ", ".join(["'v000'"] * 100) + "]", OVERLAP), + ("h", _ors(i % 26 for i in range(51)), OVERLAP), + ("h", f"pk IN [{_values(26)}] OR pk IN [{_values(26)}]", OVERLAP), + ("c", f"pk IN [{_values(26)}] AND (sk = 's' OR sk > 'a')", OVERLAP), + ], + ids=[ + "operator-count", + "scalar-key", + "key-type-last", + "key-type-first", + "sort-key-type", + "order-by-hash-key", + "order-by-other-attribute", + "overlap-51-of-26", + "overlap-60-of-51", + "overlap-100-copies", + "overlap-or", + "overlap-two-in-lists", + "overlap-sort-key", + ], +) +def test_the_refusals_checked_before_the_count( + dynamodb_client, htable, ctable, table, condition, result +): + name = htable if table == "h" else ctable + assert _count(dynamodb_client, f'SELECT * FROM "{name}" WHERE {condition}') == result + + +def test_a_missing_table_is_reported_before_the_count(dynamodb_client): + missing = f"extenddb-no-such-{uuid.uuid4().hex[:12]}" + statement = f'SELECT * FROM "{missing}" WHERE pk IN [{_values(51)}]' + with pytest.raises(ClientError) as e: + dynamodb_client.execute_statement(Statement=statement) + assert e.value.response["Error"]["Code"] == "ResourceNotFoundException" + assert e.value.response["Error"]["Message"] == "Requested resource not found" + + +@pytest.mark.parametrize( + ("table", "condition"), + [ + ("h", f"pk IN [{_values(50)}, '{BIG_HASH}']"), + ("h", f"pk IN ['{BIG_HASH}', {_values(50)}]"), + ("c", f"pk IN [{_values(51)}] AND sk = a"), + ("h", f"pk IN [{_values(51)}] ORDER BY pk"), + ], + ids=["oversized-last", "oversized-first", "key-attribute-in-the-filter", "order-by-pk"], +) +def test_the_count_comes_before_the_checks_made_when_a_disjunct_is_read( + dynamodb_client, htable, ctable, table, condition +): + name = htable if table == "h" else ctable + assert _count(dynamodb_client, f'SELECT * FROM "{name}" WHERE {condition}') == TOO_MANY + + +def test_a_write_names_its_item_with_one_value_of_each_key(dynamodb_client, htable): + statement = f'UPDATE "{htable}" SET q = 1 WHERE pk IN [{_values(2)}]' + assert _count(dynamodb_client, statement) == ( + "Where clause does not contain a mandatory equality on all key attributes" + ) diff --git a/tests/test_partiql_exists.py b/tests/test_partiql_exists.py new file mode 100644 index 000000000..7f4f6bdaa --- /dev/null +++ b/tests/test_partiql_exists.py @@ -0,0 +1,193 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""Dual-target wire tests for an EXISTS(SELECT ...) member of ExecuteTransaction +beside a member on a missing table. + +An EXISTS member on a missing table cancels the transaction with a +ResourceNotFound reason at its own position, and None for the other members. +An EXISTS member that is not a read of one item with a further condition fails +the whole request instead, even when another member names a missing table, and +so does one through an index. A projection, a projected call or an ORDER BY in +the SELECT leaves the member a plain condition check. + +Runs against real DynamoDB (EXTENDDB_TEST_ENDPOINT unset, AWS credentials) and +against an extenddb server (EXTENDDB_TEST_ENDPOINT set) with identical +assertions. Every expected value was measured on the service (2026-11-02). +Tables use the ``extenddb-measure-pqe-`` prefix and are deleted at the end. +""" + +from __future__ import annotations + +import uuid + +import pytest +from botocore.exceptions import ClientError + +from conftest import wait_for_active + +CANCELLED = "Transaction cancelled, please refer cancellation reasons for specific reasons" +NOT_FOUND = {"Code": "ResourceNotFound", "Message": "Requested resource not found"} +NONE = {"Code": "None"} + + +@pytest.fixture(scope="module") +def tables(dynamodb_client): + """A pk(S)/sk(S) table and the name of a table that does not exist.""" + prefix = f"extenddb-measure-pqe-{uuid.uuid4().hex[:12]}" + name = f"{prefix}-c" + dynamodb_client.create_table( + TableName=name, + BillingMode="PAY_PER_REQUEST", + AttributeDefinitions=[ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "sk", "AttributeType": "S"}, + ], + KeySchema=[ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"}, + ], + ) + try: + wait_for_active(dynamodb_client, name) + yield {"table": name, "absent": f"{prefix}-absent"} + finally: + try: + dynamodb_client.delete_table(TableName=name) + except ClientError: + pass + + +@pytest.fixture +def key(dynamodb_client, tables) -> str: + """The partition key of a fresh item (sk 's', n 7).""" + key = f"ex-{uuid.uuid4().hex[:8]}" + dynamodb_client.put_item( + TableName=tables["table"], + Item={"pk": {"S": key}, "sk": {"S": "s"}, "n": {"N": "7"}}, + ) + return key + + +def refusal(client, statements: list[dict]) -> dict: + with pytest.raises(ClientError) as excinfo: + client.execute_transaction(TransactStatements=statements) + return excinfo.value.response + + +def extra(client, tables, key: str): + item = client.get_item( + TableName=tables["table"], + Key={"pk": {"S": key}, "sk": {"S": "s"}}, + ConsistentRead=True, + )["Item"] + return item.get("extra") + + +@pytest.mark.parametrize("position", [0, 1]) +def test_an_exists_member_on_a_missing_table_is_not_found_at_its_position( + dynamodb_client, tables, key, position +): + exists = { + "Statement": ( + f"EXISTS(SELECT * FROM \"{tables['absent']}\" " + f"WHERE pk = '{key}' AND sk = 's' AND n = 7)" + ) + } + statements = [ + {"Statement": f"UPDATE \"{tables['table']}\" SET extra = 1 WHERE pk = '{key}' AND sk = 's'"} + ] + statements.insert(position, exists) + response = refusal(dynamodb_client, statements) + reasons = [NONE, NONE] + reasons[position] = NOT_FOUND + codes = ", ".join(reason["Code"] for reason in reasons) + assert response["Error"] == { + "Code": "TransactionCanceledException", + "Message": f"{CANCELLED} [{codes}]", + } + assert response["CancellationReasons"] == reasons + assert extra(dynamodb_client, tables, key) is None + + +def test_an_exists_member_with_a_partial_key_fails_the_request_beside_a_missing_table( + dynamodb_client, tables, key +): + response = refusal( + dynamodb_client, + [ + { + "Statement": ( + f"EXISTS(SELECT * FROM \"{tables['table']}\" WHERE pk = '{key}' AND n = 7)" + ) + }, + { + "Statement": ( + f"UPDATE \"{tables['absent']}\" SET extra = 1 WHERE pk = '{key}' AND sk = 's'" + ) + }, + ], + ) + assert response["Error"] == { + "Code": "ValidationException", + "Message": ( + "Validation failed in TransactStatements[0]: EXISTS() must contain a single item " + "read with additional condition" + ), + } + + +def test_an_exists_member_through_an_index_is_refused(dynamodb_client, tables, key): + response = refusal( + dynamodb_client, + [ + { + "Statement": ( + f"EXISTS(SELECT * FROM \"{tables['table']}\".\"idx\" " + f"WHERE pk = '{key}' AND sk = 's' AND n = 7)" + ) + }, + { + "Statement": ( + f"UPDATE \"{tables['table']}\" SET extra = 1 WHERE pk = '{key}' AND sk = 's'" + ) + }, + ], + ) + assert response["Error"] == { + "Code": "ValidationException", + "Message": ( + "Validation failed in TransactStatements[0]: Operations on indices are not " + "supported in transactions" + ), + } + assert extra(dynamodb_client, tables, key) is None + + +@pytest.mark.parametrize( + "select", + [ + "SELECT n FROM \"{t}\" WHERE pk = '{k}' AND sk = 's' AND n = 7", + "SELECT COUNT(*) FROM \"{t}\" WHERE pk = '{k}' AND sk = 's' AND n = 7", + "SELECT * FROM \"{t}\" WHERE pk = '{k}' AND sk = 's' AND n = 7 ORDER BY pk", + ], + ids=["projection", "count", "order-by"], +) +def test_an_exists_member_checks_only_its_condition(dynamodb_client, tables, key, select): + other = f"ex-{uuid.uuid4().hex[:8]}" + dynamodb_client.put_item( + TableName=tables["table"], Item={"pk": {"S": other}, "sk": {"S": "s"}} + ) + resp = dynamodb_client.execute_transaction( + TransactStatements=[ + {"Statement": f"EXISTS({select.format(t=tables['table'], k=key)})"}, + { + "Statement": ( + f"UPDATE \"{tables['table']}\" SET extra = 1 WHERE pk = '{other}' " + "AND sk = 's'" + ) + }, + ] + ) + assert resp["Responses"] == [] + assert extra(dynamodb_client, tables, other) == {"N": "1"} diff --git a/tests/test_partiql_forms.py b/tests/test_partiql_forms.py new file mode 100644 index 000000000..2dd3152b2 --- /dev/null +++ b/tests/test_partiql_forms.py @@ -0,0 +1,873 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""PartiQL statement forms and functions over the wire. + +Covers set_add and set_delete in an UPDATE SET, the FROM-first UPDATE +(``FROM "t" WHERE ... SET ... REMOVE ...``), a condition function compared with +a boolean, and ``EXISTS(SELECT ...)`` as an ExecuteTransaction member. + +Dual-target: identical assertions run against real DynamoDB and against an +ExtendDB server (selected by EXTENDDB_TEST_ENDPOINT, see conftest). Every +expected response and error Code+Message is the live service's. + +Two tables: a hash-key table pk(S), and a composite pk(S)/sk(S) table for the +partial-key cases. Each test uses its own partition key. +""" + +from __future__ import annotations + +import os +import uuid + +import boto3 +import pytest +from botocore.config import Config +from botocore.exceptions import ClientError + +from conftest import wait_for_active + +FULL_KEY = "Where clause does not contain a mandatory equality on all key attributes" +WRONG_TYPE = "An operand in the update expression has an incorrect data type" +NOT_WELL_FORMED = "Statement wasn't well formed, can't be processed: " + + +@pytest.fixture(scope="session") +def dynamodb_client(endpoint_url): + """DynamoDB client honoring EXTENDDB_TEST_PROFILE for the live-service run.""" + region = os.environ.get("AWS_DEFAULT_REGION", "us-east-1") + profile = os.environ.get("EXTENDDB_TEST_PROFILE") or None + session = boto3.Session(profile_name=profile, region_name=region) + kwargs: dict = {"service_name": "dynamodb"} + if endpoint_url: + kwargs["endpoint_url"] = endpoint_url + if endpoint_url.startswith("https://"): + kwargs["verify"] = False + kwargs["config"] = Config(parameter_validation=True) + return session.client(**kwargs) + + +def _create(client, name, key_schema, attribute_definitions): + client.create_table( + TableName=name, + AttributeDefinitions=attribute_definitions, + KeySchema=key_schema, + BillingMode="PAY_PER_REQUEST", + ) + wait_for_active(client, name) + + +def _drop(client, name): + try: + client.delete_table(TableName=name) + except client.exceptions.ResourceNotFoundException: + pass + + +@pytest.fixture(scope="module") +def htable(dynamodb_client): + """A hash-key table pk(S).""" + name = f"extenddb-pqforms-h-{uuid.uuid4().hex[:12]}" + _create( + dynamodb_client, + name, + [{"AttributeName": "pk", "KeyType": "HASH"}], + [{"AttributeName": "pk", "AttributeType": "S"}], + ) + try: + yield name + finally: + _drop(dynamodb_client, name) + + +@pytest.fixture(scope="module") +def ctable(dynamodb_client): + """A composite pk(S)/sk(S) table.""" + name = f"extenddb-pqforms-c-{uuid.uuid4().hex[:12]}" + _create( + dynamodb_client, + name, + [ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"}, + ], + [ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "sk", "AttributeType": "S"}, + ], + ) + try: + yield name + finally: + _drop(dynamodb_client, name) + + +def _exec(client, statement, parameters=None, **kwargs): + if parameters is not None: + kwargs["Parameters"] = parameters + return client.execute_statement(Statement=statement, **kwargs) + + +def _error(excinfo): + err = excinfo.value.response["Error"] + return err["Code"], err["Message"] + + +def _refusal(client, statement, parameters=None): + with pytest.raises(ClientError) as excinfo: + _exec(client, statement, parameters) + return _error(excinfo) + + +def _txn_refusal(client, statements): + with pytest.raises(ClientError) as excinfo: + client.execute_transaction(TransactStatements=statements) + return excinfo.value.response + + +def _get(client, table, pk): + return client.get_item(TableName=table, Key={"pk": {"S": pk}}, ConsistentRead=True).get( + "Item" + ) + + +def _seed_sets(client, table, pk): + client.put_item( + TableName=table, + Item={ + "pk": {"S": pk}, + "tags": {"SS": ["red", "green", "blue"]}, + "sizes": {"NS": ["10", "20", "30"]}, + "blobs": {"BS": [b"\x0a", b"\x0b"]}, + "label": {"S": "plain"}, + }, + ) + + +def _seed_plain(client, table, pk): + client.put_item( + TableName=table, + Item={"pk": {"S": pk}, "n": {"N": "7"}, "w": {"S": "word"}}, + ) + + +def _at(statement, token, start=0, length=None): + """`1::` of the first `token` at or after `start`.""" + index = statement.index(token, start) + return f"1:{index + 1}:{len(token) if length is None else length}" + + +def _numbers(values): + return sorted(float(v) for v in values) + + +# --- set_add -------------------------------------------------------------- + + +def test_set_add_adds_members_to_a_string_set(dynamodb_client, htable): + _seed_sets(dynamodb_client, htable, "sa-ss") + _exec( + dynamodb_client, + f"UPDATE \"{htable}\" SET tags = set_add(tags, <<'amber', 'white'>>) WHERE pk = 'sa-ss'", + ) + tags = _get(dynamodb_client, htable, "sa-ss")["tags"]["SS"] + assert sorted(tags) == ["amber", "blue", "green", "red", "white"] + + +def test_set_add_adds_members_to_a_number_set(dynamodb_client, htable): + _seed_sets(dynamodb_client, htable, "sa-ns") + _exec( + dynamodb_client, + f"UPDATE \"{htable}\" SET sizes = set_add(sizes, <<40, 10>>) WHERE pk = 'sa-ns'", + ) + sizes = _get(dynamodb_client, htable, "sa-ns")["sizes"]["NS"] + assert _numbers(sizes) == [10, 20, 30, 40] + + +def test_set_add_takes_a_binary_set_parameter(dynamodb_client, htable): + _seed_sets(dynamodb_client, htable, "sa-bs") + _exec( + dynamodb_client, + f"UPDATE \"{htable}\" SET blobs = set_add(blobs, ?) WHERE pk = 'sa-bs'", + [{"BS": [b"\x0b", b"\x0c"]}], + ) + blobs = _get(dynamodb_client, htable, "sa-bs")["blobs"]["BS"] + assert sorted(blobs) == [b"\x0a", b"\x0b", b"\x0c"] + + +def test_set_add_of_a_present_member_changes_nothing(dynamodb_client, htable): + _seed_sets(dynamodb_client, htable, "sa-same") + _exec( + dynamodb_client, + f"UPDATE \"{htable}\" SET tags = set_add(tags, <<'red'>>) WHERE pk = 'sa-same'", + ) + assert sorted(_get(dynamodb_client, htable, "sa-same")["tags"]["SS"]) == [ + "blue", + "green", + "red", + ] + + +def test_set_add_to_a_missing_attribute_creates_the_set(dynamodb_client, htable): + _seed_sets(dynamodb_client, htable, "sa-new") + _exec( + dynamodb_client, + f"UPDATE \"{htable}\" SET fresh = set_add(fresh, <<5>>) WHERE pk = 'sa-new'", + ) + assert _get(dynamodb_client, htable, "sa-new")["fresh"] == {"NS": ["5"]} + + +def test_set_add_name_is_case_insensitive(dynamodb_client, htable): + _seed_sets(dynamodb_client, htable, "sa-case") + _exec( + dynamodb_client, + f"UPDATE \"{htable}\" SET tags = Set_Add(tags, <<'teal'>>) WHERE pk = 'sa-case'", + ) + assert "teal" in _get(dynamodb_client, htable, "sa-case")["tags"]["SS"] + + +def test_set_add_in_two_set_clauses(dynamodb_client, htable): + _seed_sets(dynamodb_client, htable, "sa-two") + _exec( + dynamodb_client, + f"UPDATE \"{htable}\" SET tags = set_add(tags, <<'teal'>>) " + f"SET sizes = set_add(sizes, <<50>>) WHERE pk = 'sa-two'", + ) + item = _get(dynamodb_client, htable, "sa-two") + assert "teal" in item["tags"]["SS"] + assert _numbers(item["sizes"]["NS"]) == [10, 20, 30, 50] + + +def test_set_add_beside_a_set_and_a_remove(dynamodb_client, htable): + _seed_sets(dynamodb_client, htable, "sa-mixed") + _exec( + dynamodb_client, + f"UPDATE \"{htable}\" SET tags = set_add(tags, <<'teal'>>), note = 'n' " + f"REMOVE label WHERE pk = 'sa-mixed'", + ) + item = _get(dynamodb_client, htable, "sa-mixed") + assert "teal" in item["tags"]["SS"] + assert item["note"] == {"S": "n"} + assert "label" not in item + + +def test_set_add_returning_all_new(dynamodb_client, htable): + _seed_sets(dynamodb_client, htable, "sa-ret") + resp = _exec( + dynamodb_client, + f"UPDATE \"{htable}\" SET tags = set_add(tags, <<'teal'>>) WHERE pk = 'sa-ret' " + f"RETURNING ALL NEW *", + ) + assert len(resp["Items"]) == 1 + assert sorted(resp["Items"][0]["tags"]["SS"]) == ["blue", "green", "red", "teal"] + + +def test_set_add_on_a_missing_item_fails_its_condition(dynamodb_client, htable): + code, message = _refusal( + dynamodb_client, + f"UPDATE \"{htable}\" SET tags = set_add(tags, <<'teal'>>) WHERE pk = 'sa-absent'", + ) + assert (code, message) == ("ConditionalCheckFailedException", "The conditional request failed") + assert _get(dynamodb_client, htable, "sa-absent") is None + + +def test_set_add_in_execute_transaction(dynamodb_client, htable): + _seed_sets(dynamodb_client, htable, "sa-txn") + resp = dynamodb_client.execute_transaction( + TransactStatements=[ + { + "Statement": f"UPDATE \"{htable}\" SET tags = set_add(tags, <<'teal'>>) " + f"WHERE pk = 'sa-txn'" + } + ] + ) + assert resp["Responses"] == [] + assert "teal" in _get(dynamodb_client, htable, "sa-txn")["tags"]["SS"] + + +def test_set_add_in_batch_execute_statement(dynamodb_client, htable): + _seed_sets(dynamodb_client, htable, "sa-batch") + resp = dynamodb_client.batch_execute_statement( + Statements=[ + { + "Statement": f"UPDATE \"{htable}\" SET tags = set_add(tags, <<'teal'>>) " + f"WHERE pk = 'sa-batch'" + } + ] + ) + assert resp["Responses"] == [{"TableName": htable}] + assert "teal" in _get(dynamodb_client, htable, "sa-batch")["tags"]["SS"] + + +def test_set_add_of_a_number_to_a_string_set_is_refused(dynamodb_client, htable): + _seed_sets(dynamodb_client, htable, "sa-mismatch") + code, message = _refusal( + dynamodb_client, + f"UPDATE \"{htable}\" SET tags = set_add(tags, <<1>>) WHERE pk = 'sa-mismatch'", + ) + assert (code, message) == ("ValidationException", WRONG_TYPE) + + +def test_set_add_to_an_attribute_that_is_not_a_set_is_refused(dynamodb_client, htable): + _seed_sets(dynamodb_client, htable, "sa-scalar") + code, message = _refusal( + dynamodb_client, + f"UPDATE \"{htable}\" SET label = set_add(label, <<'x'>>) WHERE pk = 'sa-scalar'", + ) + assert (code, message) == ("ValidationException", WRONG_TYPE) + assert _get(dynamodb_client, htable, "sa-scalar")["label"] == {"S": "plain"} + + +def test_set_add_of_a_string_parameter_is_refused(dynamodb_client, htable): + _seed_sets(dynamodb_client, htable, "sa-sparam") + statement = f"UPDATE \"{htable}\" SET tags = set_add(tags, ?) WHERE pk = 'sa-sparam'" + code, message = _refusal(dynamodb_client, statement, [{"S": "teal"}]) + assert code == "ValidationException" + assert message == ( + "The second argument to SET_ADD must be a value with type SET at " + + _at(statement, "tags", statement.index("set_add(")) + ) + + +def test_set_add_of_a_number_parameter_to_a_missing_attribute_is_refused( + dynamodb_client, htable +): + _seed_sets(dynamodb_client, htable, "sa-nparam") + statement = f"UPDATE \"{htable}\" SET fresh = set_add(fresh, ?) WHERE pk = 'sa-nparam'" + code, message = _refusal(dynamodb_client, statement, [{"N": "5"}]) + assert code == "ValidationException" + assert message == ( + "The second argument to SET_ADD must be a value with type SET at " + + _at(statement, "fresh", statement.index("set_add(")) + ) + assert "fresh" not in _get(dynamodb_client, htable, "sa-nparam") + + +def test_set_add_of_an_attribute_is_refused(dynamodb_client, htable): + _seed_sets(dynamodb_client, htable, "sa-path") + statement = f"UPDATE \"{htable}\" SET tags = set_add(tags, label) WHERE pk = 'sa-path'" + code, message = _refusal(dynamodb_client, statement) + assert code == "ValidationException" + assert message == ( + "The second argument to SET_ADD must be a value with type SET at " + + _at(statement, "tags", statement.index("set_add(")) + ) + + +def test_set_add_with_no_arguments_is_refused(dynamodb_client, htable): + statement = f"UPDATE \"{htable}\" SET tags = set_add() WHERE pk = 'sa-noargs'" + code, message = _refusal(dynamodb_client, statement) + assert code == "ValidationException" + assert message == "SET_ADD must have exactly two arguments at " + _at(statement, "set_add") + + +# --- set_delete ----------------------------------------------------------- + + +def test_set_delete_removes_members(dynamodb_client, htable): + _seed_sets(dynamodb_client, htable, "sd-some") + _exec( + dynamodb_client, + f"UPDATE \"{htable}\" SET tags = set_delete(tags, <<'red', 'blue'>>) WHERE pk = 'sd-some'", + ) + assert _get(dynamodb_client, htable, "sd-some")["tags"] == {"SS": ["green"]} + + +def test_set_delete_removes_a_number(dynamodb_client, htable): + _seed_sets(dynamodb_client, htable, "sd-ns") + _exec( + dynamodb_client, + f"UPDATE \"{htable}\" SET sizes = set_delete(sizes, <<20>>) WHERE pk = 'sd-ns'", + ) + assert _numbers(_get(dynamodb_client, htable, "sd-ns")["sizes"]["NS"]) == [10, 30] + + +def test_set_delete_of_every_member_removes_the_attribute(dynamodb_client, htable): + _seed_sets(dynamodb_client, htable, "sd-all") + _exec( + dynamodb_client, + f"UPDATE \"{htable}\" SET tags = set_delete(tags, <<'red', 'green', 'blue'>>) " + f"WHERE pk = 'sd-all'", + ) + assert "tags" not in _get(dynamodb_client, htable, "sd-all") + + +def test_set_delete_of_an_absent_member_changes_nothing(dynamodb_client, htable): + _seed_sets(dynamodb_client, htable, "sd-absent") + _exec( + dynamodb_client, + f"UPDATE \"{htable}\" SET tags = set_delete(tags, <<'black'>>) WHERE pk = 'sd-absent'", + ) + assert sorted(_get(dynamodb_client, htable, "sd-absent")["tags"]["SS"]) == [ + "blue", + "green", + "red", + ] + + +def test_set_delete_from_a_missing_attribute_leaves_it_missing(dynamodb_client, htable): + _seed_sets(dynamodb_client, htable, "sd-missing") + _exec( + dynamodb_client, + f"UPDATE \"{htable}\" SET fresh = set_delete(fresh, <<'x'>>) WHERE pk = 'sd-missing'", + ) + assert "fresh" not in _get(dynamodb_client, htable, "sd-missing") + + +def test_set_delete_of_a_number_from_a_string_set_is_refused(dynamodb_client, htable): + _seed_sets(dynamodb_client, htable, "sd-mismatch") + code, message = _refusal( + dynamodb_client, + f"UPDATE \"{htable}\" SET tags = set_delete(tags, <<1>>) WHERE pk = 'sd-mismatch'", + ) + assert (code, message) == ("ValidationException", WRONG_TYPE) + + +def test_set_delete_of_a_string_parameter_is_refused(dynamodb_client, htable): + _seed_sets(dynamodb_client, htable, "sd-sparam") + statement = f"UPDATE \"{htable}\" SET tags = set_delete(tags, ?) WHERE pk = 'sd-sparam'" + code, message = _refusal(dynamodb_client, statement, [{"S": "red"}]) + assert code == "ValidationException" + assert message == ( + "The second argument to SET_DELETE must be a value with type SET at " + + _at(statement, "tags", statement.index("set_delete(")) + ) + assert sorted(_get(dynamodb_client, htable, "sd-sparam")["tags"]["SS"]) == [ + "blue", + "green", + "red", + ] + + +# --- FROM-first UPDATE ---------------------------------------------------- + + +def test_from_first_update_sets_an_attribute(dynamodb_client, htable): + _seed_plain(dynamodb_client, htable, "ff-set") + resp = _exec(dynamodb_client, f"FROM \"{htable}\" WHERE pk = 'ff-set' SET extra = 'e'") + assert resp["Items"] == [] + assert _get(dynamodb_client, htable, "ff-set") == { + "pk": {"S": "ff-set"}, + "n": {"N": "7"}, + "w": {"S": "word"}, + "extra": {"S": "e"}, + } + + +def test_from_first_update_removes_an_attribute(dynamodb_client, htable): + _seed_plain(dynamodb_client, htable, "ff-remove") + _exec(dynamodb_client, f"FROM \"{htable}\" WHERE pk = 'ff-remove' REMOVE w") + assert _get(dynamodb_client, htable, "ff-remove") == { + "pk": {"S": "ff-remove"}, + "n": {"N": "7"}, + } + + +def test_from_first_update_set_then_remove(dynamodb_client, htable): + _seed_plain(dynamodb_client, htable, "ff-both") + _exec(dynamodb_client, f"FROM \"{htable}\" WHERE pk = 'ff-both' SET n = n + 1 REMOVE w") + assert _get(dynamodb_client, htable, "ff-both") == { + "pk": {"S": "ff-both"}, + "n": {"N": "8"}, + } + + +def test_from_first_update_remove_then_set(dynamodb_client, htable): + _seed_plain(dynamodb_client, htable, "ff-rs") + _exec(dynamodb_client, f"FROM \"{htable}\" WHERE pk = 'ff-rs' REMOVE w SET extra = 1") + assert _get(dynamodb_client, htable, "ff-rs") == { + "pk": {"S": "ff-rs"}, + "n": {"N": "7"}, + "extra": {"N": "1"}, + } + + +def test_update_remove_then_set(dynamodb_client, htable): + _seed_plain(dynamodb_client, htable, "up-rs") + _exec(dynamodb_client, f"UPDATE \"{htable}\" REMOVE w SET extra = 1 WHERE pk = 'up-rs'") + assert _get(dynamodb_client, htable, "up-rs") == { + "pk": {"S": "up-rs"}, + "n": {"N": "7"}, + "extra": {"N": "1"}, + } + + +def test_a_comma_after_a_remove_path_is_a_leftover_token(dynamodb_client, htable): + item = {"pk": {"S": "rm-comma"}, "m": {"M": {"a": {"N": "0"}}}, "w": {"S": "word"}} + dynamodb_client.put_item(TableName=htable, Item=item) + code, message = _refusal( + dynamodb_client, f"UPDATE \"{htable}\" REMOVE m.a, m WHERE pk = 'rm-comma'" + ) + assert (code, message) == ( + "ValidationException", + NOT_WELL_FORMED + "Unexpected token after expression", + ) + assert _get(dynamodb_client, htable, "rm-comma") == item + + +@pytest.mark.parametrize( + "statement", + [ + "UPDATE \"{t}\" SET n = 1 REMOVE w, extra WHERE pk = '{pk}'", + "UPDATE \"{t}\" REMOVE w, extra SET n = 1 WHERE pk = '{pk}'", + "FROM \"{t}\" WHERE pk = '{pk}' REMOVE w, extra", + ], +) +def test_a_remove_takes_one_path(dynamodb_client, htable, statement): + pk = f"rm-one-{uuid.uuid4().hex[:8]}" + _seed_plain(dynamodb_client, htable, pk) + code, message = _refusal(dynamodb_client, statement.format(t=htable, pk=pk)) + assert (code, message) == ( + "ValidationException", + NOT_WELL_FORMED + "Unexpected token after expression", + ) + assert _get(dynamodb_client, htable, pk) == { + "pk": {"S": pk}, + "n": {"N": "7"}, + "w": {"S": "word"}, + } + + +def test_two_remove_clauses_remove_two_paths(dynamodb_client, htable): + _seed_plain(dynamodb_client, htable, "rm-two") + _exec(dynamodb_client, f"UPDATE \"{htable}\" REMOVE w REMOVE n WHERE pk = 'rm-two'") + assert _get(dynamodb_client, htable, "rm-two") == {"pk": {"S": "rm-two"}} + + +def test_from_first_update_returning_all_new(dynamodb_client, htable): + _seed_plain(dynamodb_client, htable, "ff-ret") + resp = _exec( + dynamodb_client, + f"FROM \"{htable}\" WHERE pk = 'ff-ret' SET extra = 2 RETURNING ALL NEW *", + ) + assert resp["Items"] == [ + {"pk": {"S": "ff-ret"}, "n": {"N": "7"}, "w": {"S": "word"}, "extra": {"N": "2"}} + ] + + +def test_from_first_update_keywords_are_case_insensitive(dynamodb_client, htable): + _seed_plain(dynamodb_client, htable, "ff-lower") + _exec(dynamodb_client, f"from \"{htable}\" where pk = 'ff-lower' set extra = 3") + assert _get(dynamodb_client, htable, "ff-lower")["extra"] == {"N": "3"} + + +def test_from_first_update_takes_parameters(dynamodb_client, htable): + _seed_plain(dynamodb_client, htable, "ff-params") + _exec( + dynamodb_client, + f"FROM \"{htable}\" WHERE pk = ? SET extra = ?", + [{"S": "ff-params"}, {"S": "via-param"}], + ) + assert _get(dynamodb_client, htable, "ff-params")["extra"] == {"S": "via-param"} + + +def test_from_first_update_condition_that_holds(dynamodb_client, htable): + _seed_plain(dynamodb_client, htable, "ff-cond-ok") + _exec(dynamodb_client, f"FROM \"{htable}\" WHERE pk = 'ff-cond-ok' AND n = 7 SET extra = 4") + assert _get(dynamodb_client, htable, "ff-cond-ok")["extra"] == {"N": "4"} + + +def test_from_first_update_condition_that_fails(dynamodb_client, htable): + _seed_plain(dynamodb_client, htable, "ff-cond-no") + code, message = _refusal( + dynamodb_client, f"FROM \"{htable}\" WHERE pk = 'ff-cond-no' AND n = 8 SET extra = 4" + ) + assert (code, message) == ("ConditionalCheckFailedException", "The conditional request failed") + assert "extra" not in _get(dynamodb_client, htable, "ff-cond-no") + + +def test_from_first_update_of_a_missing_item_fails_its_condition(dynamodb_client, htable): + code, message = _refusal( + dynamodb_client, f"FROM \"{htable}\" WHERE pk = 'ff-absent' SET extra = 1" + ) + assert (code, message) == ("ConditionalCheckFailedException", "The conditional request failed") + assert _get(dynamodb_client, htable, "ff-absent") is None + + +def test_from_first_update_in_execute_transaction(dynamodb_client, htable): + _seed_plain(dynamodb_client, htable, "ff-txn") + resp = dynamodb_client.execute_transaction( + TransactStatements=[ + {"Statement": f"FROM \"{htable}\" WHERE pk = 'ff-txn' SET extra = 5"} + ] + ) + assert resp["Responses"] == [] + assert _get(dynamodb_client, htable, "ff-txn")["extra"] == {"N": "5"} + + +def test_from_first_update_in_batch_execute_statement(dynamodb_client, htable): + _seed_plain(dynamodb_client, htable, "ff-batch") + resp = dynamodb_client.batch_execute_statement( + Statements=[{"Statement": f"FROM \"{htable}\" WHERE pk = 'ff-batch' SET extra = 6"}] + ) + assert resp["Responses"] == [{"TableName": htable}] + assert _get(dynamodb_client, htable, "ff-batch")["extra"] == {"N": "6"} + + +@pytest.mark.parametrize("clause", ["SET extra = 1", "REMOVE w"]) +def test_from_first_update_without_where_is_refused(dynamodb_client, htable, clause): + code, message = _refusal(dynamodb_client, f"FROM \"{htable}\" {clause}") + assert (code, message) == ("ValidationException", FULL_KEY) + + +def test_from_first_update_without_where_on_a_missing_table(dynamodb_client, htable): + code, message = _refusal(dynamodb_client, f"FROM \"{htable}-absent\" SET extra = 1") + assert (code, message) == ("ResourceNotFoundException", "Requested resource not found") + + +def test_from_first_update_with_a_non_key_where_is_refused(dynamodb_client, htable): + code, message = _refusal(dynamodb_client, f"FROM \"{htable}\" WHERE n = 7 SET extra = 1") + assert (code, message) == ("ValidationException", FULL_KEY) + + +def test_from_first_update_with_a_partial_key_is_refused(dynamodb_client, ctable): + code, message = _refusal(dynamodb_client, f"FROM \"{ctable}\" WHERE pk = 'ff-p' SET extra = 1") + assert (code, message) == ("ValidationException", FULL_KEY) + + +def test_from_first_update_on_a_missing_table(dynamodb_client, htable): + code, message = _refusal( + dynamodb_client, f"FROM \"{htable}-absent\" WHERE pk = 'ff-x' SET extra = 1" + ) + assert (code, message) == ("ResourceNotFoundException", "Requested resource not found") + + +def test_from_first_update_without_a_clause_is_refused(dynamodb_client, htable): + code, message = _refusal(dynamodb_client, f"FROM \"{htable}\" WHERE pk = 'ff-none'") + assert (code, message) == ("ValidationException", NOT_WELL_FORMED + "Expected data manipulation") + + +# --- a condition function compared with a boolean ------------------------- + + +@pytest.mark.parametrize( + ("condition", "matches"), + [ + ("attribute_exists(w) = true", True), + ("attribute_exists(absent) = true", False), + ("attribute_exists(w) = false", False), + ("attribute_exists(absent) = false", True), + ("attribute_exists(w) <> true", False), + ("attribute_exists(w) <> false", True), + ("attribute_not_exists(absent) = true", True), + ("begins_with(w, 'wo') = true", True), + ("begins_with(w, 'xx') = true", False), + ], +) +def test_condition_function_compared_with_a_boolean(dynamodb_client, htable, condition, matches): + _seed_plain(dynamodb_client, htable, "cb-item") + resp = _exec( + dynamodb_client, + f"SELECT pk FROM \"{htable}\" WHERE pk = 'cb-item' AND {condition}", + ConsistentRead=True, + ) + assert resp["Items"] == ([{"pk": {"S": "cb-item"}}] if matches else []) + + +def test_update_condition_compared_with_true(dynamodb_client, htable): + _seed_plain(dynamodb_client, htable, "cb-upd") + _exec( + dynamodb_client, + f"UPDATE \"{htable}\" SET extra = 1 WHERE pk = 'cb-upd' AND attribute_exists(w) = true", + ) + assert _get(dynamodb_client, htable, "cb-upd")["extra"] == {"N": "1"} + code, message = _refusal( + dynamodb_client, + f"UPDATE \"{htable}\" SET extra = 2 WHERE pk = 'cb-upd' " + f"AND attribute_exists(absent) = true", + ) + assert (code, message) == ("ConditionalCheckFailedException", "The conditional request failed") + + +# --- function operands in a WHERE and an INSERT --------------------------- + + +def test_attribute_not_exists_with_a_string_is_refused(dynamodb_client, htable): + code, message = _refusal( + dynamodb_client, + f"SELECT * FROM \"{htable}\" WHERE pk = 'fo-x' AND attribute_not_exists('w')", + ) + assert (code, message) == ( + "ValidationException", + "Operator or function requires a document path; " + "operator or function: attribute_not_exists", + ) + + +def test_an_unknown_function_on_the_right_of_a_comparison_is_refused(dynamodb_client, htable): + statement = f"SELECT * FROM \"{htable}\" WHERE pk = 'fo-x' AND w = upper(w)" + code, message = _refusal(dynamodb_client, statement) + assert (code, message) == ( + "ValidationException", + "Unrecognized function: upper at " + _at(statement, "upper"), + ) + + +def test_list_append_as_an_insert_value_is_refused(dynamodb_client, htable): + statement = f"INSERT INTO \"{htable}\" VALUE {{'pk': 'fo-ins', 'l': list_append([1], [2])}}" + code, message = _refusal(dynamodb_client, statement) + assert (code, message) == ( + "ValidationException", + "Unsupported data type: list_append under key root at " + _at(statement, "list_append"), + ) + assert _get(dynamodb_client, htable, "fo-ins") is None + + +def test_a_name_as_an_insert_value_is_refused_naming_itself(dynamodb_client, htable): + statement = f"INSERT INTO \"{htable}\" VALUE {{'pk': pk}}" + code, message = _refusal(dynamodb_client, statement) + assert (code, message) == ( + "ValidationException", + "Unsupported data type: pk under key root at " + _at(statement, "pk}", length=2), + ) + + +# --- EXISTS(SELECT ...) in ExecuteTransaction ----------------------------- + +EXISTS_REFUSAL = "EXISTS() must contain a single item read with additional condition" + + +def test_exists_without_a_further_condition_is_refused(dynamodb_client, htable): + _seed_plain(dynamodb_client, htable, "ex-bare") + response = _txn_refusal( + dynamodb_client, + [ + {"Statement": f"EXISTS(SELECT * FROM \"{htable}\" WHERE pk = 'ex-bare')"}, + {"Statement": f"UPDATE \"{htable}\" SET extra = 1 WHERE pk = 'ex-bare'"}, + ], + ) + assert response["Error"]["Code"] == "ValidationException" + assert ( + response["Error"]["Message"] == f"Validation failed in TransactStatements[0]: {EXISTS_REFUSAL}" + ) + assert "extra" not in _get(dynamodb_client, htable, "ex-bare") + + +def test_exists_refusal_names_its_position(dynamodb_client, htable): + _seed_plain(dynamodb_client, htable, "ex-pos") + response = _txn_refusal( + dynamodb_client, + [ + {"Statement": f"UPDATE \"{htable}\" SET extra = 1 WHERE pk = 'ex-pos'"}, + {"Statement": f"EXISTS(SELECT * FROM \"{htable}\" WHERE pk = 'ex-other')"}, + ], + ) + assert response["Error"]["Code"] == "ValidationException" + assert ( + response["Error"]["Message"] == f"Validation failed in TransactStatements[1]: {EXISTS_REFUSAL}" + ) + + +def test_exists_with_a_partial_key_is_refused(dynamodb_client, ctable): + response = _txn_refusal( + dynamodb_client, + [ + {"Statement": f"EXISTS(SELECT * FROM \"{ctable}\" WHERE pk = 'ex-p' AND n = 7)"}, + {"Statement": f"UPDATE \"{ctable}\" SET extra = 1 WHERE pk = 'ex-p' AND sk = 's'"}, + ], + ) + assert response["Error"]["Code"] == "ValidationException" + assert ( + response["Error"]["Message"] == f"Validation failed in TransactStatements[0]: {EXISTS_REFUSAL}" + ) + + +def test_exists_that_holds_lets_the_transaction_write(dynamodb_client, htable): + _seed_plain(dynamodb_client, htable, "ex-ok") + _seed_plain(dynamodb_client, htable, "ex-ok-target") + resp = dynamodb_client.execute_transaction( + TransactStatements=[ + {"Statement": f"EXISTS(SELECT * FROM \"{htable}\" WHERE pk = 'ex-ok' AND n = 7)"}, + {"Statement": f"UPDATE \"{htable}\" SET extra = 1 WHERE pk = 'ex-ok-target'"}, + ] + ) + assert resp["Responses"] == [] + assert _get(dynamodb_client, htable, "ex-ok-target")["extra"] == {"N": "1"} + + +def test_exists_that_fails_cancels_the_transaction(dynamodb_client, htable): + _seed_plain(dynamodb_client, htable, "ex-no") + _seed_plain(dynamodb_client, htable, "ex-no-target") + response = _txn_refusal( + dynamodb_client, + [ + {"Statement": f"EXISTS(SELECT * FROM \"{htable}\" WHERE pk = 'ex-no' AND n = 8)"}, + {"Statement": f"UPDATE \"{htable}\" SET extra = 1 WHERE pk = 'ex-no-target'"}, + ], + ) + assert response["Error"]["Code"] == "TransactionCanceledException" + assert response["Error"]["Message"] == ( + "Transaction cancelled, please refer cancellation reasons for specific reasons " + "[ConditionalCheckFailed, None]" + ) + assert [r["Code"] for r in response["CancellationReasons"]] == [ + "ConditionalCheckFailed", + "None", + ] + assert "extra" not in _get(dynamodb_client, htable, "ex-no-target") + + +def test_exists_on_a_missing_item_whose_condition_holds_lets_the_write_run(dynamodb_client, htable): + _seed_plain(dynamodb_client, htable, "ex-gone-target") + dynamodb_client.execute_transaction( + TransactStatements=[ + { + "Statement": f"EXISTS(SELECT * FROM \"{htable}\" WHERE pk = 'ex-gone' " + f"AND w IS MISSING)" + }, + {"Statement": f"UPDATE \"{htable}\" SET extra = 1 WHERE pk = 'ex-gone-target'"}, + ] + ) + assert _get(dynamodb_client, htable, "ex-gone-target")["extra"] == {"N": "1"} + + +def test_exists_alone_is_a_transaction(dynamodb_client, htable): + _seed_plain(dynamodb_client, htable, "ex-alone") + resp = dynamodb_client.execute_transaction( + TransactStatements=[ + {"Statement": f"EXISTS(SELECT * FROM \"{htable}\" WHERE pk = 'ex-alone' AND n = 7)"} + ] + ) + assert resp["Responses"] == [] + + +def test_exists_beside_a_select_mixes_reads_and_writes(dynamodb_client, htable): + _seed_plain(dynamodb_client, htable, "ex-mix") + response = _txn_refusal( + dynamodb_client, + [ + {"Statement": f"EXISTS(SELECT * FROM \"{htable}\" WHERE pk = 'ex-mix' AND n = 7)"}, + {"Statement": f"SELECT * FROM \"{htable}\" WHERE pk = 'ex-mix'"}, + ], + ) + assert response["Error"]["Code"] == "ValidationException" + assert response["Error"]["Message"] == ( + "ExecuteTransaction API does not support both read and write operations in the same " + "request." + ) + + +def test_exists_in_execute_statement_is_refused(dynamodb_client, htable): + code, message = _refusal( + dynamodb_client, f"EXISTS(SELECT * FROM \"{htable}\" WHERE pk = 'ex-single' AND n = 7)" + ) + assert (code, message) == ( + "ValidationException", + "EXISTS can only be used in ExecuteTransaction write requests.", + ) + + +def test_exists_in_batch_execute_statement_is_refused(dynamodb_client, htable): + resp = dynamodb_client.batch_execute_statement( + Statements=[ + {"Statement": f"EXISTS(SELECT * FROM \"{htable}\" WHERE pk = 'ex-batch' AND n = 7)"} + ] + ) + assert resp["Responses"] == [ + { + "Error": { + "Code": "ValidationError", + "Message": "EXISTS can only be used in ExecuteTransaction write requests.", + } + } + ] diff --git a/tests/test_partiql_grammar.py b/tests/test_partiql_grammar.py new file mode 100644 index 000000000..b89de1d2d --- /dev/null +++ b/tests/test_partiql_grammar.py @@ -0,0 +1,1497 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""Dual-target wire tests for how PartiQL statements are read: comments, +number text, IS tests, the syntax error a statement gets where it breaks off, +table aliases, reserved words in paths, OR branches that overlap on the sort +key, statement ends, unquoted and empty names, the table name rule, operators +a condition refuses, and bare expressions. + +Dual-target: EXTENDDB_TEST_ENDPOINT selects an extenddb server; unset, the +tests run against the live service with the default credential chain. The +assertions are identical on both. +""" + +from __future__ import annotations + +import uuid + +import pytest +from botocore.exceptions import ClientError + +from conftest import scoped_table, wait_for_active + +WF = "Statement wasn't well formed, can't be processed: " +ALIASING = "Aliasing is not supported" +OVERLAP = "Overlapping conditions with range keys are not supported in where clause" + + +@pytest.fixture(scope="module") +def stable(dynamodb_client): + """A table keyed on a string pk.""" + with scoped_table(dynamodb_client) as name: + yield name + + +@pytest.fixture(scope="module") +def ntable(dynamodb_client): + """A table keyed on a number pk.""" + with scoped_table( + dynamodb_client, + attribute_definitions=[{"AttributeName": "pk", "AttributeType": "N"}], + key_schema=[{"AttributeName": "pk", "KeyType": "HASH"}], + ) as name: + yield name + + +@pytest.fixture(scope="module") +def ctable(dynamodb_client): + """A table keyed on a string pk and a string sk.""" + with scoped_table( + dynamodb_client, + attribute_definitions=[ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "sk", "AttributeType": "S"}, + ], + key_schema=[ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"}, + ], + ) as name: + yield name + + +def _select(client, statement, **kwargs): + return client.execute_statement(Statement=statement, ConsistentRead=True, **kwargs).get( + "Items", [] + ) + + +def _err(client, statement, **kwargs): + with pytest.raises(ClientError) as e: + client.execute_statement(Statement=statement, **kwargs) + err = e.value.response["Error"] + return err["Code"], err["Message"] + + +def _message(client, statement, **kwargs): + code, message = _err(client, statement, **kwargs) + assert code == "ValidationException", message + return message + + +# ---- comments ---------------------------------------------------------------- + + +@pytest.mark.parametrize( + "template", + [ + "SELECT * FROM \"{t}\" WHERE pk = 'cm-a' -- a note", + "SELECT * FROM \"{t}\" -- a note\nWHERE pk = 'cm-a'", + "-- a note\nSELECT * FROM \"{t}\" WHERE pk = 'cm-a'", + "SELECT * FROM \"{t}\" /* a note */ WHERE pk = 'cm-a'", + "SELECT * FROM \"{t}\" WHERE pk = 'cm-a' /* a note */", + "SELECT * FROM \"{t}\" /* a\nlonger note */ WHERE pk = 'cm-a'", + ], +) +def test_comments_read_as_whitespace(dynamodb_client, stable, template): + dynamodb_client.put_item(TableName=stable, Item={"pk": {"S": "cm-a"}}) + dynamodb_client.put_item(TableName=stable, Item={"pk": {"S": "cm-b"}}) + items = _select(dynamodb_client, template.format(t=stable)) + assert [i["pk"]["S"] for i in items] == ["cm-a"] + + +@pytest.mark.parametrize( + "tail", + [ + "# note", + "/* note", + "'unterminated", + '"unterminated', + "& 1", + "~ 1", + "\\ 1", + ], +) +def test_text_the_lexer_cannot_read_has_nothing_after_the_prefix(dynamodb_client, stable, tail): + statement = f"SELECT * FROM \"{stable}\" WHERE pk = 'x' AND a = 1 {tail}" + assert _message(dynamodb_client, statement) == WF + + +# ---- numbers ----------------------------------------------------------------- + + +def test_numbers_written_from_the_decimal_point(dynamodb_client, ntable): + for n in ("7.3", "-7.3"): + dynamodb_client.put_item(TableName=ntable, Item={"pk": {"N": n}}) + assert [i["pk"]["N"] for i in _select(dynamodb_client, f'SELECT * FROM "{ntable}" WHERE pk = .73e1')] == ["7.3"] + assert [i["pk"]["N"] for i in _select(dynamodb_client, f'SELECT * FROM "{ntable}" WHERE pk = -.73e1')] == ["-7.3"] + dynamodb_client.execute_statement(Statement=f"INSERT INTO \"{ntable}\" VALUE {{'pk': .7325e1}}") + got = dynamodb_client.get_item(TableName=ntable, Key={"pk": {"N": "7.325"}}, ConsistentRead=True) + assert "Item" in got + + +def test_a_sign_before_a_number_belongs_to_the_number(dynamodb_client, ntable): + dynamodb_client.execute_statement( + Statement=f"INSERT INTO \"{ntable}\" VALUE {{'pk': -41, 'm': {{'x': -2}}, 'l': [-3, + 4]}}" + ) + item = dynamodb_client.get_item(TableName=ntable, Key={"pk": {"N": "-41"}}, ConsistentRead=True)["Item"] + assert item["m"] == {"M": {"x": {"N": "-2"}}} + assert item["l"] == {"L": [{"N": "-3"}, {"N": "4"}]} + dynamodb_client.execute_statement(Statement=f'UPDATE "{ntable}" SET a = - 5 WHERE pk = -41') + items = _select(dynamodb_client, f'SELECT a FROM "{ntable}" WHERE pk = - 41 AND a = -5') + assert items == [{"a": {"N": "-5"}}] + + +# ---- IS ---------------------------------------------------------------------- + + +@pytest.fixture(scope="module") +def typed_item(dynamodb_client, stable): + dynamodb_client.put_item( + TableName=stable, + Item={ + "pk": {"S": "is-item"}, + "nul": {"NULL": True}, + "s": {"S": "str"}, + "n": {"N": "1"}, + "b": {"B": b"\x01"}, + "flag": {"BOOL": True}, + "m": {"M": {"x": {"S": "y"}}}, + "l": {"L": [{"S": "y"}]}, + }, + ) + return "is-item" + + +@pytest.mark.parametrize( + ("condition", "matches"), + [ + ("nul IS NULL", True), + ("s IS NULL", False), + ("absent IS NULL", False), + ("s IS NOT NULL", True), + ("nul IS NOT NULL", False), + ("absent IS NOT NULL", True), + ("flag IS BOOL", True), + ("flag IS BOOLEAN", True), + ("s IS STRING", True), + ("s IS BOOL", False), + ("s IS NOT BOOL", True), + ("n IS NUMERIC", True), + ("b IS BLOB", True), + ("m IS STRUCT", True), + ("m IS TUPLE", True), + ("l IS LIST", True), + ("absent IS MISSING", True), + ("s IS NOT MISSING", True), + ], +) +def test_is_filters(dynamodb_client, stable, typed_item, condition, matches): + items = _select(dynamodb_client, f"SELECT pk FROM \"{stable}\" WHERE pk = '{typed_item}' AND {condition}") + assert items == ([{"pk": {"S": typed_item}}] if matches else []) + + +@pytest.mark.parametrize( + ("type_name", "service_name"), + [ + ("INT4", "Integer4Type"), + ("INT", "IntegerType"), + ("SMALLINT", "SmallintType"), + ("BIGINT", "Integer8Type"), + ("FLOAT", "FloatType"), + ("DECIMAL", "DecimalType"), + ("TIMESTAMP", "TimestampType"), + ("VARCHAR", "CharacterVaryingType"), + ("SYMBOL", "SymbolType"), + ("BAG", "BagType"), + ], +) +def test_an_is_filter_on_a_type_with_no_filter(dynamodb_client, stable, type_name, service_name): + statement = f"SELECT * FROM \"{stable}\" WHERE pk = 'x' AND a IS {type_name}" + # The refusal carries the position of the IS test's left side. + column = statement.index(" a IS ") + 2 + assert _message(dynamodb_client, statement) == ( + f"Unsupported data type for IS operator: {service_name}. Supported data types are " + f"[MISSING, NULL, BOOLEAN, BLOB, STRING, NUMERIC, STRUCT, TUPLE, LIST] at 1:{column}:1" + ) + + +def test_an_is_filter_reads_the_length_after_the_type(dynamodb_client, stable): + statement = f"SELECT * FROM \"{stable}\" WHERE pk = 'x' AND sv IS VARCHAR(10)" + column = statement.index(" sv IS ") + 2 + assert _message(dynamodb_client, statement) == ( + "Unsupported data type for IS operator: CharacterVaryingType. Supported data types are " + f"[MISSING, NULL, BOOLEAN, BLOB, STRING, NUMERIC, STRUCT, TUPLE, LIST] at 1:{column}:2" + ) + + +def test_an_is_test_on_a_parameter(dynamodb_client, stable): + statement = f"SELECT * FROM \"{stable}\" WHERE pk = 'x' AND ? IS NULL" + assert _message(dynamodb_client, statement, Parameters=[{"S": "v"}]) == ( + "The left hand side argument to IS must be a Variable Reference, Literal, or Path" + ) + + +def test_an_is_test_on_a_literal_runs(dynamodb_client, stable): + dynamodb_client.put_item(TableName=stable, Item={"pk": {"S": "is-lit"}}) + items = _select(dynamodb_client, f"SELECT pk FROM \"{stable}\" WHERE pk = 'is-lit' AND 'a' IS STRING") + assert items == [{"pk": {"S": "is-lit"}}] + + +def test_an_is_test_as_a_set_value(dynamodb_client, stable): + statement = f"UPDATE \"{stable}\" SET a = b IS NULL WHERE pk = 'x'" + assert _message(dynamodb_client, statement) == "IS operator is not supported in update expressions" + + +@pytest.mark.parametrize( + "type_name", + [ + "MISSING", "NULL", "BOOL", "BOOLEAN", "SMALLINT", "INT2", "INTEGER2", "INT4", + "INTEGER4", "INT8", "INTEGER8", "BIGINT", "INT", "INTEGER", "FLOAT", "REAL", + "DOUBLE PRECISION", "DECIMAL", "DEC", "NUMERIC", "TIMESTAMP", "DATE", "TIME", + "CHARACTER", "CHAR", "CHARACTER VARYING", "VARCHAR", "STRING", "SYMBOL", "CLOB", + "BLOB", "STRUCT", "TUPLE", "LIST", "SEXP", "BAG", "ANY", + ], +) +def test_a_bare_is_type_test_is_refused_by_name(dynamodb_client, type_name): + assert _message(dynamodb_client, f"x IS {type_name}") == "Unsupported operation: IsType" + + +@pytest.mark.parametrize("statement", ["x IS FOO", "x IS DOUBLE", "x IS VARYING"]) +def test_is_with_a_name_that_is_not_a_type(dynamodb_client, statement): + assert _message(dynamodb_client, statement) == f"{WF}Expected type name" + + +# ---- where a statement breaks off -------------------------------------------- + + +def test_an_empty_where_in_a_transaction_and_a_batch(dynamodb_client, stable): + statement = f'SELECT * FROM "{stable}" WHERE' + with pytest.raises(ClientError) as e: + dynamodb_client.execute_transaction(TransactStatements=[{"Statement": statement}]) + assert e.value.response["Error"]["Code"] == "ValidationException" + assert e.value.response["Error"]["Message"] == ( + f"Validation failed in TransactStatements[0]: {WF}Unexpected term" + ) + resp = dynamodb_client.batch_execute_statement(Statements=[{"Statement": statement}]) + assert resp["Responses"] == [{"Error": {"Code": "ValidationError", "Message": f"{WF}Unexpected term"}}] + + +def test_an_alias_in_a_transaction_carries_the_statement_position(dynamodb_client, stable): + with pytest.raises(ClientError) as e: + dynamodb_client.execute_transaction( + TransactStatements=[ + {"Statement": f"SELECT * FROM \"{stable}\" WHERE pk = 'a'"}, + {"Statement": f"SELECT * FROM \"{stable}\" t WHERE pk = 'b'"}, + ] + ) + assert e.value.response["Error"]["Message"] == f"Validation failed in TransactStatements[1]: {ALIASING}" + + +@pytest.mark.parametrize( + "condition", + ["pk = 'a' AND", "pk = 'a' OR", "pk =", "pk = 'a' AND n <", "pk = 'a' AND n = 1 +", "pk = 'a' AND n IN", "pk = 'a' AND n BETWEEN"], +) +def test_an_infix_operator_with_nothing_after_it(dynamodb_client, stable, condition): + statement = f'SELECT * FROM "{stable}" WHERE {condition}' + assert _message(dynamodb_client, statement) == f"{WF}Missing right-hand side expression of infix operator" + + +@pytest.mark.parametrize( + ("condition", "reason"), + [ + ("ORDER BY pk", "Unexpected keyword"), + ("pk = 'a' AND value = 1", "Unexpected keyword"), + ("IS NULL", "Unexpected operator"), + ("pk = LIKE", "Unexpected operator"), + ("= 1", "Unexpected operator"), + ("pk = 'a' AND , 1", "Unexpected term"), + ("pk = 'a' AND asc = 1", "Unexpected term"), + ("pk = 'a' AND;", "Unexpected term"), + ], +) +def test_a_term_that_cannot_start(dynamodb_client, stable, condition, reason): + assert _message(dynamodb_client, f'SELECT * FROM "{stable}" WHERE {condition}') == f"{WF}{reason}" + + +@pytest.mark.parametrize( + "statement", + [ + "DELETE FROM", + "DELETE FROM WHERE pk = 'a'", + "UPDATE SET a = 1 WHERE pk = 'a'", + "UPDATE WHERE pk = 'a'", + "INSERT INTO VALUE {'pk': 'a'}", + "INSERT INTO", + ], +) +def test_a_write_with_no_table(dynamodb_client, statement): + assert _message(dynamodb_client, statement) == f"{WF}Expected identifier for simple path" + + +@pytest.mark.parametrize( + ("statement", "reason"), + [ + ("SELECT * FROM", "Unexpected term"), + ("SELECT * FROM;", "Unexpected term"), + ("SELECT * FROM WHERE pk = 'a'", "Unexpected keyword"), + ], +) +def test_a_select_with_no_table(dynamodb_client, statement, reason): + assert _message(dynamodb_client, statement) == f"{WF}{reason}" + + +# ---- aliases ----------------------------------------------------------------- + + +@pytest.mark.parametrize( + "template", + [ + "SELECT * FROM \"{t}\" t WHERE pk = 'al'", + "SELECT * FROM \"{t}\" AS t WHERE pk = 'al'", + "SELECT * FROM \"{t}\" \"t\" WHERE pk = 'al'", + "SELECT pk AS k FROM \"{t}\" WHERE pk = 'al'", + "SELECT pk k FROM \"{t}\" WHERE pk = 'al'", + "UPDATE \"{t}\" t SET a = 3 WHERE pk = 'al'", + "UPDATE \"{t}\" t SET a = 3", + "DELETE FROM \"{t}\" t WHERE pk = 'al'", + "DELETE FROM \"{t}\" AS t WHERE pk = 'al'", + ], +) +def test_an_alias_is_refused_and_nothing_runs(dynamodb_client, stable, template): + dynamodb_client.put_item(TableName=stable, Item={"pk": {"S": "al"}}) + assert _message(dynamodb_client, template.format(t=stable)) == ALIASING + item = dynamodb_client.get_item(TableName=stable, Key={"pk": {"S": "al"}}, ConsistentRead=True) + assert item["Item"] == {"pk": {"S": "al"}} + + +def test_an_alias_on_a_table_that_does_not_exist(dynamodb_client, stable): + assert _message(dynamodb_client, f"SELECT * FROM \"{stable}-absent\" t WHERE pk = 'x'") == ALIASING + + +def test_a_syntax_error_is_reported_ahead_of_an_alias(dynamodb_client, stable): + assert _message(dynamodb_client, f'SELECT * FROM "{stable}" t WHERE') == f"{WF}Unexpected term" + + +def test_an_alias_on_an_insert_is_a_leftover_token(dynamodb_client, stable): + statement = f"INSERT INTO \"{stable}\" t VALUE {{'pk': 'al-ins'}}" + assert _message(dynamodb_client, statement) == f"{WF}Unexpected token after expression" + item = dynamodb_client.get_item(TableName=stable, Key={"pk": {"S": "al-ins"}}, ConsistentRead=True) + assert "Item" not in item + + +# ---- reserved words in paths ------------------------------------------------- + +# Every reserved word the parser knows, read as an attribute name after a dot. +RESERVED = """ +absolute action add all allocate alter any are assertion authorization avg bag begin +bigint bit bit_length blob bool boolean can_cast can_lossless_cast cascade cascaded +case cast catalog char char_length character character_length check clob close +coalesce collate collation column commit conflict connect connection constraint +constraints continue convert corresponding count create cross current current_date +current_time current_timestamp current_user cursor date date_add date_diff deallocate +dec decimal declare default deferrable deferred delete describe descriptor diagnostics +disconnect distinct do domain double drop else end escape exception exec execute +exists external extract fetch float foreign found from full get global go goto grant +group having identity immediate index indicator initially inner input insensitive +insert int int2 int4 int8 integer integer2 integer4 integer8 interval into isolation +join key language left let level limit list local lower match max min modified module +names national natural nchar new next no nothing nullif numeric octet_length of offset +old on only open option order outer output overlaps pad partial pivot position +precision prepare preserve primary prior privileges procedure public read real +references relative remove restrict returning revoke right rollback rows schema scroll +section select session session_user set sexp size smallint some space sql sqlcode +sqlerror sqlstate string struct substring sum symbol system_user table temporary then +time timestamp to transaction translate translation trim tuple unique unknown unpivot +update upper usage user using value values varchar varying view when whenever where +with work write zone +and or not is in like between union except intersect +as at by asc desc for nulls first last +true false null missing +""".split() + + +@pytest.mark.parametrize("word", RESERVED) +def test_a_reserved_word_after_a_dot(dynamodb_client, stable, word): + statement = f"SELECT * FROM \"{stable}\" WHERE pk = 'x' AND attribute_exists(m.{word})" + assert _message(dynamodb_client, statement) == f"{WF}Invalid path dot component" + + +@pytest.mark.parametrize("word", ["name", "type", "status", "data", "year", "month", "hour", "item"]) +def test_an_ordinary_word_after_a_dot_is_a_name(dynamodb_client, stable, word): + statement = f"SELECT * FROM \"{stable}\" WHERE pk = 'x' AND attribute_exists(m.{word})" + assert _select(dynamodb_client, statement) == [] + + +def test_a_quoted_reserved_word_after_a_dot_is_a_name(dynamodb_client, stable): + dynamodb_client.put_item(TableName=stable, Item={"pk": {"S": "rw"}, "m": {"M": {"inner": {"S": "v"}}}}) + items = _select(dynamodb_client, f"SELECT pk FROM \"{stable}\" WHERE pk = 'rw' AND attribute_exists(m.\"inner\")") + assert items == [{"pk": {"S": "rw"}}] + + +@pytest.mark.parametrize( + ("template", "reason"), + [ + # SELECT VALUE reads an expression, here a FROM-led form with no + # data manipulation after it. + ("SELECT value FROM \"{t}\" WHERE pk = 'x'", "Expected data manipulation"), + ("UPDATE \"{t}\" SET value = 1 WHERE pk = 'x'", "Expected identifier for simple path"), + ("SELECT * FROM \"{t}\" WHERE pk = 'x' AND attribute_exists(value)", "Unexpected keyword"), + ], +) +def test_a_reserved_word_where_an_attribute_name_starts(dynamodb_client, stable, template, reason): + assert _message(dynamodb_client, template.format(t=stable)) == f"{WF}{reason}" + + +# ---- empty FROM names -------------------------------------------------------- + + +def test_an_empty_table_name_is_refused_before_the_table_is_looked_up(dynamodb_client): + assert _message(dynamodb_client, "SELECT * FROM \"\" WHERE pk = 'x'") == "Path component cannot be an empty string" + + +# ---- OR branches that overlap on the sort key -------------------------------- + + +@pytest.fixture(scope="module") +def overlap_items(dynamodb_client, ctable): + for sk in ("1", "2", "3"): + dynamodb_client.put_item(TableName=ctable, Item={"pk": {"S": "ov"}, "sk": {"S": sk}}) + dynamodb_client.put_item(TableName=ctable, Item={"pk": {"S": "ov2"}, "sk": {"S": "1"}}) + return "ov" + + +@pytest.mark.parametrize( + "condition", + [ + "pk = 'ov' AND x = '1' OR pk = 'ov' AND sk = '2'", + "pk = 'ov' AND x <> '1' OR pk = 'ov' AND sk = '2'", + "pk = 'ov' OR pk = 'ov' AND sk = '2'", + "pk = 'ov' AND sk <= '2' OR pk = 'ov' AND sk >= '2'", + "pk = 'ov' AND sk BETWEEN '1' AND '2' OR pk = 'ov' AND sk = '2'", + "pk = 'ov' AND sk > '1' OR pk = 'ov' AND sk = '3'", + "pk = 'ov' AND begins_with(sk, '2') OR pk = 'ov' AND sk = '2'", + "pk = 'ov' AND (sk = '1' OR x = '2')", + "pk IN ['ov', 'ov']", + ], +) +def test_overlapping_branches_are_refused(dynamodb_client, ctable, overlap_items, condition): + assert _message(dynamodb_client, f'SELECT * FROM "{ctable}" WHERE {condition}') == OVERLAP + + +@pytest.mark.parametrize( + ("condition", "expected"), + [ + ("pk = 'ov' AND sk < '2' OR pk = 'ov' AND sk > '2'", [("ov", "1"), ("ov", "3")]), + ("pk = 'ov' AND sk = '1' OR pk = 'ov' AND sk = '3'", [("ov", "1"), ("ov", "3")]), + ("pk = 'ov' AND sk IN ['1', '3']", [("ov", "1"), ("ov", "3")]), + ("pk = 'ov' AND sk = '1' OR pk = 'ov2' AND sk = '1'", [("ov", "1"), ("ov2", "1")]), + ("pk = 'ov' OR pk = 'ov2'", [("ov", "1"), ("ov", "2"), ("ov", "3"), ("ov2", "1")]), + ("pk = 'ov' AND sk = '1' OR x = '2'", [("ov", "1")]), + ("pk = 'ov' AND begins_with(sk, '1') OR pk = 'ov' AND sk = '2'", [("ov", "1"), ("ov", "2")]), + ], +) +def test_branches_that_do_not_overlap_run(dynamodb_client, ctable, overlap_items, condition, expected): + items = _select(dynamodb_client, f'SELECT * FROM "{ctable}" WHERE {condition}') + assert sorted((i["pk"]["S"], i["sk"]["S"]) for i in items) == expected + + +# ---- statement ends, unquoted names, INSERT forms ---------------------------- + + +@pytest.fixture(scope="module") +def word_table(dynamodb_client): + """A table whose name is a plain word, for unquoted references.""" + name = f"extenddb_test_{uuid.uuid4().hex[:12]}" + dynamodb_client.create_table( + TableName=name, + AttributeDefinitions=[{"AttributeName": "pk", "AttributeType": "S"}], + KeySchema=[{"AttributeName": "pk", "KeyType": "HASH"}], + BillingMode="PAY_PER_REQUEST", + ) + wait_for_active(dynamodb_client, name) + try: + yield name + finally: + dynamodb_client.delete_table(TableName=name) + + +def test_unquoted_table_names(dynamodb_client, word_table): + t = word_table + dynamodb_client.execute_statement(Statement=f"INSERT INTO {t} VALUE {{'pk': 'uq'}}") + assert _select(dynamodb_client, f"SELECT pk FROM {t} WHERE pk = 'uq'") == [{"pk": {"S": "uq"}}] + dynamodb_client.execute_statement(Statement=f"UPDATE {t} SET a = 1 WHERE pk = 'uq'") + assert _select(dynamodb_client, f"SELECT a FROM {t} WHERE pk = 'uq'") == [{"a": {"N": "1"}}] + dynamodb_client.execute_statement(Statement=f"DELETE FROM {t} WHERE pk = 'uq'") + assert _select(dynamodb_client, f"SELECT pk FROM {t} WHERE pk = 'uq'") == [] + assert _err(dynamodb_client, f"SELECT * FROM {t}_absent WHERE pk = 'uq'") == ( + "ResourceNotFoundException", + "Requested resource not found", + ) + + +def test_a_statement_may_end_with_a_semicolon(dynamodb_client, stable): + dynamodb_client.put_item(TableName=stable, Item={"pk": {"S": "semi"}}) + assert _select(dynamodb_client, f"SELECT pk FROM \"{stable}\" WHERE pk = 'semi';") == [ + {"pk": {"S": "semi"}} + ] + dynamodb_client.execute_statement(Statement=f"UPDATE \"{stable}\" SET a = 2 WHERE pk = 'semi';") + assert _message(dynamodb_client, "x IS INT4;") == "Unsupported operation: IsType" + + +@pytest.mark.parametrize("tail", ["{'pk': 'x'}", "t", "'x'"]) +def test_an_insert_without_value_is_refused_as_many_items(dynamodb_client, stable, tail): + assert _message(dynamodb_client, f"INSERT INTO \"{stable}\" {tail}") == ( + "Unsupported operation: Inserting multiple items in a single statement is not " + "supported, use \"INSERT INTO tableName VALUE item\" instead" + ) + + +def test_an_insert_with_nothing_after_the_table(dynamodb_client, stable): + assert _message(dynamodb_client, f"INSERT INTO \"{stable}\"") == f"{WF}Unexpected term" + + +# ---- aliases, ranges and operators ------------------------------------------- + + +@pytest.mark.parametrize( + "template", + ["SELECT pk AS FROM \"{t}\" WHERE pk = 'p'", "SELECT * FROM \"{t}\" AS WHERE pk = 'p'"], +) +def test_as_with_no_name_after_it(dynamodb_client, stable, template): + assert _message(dynamodb_client, template.format(t=stable)) == f"{WF}Expected identifier for AS-alias" + + +@pytest.mark.parametrize("word", ["AT", "BY"]) +def test_at_and_by_after_a_table_are_aliases(dynamodb_client, stable, word): + statement = f"SELECT * FROM \"{stable}\" {word} x WHERE pk = 'p'" + assert _message(dynamodb_client, statement) == ALIASING + + +def test_an_alias_is_refused_ahead_of_an_empty_index_name(dynamodb_client, stable): + assert _message(dynamodb_client, f"SELECT * FROM \"{stable}\".\"\" t WHERE pk = 'p'") == ALIASING + + +@pytest.mark.parametrize( + ("tail", "reason"), + [("n BETWEEN 1 AND", "Expected expression after AND"), ("n BETWEEN 1", "Expected AND keyword")], +) +def test_a_between_that_breaks_off(dynamodb_client, stable, tail, reason): + statement = f"SELECT * FROM \"{stable}\" WHERE pk = 'p' AND {tail}" + assert _message(dynamodb_client, statement) == f"{WF}{reason}" + + +def test_like_with_nothing_after_it(dynamodb_client, stable): + statement = f"SELECT * FROM \"{stable}\" WHERE pk = 'p' AND s LIKE" + assert _message(dynamodb_client, statement) == f"{WF}Missing right-hand side expression of infix operator" + + +@pytest.mark.parametrize( + ("condition", "operator"), + [ + ("pk = 'p' AND n = -n", "-"), + ("pk = 'p' AND n = +n", "+"), + ("pk = 1-2", "-"), + ("pk = 1+2", "+"), + ("pk = 'p' AND n % 2 = 1", "%"), + ("pk = 'p' AND n / 2 = 1", "/"), + ("pk = 'p' AND s || 'x' = 'y'", "||"), + ], +) +def test_a_condition_takes_no_arithmetic(dynamodb_client, stable, condition, operator): + assert _message(dynamodb_client, f"SELECT * FROM \"{stable}\" WHERE {condition}") == ( + f"Unsupported operator in Condition Expression. Operator: {operator}" + ) + + +def test_a_sign_before_a_path_in_a_set(dynamodb_client, stable): + assert _message(dynamodb_client, f"UPDATE \"{stable}\" SET a = -b WHERE pk = 'p'") == ( + "Incorrect number of operands for operator or function; operator or function: -, " + "number of operands: 1" + ) + + +# ---- words and characters ---------------------------------------------------- + + +def test_at_and_dollar_words_are_names(dynamodb_client, stable): + dynamodb_client.put_item(TableName=stable, Item={"pk": {"S": "at"}, "n": {"N": "1"}}) + assert _select(dynamodb_client, f"SELECT pk FROM \"{stable}\" WHERE pk = 'at' AND @n = 1") == [ + {"pk": {"S": "at"}} + ] + assert _select(dynamodb_client, f"SELECT pk FROM \"{stable}\" WHERE pk = 'at' AND $x = 1") == [] + + +def test_a_word_with_a_letter_outside_ascii_does_not_lex(dynamodb_client, stable): + statement = f"SELECT * FROM \"{stable}\" WHERE pk = 'p' AND café = 1" + assert _message(dynamodb_client, statement) == WF + + +def test_a_backtick_literal_is_refused_by_its_ion_type(dynamodb_client, stable): + statement = f"SELECT * FROM \"{stable}\" WHERE pk = 'p' AND s = `1`" + column = statement.index("`") + 1 + assert _message(dynamodb_client, statement) == ( + f"Ion Literals are not supported: INT under key root at 1:{column}:3" + ) + + +# ---- bare expressions ---------------------------------------------------------- + +CHECK = "EXISTS can only be used in ExecuteTransaction write requests." + + +@pytest.mark.parametrize( + ("statement", "message"), + [ + ("x", "Unsupported operation: Id"), + ("1", "Unsupported operation: Lit"), + ("x = 1", CHECK), + ("NOT x", CHECK), + ("x IS NOT INT4", CHECK), + ("x IS VARCHAR(10)", "Unsupported operation: IsType"), + ("x IS TIME WITH TIME ZONE", "Unsupported operation: IsType"), + ("'a' IS STRING", "Unsupported operation: IsType"), + ], +) +def test_a_bare_expression_is_refused_by_what_it_is(dynamodb_client, statement, message): + assert _message(dynamodb_client, statement) == message + + +def test_a_bare_is_test_on_a_parameter(dynamodb_client): + assert _message(dynamodb_client, "? IS NULL", Parameters=[{"S": "v"}]) == "Unsupported operation: IsType" + + +# ---- empty names and the table name rule ------------------------------------- + +EMPTY = "Path component cannot be an empty string" +NAME_RULE = ( + "Table name should be within the length [3, 255] and only contain 'a-z', 'A-Z', '0-9', " + "'-', '_', '.'." +) + + +@pytest.mark.parametrize( + "statement", + ["UPDATE \"\" SET a = 1 WHERE pk = 'p'", "DELETE FROM \"\" WHERE pk = 'p'"], +) +def test_an_empty_write_target(dynamodb_client, statement): + assert _message(dynamodb_client, statement) == EMPTY + + +@pytest.mark.parametrize("condition", ["\"\" = 1", "m.\"\" = 1"]) +def test_an_empty_name_in_a_condition(dynamodb_client, stable, condition): + statement = f"SELECT * FROM \"{stable}\" WHERE pk = 'p' AND {condition}" + assert _message(dynamodb_client, statement) == EMPTY + + +@pytest.mark.parametrize("name", ["", "a", "ab", "bad!name", "x" * 256]) +@pytest.mark.parametrize( + "template", + [ + "SELECT * FROM \"{n}\" WHERE pk = 'p'", + "UPDATE \"{n}\" SET a = 1 WHERE pk = 'p'", + "DELETE FROM \"{n}\" WHERE pk = 'p'", + "INSERT INTO \"{n}\" VALUE {{'pk': 'p'}}", + ], + ids=["select", "update", "delete", "insert"], +) +def test_a_table_name_outside_the_name_rule(dynamodb_client, name, template): + if name == "" and not template.startswith("INSERT"): + pytest.skip("an empty FROM path component has its own refusal") + assert _message(dynamodb_client, template.format(n=name)) == NAME_RULE + + +@pytest.mark.parametrize("index", ["i", "ix", "bad!name", "x" * 256]) +@pytest.mark.parametrize("verb", ["select", "update", "delete"]) +def test_an_index_name_outside_the_name_rule(dynamodb_client, stable, index, verb): + q = f"\"{stable}\".\"{index}\"" + statement = { + "select": f"SELECT * FROM {q} WHERE pk = 'p'", + "update": f"UPDATE {q} SET a = 1 WHERE pk = 'p'", + "delete": f"DELETE FROM {q} WHERE pk = 'p'", + }[verb] + assert _message(dynamodb_client, statement) == NAME_RULE + + +@pytest.mark.parametrize("index", ["idx", "x" * 255]) +def test_an_index_name_inside_the_name_rule(dynamodb_client, stable, index): + q = f"\"{stable}\".\"{index}\"" + assert _message(dynamodb_client, f"SELECT * FROM {q} WHERE pk = 'p'") == ( + "The table does not have the specified index" + ) + for statement in [f"UPDATE {q} SET a = 1 WHERE pk = 'p'", f"DELETE FROM {q} WHERE pk = 'p'"]: + assert _message(dynamodb_client, statement) == "This operation is not supported on an index" + + +# ---- number text, FROM path steps, projections -------------------------------- + + +@pytest.mark.parametrize( + ("number", "reason"), + [ + ("1.2.3", "Unexpected token after expression"), + ("1e5e5", "Unexpected token after expression"), + ("1_000", "Unexpected token after expression"), + ("1..2", ""), + ("1e+", ""), + ("5.e", ""), + ("1.2.", "Invalid path dot component"), + ("1.2..3", "Invalid path dot component"), + ("..5", "Unexpected term"), + ], +) +def test_number_text_in_a_condition(dynamodb_client, stable, number, reason): + statement = f"SELECT * FROM \"{stable}\" WHERE pk = {number}" + assert _message(dynamodb_client, statement) == f"{WF}{reason}" + + +def test_a_path_from_a_number_is_refused_at_the_number(dynamodb_client, stable): + statement = f"SELECT * FROM \"{stable}\" WHERE pk = 1.2.e3" + column = statement.index("1.2") + 1 + assert _message(dynamodb_client, statement) == f"Unexpected path component at 1:{column}:3" + + +@pytest.mark.parametrize("number", ["1.e3", "1.", "00.5", "1e05", "1E5"]) +def test_number_text_the_service_reads(dynamodb_client, ntable, number): + assert _select(dynamodb_client, f"SELECT * FROM \"{ntable}\" WHERE pk = {number}") == [] + + +@pytest.mark.parametrize("tail", [".index", ". WHERE pk = 'p'"]) +def test_a_from_path_step_that_is_not_a_name(dynamodb_client, stable, tail): + assert _message(dynamodb_client, f"SELECT * FROM \"{stable}\"{tail}") == f"{WF}Invalid path dot component" + + +def test_a_projected_path_with_two_steps_runs(dynamodb_client, stable): + dynamodb_client.put_item( + TableName=stable, + Item={"pk": {"S": "deep"}, "m": {"M": {"a": {"M": {"b": {"S": "c"}}}}}}, + ) + items = _select(dynamodb_client, f"SELECT m.a.b FROM \"{stable}\" WHERE pk = 'deep'") + assert items == [{"b": {"S": "c"}}] + + +@pytest.mark.parametrize( + ("target", "message"), + [ + ("\"{t}\".\"idx\"", "This operation is not supported on an index"), + ("\"{t}\".\"i\"", NAME_RULE), + ("\"{t}\".\"a\".\"b\"", "A path may contain at most 2 components in the FROM clause"), + ("\"{t}\".\"\"", EMPTY), + ("\"{t}\" t", ALIASING), + ("\"{t}\" AS t", ALIASING), + ], +) +def test_a_from_first_update_target(dynamodb_client, stable, target, message): + statement = f"FROM {target.format(t=stable)} WHERE pk = 'p' SET c = 1" + assert _message(dynamodb_client, statement) == message + + +@pytest.mark.parametrize("function", ["set_add", "set_delete"]) +def test_a_bag_member_followed_by_a_second_number(dynamodb_client, stable, function): + statement = f"UPDATE \"{stable}\" SET ss = {function}(ss, <<1.2.3>>) WHERE pk = 'p'" + assert _message(dynamodb_client, statement) == f"{WF}Expected RIGHT_DOUBLE_ANGLE_BRACKET" + + +def test_an_update_without_where(dynamodb_client, stable): + statement = f"UPDATE \"{stable}\" SET a = 1" + assert _message(dynamodb_client, statement) == ( + "Where clause does not contain a mandatory equality on all key attributes" + ) + with pytest.raises(ClientError) as e: + dynamodb_client.execute_transaction(TransactStatements=[{"Statement": statement}]) + assert e.value.response["Error"]["Code"] == "TransactionCanceledException" + assert e.value.response["Error"]["Message"] == ( + "Transaction cancelled, please refer cancellation reasons for specific reasons [ValidationError]" + ) + + +# ---- shapes read by the same rules as the measured ones --------------------- + + +@pytest.mark.parametrize( + ("statement", "message"), + [ + ("x.y", "Unsupported operation: Path"), + ("size(x)", CHECK), + ("attribute_exists(x)", CHECK), + ("foo(x)", CHECK), + ("{'a': 1}", "Unsupported operation: Struct"), + ("[1]", "Unsupported operation: List"), + ("1 + 2", CHECK), + ("x AND y = 1", CHECK), + ("x BETWEEN 1 AND 2", CHECK), + ], +) +def test_a_bare_expression_of_another_kind(dynamodb_client, statement, message): + assert _message(dynamodb_client, statement) == message + + +def test_a_bare_parameter(dynamodb_client): + assert _message(dynamodb_client, "?", Parameters=[{"S": "v"}]) == "Unsupported operation: Parameter" + + +@pytest.mark.parametrize( + ("literal", "ion_type"), + [("\"a\"", "STRING"), ("1.5", "DECIMAL"), ("1e0", "FLOAT"), ("{a: 1}", "STRUCT"), ("[1]", "LIST")], +) +def test_a_backtick_literal_of_another_ion_type(dynamodb_client, stable, literal, ion_type): + statement = f"SELECT * FROM \"{stable}\" WHERE pk = 'p' AND s = `{literal}`" + column = statement.index("`") + 1 + assert _message(dynamodb_client, statement) == ( + f"Ion Literals are not supported: {ion_type} under key root at 1:{column}:{len(literal) + 2}" + ) + + +def test_a_condition_takes_no_multiplication(dynamodb_client, stable): + statement = f"SELECT * FROM \"{stable}\" WHERE pk = 'p' AND n * 2 = 1" + assert _message(dynamodb_client, statement) == ( + "Unsupported operator in Condition Expression. Operator: *" + ) + + +@pytest.mark.parametrize(("literal", "length"), [("1", 1), ("'a'", 3)]) +def test_a_projected_number_or_string(dynamodb_client, stable, literal, length): + statement = f"SELECT {literal} FROM \"{stable}\" WHERE pk = 'p'" + assert _message(dynamodb_client, statement) == f"Unexpected path component at 1:8:{length}" + + +def test_a_plus_sign_before_a_path_in_a_set(dynamodb_client, stable): + assert _message(dynamodb_client, f"UPDATE \"{stable}\" SET a = +b WHERE pk = 'p'") == ( + "Incorrect number of operands for operator or function; operator or function: +, " + "number of operands: 1" + ) + + +def test_an_update_without_where_on_a_missing_table(dynamodb_client, stable): + assert _err(dynamodb_client, f"UPDATE \"{stable}-absent\" SET a = 1") == ( + "ResourceNotFoundException", + "Requested resource not found", + ) + + +# ---- key values of the wrong type -------------------------------------------- + +KEY_TYPE = "Key attribute's data type should match its data type in table's schema: Key {}" + + +@pytest.fixture(scope="module") +def sntable(dynamodb_client): + """A table keyed on a string pk and a number sk.""" + with scoped_table( + dynamodb_client, + attribute_definitions=[ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "sk", "AttributeType": "N"}, + ], + key_schema=[ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"}, + ], + ) as name: + yield name + + +@pytest.mark.parametrize( + "template", + [ + "SELECT * FROM \"{t}\" WHERE pk = {v}", + "UPDATE \"{t}\" SET a = 1 WHERE pk = {v}", + "DELETE FROM \"{t}\" WHERE pk = {v}", + ], + ids=["select", "update", "delete"], +) +def test_a_number_against_a_string_key(dynamodb_client, stable, template): + assert _message(dynamodb_client, template.format(t=stable, v="1")) == KEY_TYPE.format("pk") + + +@pytest.mark.parametrize( + "template", + [ + "SELECT * FROM \"{t}\" WHERE pk = {v}", + "UPDATE \"{t}\" SET a = 1 WHERE pk = {v}", + "DELETE FROM \"{t}\" WHERE pk = {v}", + ], + ids=["select", "update", "delete"], +) +def test_a_string_against_a_number_key(dynamodb_client, ntable, template): + assert _message(dynamodb_client, template.format(t=ntable, v="'a'")) == KEY_TYPE.format("pk") + + +def test_a_parameter_of_the_wrong_type_for_a_key(dynamodb_client, stable): + statement = f"SELECT * FROM \"{stable}\" WHERE pk = ?" + assert _message(dynamodb_client, statement, Parameters=[{"N": "1"}]) == KEY_TYPE.format("pk") + + +@pytest.mark.parametrize( + "template", + [ + "SELECT * FROM \"{t}\" WHERE pk = 'a' AND sk = 'b'", + "UPDATE \"{t}\" SET a = 1 WHERE pk = 'a' AND sk = 'b'", + "DELETE FROM \"{t}\" WHERE pk = 'a' AND sk = 'b'", + ], + ids=["select", "update", "delete"], +) +def test_a_string_against_a_number_sort_key(dynamodb_client, sntable, template): + assert _message(dynamodb_client, template.format(t=sntable)) == KEY_TYPE.format("sk") + + +def test_a_wrong_type_on_a_key_in_an_or_branch_is_refused(dynamodb_client, stable): + statement = f"SELECT * FROM \"{stable}\" WHERE pk = 1 OR a = 'no-such-value'" + assert _message(dynamodb_client, statement) == KEY_TYPE.format("pk") + + +# ---- parameters inside literals ---------------------------------------------- + + +@pytest.mark.parametrize( + ("template", "path"), + [ + ("INSERT INTO \"{t}\" VALUE {{'pk':'m1','w':{{'k':?}}}}", "root.k"), + ("INSERT INTO \"{t}\" VALUE {{'pk':'m2','w':[?]}}", "root[0]"), + ("INSERT INTO \"{t}\" VALUE {{'pk':'m3','w':[1,?]}}", "root[1]"), + ("INSERT INTO \"{t}\" VALUE {{'pk':'m4','w':{{'a':{{'b':?}}}}}}", "root.a.b"), + ("INSERT INTO \"{t}\" VALUE {{'pk':'m7','w':{{'k':?, 'j':?}}}}", "root.k"), + ("UPDATE \"{t}\" SET w = {{'k':?}} WHERE pk = 'p'", "root.k"), + ("UPDATE \"{t}\" SET w = [?] WHERE pk = 'p'", "root[0]"), + ("SELECT * FROM \"{t}\" WHERE pk = 'p' AND w = {{'k':?}}", "root.k"), + ("SELECT * FROM \"{t}\" WHERE pk = 'p' AND w = [?]", "root[0]"), + ], +) +def test_a_parameter_inside_a_literal(dynamodb_client, stable, template, path): + statement = template.format(t=stable) + column = statement.index("?") + 1 + params = [{"S": "v"}] * statement.count("?") + assert _message(dynamodb_client, statement, Parameters=params) == ( + f"Unsupported data type: Parameter under key {path} at 1:{column}:1" + ) + + +def test_a_parameter_inside_a_bag(dynamodb_client, stable): + statement = f"INSERT INTO \"{stable}\" VALUE {{'pk':'m5','w':<>}}" + column = statement.index("?") + 1 + assert _message(dynamodb_client, statement, Parameters=[{"S": "v"}]) == ( + "Unsupported data type in Bag. DynamoDB only supports either numbers or strings in bags " + f"at 1:{column}:1" + ) + + +def test_a_parameter_as_a_whole_value(dynamodb_client, stable): + statement = f"INSERT INTO \"{stable}\" VALUE {{'pk':'m6','w':?}}" + dynamodb_client.execute_statement(Statement=statement, Parameters=[{"S": "v"}]) + item = dynamodb_client.get_item(TableName=stable, Key={"pk": {"S": "m6"}}, ConsistentRead=True) + assert item["Item"]["w"] == {"S": "v"} + + +# ---- items for the path and parenthesis tests ---------------------------------- + +OVERFLOW = ( + "Number overflow. Attempting to store a number with magnitude larger than supported " + "range under {}" +) + + +@pytest.fixture(scope="module") +def pair(dynamodb_client, ctable): + """The partition key of two items P/b and P/c in a partition of their own, + each with a = 1, b = 2, x = 0 and m = {k: 1, 'my key': 1}.""" + pk = f"pp-{uuid.uuid4().hex[:8]}" + for sk in ("b", "c"): + dynamodb_client.put_item( + TableName=ctable, + Item={ + "pk": {"S": pk}, + "sk": {"S": sk}, + "a": {"N": "1"}, + "b": {"N": "2"}, + "x": {"N": "0"}, + "m": {"M": {"k": {"N": "1"}, "my key": {"N": "1"}}}, + }, + ) + return pk + + +def _sorts(client, statement, **kwargs): + return sorted(i["sk"]["S"] for i in _select(client, statement, **kwargs)) + + +# ---- a string or a name in brackets --------------------------------------------- +# +# Measured 2026-11-05. A string, a quoted name or a name in brackets after a +# path is a map member, as after a dot: m['k'], m["k"] and m[k] are m.k, in a +# condition, a SET target, a REMOVE, a SET value and the select list. A +# number is a list index as before. Anything else in brackets parses and is +# refused once the statement parses; a reserved word there is a syntax +# error, and so is a bracket left open. + + +@pytest.mark.parametrize( + ("condition", "sorts"), + [ + ("m['k'] = 1", ["b", "c"]), + ("m['my key'] = 1", ["b", "c"]), + ("m[k] = 1", ["b", "c"]), + ('m["k"] = 1', ["b", "c"]), + ("m[ 'k' ] = 1", ["b", "c"]), + ("m['k'] = 1 AND m.k = 1", ["b", "c"]), + ("m['k'].z IS MISSING", ["b", "c"]), + ("m[k].j IS MISSING", ["b", "c"]), + ("m['0'] = 1", []), + ("m['a.b'] = 1", []), + ("m.k = 1", ["b", "c"]), + ], +) +def test_a_string_or_a_name_in_brackets_is_a_map_member(dynamodb_client, ctable, pair, condition, sorts): + statement = f"SELECT * FROM \"{ctable}\" WHERE pk = '{pair}' AND {condition}" + assert _sorts(dynamodb_client, statement) == sorts + + +def test_a_member_in_brackets_in_the_select_list(dynamodb_client, ctable, pair): + for path in ("m['k']", "m[k]"): + statement = f"SELECT {path} FROM \"{ctable}\" WHERE pk = '{pair}' AND sk = 'b'" + assert _select(dynamodb_client, statement) == [{"k": {"N": "1"}}] + + +@pytest.mark.parametrize( + ("clause", "m", "x"), + [ + ("SET m['k'] = 5", {"k": {"N": "5"}, "my key": {"N": "1"}}, {"N": "0"}), + ("SET m['new key'] = 7", {"k": {"N": "1"}, "my key": {"N": "1"}, "new key": {"N": "7"}}, {"N": "0"}), + ("REMOVE m['k']", {"my key": {"N": "1"}}, {"N": "0"}), + ("SET x = m['my key']", {"k": {"N": "1"}, "my key": {"N": "1"}}, {"N": "1"}), + ], +) +def test_a_member_in_brackets_in_an_update(dynamodb_client, ctable, clause, m, x): + pk = f"pb-{uuid.uuid4().hex[:8]}" + dynamodb_client.put_item( + TableName=ctable, + Item={ + "pk": {"S": pk}, + "sk": {"S": "b"}, + "x": {"N": "0"}, + "m": {"M": {"k": {"N": "1"}, "my key": {"N": "1"}}}, + }, + ) + dynamodb_client.execute_statement( + Statement=f"UPDATE \"{ctable}\" {clause} WHERE pk = '{pk}' AND sk = 'b'" + ) + item = dynamodb_client.get_item( + TableName=ctable, Key={"pk": {"S": pk}, "sk": {"S": "b"}}, ConsistentRead=True + )["Item"] + assert item["m"] == {"M": m} + assert item["x"] == x + + +def test_a_member_in_brackets_and_after_a_dot_is_one_path(dynamodb_client, ctable): + statement = f"UPDATE \"{ctable}\" SET m['k'] = 5, m.k = 6 WHERE pk = 'pb' AND sk = 'b'" + assert _message(dynamodb_client, statement) == ( + "Two document paths overlap with each other; must remove or rewrite one of these paths; " + "path one: [m, k], path two: [m, k]" + ) + + +@pytest.mark.parametrize( + ("brackets", "refusal", "at"), + [ + ("['']", "Path component cannot be an empty string", None), + ('[""]', "Path component cannot be an empty string", None), + ("[?]", "Unexpected path component", "?"), + ("[1.5]", "Unexpected path component", "1.5"), + ("[1e0]", "Unexpected path component", "1e0"), + ("[true]", "Unexpected path component", "true"), + ("[MISSING]", "Unexpected path component", "MISSING"), + ("[-1]", "List index is not within the allowable range; index: [-1]", "1"), + ("[- 1]", "List index is not within the allowable range; index: [-1]", "1"), + ("[-1.5]", "Unexpected path component", "1.5"), + ("[k.j]", "Unexpected path component", "k"), + ("[k[0]]", "Unexpected path component", "k"), + ('["k".x]', "Unexpected path component", '"k"'), + ("['k'.x]", "Unexpected path component", "'k'"), + ("[select]", WF + "Unexpected term", None), + ("[from]", WF + "Unexpected term", None), + ("[desc]", WF + "Unexpected term", None), + ("[and]", WF + "Unexpected operator", None), + ("['k' 'j']", WF + "Expected RIGHT_BRACKET", None), + ("[k j]", WF + "Expected RIGHT_BRACKET", None), + ], +) +def test_other_contents_of_brackets_are_refused(dynamodb_client, ctable, brackets, refusal, at): + statement = f"SELECT * FROM \"{ctable}\" WHERE pk = 'pb' AND m{brackets} = 1" + kwargs = {"Parameters": [{"S": "k"}]} if "?" in statement else {} + expected = refusal + if at is not None: + column = statement.index(at, statement.index("m[") + 2) + 1 + expected = f"{refusal} at 1:{column}:{len(at)}" + assert _message(dynamodb_client, statement, **kwargs) == expected + + +@pytest.mark.parametrize("tail", ["m['k'", "m[k", "m[0", "m[-1", "m[1.5", "m[?", "m[k.j"]) +def test_a_bracket_left_open_is_a_syntax_error(dynamodb_client, ctable, tail): + statement = f"SELECT * FROM \"{ctable}\" WHERE pk = 'pb' AND {tail}" + kwargs = {"Parameters": [{"S": "k"}]} if "?" in statement else {} + assert _message(dynamodb_client, statement, **kwargs) == WF + "Expected RIGHT_BRACKET" + + +@pytest.mark.parametrize( + ("condition", "first"), + [ + ("a = 1e126 AND m[''] = 1", "overflow"), + ("m[''] = 1 AND a = 1e126", "empty"), + ("a = 1e126 AND m[k.j] = 1", "overflow"), + ("m[k.j] = 1e126", "path"), + ], +) +def test_a_refusal_in_brackets_keeps_its_place_in_the_condition(dynamodb_client, ctable, condition, first): + statement = f"SELECT * FROM \"{ctable}\" WHERE pk = 'pb' AND {condition}" + if first == "overflow": + expected = OVERFLOW.format("root") + elif first == "empty": + expected = "Path component cannot be an empty string" + else: + expected = f"Unexpected path component at 1:{statement.index('k.j') + 1}:1" + assert _message(dynamodb_client, statement) == expected + + +def test_a_path_into_a_key_in_brackets_is_refused(dynamodb_client, ctable): + statement = f"SELECT * FROM \"{ctable}\" WHERE pk = 'pb' AND sk['x'] = 'b'" + assert _message(dynamodb_client, statement) == ( + "Key attributes must be scalars; list random access '[]' and map lookup '.' are not " + "allowed: Key sk" + ) + + +# ---- one value or path in parentheses ------------------------------------------ +# +# Measured 2026-11-05. Parentheses around one value or path only group it, in +# a condition, a SET value, a call's argument and an INSERT value; a comma +# inside them makes a list. A sign before a number in parentheses belongs to +# the number. An operand alone in parentheses where a condition belongs is +# refused as the operand alone is, also inside a group and after a NOT. + + +@pytest.mark.parametrize( + ("condition", "sorts"), + [ + ("pk = ('{p}')", ["b", "c"]), + ("(pk) = '{p}'", ["b", "c"]), + ("((pk)) = (('{p}'))", ["b", "c"]), + ("pk = '{p}' AND (a) = 1", ["b", "c"]), + ("pk = '{p}' AND (a) = (1)", ["b", "c"]), + ("pk = '{p}' AND ('x') = 'x'", ["b", "c"]), + ("pk = '{p}' AND a = -(-1)", ["b", "c"]), + ("pk = '{p}' AND x = (-0)", ["b", "c"]), + ("pk = '{p}' AND (a) BETWEEN 0 AND 2", ["b", "c"]), + ("pk = '{p}' AND (m) IS NOT MISSING", ["b", "c"]), + ("pk = '{p}' AND (a) IS MISSING", []), + ("pk = '{p}' AND (a) IN [1]", ["b", "c"]), + ("pk = '{p}' AND (a) NOT IN [2]", ["b", "c"]), + ("pk = '{p}' AND NOT (a) = 2", ["b", "c"]), + ("pk = '{p}' AND (a) = 1 = true", ["b", "c"]), + ("pk = '{p}' AND (a) > (b)", []), + ("pk = '{p}' AND begins_with((sk), 'b')", ["b"]), + ("pk = '{p}' AND size((m)) > 1", ["b", "c"]), + ("pk = '{p}' AND attribute_exists((a))", ["b", "c"]), + ("pk = '{p}' AND (m).k = 1", ["b", "c"]), + ("pk = '{p}' AND (m)['k'] = 1", ["b", "c"]), + ("(pk) = '{p}' AND (sk) > 'b'", ["c"]), + ("(pk = '{p}') AND (a) = 1", ["b", "c"]), + ], +) +def test_parentheses_around_one_value_or_path_group_it(dynamodb_client, ctable, pair, condition, sorts): + statement = f'SELECT * FROM "{ctable}" WHERE ' + condition.format(p=pair) + assert _sorts(dynamodb_client, statement) == sorts + + +def test_a_parameter_in_parentheses_and_a_key_in_parentheses_read_by_key(dynamodb_client, ctable, pair): + statement = f'SELECT * FROM "{ctable}" WHERE pk = (?)' + assert _sorts(dynamodb_client, statement, Parameters=[{"S": pair}]) == ["b", "c"] + statement = f"SELECT * FROM \"{ctable}\" WHERE (pk) = '{pair}' ORDER BY sk DESC" + assert [i["sk"]["S"] for i in _select(dynamodb_client, statement)] == ["c", "b"] + + +@pytest.mark.parametrize( + ("assignment", "stored"), + [ + ("x = -(5)", {"N": "-5"}), + ("x = - (5)", {"N": "-5"}), + ("x = -(-(5))", {"N": "5"}), + ("x = (5)", {"N": "5"}), + ("x = (a + 1)", {"N": "2"}), + ("x = (a) + 1", {"N": "2"}), + ("x = (m).k", {"N": "1"}), + ("x = ('a', 'b')", {"L": [{"S": "a"}, {"S": "b"}]}), + ], +) +def test_a_set_value_in_parentheses(dynamodb_client, ctable, assignment, stored): + pk = f"pp-{uuid.uuid4().hex[:8]}" + dynamodb_client.put_item( + TableName=ctable, + Item={"pk": {"S": pk}, "sk": {"S": "b"}, "a": {"N": "1"}, "m": {"M": {"k": {"N": "1"}}}}, + ) + dynamodb_client.execute_statement( + Statement=f"UPDATE \"{ctable}\" SET {assignment} WHERE pk = '{pk}' AND sk = 'b'" + ) + item = dynamodb_client.get_item( + TableName=ctable, Key={"pk": {"S": pk}, "sk": {"S": "b"}}, ConsistentRead=True + )["Item"] + assert item["x"] == stored + + +def test_an_insert_value_in_parentheses(dynamodb_client, ctable): + pk = f"pp-{uuid.uuid4().hex[:8]}" + dynamodb_client.execute_statement( + Statement=( + f"INSERT INTO \"{ctable}\" VALUE {{'pk': ('{pk}'), 'sk': 'b', 'x': (5), " + "'l': [(5)], 'p': (1, 2)}" + ) + ) + item = dynamodb_client.get_item( + TableName=ctable, Key={"pk": {"S": pk}, "sk": {"S": "b"}}, ConsistentRead=True + )["Item"] + assert item["x"] == {"N": "5"} + assert item["l"] == {"L": [{"N": "5"}]} + assert item["p"] == {"L": [{"N": "1"}, {"N": "2"}]} + + +@pytest.mark.parametrize( + ("statement", "message"), + [ + ("UPDATE {t} SET x = -(a) WHERE pk = 'pp' AND sk = 'b'", "minus"), + ("UPDATE {t} SET x = -(?) WHERE pk = 'pp' AND sk = 'b'", "minus"), + ("SELECT * FROM {t} WHERE (a)", "WHERE clause must contain an expression that resolves to a boolean"), + ("SELECT * FROM {t} WHERE ((a))", "WHERE clause must contain an expression that resolves to a boolean"), + ("SELECT * FROM {t} WHERE pk = 'pp' AND (a)", "The arguments to AND/OR must be Expressions"), + ("SELECT * FROM {t} WHERE pk = 'pp' AND (a) OR pk = 'b'", "The arguments to AND/OR must be Expressions"), + ("SELECT * FROM {t} WHERE pk = 'pp' AND ((a) AND b = 2)", "The arguments to AND/OR must be Expressions"), + ("SELECT * FROM {t} WHERE pk = 'pp' AND (a AND b = 2)", "The arguments to AND/OR must be Expressions"), + ("SELECT * FROM {t} WHERE pk = 'pp' AND (b = 2 AND a)", "The arguments to AND/OR must be Expressions"), + ("SELECT * FROM {t} WHERE pk = 'pp' AND NOT (a)", "The argument to NOT must be a Type or Expression"), + ("SELECT * FROM {t} WHERE pk = 'pp' AND NOT a", "The argument to NOT must be a Type or Expression"), + ("SELECT * FROM {t} WHERE (NOT a)", "The argument to NOT must be a Type or Expression"), + ("SELECT * FROM {t} WHERE NOT ((a))", "The argument to NOT must be a Type or Expression"), + ("SELECT * FROM {t} WHERE NOT (a) AND pk = 'pp'", "The argument to NOT must be a Type or Expression"), + ("SELECT * FROM {t} WHERE pk = 'pp' AND (a) AND b = 1e126", "The arguments to AND/OR must be Expressions"), + ("SELECT * FROM {t} WHERE pk = 'pp' AND (a + 1) = 2", "Unsupported operator in Condition Expression. Operator: +"), + ("SELECT * FROM {t} WHERE pk = 'pp' AND (a) LIKE 'x'", "Unsupported operator in Condition Expression. Operator: like"), + ("SELECT * FROM {t} WHERE pk = 'pp' AND (a) = 1 IS NULL", "The left hand side argument to IS must be a Variable Reference, Literal, or Path"), + ("SELECT * FROM {t} WHERE pk = 'pp' AND (sk.x) = 1", "Key attributes must be scalars; list random access '[]' and map lookup '.' are not allowed: Key sk"), + ("SELECT * FROM {t} WHERE pk = 'pp' AND a = -(1e126)", OVERFLOW.format("root")), + ("SELECT * FROM {t} WHERE pk = 'pp' AND (1e126) = a", OVERFLOW.format("root")), + ("SELECT * FROM {t} WHERE pk = ('pp'", WF + "Expected RIGHT_PAREN"), + ("SELECT * FROM {t} WHERE pk = 'pp' AND a = (1", WF + "Expected RIGHT_PAREN"), + ("SELECT * FROM {t} WHERE pk = 'pp' AND a = (1))", WF + "Unexpected token after expression"), + ("SELECT * FROM {t} WHERE pk = 'pp' AND a = ()", WF + "Unexpected term"), + ], +) +def test_parentheses_around_one_operand_keep_its_refusals(dynamodb_client, ctable, statement, message): + statement = statement.format(t=f'"{ctable}"') + if message == "minus": + message = ( + "Incorrect number of operands for operator or function; operator or function: -, " + "number of operands: 1" + ) + kwargs = {"Parameters": [{"N": "5"}]} if "?" in statement else {} + assert _message(dynamodb_client, statement, **kwargs) == message + + +def test_an_is_test_on_an_operand_in_parentheses_names_the_operand(dynamodb_client, ctable): + statement = f"SELECT * FROM \"{ctable}\" WHERE pk = 'pp' AND (a) IS INT4" + column = statement.index("(a)") + 2 + assert _message(dynamodb_client, statement) == ( + "Unsupported data type for IS operator: Integer4Type. Supported data types are [MISSING, " + f"NULL, BOOLEAN, BLOB, STRING, NUMERIC, STRUCT, TUPLE, LIST] at 1:{column}:1" + ) + + +# Measured 2026-11-05: an operand followed by a token that does not go on with +# it ends there, and the part of the statement around the operand reads that +# token. A syntax error there comes ahead of the operand's own refusal. +@pytest.mark.parametrize( + ("statement", "message"), + [ + ("SELECT * FROM {t} WHERE a b", WF + "Unexpected token after expression"), + ("SELECT * FROM {t} WHERE pk = 'pp' AND a b", WF + "Unexpected token after expression"), + ( + "DELETE FROM {t} WHERE pk = 'pp' AND sk = 'b' AND a b", + WF + "Unexpected token after expression", + ), + ("SELECT * FROM {t} WHERE (a b)", WF + "Expected RIGHT_PAREN"), + ("SELECT * FROM {t} WHERE pk = 'pp' AND (a b)", WF + "Expected RIGHT_PAREN"), + ("SELECT * FROM {t} WHERE (pk = 'pp' AND a LIMIT 1)", WF + "Expected RIGHT_PAREN"), + ( + "SELECT * FROM {t} WHERE a ORDER BY pk", + "WHERE clause must contain an expression that resolves to a boolean", + ), + ], +) +def test_a_token_after_an_operand_ends_it(dynamodb_client, ctable, statement, message): + assert _message(dynamodb_client, statement.format(t=f'"{ctable}"')) == message + + +def test_a_limit_after_an_operand_is_refused_for_the_clause(dynamodb_client, ctable): + statement = f"SELECT * FROM \"{ctable}\" WHERE a LIMIT 1" + # The refusal names the clause's value, the last character. + assert _message(dynamodb_client, statement) == ( + f"Unsupported clause: LIMIT at 1:{len(statement)}:1" + ) + + +# ---- negative-zero decimals ---------------------------------------------------- +# +# Measured 2026-11-05. A zero written with a decimal point or an exponent and a +# minus sign before it, anywhere in the run of signs and parentheses before it, +# is refused at the number, naming where it sits in its value. Among the other +# refusals of a value it comes in statement order with the number rules and a +# parameter inside a literal, after the condition's, and ahead of a sign +# before a SET value that is not a number and of the table lookup. An integer +# zero and a parameter are stored as 0. + +NEGATIVE_ZERO = "Negative zero decimal is not supported under key {} at 1:{}:{}" + + +def _negative_zero(statement, number, path, start=0): + column = statement.index(number, start) + 1 + return NEGATIVE_ZERO.format(path, column, len(number)) + + +@pytest.mark.parametrize( + ("value", "number", "path"), + [ + ("-0.0", "0.0", "root"), + ("- 0.0", "0.0", "root"), + ("- -0.0", "0.0", "root"), + ("-+0.0", "0.0", "root"), + ("+-0.0", "0.0", "root"), + ("-0e0", "0e0", "root"), + ("-0E-3", "0E-3", "root"), + ("-.0", ".0", "root"), + ("-0.", "0.", "root"), + ("-00.0", "00.0", "root"), + ("-0.0e0", "0.0e0", "root"), + ("[1, -0.0]", "0.0", "root[1]"), + ("{'y': -0.0}", "0.0", "root.y"), + ("<<-0.0, 1>>", "0.0", "root"), + ], +) +def test_a_negative_zero_decimal_in_an_insert_is_refused(dynamodb_client, stable, value, number, path): + pk = f"nz-{uuid.uuid4().hex[:8]}" + statement = f"INSERT INTO \"{stable}\" VALUE {{'pk': '{pk}', 'x': {value}}}" + start = statement.index("'x'") + assert _message(dynamodb_client, statement) == _negative_zero(statement, number, path, start) + got = dynamodb_client.get_item(TableName=stable, Key={"pk": {"S": pk}}, ConsistentRead=True) + assert "Item" not in got + + +@pytest.mark.parametrize( + ("template", "number", "path"), + [ + ("UPDATE {t} SET x = -0e0 WHERE pk = '{k}'", "0e0", "root"), + ("UPDATE {t} SET x = -(0.0) WHERE pk = '{k}'", "0.0", "root"), + ("UPDATE {t} SET x = (-0.0) WHERE pk = '{k}'", "0.0", "root"), + ("UPDATE {t} SET x = -(+0.0) WHERE pk = '{k}'", "0.0", "root"), + ("UPDATE {t} SET x = {{'y': -0.0}} WHERE pk = '{k}'", "0.0", "root.y"), + ("UPDATE {t} SET x = set_add(x, <<-0.0>>) WHERE pk = '{k}'", "0.0", "root"), + ("SELECT * FROM {t} WHERE pk = '{k}' AND x = -0.0", "0.0", "root"), + ("SELECT * FROM {t} WHERE pk = '{k}' AND x = - -0.0", "0.0", "root"), + ("SELECT * FROM {t} WHERE pk = '{k}' AND x IN [-0.0]", "0.0", "root"), + ("SELECT * FROM {t} WHERE pk = '{k}' AND x = [-0.0]", "0.0", "root[0]"), + ("SELECT * FROM {t}-absent WHERE pk = '{k}' AND x = -0.0", "0.0", "root"), + ], + ids=[ + "set-exponent", + "set-sign-before-parentheses", + "set-sign-in-parentheses", + "set-minus-before-a-plus", + "set-map-member", + "set-add-bag", + "where", + "where-two-minus-signs", + "where-in", + "where-list", + "ahead-of-the-table-lookup", + ], +) +def test_a_negative_zero_decimal_in_a_set_or_a_condition_is_refused( + dynamodb_client, stable, template, number, path +): + pk = f"nz-{uuid.uuid4().hex[:8]}" + dynamodb_client.put_item(TableName=stable, Item={"pk": {"S": pk}, "x": {"N": "0"}}) + statement = template.format(t=f'"{stable}"', k=pk).replace('"-absent', '-absent"') + start = statement.index("x =") if "x =" in statement else statement.index("x IN") + assert _message(dynamodb_client, statement) == _negative_zero(statement, number, path, start) + item = dynamodb_client.get_item(TableName=stable, Key={"pk": {"S": pk}}, ConsistentRead=True) + assert item["Item"]["x"] == {"N": "0"} + + +def test_an_integer_zero_and_a_negative_zero_parameter_are_zero(dynamodb_client, stable): + first, second = (f"nz-{uuid.uuid4().hex[:8]}" for _ in range(2)) + dynamodb_client.execute_statement( + Statement=f"INSERT INTO \"{stable}\" VALUE {{'pk': '{first}', 'x': -0, 'y': +0.0}}" + ) + dynamodb_client.execute_statement( + Statement=f"INSERT INTO \"{stable}\" VALUE {{'pk': '{second}', 'x': ?}}", + Parameters=[{"N": "-0.0"}], + ) + for pk in (first, second): + item = dynamodb_client.get_item(TableName=stable, Key={"pk": {"S": pk}}, ConsistentRead=True) + assert item["Item"]["x"] == {"N": "0"} + statement = f"SELECT x FROM \"{stable}\" WHERE pk = '{second}' AND x = ?" + assert _select(dynamodb_client, statement, Parameters=[{"N": "-0.0"}]) == [{"x": {"N": "0"}}] + + +@pytest.mark.parametrize( + ("value", "first"), + [ + ("'x': 1e126, 'y': -0.0", "overflow"), + ("'x': -0.0, 'y': 1e126", "zero"), + ("'x': {'b': -0.0, 'a': 1e126}", "zero"), + ("'x': [-0.0, ?]", "zero"), + ("'x': [?, -0.0]", "parameter"), + ], +) +def test_a_negative_zero_in_an_insert_keeps_statement_order(dynamodb_client, stable, value, first): + statement = f"INSERT INTO \"{stable}\" VALUE {{'pk': 'nz-order', {value}}}" + parameters = [{"N": "1"}] if "?" in statement else None + kwargs = {"Parameters": parameters} if parameters else {} + if first == "overflow": + expected = OVERFLOW.format("root") + elif first == "zero": + path = "root.b" if "'b'" in value else ("root[0]" if "[" in value else "root") + expected = _negative_zero(statement, "0.0", path, statement.index("-0.0")) + else: + column = statement.index("?") + 1 + expected = f"Unsupported data type: Parameter under key root[0] at 1:{column}:1" + assert _message(dynamodb_client, statement, **kwargs) == expected + + +@pytest.mark.parametrize( + ("assignment", "condition", "first"), + [ + ("x = -0.0, y = 1e126", "", "zero"), + ("x = 1e126, y = -0.0", "", "overflow"), + ("x = +b, y = -0.0", "", "zero"), + ("x = -0.0", " AND a = 1e126", "overflow"), + ], + ids=["zero-first", "overflow-first", "ahead-of-a-sign", "after-the-condition"], +) +def test_a_negative_zero_in_a_set_keeps_its_place(dynamodb_client, stable, assignment, condition, first): + statement = f"UPDATE \"{stable}\" SET {assignment} WHERE pk = 'nz-order'{condition}" + if first == "zero": + expected = _negative_zero(statement, "0.0", "root", statement.index("-0.0")) + else: + expected = OVERFLOW.format("root") + assert _message(dynamodb_client, statement) == expected + + +def test_a_negative_zero_in_a_batch_and_a_transaction(dynamodb_client, stable): + statement = f"INSERT INTO \"{stable}\" VALUE {{'pk': 'nz-batch', 'x': -0.0}}" + expected = _negative_zero(statement, "0.0", "root", statement.index("'x'")) + responses = dynamodb_client.batch_execute_statement(Statements=[{"Statement": statement}])[ + "Responses" + ] + assert responses == [{"Error": {"Code": "ValidationError", "Message": expected}}] + with pytest.raises(ClientError) as e: + dynamodb_client.execute_transaction(TransactStatements=[{"Statement": statement}]) + assert e.value.response["Error"]["Message"] == ( + f"Validation failed in TransactStatements[0]: {expected}" + ) diff --git a/tests/test_partiql_index_disjuncts.py b/tests/test_partiql_index_disjuncts.py new file mode 100644 index 000000000..3508ca993 --- /dev/null +++ b/tests/test_partiql_index_disjuncts.py @@ -0,0 +1,126 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""Dual-target wire tests for a PartiQL read of an index by several +disjuncts, one of which fails a check made when it is read. + +The service reads the other disjuncts first and refuses the request with the +NextToken of their last page, on an index as on a table (measured +2026-11-03: `g IN ['a', <2049 bytes>]` on a GSI keyed on g gives the items of +partition a and a NextToken, and the request with that token gets the hash +key size refusal). Beside a long filter, 50 disjuncts read 10 a page, the +held-back one in its place, and more than 50 are refused on the first +request, the held-back one among them (measured 2026-11-03). + +Dual-target: EXTENDDB_TEST_ENDPOINT selects an extenddb server; unset, the +tests run against the live service with the default credential chain. The +assertions are identical on both. +""" + +from __future__ import annotations + +import time + +import pytest +from botocore.exceptions import ClientError + +from conftest import scoped_table + +HASH_SIZE = ( + "One or more parameter values were invalid: Size of hashkey has exceeded the maximum " + "size limit of2048 bytes" +) +TOO_MANY = "Too many decomposed read operations for a given query." +BIG_HASH = "z" * 2049 +FILTER = [f"x{i:02}" for i in range(1, 100)] + + +@pytest.fixture(scope="module") +def gtable(dynamodb_client): + """A table keyed on pk with a GSI gix keyed on g. p1 (g a00), p2 (g a50) + and p3 (g a98) sit in partitions the statements read, p4 (g b) in one + they do not; p3 lacks x05.""" + with scoped_table( + dynamodb_client, + attribute_definitions=[ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "g", "AttributeType": "S"}, + ], + GlobalSecondaryIndexes=[ + { + "IndexName": "gix", + "KeySchema": [{"AttributeName": "g", "KeyType": "HASH"}], + "Projection": {"ProjectionType": "ALL"}, + } + ], + ) as name: + ones = {attribute: {"N": "1"} for attribute in FILTER} + for pk, g in [("p1", "a00"), ("p2", "a50"), ("p3", "a98"), ("p4", "b")]: + item = {"pk": {"S": pk}, "g": {"S": g}, **ones} + if pk == "p3": + del item["x05"] + dynamodb_client.put_item(TableName=name, Item=item) + for _ in range(240): + scan = dynamodb_client.scan(TableName=name, IndexName="gix", Select="COUNT") + if scan["Count"] == 4: + break + time.sleep(0.5) + yield name + + +def _pages(client, statement): + """The sorted keys of every page of `statement`, following NextToken, and + the refusal that ends the paging when there is one.""" + pages, token = [], None + for _ in range(8): + args = {"NextToken": token} if token else {} + try: + response = client.execute_statement(Statement=statement, **args) + except ClientError as e: + assert e.response["Error"]["Code"] == "ValidationException", e.response + pages.append(e.response["Error"]["Message"]) + break + pages.append(sorted(item["pk"]["S"] for item in response["Items"])) + token = response.get("NextToken") + if not token: + break + return pages + + +def test_an_oversized_disjunct_on_an_index_is_refused_after_the_others(dynamodb_client, gtable): + statement = f"SELECT pk FROM \"{gtable}\".\"gix\" WHERE g IN ['a00', '{BIG_HASH}']" + assert _pages(dynamodb_client, statement) == [["p1"], HASH_SIZE] + + +@pytest.mark.parametrize( + ("held_back", "pages"), + [ + (True, [["p1"], [], [], [], ["p2"], HASH_SIZE]), + (False, [["p1"], [], [], [], ["p2"]]), + ], + ids=["50-disjuncts", "49-disjuncts"], +) +def test_50_disjuncts_beside_a_long_filter_on_an_index_read_10_a_page( + dynamodb_client, gtable, held_back, pages +): + # a00 to a46, then the partitions of p2 and p3: p3 lacks x05. + values = [f"'a{i:02}'" for i in range(47)] + ["'a50'", "'a98'"] + if held_back: + values.append(f"'{BIG_HASH}'") + condition = " AND ".join(f"{attribute} = 1" for attribute in FILTER) + statement = ( + f"SELECT pk FROM \"{gtable}\".\"gix\" WHERE g IN [{', '.join(values)}] AND {condition}" + ) + assert _pages(dynamodb_client, statement) == pages + + +@pytest.mark.parametrize("count", [99, 50], ids=["100-disjuncts", "51-disjuncts"]) +def test_more_than_50_disjuncts_beside_a_long_filter_on_an_index_are_refused( + dynamodb_client, gtable, count +): + values = ", ".join(f"'a{i:02}'" for i in range(count)) + condition = " AND ".join(f"{attribute} = 1" for attribute in FILTER) + statement = ( + f"SELECT pk FROM \"{gtable}\".\"gix\" WHERE g IN [{values}, '{BIG_HASH}'] AND {condition}" + ) + assert _pages(dynamodb_client, statement) == [TOO_MANY] diff --git a/tests/test_partiql_index_keys.py b/tests/test_partiql_index_keys.py new file mode 100644 index 000000000..4c0e22245 --- /dev/null +++ b/tests/test_partiql_index_keys.py @@ -0,0 +1,328 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""PartiQL behaviour on secondary indexes and multi-attribute key schemas, +measured on the service and asserted on both targets. + +Three service rules this file pins: + * a PartiQL statement against a table or index with more than one HASH or + more than one RANGE key attribute is refused with one fixed message; + * a BatchExecuteStatement SELECT that names an index can never satisfy the + primary-key rule, whatever the WHERE names; + * an ExecuteTransaction SELECT that names any index is refused, and an + unknown table cancels the whole read transaction. + +Dual-target: EXTENDDB_TEST_ENDPOINT selects a fix server, EXTENDDB_TEST_PROFILE +selects the live-service profile. Every expected value was measured on the +service; see conf-run/partiql/probe_multiattr_gsi.py, +probe_index_batch_txn.py, probe_index_order.py and probe_vector_batch_txn.py. +""" + +from __future__ import annotations + +import os +import uuid + +import boto3 +import pytest +from botocore.config import Config +from botocore.exceptions import ClientError + +from conftest import wait_for_active + +MULTI_ATTR_MSG = ( + "PartiQL statements cannot be executed on tables with multi-attribute " + "HASH keys or multi-attribute RANGE keys." +) +BATCH_KEY_MSG = ( + "Select statements within BatchExecuteStatement must specify the primary " + "key in the where clause." +) +TXN_INDEX_MSG = "Reads on indices are not supported within transactions." + + +@pytest.fixture +def dynamodb_client(endpoint_url): + region = os.environ.get("AWS_DEFAULT_REGION", "us-east-1") + profile = os.environ.get("EXTENDDB_TEST_PROFILE") or None + session = boto3.Session(profile_name=profile, region_name=region) + kwargs: dict = {"service_name": "dynamodb"} + if endpoint_url: + kwargs["endpoint_url"] = endpoint_url + if endpoint_url.startswith("https://"): + kwargs["verify"] = False + kwargs["config"] = Config(parameter_validation=True) + return session.client(**kwargs) + + +def _create(client, name, gsi_key): + all_defs = { + "pk": "S", + "g1": "S", + "g2": "S", + "g3": "N", + } + used = {"pk"} | {k["AttributeName"] for k in gsi_key} + client.create_table( + TableName=name, + BillingMode="PAY_PER_REQUEST", + AttributeDefinitions=[ + {"AttributeName": n, "AttributeType": all_defs[n]} + for n in all_defs + if n in used + ], + KeySchema=[{"AttributeName": "pk", "KeyType": "HASH"}], + GlobalSecondaryIndexes=[ + { + "IndexName": "gsi", + "KeySchema": gsi_key, + "Projection": {"ProjectionType": "ALL"}, + } + ], + ) + wait_for_active(client, name) + client.put_item( + TableName=name, + Item={ + "pk": {"S": "p1"}, + "g1": {"S": "a"}, + "g2": {"S": "b1"}, + "g3": {"N": "1"}, + }, + ) + + +@pytest.fixture +def multi_attr_table(dynamodb_client): + """A table whose GSI key has two RANGE attributes. A backend that cannot + store a multi-attribute GSI skips the group rather than failing it.""" + name = f"extenddb-measure-mak-{uuid.uuid4().hex[:12]}" + gsi_key = [ + {"AttributeName": "g1", "KeyType": "HASH"}, + {"AttributeName": "g2", "KeyType": "RANGE"}, + {"AttributeName": "g3", "KeyType": "RANGE"}, + ] + try: + _create(dynamodb_client, name, gsi_key) + except ClientError as exc: + try: + dynamodb_client.delete_table(TableName=name) + except ClientError: + pass + pytest.skip(f"multi-attribute GSI not supported here: {exc}") + try: + yield name + finally: + try: + dynamodb_client.delete_table(TableName=name) + except ClientError: + pass + + +@pytest.fixture +def plain_index_table(dynamodb_client): + name = f"extenddb-measure-idx-{uuid.uuid4().hex[:12]}" + gsi_key = [ + {"AttributeName": "g1", "KeyType": "HASH"}, + {"AttributeName": "g2", "KeyType": "RANGE"}, + ] + _create(dynamodb_client, name, gsi_key) + try: + yield name + finally: + try: + dynamodb_client.delete_table(TableName=name) + except ClientError: + pass + + +class TestMultiAttributeKeyRefusal: + @pytest.mark.parametrize( + "where", + [ + "", + " WHERE g1 = 'a'", + " WHERE g1 = 'a' AND g2 = 'b1' AND g3 = 1", + ], + ) + def test_single_statement_refused(self, dynamodb_client, multi_attr_table, where): + with pytest.raises(ClientError) as ei: + dynamodb_client.execute_statement( + Statement=f'SELECT * FROM "{multi_attr_table}"."gsi"{where}' + ) + err = ei.value.response["Error"] + assert err["Code"] == "ValidationException" + assert err["Message"] == MULTI_ATTR_MSG + + def test_multi_attribute_beats_consistent_read(self, dynamodb_client, multi_attr_table): + with pytest.raises(ClientError) as ei: + dynamodb_client.execute_statement( + Statement=f'SELECT * FROM "{multi_attr_table}"."gsi" WHERE g1 = \'a\'', + ConsistentRead=True, + ) + assert ei.value.response["Error"]["Message"] == MULTI_ATTR_MSG + + def test_batch_statement_refused(self, dynamodb_client, multi_attr_table): + out = dynamodb_client.batch_execute_statement( + Statements=[ + {"Statement": f'SELECT * FROM "{multi_attr_table}"."gsi" WHERE g1 = \'a\' AND g2 = \'b1\' AND g3 = 1'} + ] + ) + err = out["Responses"][0]["Error"] + assert err["Code"] == "ValidationError" + assert err["Message"] == MULTI_ATTR_MSG + + def test_base_table_write_unaffected(self, dynamodb_client, multi_attr_table): + # The base table key is single-attribute, so a write to it is accepted + # even though the table carries a multi-attribute GSI. + dynamodb_client.execute_statement( + Statement=( + f"INSERT INTO \"{multi_attr_table}\" " + "VALUE {'pk': 'w1', 'g1': 'a', 'g2': 'b', 'g3': 2}" + ) + ) + got = dynamodb_client.execute_statement( + Statement=f"SELECT * FROM \"{multi_attr_table}\" WHERE pk = 'w1'" + ) + assert got["Items"][0]["pk"]["S"] == "w1" + + +class TestBatchIndexReads: + def test_index_with_base_key_still_needs_primary_key(self, dynamodb_client, plain_index_table): + # Regression: a batch SELECT naming an index used to run as a base-table + # point read when the WHERE happened to name the base key. The service + # rejects any index SELECT in a batch. + out = dynamodb_client.batch_execute_statement( + Statements=[{"Statement": f'SELECT * FROM "{plain_index_table}"."gsi" WHERE pk = \'p1\''}] + ) + err = out["Responses"][0]["Error"] + assert err["Code"] == "ValidationError" + assert err["Message"] == BATCH_KEY_MSG + + def test_index_with_index_key_needs_primary_key(self, dynamodb_client, plain_index_table): + out = dynamodb_client.batch_execute_statement( + Statements=[{"Statement": f'SELECT * FROM "{plain_index_table}"."gsi" WHERE g1 = \'a\' AND g2 = \'b1\''}] + ) + err = out["Responses"][0]["Error"] + assert err["Code"] == "ValidationError" + assert err["Message"] == BATCH_KEY_MSG + + def test_missing_index_is_per_statement(self, dynamodb_client, plain_index_table): + out = dynamodb_client.batch_execute_statement( + Statements=[{"Statement": f'SELECT * FROM "{plain_index_table}"."nope" WHERE pk = \'p1\''}] + ) + err = out["Responses"][0]["Error"] + assert err["Code"] == "ValidationError" + assert err["Message"] == "The table does not have the specified index" + + +@pytest.fixture +def vector_table(dynamodb_client): + name = f"extenddb-measure-vix-{uuid.uuid4().hex[:12]}" + try: + dynamodb_client.create_table( + TableName=name, + BillingMode="PAY_PER_REQUEST", + AttributeDefinitions=[{"AttributeName": "pk", "AttributeType": "S"}], + KeySchema=[{"AttributeName": "pk", "KeyType": "HASH"}], + VectorIndexes=[ + { + "IndexName": "vix", + "VectorAttribute": {"AttributeName": "embedding"}, + "Dimensions": 3, + "DistanceFunction": "COSINE", + "Projection": {"ProjectionType": "ALL"}, + } + ], + ) + except ClientError as exc: + try: + dynamodb_client.delete_table(TableName=name) + except ClientError: + pass + msg = exc.response["Error"]["Message"].lower() + if "vector" in msg or exc.response["Error"]["Code"] == "ValidationException": + pytest.skip("vector indexes not supported by this backend") + raise + wait_for_active(dynamodb_client, name) + dynamodb_client.put_item( + TableName=name, + Item={"pk": {"S": "p1"}, "embedding": {"L": [{"N": "1"}, {"N": "0"}, {"N": "0"}]}}, + ) + try: + yield name + finally: + try: + dynamodb_client.delete_table(TableName=name) + except ClientError: + pass + + +class TestVectorIndexReads: + def test_single_consistent_read_beats_vector_refusal(self, dynamodb_client, vector_table): + # On a vector index the strong-read refusal is measured to win over the + # vector refusal. + with pytest.raises(ClientError) as ei: + dynamodb_client.execute_statement( + Statement=f'SELECT * FROM "{vector_table}"."vix"', ConsistentRead=True + ) + assert ( + ei.value.response["Error"]["Message"] + == "Strongly consistent read is not supported on Global Secondary Indexes" + ) + + def test_single_vector_refusal(self, dynamodb_client, vector_table): + with pytest.raises(ClientError) as ei: + dynamodb_client.execute_statement(Statement=f'SELECT * FROM "{vector_table}"."vix"') + assert ( + ei.value.response["Error"]["Message"] + == "Scan operation not supported on this index type" + ) + + def test_batch_vector_index_needs_primary_key(self, dynamodb_client, vector_table): + # Regression: a vector-index SELECT in a batch with the base key used to + # return the item; the service answers with the primary-key message. + out = dynamodb_client.batch_execute_statement( + Statements=[{"Statement": f'SELECT * FROM "{vector_table}"."vix" WHERE pk = \'p1\''}] + ) + err = out["Responses"][0]["Error"] + assert err["Code"] == "ValidationError" + assert err["Message"] == BATCH_KEY_MSG + + def test_transaction_vector_index_refused(self, dynamodb_client, vector_table): + with pytest.raises(ClientError) as ei: + dynamodb_client.execute_transaction( + TransactStatements=[{"Statement": f'SELECT * FROM "{vector_table}"."vix" WHERE pk = \'p1\''}] + ) + assert TXN_INDEX_MSG in ei.value.response["Error"]["Message"] + + +class TestTransactionIndexReads: + @pytest.mark.parametrize("where", [" WHERE pk = 'p1'", " WHERE g1 = 'a' AND g2 = 'b1'"]) + def test_index_read_refused(self, dynamodb_client, plain_index_table, where): + with pytest.raises(ClientError) as ei: + dynamodb_client.execute_transaction( + TransactStatements=[{"Statement": f'SELECT * FROM "{plain_index_table}"."gsi"{where}'}] + ) + err = ei.value.response["Error"] + assert err["Code"] == "ValidationException" + assert f"TransactStatements[0]: {TXN_INDEX_MSG}" in err["Message"] + + def test_missing_index_refused(self, dynamodb_client, plain_index_table): + with pytest.raises(ClientError) as ei: + dynamodb_client.execute_transaction( + TransactStatements=[{"Statement": f'SELECT * FROM "{plain_index_table}"."nope" WHERE pk = \'p1\''}] + ) + assert TXN_INDEX_MSG in ei.value.response["Error"]["Message"] + + def test_unknown_table_cancels_the_read(self, dynamodb_client): + missing = f"extenddb-no-such-{uuid.uuid4().hex[:8]}" + with pytest.raises(ClientError) as ei: + dynamodb_client.execute_transaction( + TransactStatements=[{"Statement": f'SELECT * FROM "{missing}" WHERE pk = \'a\''}] + ) + resp = ei.value.response + assert resp["Error"]["Code"] == "TransactionCanceledException" + reasons = resp["CancellationReasons"] + assert reasons[0]["Code"] == "ResourceNotFound" + assert reasons[0]["Message"] == "Requested resource not found" diff --git a/tests/test_partiql_index_reads.py b/tests/test_partiql_index_reads.py new file mode 100644 index 000000000..5c327341d --- /dev/null +++ b/tests/test_partiql_index_reads.py @@ -0,0 +1,551 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""Dual-target wire tests for PartiQL reads through an index that does not +project every attribute, and for the FROM path rules every statement shares. + +Rules pinned here, each measured on the service (2026-11-02): + * a keyed read of the index (an equality or IN on its partition key at the + top of the WHERE) is refused when its filter names an attribute the index + does not project, on a GSI and on an LSI; an unkeyed read is a scan that + evaluates the filter against the index rows; + * a GSI read whose projection names such an attribute is refused, keyed or + not, and the filter refusal is reported first; + * an LSI read whose projection names such an attribute reads every row it + walks back from the base table, serves the projection and the filter from + that item, and charges each read on the table arm of ConsumedCapacity; + * a begins_with with a number prefix gets the operand-type message, in a + filter and on the LSI sort key, whether or not the read reaches back; + * a FROM path with an empty component or more than two components is + refused once the statement has parsed and before the table is looked up, + and an UPDATE, DELETE or INSERT naming an index is refused the same way. + +Dual-target: EXTENDDB_TEST_ENDPOINT selects an ExtendDB server; unset, the +tests run against the live service. The table carries an +``extenddb-measure-ixr-`` prefix and is deleted in a finally. +""" + +from __future__ import annotations + +import os +import time +import uuid + +import pytest +from botocore.exceptions import ClientError + +NOT_WELL_FORMED = "Statement wasn't well formed, can't be processed: " +EMPTY_COMPONENT = "Path component cannot be an empty string" +TOO_MANY_COMPONENTS = "A path may contain at most 2 components in the FROM clause" +ON_INDEX = "This operation is not supported on an index" +INVALID = "One or more parameter values were invalid: " + + +def _filter_refusal(index: str, names: str) -> str: + return f"{INVALID}Secondary index {index} does not project one or more filter attributes: [{names}]" + + +def _projection_refusal(index: str, names: str) -> str: + return f"{INVALID}Global secondary index {index} does not project [{names}]" + + +def _insert_refusal(table: str) -> str: + # The span is the table name token: "INSERT INTO " is 12 characters, and + # the token includes its two double quotes. + return ( + "FROM clause may only contain a single table name in data manipulation " + f"statements at 1:13:{len(table) + 2}" + ) + + +def _is_service() -> bool: + return not os.environ.get("EXTENDDB_TEST_ENDPOINT", "").strip() + + +def _interval() -> float: + return 0.3 if _is_service() else 0.03 + + +# Items. Partition "a" holds the rows the LSIs index (a/1, a/2) and one row no +# index holds (a/3); b/1 sits in a second GSI partition. "note" and "label" are +# projected by no index. +ITEMS = [ + { + "pk": {"S": "a"}, "sk": {"S": "1"}, "color": {"S": "red"}, + "rank": {"S": "r1"}, "weight": {"S": "w1"}, "dims": {"S": "S"}, + "note": {"S": "n1"}, "label": {"S": "l1"}, + }, + { + "pk": {"S": "a"}, "sk": {"S": "2"}, "color": {"S": "red"}, + "rank": {"S": "r2"}, "weight": {"S": "w2"}, "dims": {"S": "M"}, + "note": {"S": "n2"}, "label": {"S": "l2"}, + }, + {"pk": {"S": "a"}, "sk": {"S": "3"}, "note": {"S": "n3"}}, + { + "pk": {"S": "b"}, "sk": {"S": "1"}, "color": {"S": "blue"}, + "dims": {"S": "L"}, "note": {"S": "n4"}, + }, +] + + +def _wait_active(client, name: str, timeout: float = 300.0) -> None: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + d = client.describe_table(TableName=name)["Table"] + gsis = d.get("GlobalSecondaryIndexes", []) + if d["TableStatus"] == "ACTIVE" and all(g.get("IndexStatus") == "ACTIVE" for g in gsis): + return + time.sleep(_interval()) + raise TimeoutError(f"{name} not ACTIVE within {timeout}s") + + +def _wait_deleted(client, name: str, timeout: float = 300.0) -> None: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + client.describe_table(TableName=name) + except client.exceptions.ResourceNotFoundException: + return + time.sleep(_interval()) + + +def _wait_gsi(client, name: str, color: str, want: int, timeout: float = 60.0) -> None: + """A GSI fills asynchronously on both targets; poll until it holds the rows.""" + statement = f"SELECT pk FROM \"{name}\".\"by-color\" WHERE color = '{color}'" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if len(client.execute_statement(Statement=statement).get("Items", [])) >= want: + return + time.sleep(_interval()) + raise TimeoutError(f"by-color did not reach {want} rows for {color}") + + +@pytest.fixture(scope="module") +def table(dynamodb_client): + name = f"extenddb-measure-ixr-{uuid.uuid4().hex[:12]}" + dynamodb_client.create_table( + TableName=name, + BillingMode="PAY_PER_REQUEST", + AttributeDefinitions=[ + {"AttributeName": n, "AttributeType": "S"} + for n in ("pk", "sk", "color", "rank", "weight") + ], + KeySchema=[ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"}, + ], + GlobalSecondaryIndexes=[ + { + "IndexName": "by-color", + "KeySchema": [{"AttributeName": "color", "KeyType": "HASH"}], + "Projection": {"ProjectionType": "INCLUDE", "NonKeyAttributes": ["dims"]}, + } + ], + LocalSecondaryIndexes=[ + { + "IndexName": "by-rank", + "KeySchema": [ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "rank", "KeyType": "RANGE"}, + ], + "Projection": {"ProjectionType": "KEYS_ONLY"}, + }, + { + "IndexName": "by-weight", + "KeySchema": [ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "weight", "KeyType": "RANGE"}, + ], + "Projection": {"ProjectionType": "INCLUDE", "NonKeyAttributes": ["dims"]}, + }, + ], + ) + try: + _wait_active(dynamodb_client, name) + for item in ITEMS: + dynamodb_client.put_item(TableName=name, Item=item) + _wait_gsi(dynamodb_client, name, "red", 2) + _wait_gsi(dynamodb_client, name, "blue", 1) + yield name + finally: + try: + dynamodb_client.delete_table(TableName=name) + _wait_deleted(dynamodb_client, name) + except ClientError: + pass + + +def _refusal(client, statement: str, **kwargs) -> str: + with pytest.raises(ClientError) as ei: + client.execute_statement(Statement=statement, **kwargs) + err = ei.value.response["Error"] + assert err["Code"] == "ValidationException", err + return err["Message"] + + +def _items(client, statement: str, **kwargs) -> list: + return client.execute_statement(Statement=statement, **kwargs).get("Items", []) + + +def _keys(items) -> list: + return sorted(f"{i['pk']['S']}/{i['sk']['S']}" for i in items) + + +# ---- the filter of a keyed read -------------------------------------------- + +@pytest.mark.parametrize( + "where", + [ + "color = 'red' AND note = 'n1'", + "color = 'red' AND NOT note = 'n1'", + "color = 'red' AND note IS MISSING", + "color = 'red' AND note IS NOT MISSING", + "color = 'red' AND begins_with(note, 'n')", + "color IN ['red', 'blue'] AND note = 'n1'", + "color = 'red' AND (note = 'n1' OR dims = 'S')", + "color = 'red' AND note.child = 'x'", + "'red' = color AND note = 'n1'", + ], +) +def test_keyed_gsi_filter_on_an_unprojected_attribute_is_refused(dynamodb_client, table, where): + message = _refusal(dynamodb_client, f'SELECT pk FROM "{table}"."by-color" WHERE {where}') + assert message == _filter_refusal("by-color", "note") + + +@pytest.mark.parametrize( + "where", + [ + "pk = 'a' AND note = 'n1'", + "pk = 'a' AND rank BETWEEN 'r1' AND 'r2' AND note = 'n1'", + "pk = 'a' AND rank > 'r0' AND note = 'n1'", + "pk = 'a' AND begins_with(rank, 'r') AND note = 'n1'", + "pk = 'a' AND rank IN ['r1', 'r2'] AND note = 'n1'", + ], +) +def test_keyed_lsi_filter_on_an_unprojected_attribute_is_refused(dynamodb_client, table, where): + message = _refusal(dynamodb_client, f'SELECT pk FROM "{table}"."by-rank" WHERE {where}') + assert message == _filter_refusal("by-rank", "note") + + +def test_a_not_equal_sort_condition_makes_the_lsi_read_a_scan(dynamodb_client, table): + statement = f"SELECT pk FROM \"{table}\".\"by-rank\" WHERE pk = 'a' AND rank <> 'r1' AND note = 'n1'" + assert _items(dynamodb_client, statement) == [] + + +def test_filter_refusal_lists_each_attribute_in_statement_order(dynamodb_client, table): + message = _refusal( + dynamodb_client, + f"SELECT pk FROM \"{table}\".\"by-color\" WHERE color = 'red' AND note = 'n1' AND label = 'l1'", + ) + assert message == _filter_refusal("by-color", "note, label") + + +@pytest.mark.parametrize( + "where", + [ + "note = 'n1'", + "note = 'n1' AND note = 'n2'", + "color <> 'blue' AND note = 'n1'", + "begins_with(color, 're') AND note = 'n1'", + "color = 'red' AND color = 'blue' AND note = 'n1'", + "color = 'red' AND NOT color = 'blue' AND note = 'n1'", + "pk = 'a' AND note = 'n1'", + ], +) +def test_unkeyed_gsi_filter_is_evaluated_on_index_rows(dynamodb_client, table, where): + # The index rows carry no "note", so nothing matches. + assert _items(dynamodb_client, f'SELECT pk FROM "{table}"."by-color" WHERE {where}') == [] + + +def test_unprojected_filter_inside_an_or_branch_is_accepted(dynamodb_client, table): + items = _items( + dynamodb_client, + f"SELECT pk, sk FROM \"{table}\".\"by-color\" " + "WHERE (color = 'red' AND note = 'n1') OR color = 'blue'", + ) + assert _keys(items) == ["b/1"] + + +def test_keyed_filter_on_a_projected_attribute_is_accepted(dynamodb_client, table): + items = _items( + dynamodb_client, + f"SELECT pk, sk FROM \"{table}\".\"by-color\" WHERE color = 'red' AND dims = 'S'", + ) + assert _keys(items) == ["a/1"] + + +# ---- the projection of a GSI read ------------------------------------------ + +@pytest.mark.parametrize( + "statement", + [ + "SELECT note FROM \"{t}\".\"by-color\" WHERE color = 'red'", + "SELECT note FROM \"{t}\".\"by-color\"", + "SELECT pk, note FROM \"{t}\".\"by-color\" WHERE note = 'n1'", + "SELECT note.child FROM \"{t}\".\"by-color\" WHERE color = 'red'", + ], +) +def test_gsi_projection_of_an_unprojected_attribute_is_refused(dynamodb_client, table, statement): + message = _refusal(dynamodb_client, statement.format(t=table)) + assert message == _projection_refusal("by-color", "note") + + +def test_projection_refusal_lists_each_attribute_in_statement_order(dynamodb_client, table): + message = _refusal( + dynamodb_client, + f"SELECT note, label FROM \"{table}\".\"by-color\" WHERE color = 'red'", + ) + assert message == _projection_refusal("by-color", "note, label") + + +def test_keyed_read_reports_the_filter_refusal_first(dynamodb_client, table): + message = _refusal( + dynamodb_client, + f"SELECT note FROM \"{table}\".\"by-color\" WHERE color = 'red' AND label = 'l1'", + ) + assert message == _filter_refusal("by-color", "label") + + +# ---- an LSI read that reaches back to the base table ------------------------ + +def _capacity(resp) -> tuple: + cc = resp["ConsumedCapacity"] + lsi = cc.get("LocalSecondaryIndexes", {}) + return ( + cc["CapacityUnits"], + cc.get("Table", {}).get("CapacityUnits"), + {name: arm["CapacityUnits"] for name, arm in lsi.items()}, + ) + + +def test_lsi_serves_an_unprojected_attribute_from_the_base_table(dynamodb_client, table): + resp = dynamodb_client.execute_statement( + Statement=f"SELECT note FROM \"{table}\".\"by-rank\" WHERE pk = 'a'", + ReturnConsumedCapacity="INDEXES", + ) + assert resp["Items"] == [{"note": {"S": "n1"}}, {"note": {"S": "n2"}}] + # Two rows walked: half a read unit each on the table arm, eventually + # consistent; the index arm is the ordinary read of the two index rows. + assert _capacity(resp) == (1.5, 1.0, {"by-rank": 0.5}) + + +def test_reach_back_charge_follows_the_rows_walked(dynamodb_client, table): + resp = dynamodb_client.execute_statement( + Statement=f"SELECT note FROM \"{table}\".\"by-rank\" WHERE pk = 'a'", + ReturnConsumedCapacity="INDEXES", + Limit=1, + ) + assert resp["Items"] == [{"note": {"S": "n1"}}] + assert "NextToken" in resp + assert _capacity(resp) == (1.0, 0.5, {"by-rank": 0.5}) + + +def test_reach_back_charges_rows_the_filter_drops(dynamodb_client, table): + resp = dynamodb_client.execute_statement( + Statement=f"SELECT note FROM \"{table}\".\"by-weight\" WHERE pk = 'a' AND dims = 'XL'", + ReturnConsumedCapacity="INDEXES", + ) + assert resp["Items"] == [] + assert _capacity(resp) == (1.5, 1.0, {"by-weight": 0.5}) + + +def test_reach_back_strong_read_doubles_both_arms(dynamodb_client, table): + resp = dynamodb_client.execute_statement( + Statement=f"SELECT note FROM \"{table}\".\"by-rank\" WHERE pk = 'a'", + ReturnConsumedCapacity="INDEXES", + ConsistentRead=True, + ) + assert _capacity(resp) == (3.0, 2.0, {"by-rank": 1.0}) + + +def test_reach_back_total_capacity_includes_the_table_reads(dynamodb_client, table): + resp = dynamodb_client.execute_statement( + Statement=f"SELECT note FROM \"{table}\".\"by-rank\" WHERE pk = 'a'", + ReturnConsumedCapacity="TOTAL", + ) + assert resp["ConsumedCapacity"]["CapacityUnits"] == 1.5 + assert "Table" not in resp["ConsumedCapacity"] + + +def test_reach_back_serves_key_and_unprojected_attributes_together(dynamodb_client, table): + items = _items( + dynamodb_client, + f"SELECT sk, rank, label FROM \"{table}\".\"by-rank\" WHERE pk = 'a'", + ) + assert items == [ + {"sk": {"S": "1"}, "rank": {"S": "r1"}, "label": {"S": "l1"}}, + {"sk": {"S": "2"}, "rank": {"S": "r2"}, "label": {"S": "l2"}}, + ] + + +def test_select_star_on_a_keys_only_lsi_does_not_reach_back(dynamodb_client, table): + resp = dynamodb_client.execute_statement( + Statement=f"SELECT * FROM \"{table}\".\"by-rank\" WHERE pk = 'a'", + ReturnConsumedCapacity="INDEXES", + ) + assert sorted(resp["Items"][0]) == ["pk", "rank", "sk"] + assert _capacity(resp) == (0.5, 0.0, {"by-rank": 0.5}) + + +def test_keyed_reach_back_still_refuses_an_unprojected_filter(dynamodb_client, table): + message = _refusal( + dynamodb_client, + f"SELECT note FROM \"{table}\".\"by-rank\" WHERE pk = 'a' AND note = 'n1'", + ) + assert message == _filter_refusal("by-rank", "note") + + +def test_unkeyed_reach_back_serves_the_base_items(dynamodb_client, table): + items = _items(dynamodb_client, f'SELECT note FROM "{table}"."by-rank"') + assert sorted(i["note"]["S"] for i in items) == ["n1", "n2"] + + +def test_unkeyed_reach_back_filters_the_base_items(dynamodb_client, table): + items = _items( + dynamodb_client, + f"SELECT note FROM \"{table}\".\"by-rank\" WHERE note = 'n1'", + ) + assert items == [{"note": {"S": "n1"}}] + + +def test_unkeyed_read_without_reach_back_filters_the_index_rows(dynamodb_client, table): + items = _items( + dynamodb_client, + f"SELECT pk FROM \"{table}\".\"by-rank\" WHERE note = 'n1'", + ) + assert items == [] + + +BEGINS_WITH_NUMBER = ( + "Incorrect operand type for operator or function; operator or function: begins_with, " + "operand type: N" +) + + +@pytest.mark.parametrize("projection", ["note", "*"], ids=["reach-back", "index-rows"]) +def test_a_number_begins_with_prefix_in_an_lsi_filter(dynamodb_client, table, projection): + message = _refusal( + dynamodb_client, + f"SELECT {projection} FROM \"{table}\".\"by-rank\" WHERE begins_with(note, 1)", + ) + assert message == BEGINS_WITH_NUMBER + + +@pytest.mark.parametrize("projection", ["note", "*"], ids=["reach-back", "index-rows"]) +def test_a_number_begins_with_prefix_on_the_lsi_sort_key(dynamodb_client, table, projection): + message = _refusal( + dynamodb_client, + f"SELECT {projection} FROM \"{table}\".\"by-rank\" " + "WHERE pk = 'a' AND begins_with(rank, 1)", + ) + assert message == BEGINS_WITH_NUMBER + + +# ---- the FROM path ---------------------------------------------------------- + +def test_three_component_path_is_refused(dynamodb_client, table): + message = _refusal( + dynamodb_client, f"SELECT * FROM \"{table}\".\"by-color\".\"x\" WHERE color = 'red'" + ) + assert message == TOO_MANY_COMPONENTS + + +def test_empty_index_component_is_refused(dynamodb_client, table): + message = _refusal(dynamodb_client, f"SELECT * FROM \"{table}\".\"\" WHERE pk = 'a'") + assert message == EMPTY_COMPONENT + + +def test_empty_table_component_is_refused(dynamodb_client): + assert _refusal(dynamodb_client, "SELECT * FROM \"\" WHERE pk = 'a'") == EMPTY_COMPONENT + + +def test_empty_component_is_reported_ahead_of_the_length(dynamodb_client, table): + message = _refusal(dynamodb_client, f"SELECT * FROM \"{table}\".\"by-color\".\"\" WHERE pk = 'a'") + assert message == EMPTY_COMPONENT + + +def test_path_is_refused_before_the_table_is_looked_up(dynamodb_client): + missing = f"extenddb-no-such-{uuid.uuid4().hex[:8]}" + message = _refusal(dynamodb_client, f"SELECT * FROM \"{missing}\".\"a\".\"b\" WHERE pk = 'a'") + assert message == TOO_MANY_COMPONENTS + + +def test_syntax_error_is_reported_ahead_of_the_path(dynamodb_client, table): + message = _refusal( + dynamodb_client, f"SELECT * FROM \"{table}\".\"a\".\"b\" WHERE pk = 'a'; SELECT 1" + ) + assert message.startswith(NOT_WELL_FORMED) + + +def test_path_is_refused_ahead_of_the_limit_clause(dynamodb_client, table): + message = _refusal(dynamodb_client, f"SELECT * FROM \"{table}\".\"a\".\"b\" LIMIT 1") + assert message == TOO_MANY_COMPONENTS + + +# ---- write statements naming an index --------------------------------------- + +@pytest.mark.parametrize( + "statement", + [ + "UPDATE \"{t}\".\"by-color\" SET dims = 'Z' WHERE pk = 'a' AND sk = '1'", + "UPDATE \"{t}\".\"no-such-index\" SET dims = 'Z' WHERE pk = 'a' AND sk = '1'", + "DELETE FROM \"{t}\".\"by-color\" WHERE pk = 'a' AND sk = '1'", + "DELETE FROM \"{t}\".\"by-rank\" WHERE pk = 'a' AND sk = '1'", + ], +) +def test_update_and_delete_on_an_index_are_refused(dynamodb_client, table, statement): + assert _refusal(dynamodb_client, statement.format(t=table)) == ON_INDEX + + +@pytest.mark.parametrize( + "statement", + [ + "UPDATE \"{t}\".\"by-color\" WHERE pk = 'a' AND sk = '1'", + "INSERT INTO \"{t}\".\"by-color\" VALUE {{'pk': 'w', 'sk': 'w'", + ], +) +def test_write_syntax_error_is_reported_ahead_of_the_index(dynamodb_client, table, statement): + assert _refusal(dynamodb_client, statement.format(t=table)).startswith(NOT_WELL_FORMED) + + +def test_update_on_an_index_is_refused_before_the_table_is_looked_up(dynamodb_client): + missing = f"extenddb-no-such-{uuid.uuid4().hex[:8]}" + message = _refusal( + dynamodb_client, f"UPDATE \"{missing}\".\"idx\" SET a = 1 WHERE pk = 'a' AND sk = '1'" + ) + assert message == ON_INDEX + + +@pytest.mark.parametrize( + ("statement", "want"), + [ + ("UPDATE \"{t}\".\"a\".\"b\" SET a = 1 WHERE pk = 'a' AND sk = '1'", TOO_MANY_COMPONENTS), + ("DELETE FROM \"{t}\".\"\" WHERE pk = 'a' AND sk = '1'", EMPTY_COMPONENT), + ("UPDATE \"\" SET a = 1 WHERE pk = 'a' AND sk = '1'", EMPTY_COMPONENT), + ], +) +def test_write_paths_take_the_select_path_refusals(dynamodb_client, table, statement, want): + assert _refusal(dynamodb_client, statement.format(t=table)) == want + + +@pytest.mark.parametrize("target", ['"{t}"."by-color"', '"{t}"."no-such-index"', '"{t}"."a"."b"']) +def test_insert_into_a_path_is_refused_at_the_table_name(dynamodb_client, table, target): + statement = f"INSERT INTO {target.format(t=table)} VALUE {{'pk': 'w', 'sk': 'w'}}" + assert _refusal(dynamodb_client, statement) == _insert_refusal(table) + + +def test_insert_into_a_path_is_refused_before_the_table_is_looked_up(dynamodb_client): + missing = f"extenddb-no-such-{uuid.uuid4().hex[:8]}" + statement = f"INSERT INTO \"{missing}\".\"i\" VALUE {{'pk': 'w', 'sk': 'w'}}" + assert _refusal(dynamodb_client, statement) == _insert_refusal(missing) + + +def test_refused_writes_leave_the_table_unchanged(dynamodb_client, table): + got = dynamodb_client.get_item( + TableName=table, Key={"pk": {"S": "a"}, "sk": {"S": "1"}}, ConsistentRead=True + )["Item"] + assert got == ITEMS[0] + absent = dynamodb_client.get_item( + TableName=table, Key={"pk": {"S": "w"}, "sk": {"S": "w"}}, ConsistentRead=True + ) + assert "Item" not in absent diff --git a/tests/test_partiql_item_depth.py b/tests/test_partiql_item_depth.py new file mode 100644 index 000000000..4ae3f3501 --- /dev/null +++ b/tests/test_partiql_item_depth.py @@ -0,0 +1,455 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""Dual-target wire tests for the item nesting check on PartiQL writes. + +A parameter may hold 32 maps around its leaf and a stored attribute 31. A +PartiQL INSERT or UPDATE of a 32-level parameter passes the parameter check and +meets the item check, which answers a shorter message than PutItem and +UpdateItem give the same value. An UPDATE meets the item check only once its +condition holds and its SET applies: on a missing item it fails its condition, +and a SET through a path the item does not have answers the update's own +message. A SET of a key attribute is refused ahead of all of this, and a +32-level value the statement does not store (an if_not_exists value the item +does not take, a value only in the WHERE clause) is accepted. In +ExecuteTransaction two members on one item are refused first; otherwise the +item check cancels the transaction with a ValidationError reason at the +member's position, beside each other member's own reason. A 33-level +parameter is refused by the parameter check in every form. + +Runs against real DynamoDB (EXTENDDB_TEST_ENDPOINT unset, AWS credentials) and +against an extenddb server (EXTENDDB_TEST_ENDPOINT set) with identical +assertions. Every expected value was measured on the service (2026-11-02). +Tables use the ``extenddb-measure-pqd-`` prefix and are deleted at the end. +""" + +from __future__ import annotations + +import os +import uuid + +import pytest +from botocore.exceptions import ClientError + +from conftest import wait_for_active + +ITEM_CHECK = "Nesting Levels have exceeded supported limits" +NESTED = f"{ITEM_CHECK}: Attributes in the item have nested levels beyond supported limit" +BAD_PATH = "The document path provided in the update expression is invalid for update" +CONDITION_FAILED = "The conditional request failed" +CANCELLED = "Transaction cancelled, please refer cancellation reasons for specific reasons" +ITEM_REASON = {"Code": "ValidationError", "Message": ITEM_CHECK} +FAILED_REASON = {"Code": "ConditionalCheckFailed", "Message": CONDITION_FAILED} +KEY_UPDATE = ( + "One or more parameter values were invalid: Cannot update attribute pk. This attribute " + "is part of the key" +) + +# ExtendDB hands a PartiQL parameter to UpdateItem or DeleteItem as an +# ExpressionAttributeValue, and those refuse every value nested 32 levels. The +# service runs a statement that does not store such a value (measured +# 2026-11-02). +UNSTORED_DEEP_VALUE = pytest.mark.xfail( + bool(os.environ.get("EXTENDDB_TEST_ENDPOINT", "").strip()), + reason="ExtendDB refuses a 32-level parameter the statement does not store", + raises=ClientError, + strict=True, +) + + +def deep(levels: int) -> dict: + """``levels`` maps wrapped around a string.""" + value: dict = {"S": "leaf"} + for _ in range(levels): + value = {"M": {"n": value}} + return value + + +@pytest.fixture(scope="module") +def table(dynamodb_client): + name = f"extenddb-measure-pqd-{uuid.uuid4().hex[:12]}" + dynamodb_client.create_table( + TableName=name, + BillingMode="PAY_PER_REQUEST", + AttributeDefinitions=[{"AttributeName": "pk", "AttributeType": "S"}], + KeySchema=[{"AttributeName": "pk", "KeyType": "HASH"}], + ) + try: + wait_for_active(dynamodb_client, name) + yield name + finally: + try: + dynamodb_client.delete_table(TableName=name) + except ClientError: + pass + + +@pytest.fixture +def stored(dynamodb_client, table) -> str: + """The key of a fresh item that has no ``x`` and no ``y``.""" + key = f"exists-{uuid.uuid4().hex[:8]}" + dynamodb_client.put_item(TableName=table, Item={"pk": {"S": key}, "a": {"N": "1"}}) + return key + + +def error(call) -> dict: + with pytest.raises(ClientError) as excinfo: + call() + return excinfo.value.response["Error"] + + +def validation_message(call) -> str: + err = error(call) + assert err["Code"] == "ValidationException", err + return err["Message"] + + +def read_x(client, table: str, key: str) -> dict | None: + item = client.get_item(TableName=table, Key={"pk": {"S": key}}, ConsistentRead=True)["Item"] + return item.get("x") + + +def stored_item(client, table: str, key: str) -> dict | None: + return client.get_item(TableName=table, Key={"pk": {"S": key}}, ConsistentRead=True).get( + "Item" + ) + + +def cancellation(client, statements: list[dict]) -> list: + """The reasons an ExecuteTransaction is cancelled with, after checking the + message lists their codes in order.""" + with pytest.raises(ClientError) as excinfo: + client.execute_transaction(TransactStatements=statements) + response = excinfo.value.response + reasons = response["CancellationReasons"] + codes = ", ".join(reason["Code"] for reason in reasons) + assert response["Error"] == { + "Code": "TransactionCanceledException", + "Message": f"{CANCELLED} [{codes}]", + } + return reasons + + +class TestPartiqlInsert: + def insert(self, client, table: str, key: str, levels: int): + return client.execute_statement( + Statement=f"INSERT INTO \"{table}\" VALUE {{'pk': ?, 'x': ?}}", + Parameters=[{"S": key}, deep(levels)], + ) + + @pytest.mark.parametrize("levels", [30, 31]) + def test_a_value_the_item_holds_is_stored(self, dynamodb_client, table, levels): + key = f"insert-{levels}-{uuid.uuid4().hex[:8]}" + self.insert(dynamodb_client, table, key, levels) + assert read_x(dynamodb_client, table, key) == deep(levels) + + def test_32_levels_meet_the_item_check(self, dynamodb_client, table): + key = f"insert-32-{uuid.uuid4().hex[:8]}" + assert ( + validation_message(lambda: self.insert(dynamodb_client, table, key, 32)) + == ITEM_CHECK + ) + + def test_33_levels_meet_the_parameter_check(self, dynamodb_client, table): + key = f"insert-33-{uuid.uuid4().hex[:8]}" + assert ( + validation_message(lambda: self.insert(dynamodb_client, table, key, 33)) + == NESTED + ) + + +class TestPartiqlUpdate: + def set_x(self, client, table: str, key: str, levels: int): + return client.execute_statement( + Statement=f"UPDATE \"{table}\" SET x = ? WHERE pk = '{key}'", + Parameters=[deep(levels)], + ) + + def set_y_z(self, client, table: str, key: str, levels: int): + return client.execute_statement( + Statement=f"UPDATE \"{table}\" SET y.z = ? WHERE pk = '{key}'", + Parameters=[deep(levels)], + ) + + @pytest.mark.parametrize("levels", [30, 31]) + def test_a_value_the_item_holds_is_applied(self, dynamodb_client, table, stored, levels): + self.set_x(dynamodb_client, table, stored, levels) + assert read_x(dynamodb_client, table, stored) == deep(levels) + + def test_32_levels_meet_the_item_check(self, dynamodb_client, table, stored): + assert ( + validation_message(lambda: self.set_x(dynamodb_client, table, stored, 32)) + == ITEM_CHECK + ) + assert read_x(dynamodb_client, table, stored) is None + + def test_33_levels_meet_the_parameter_check(self, dynamodb_client, table, stored): + assert ( + validation_message(lambda: self.set_x(dynamodb_client, table, stored, 33)) + == NESTED + ) + + @pytest.mark.parametrize("levels", [30, 31, 32]) + def test_a_path_the_item_lacks_answers_ahead_of_the_item_check( + self, dynamodb_client, table, stored, levels + ): + assert ( + validation_message(lambda: self.set_y_z(dynamodb_client, table, stored, levels)) + == BAD_PATH + ) + + def test_a_path_the_item_lacks_meets_the_parameter_check_at_33_levels( + self, dynamodb_client, table, stored + ): + assert ( + validation_message(lambda: self.set_y_z(dynamodb_client, table, stored, 33)) + == NESTED + ) + + @pytest.mark.parametrize("levels", [31, 32]) + def test_a_missing_item_fails_its_condition_ahead_of_the_item_check( + self, dynamodb_client, table, levels + ): + key = f"absent-{uuid.uuid4().hex[:8]}" + err = error(lambda: self.set_x(dynamodb_client, table, key, levels)) + assert err == {"Code": "ConditionalCheckFailedException", "Message": CONDITION_FAILED} + + def test_a_missing_item_meets_the_parameter_check_at_33_levels(self, dynamodb_client, table): + key = f"absent-{uuid.uuid4().hex[:8]}" + assert ( + validation_message(lambda: self.set_x(dynamodb_client, table, key, 33)) + == NESTED + ) + + @pytest.mark.parametrize("levels", [31, 32]) + def test_a_set_of_a_key_attribute_is_refused_at_any_depth( + self, dynamodb_client, table, stored, levels + ): + statement = f"UPDATE \"{table}\" SET pk = ? WHERE pk = '{stored}'" + assert ( + validation_message( + lambda: dynamodb_client.execute_statement( + Statement=statement, Parameters=[deep(levels)] + ) + ) + == KEY_UPDATE + ) + + @UNSTORED_DEEP_VALUE + def test_an_if_not_exists_value_the_item_does_not_take_is_accepted( + self, dynamodb_client, table, stored + ): + dynamodb_client.execute_statement( + Statement=f"UPDATE \"{table}\" SET a = if_not_exists(a, ?) WHERE pk = '{stored}'", + Parameters=[deep(32)], + ) + assert stored_item(dynamodb_client, table, stored) == { + "pk": {"S": stored}, + "a": {"N": "1"}, + } + + @UNSTORED_DEEP_VALUE + def test_a_deep_value_only_in_the_where_clause_is_accepted( + self, dynamodb_client, table, stored + ): + dynamodb_client.execute_statement( + Statement=f"UPDATE \"{table}\" SET y = 1 WHERE pk = '{stored}' AND a <> ?", + Parameters=[deep(32)], + ) + assert stored_item(dynamodb_client, table, stored)["y"] == {"N": "1"} + + +class TestPartiqlDeleteAndSelect: + """A 32-level value in the WHERE clause of a DELETE or a SELECT.""" + + @UNSTORED_DEEP_VALUE + def test_a_delete_with_a_deep_value_in_its_where_clause_is_accepted( + self, dynamodb_client, table, stored + ): + dynamodb_client.execute_statement( + Statement=f"DELETE FROM \"{table}\" WHERE pk = '{stored}' AND a <> ?", + Parameters=[deep(32)], + ) + assert stored_item(dynamodb_client, table, stored) is None + + def test_a_select_compares_a_deep_value_like_any_other( + self, dynamodb_client, table, stored + ): + items = dynamodb_client.execute_statement( + Statement=f"SELECT * FROM \"{table}\" WHERE pk = '{stored}' AND a <> ?", + Parameters=[deep(32)], + ConsistentRead=True, + )["Items"] + assert items == [{"pk": {"S": stored}, "a": {"N": "1"}}] + + +class TestPartiqlBatch: + """The item check on a member statement of a BatchExecuteStatement.""" + + def test_an_insert_of_32_levels_fails_only_that_statement(self, dynamodb_client, table): + good = f"batch-good-{uuid.uuid4().hex[:8]}" + bad = f"batch-bad-{uuid.uuid4().hex[:8]}" + statement = f"INSERT INTO \"{table}\" VALUE {{'pk': ?, 'x': ?}}" + resp = dynamodb_client.batch_execute_statement( + Statements=[ + {"Statement": statement, "Parameters": [{"S": good}, deep(31)]}, + {"Statement": statement, "Parameters": [{"S": bad}, deep(32)]}, + ] + ) + assert [entry.get("Error") for entry in resp["Responses"]] == [ + None, + {"Code": "ValidationError", "Message": ITEM_CHECK}, + ] + assert read_x(dynamodb_client, table, good) == deep(31) + + def test_an_update_of_32_levels_fails_only_that_statement( + self, dynamodb_client, table, stored + ): + resp = dynamodb_client.batch_execute_statement( + Statements=[ + { + "Statement": f"UPDATE \"{table}\" SET x = ? WHERE pk = '{stored}'", + "Parameters": [deep(32)], + } + ] + ) + assert [entry.get("Error") for entry in resp["Responses"]] == [ + {"Code": "ValidationError", "Message": ITEM_CHECK} + ] + + +class TestPartiqlTransaction: + """A 32-level value in an ExecuteTransaction member statement cancels the + transaction with the item check's reason at that member's position. An + UPDATE of a missing item fails its condition first.""" + + def insert(self, key: str, levels: int, table: str) -> dict: + return { + "Statement": f"INSERT INTO \"{table}\" VALUE {{'pk': ?, 'x': ?}}", + "Parameters": [{"S": key}, deep(levels)], + } + + def update(self, key: str, levels: int, table: str) -> dict: + return { + "Statement": f"UPDATE \"{table}\" SET x = ? WHERE pk = '{key}'", + "Parameters": [deep(levels)], + } + + def test_an_insert_of_32_levels_cancels_the_transaction(self, dynamodb_client, table): + key = f"txn-insert-32-{uuid.uuid4().hex[:8]}" + statements = [self.insert(key, 32, table)] + assert cancellation(dynamodb_client, statements) == [ITEM_REASON] + assert stored_item(dynamodb_client, table, key) is None + + def test_an_update_of_32_levels_cancels_the_transaction(self, dynamodb_client, table, stored): + statements = [self.update(stored, 32, table)] + assert cancellation(dynamodb_client, statements) == [ITEM_REASON] + assert read_x(dynamodb_client, table, stored) is None + + def test_only_the_member_past_the_limit_has_a_reason(self, dynamodb_client, table): + fits = f"txn-insert-31-{uuid.uuid4().hex[:8]}" + past = f"txn-insert-32-{uuid.uuid4().hex[:8]}" + statements = [self.insert(fits, 31, table), self.insert(past, 32, table)] + assert cancellation(dynamodb_client, statements) == [{"Code": "None"}, ITEM_REASON] + assert stored_item(dynamodb_client, table, fits) is None + + def test_an_update_of_a_missing_item_fails_its_condition_first(self, dynamodb_client, table): + key = f"txn-absent-{uuid.uuid4().hex[:8]}" + statements = [self.update(key, 32, table)] + assert cancellation(dynamodb_client, statements) == [ + {"Code": "ConditionalCheckFailed", "Message": CONDITION_FAILED} + ] + + def test_another_member_reports_its_failed_condition_beside_the_item_check( + self, dynamodb_client, table, stored + ): + key = f"txn-insert-32-{uuid.uuid4().hex[:8]}" + statements = [ + {"Statement": f"UPDATE \"{table}\" SET a = 2 WHERE pk = '{stored}' AND a = 99"}, + self.insert(key, 32, table), + ] + assert cancellation(dynamodb_client, statements) == [FAILED_REASON, ITEM_REASON] + assert stored_item(dynamodb_client, table, key) is None + + def test_every_member_reports_its_own_failed_condition(self, dynamodb_client, table, stored): + absent = f"txn-absent-{uuid.uuid4().hex[:8]}" + statements = [ + self.update(absent, 32, table), + {"Statement": f"UPDATE \"{table}\" SET a = 2 WHERE pk = '{stored}' AND a = 99"}, + ] + assert cancellation(dynamodb_client, statements) == [FAILED_REASON, FAILED_REASON] + + def test_two_members_on_one_item_are_refused_ahead_of_the_item_check( + self, dynamodb_client, table, stored + ): + statements = [ + self.update(stored, 32, table), + {"Statement": f"UPDATE \"{table}\" SET a = 3 WHERE pk = '{stored}'"}, + ] + err = error(lambda: dynamodb_client.execute_transaction(TransactStatements=statements)) + assert err == { + "Code": "ValidationException", + "Message": "Transaction request cannot include multiple operations on one item", + } + assert stored_item(dynamodb_client, table, stored) == { + "pk": {"S": stored}, + "a": {"N": "1"}, + } + + def test_a_missing_table_is_reported_ahead_of_the_item_check(self, dynamodb_client, table): + key = f"txn-insert-32-{uuid.uuid4().hex[:8]}" + statements = [ + {"Statement": f"UPDATE \"{table}-missing\" SET a = 1 WHERE pk = 'q'"}, + self.insert(key, 32, table), + ] + assert cancellation(dynamodb_client, statements) == [ + {"Code": "ResourceNotFound", "Message": "Requested resource not found"}, + {"Code": "None"}, + ] + + +class TestClassicWrites: + """PutItem and UpdateItem refuse a 32-level value as part of the request. + The service puts "1 validation error detected: " in front of the sentence + these tests compare (measured 2026-11-02).""" + + def put(self, client, table: str, levels: int): + key = f"put-{levels}-{uuid.uuid4().hex[:8]}" + client.put_item(TableName=table, Item={"pk": {"S": key}, "x": deep(levels)}) + return key + + def update(self, client, table: str, key: str, levels: int): + client.update_item( + TableName=table, + Key={"pk": {"S": key}}, + UpdateExpression="SET x = :v", + ExpressionAttributeValues={":v": deep(levels)}, + ) + + @pytest.mark.parametrize("levels", [30, 31]) + def test_put_item_stores_a_value_the_item_holds(self, dynamodb_client, table, levels): + key = self.put(dynamodb_client, table, levels) + assert read_x(dynamodb_client, table, key) == deep(levels) + + @pytest.mark.parametrize("levels", [32, 33]) + def test_put_item_refuses_32_levels_with_the_long_message( + self, dynamodb_client, table, levels + ): + message = validation_message(lambda: self.put(dynamodb_client, table, levels)) + assert message.endswith(NESTED), message + + @pytest.mark.parametrize("levels", [30, 31]) + def test_update_item_applies_a_value_the_item_holds( + self, dynamodb_client, table, stored, levels + ): + self.update(dynamodb_client, table, stored, levels) + assert read_x(dynamodb_client, table, stored) == deep(levels) + + @pytest.mark.parametrize("levels", [32, 33]) + def test_update_item_refuses_32_levels_with_the_long_message( + self, dynamodb_client, table, stored, levels + ): + message = validation_message( + lambda: self.update(dynamodb_client, table, stored, levels) + ) + assert message.endswith(NESTED), message diff --git a/tests/test_partiql_item_size.py b/tests/test_partiql_item_size.py new file mode 100644 index 000000000..8e630ca70 --- /dev/null +++ b/tests/test_partiql_item_size.py @@ -0,0 +1,214 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""Dual-target wire tests for the item size limit of a PartiQL UPDATE. + +The service sizes a PartiQL UPDATE by its finished item alone, as it sizes a +transacted Update: the item may grow to exactly 409,600 bytes, and one byte +more is refused with "Item size to update has exceeded the maximum allowed +size". A standalone UpdateItem is sized by its statement instead, so at the +same key it refuses a finished item of 409,600 bytes. More SET clauses and +an if_not_exists add nothing to a PartiQL UPDATE. A WHERE term that fails, and +the implicit check that the item exists, are answered before the size: an +UPDATE over the limit that would not run gets ConditionalCheckFailedException. +BatchExecuteStatement gives the refusal as the statement's ValidationError +entry, and ExecuteTransaction cancels with a ValidationError reason. + +Item size: each attribute's name bytes plus its value bytes. Every item here +holds string attributes only, at a three-byte key, so {pk, p} is +2 + 3 + 1 + len(p) bytes. + +Dual-target: EXTENDDB_TEST_ENDPOINT selects an extenddb server; unset, the +tests run against the live service with the default credential chain. The +assertions are identical on both. Every expected value was measured on the +service (us-east-1, 2026-11-04). +""" + +from __future__ import annotations + +import os + +import pytest +from botocore.exceptions import ClientError + +from conftest import scoped_table + +LIMIT = 409_600 +UPDATE_WORDING = "Item size to update has exceeded the maximum allowed size" + +# The PostgreSQL and SQLite storage layers size updates the way the service +# does. The Cassandra layer sizes every update against a fixed 409,600 bytes +# with the put wording, and the MongoDB layer applies no limit to updates (see +# docs/differences-from-dynamodb.md), so the cases that need a refusal are +# skipped on those two backends. +refused_on_default_backends_only = pytest.mark.skipif( + os.environ.get("EXTENDDB_TEST_BACKEND", "") in {"mongodb", "cassandra"}, + reason="updates are sized against a fixed limit (Cassandra) or not at all (MongoDB); " + "see docs/differences-from-dynamodb.md", +) + + +@pytest.fixture(scope="module") +def table(dynamodb_client): + """A table keyed on a string pk.""" + with scoped_table( + dynamodb_client, + attribute_definitions=[{"AttributeName": "pk", "AttributeType": "S"}], + key_schema=[{"AttributeName": "pk", "KeyType": "HASH"}], + ) as name: + yield name + + +def _p(key: str, finished: int, other: int = 0) -> str: + """A value for p that makes {pk: key, p, ...} exactly `finished` bytes, + where the other attributes take `other` bytes.""" + return "x" * (finished - (2 + len(key)) - 1 - other) + + +def _base(client, table: str, key: str, **attrs: str) -> None: + item = {"pk": {"S": key}, "p": {"S": "a"}} + item.update({name: {"S": value} for name, value in attrs.items()}) + client.put_item(TableName=table, Item=item) + + +def _stored(client, table: str, key: str) -> int | None: + item = client.get_item(TableName=table, Key={"pk": {"S": key}}, ConsistentRead=True).get("Item") + if item is None: + return None + return sum(len(name.encode()) + len(value["S"].encode()) for name, value in item.items()) + + +def _update(client, table: str, key: str, value: str, sets: str = "p = ?", where: str = "", + more: tuple = ()): + return client.execute_statement( + Statement=f'UPDATE "{table}" SET {sets} WHERE pk = ?{where}', + Parameters=[{"S": value}, *more, {"S": key}], + ) + + +def _error(call) -> dict: + with pytest.raises(ClientError) as e: + call() + return e.value.response["Error"] + + +def test_an_update_writes_a_finished_item_of_the_limit(dynamodb_client, table): + _base(dynamodb_client, table, "e01") + _update(dynamodb_client, table, "e01", _p("e01", LIMIT)) + assert _stored(dynamodb_client, table, "e01") == LIMIT + + +@refused_on_default_backends_only +def test_updateitem_refuses_the_same_finished_item(dynamodb_client, table): + # The statement figure of SET p = :p is 23 bytes more than p, 17 more + # than the finished item at this key, so UpdateItem stops 17 bytes short + # of the limit. + _base(dynamodb_client, table, "u01") + err = _error(lambda: dynamodb_client.update_item( + TableName=table, + Key={"pk": {"S": "u01"}}, + UpdateExpression="SET p = :p", + ExpressionAttributeValues={":p": {"S": _p("u01", LIMIT)}}, + )) + assert (err["Code"], err["Message"]) == ("ValidationException", UPDATE_WORDING) + assert _stored(dynamodb_client, table, "u01") == 7 + + +@refused_on_default_backends_only +def test_an_update_one_byte_over_is_refused_with_the_update_wording(dynamodb_client, table): + _base(dynamodb_client, table, "e02") + err = _error(lambda: _update(dynamodb_client, table, "e02", _p("e02", LIMIT + 1))) + assert (err["Code"], err["Message"]) == ("ValidationException", UPDATE_WORDING) + assert _stored(dynamodb_client, table, "e02") == 7 + + +@pytest.mark.parametrize( + ("key", "sets"), + [("e04", "p = ?, q = ?"), ("e05", "p = ?, q = if_not_exists(q, ?)")], + ids=["two-clauses", "if-not-exists"], +) +def test_more_clauses_add_nothing_to_the_size(dynamodb_client, table, key, sets): + _base(dynamodb_client, table, key, q="b" * 10) + _update(dynamodb_client, table, key, _p(key, LIMIT, other=11), sets=sets, + more=({"S": "c" * 10},)) + assert _stored(dynamodb_client, table, key) == LIMIT + + +@refused_on_default_backends_only +def test_two_clauses_one_byte_over_are_refused(dynamodb_client, table): + _base(dynamodb_client, table, "e06", q="b" * 10) + err = _error(lambda: _update(dynamodb_client, table, "e06", _p("e06", LIMIT + 1, other=11), + sets="p = ?, q = ?", more=({"S": "c" * 10},))) + assert (err["Code"], err["Message"]) == ("ValidationException", UPDATE_WORDING) + assert _stored(dynamodb_client, table, "e06") == 18 + + +def test_a_where_term_that_fails_is_answered_before_the_size(dynamodb_client, table): + _base(dynamodb_client, table, "e07") + err = _error(lambda: _update(dynamodb_client, table, "e07", _p("e07", LIMIT + 1), + where=" AND q = 'zz'")) + assert err["Code"] == "ConditionalCheckFailedException" + assert _stored(dynamodb_client, table, "e07") == 7 + + +def test_a_missing_item_is_answered_before_the_size(dynamodb_client, table): + err = _error(lambda: _update(dynamodb_client, table, "e08", _p("e08", LIMIT + 1))) + assert err["Code"] == "ConditionalCheckFailedException" + assert _stored(dynamodb_client, table, "e08") is None + + +def test_returning_reads_back_an_update_at_the_limit(dynamodb_client, table): + _base(dynamodb_client, table, "e10") + resp = dynamodb_client.execute_statement( + Statement=f'UPDATE "{table}" SET p = ? WHERE pk = ? RETURNING MODIFIED OLD *', + Parameters=[{"S": _p("e10", LIMIT)}, {"S": "e10"}], + ) + assert resp["Items"] == [{"p": {"S": "a"}}] + assert _stored(dynamodb_client, table, "e10") == LIMIT + + +def _batch(client, table: str, key: str, value: str) -> dict: + return client.batch_execute_statement(Statements=[{ + "Statement": f'UPDATE "{table}" SET p = ? WHERE pk = ?', + "Parameters": [{"S": value}, {"S": key}], + }])["Responses"][0] + + +def test_a_batch_update_writes_a_finished_item_of_the_limit(dynamodb_client, table): + _base(dynamodb_client, table, "b01") + assert "Error" not in _batch(dynamodb_client, table, "b01", _p("b01", LIMIT)) + assert _stored(dynamodb_client, table, "b01") == LIMIT + + +@refused_on_default_backends_only +def test_a_batch_update_one_byte_over_is_its_validation_error_entry(dynamodb_client, table): + _base(dynamodb_client, table, "b02") + entry = _batch(dynamodb_client, table, "b02", _p("b02", LIMIT + 1)) + assert entry["Error"] == {"Code": "ValidationError", "Message": UPDATE_WORDING} + assert _stored(dynamodb_client, table, "b02") == 7 + + +def _transact(client, table: str, key: str, value: str): + return client.execute_transaction(TransactStatements=[{ + "Statement": f'UPDATE "{table}" SET p = ? WHERE pk = ?', + "Parameters": [{"S": value}, {"S": key}], + }]) + + +def test_a_transacted_update_writes_a_finished_item_of_the_limit(dynamodb_client, table): + _base(dynamodb_client, table, "t01") + _transact(dynamodb_client, table, "t01", _p("t01", LIMIT)) + assert _stored(dynamodb_client, table, "t01") == LIMIT + + +@refused_on_default_backends_only +def test_a_transacted_update_one_byte_over_cancels_the_transaction(dynamodb_client, table): + _base(dynamodb_client, table, "t02") + with pytest.raises(ClientError) as e: + _transact(dynamodb_client, table, "t02", _p("t02", LIMIT + 1)) + resp = e.value.response + assert resp["Error"]["Code"] == "TransactionCanceledException" + assert [(r.get("Code"), r.get("Message")) for r in resp["CancellationReasons"]] == [ + ("ValidationError", UPDATE_WORDING) + ] + assert _stored(dynamodb_client, table, "t02") == 7 diff --git a/tests/test_partiql_key_conditions.py b/tests/test_partiql_key_conditions.py new file mode 100644 index 000000000..3a4c1112f --- /dev/null +++ b/tests/test_partiql_key_conditions.py @@ -0,0 +1,1318 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""Dual-target wire tests for the key conditions of a PartiQL WHERE clause. + +The service reads a WHERE clause as an OR of disjuncts (an OR with a key +attribute on either side, and an IN on a key attribute, split it; AND +distributes over the split) and takes the disjuncts in order. A disjunct +reads by key when it holds one distinct equality of the partition key with a +value and no other predicate on the partition key, and on the sort key either +nothing or one distinct key condition and nothing else. The first disjunct +that is a scan ends the reading. What follows from that, pinned here: + + * a key value of the wrong type is refused only inside a disjunct read by + key before the first scan, the sort key value before the partition key + value, and before the overlap refusal; + * a value on the left is the mirrored comparison, on every operation; + * an UPDATE or DELETE naming a key attribute with two values is refused, + and an EXISTS check doing so fails; a failed EXISTS check returns no + item for ALL_OLD; + * a begins_with key condition takes a prefix of the sort key's type; + * the filter of a read by key may not compare a key attribute with another + path or read a path into one (refused at once, ahead of the key type and + overlap refusals, beside other disjuncts and when another makes a scan), + and a NOT over an AND or an OR stays in that filter, while an IN with a + path in its list makes a scan; + * a path into a key attribute is refused wherever it is in the WHERE of a + table read, in a scan too; + * a read of one disjunct has its key value sizes checked, and a batch + member whose key value is too long names its table; + * a projected call through a missing index is refused before the index is + looked up. + +Dual-target: EXTENDDB_TEST_ENDPOINT selects an extenddb server; unset, the +tests run against the live service with the default credential chain. The +assertions are identical on both. Every expected value was measured on the +service (2026-11-02). +""" + +from __future__ import annotations + +import uuid + +import pytest +from botocore.exceptions import ClientError + +import partiql_gsi_table as gsi +from conftest import scoped_table + +KEY_TYPE = "Key attribute's data type should match its data type in table's schema: Key {}" +OVERLAP = "Overlapping conditions with range keys are not supported in where clause" +FILTER_KEY = ( + "Filter Expression can only contain non-primary key attributes: Primary key attribute: {}" +) +SCALAR_KEY = ( + "Key attributes must be scalars; list random access '[]' and map lookup '.' are not " + "allowed: Key {}" +) +MULTIPLE = ( + "Multiple conditions on same key {}. Only single item Update/Insert/Delete are supported" +) +HASH_SIZE = ( + "One or more parameter values were invalid: Size of hashkey has exceeded the maximum " + "size limit of2048 bytes" +) +RANGE_SIZE = ( + "One or more parameter values were invalid: Aggregated size of all range keys has " + "exceeded the size limit of 1024 bytes" +) +BEGINS_WITH_NUMBER = ( + "Incorrect operand type for operator or function; operator or function: begins_with, " + "operand type: N" +) +CANCELLED = "Transaction cancelled, please refer cancellation reasons for specific reasons" +KEY_SCHEMA_MISMATCH = { + "Code": "ValidationError", + "Message": "The provided key element does not match the schema", +} + + +def _keyed_table(client, sort_type: str): + return scoped_table( + client, + attribute_definitions=[ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "sk", "AttributeType": sort_type}, + ], + key_schema=[ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"}, + ], + ) + + +@pytest.fixture(scope="module") +def htable(dynamodb_client): + """A table keyed on a string pk, holding the item pk = 'a'.""" + with scoped_table(dynamodb_client) as name: + dynamodb_client.put_item(TableName=name, Item={"pk": {"S": "a"}, "a": {"N": "1"}}) + yield name + + +@pytest.fixture(scope="module") +def ctable(dynamodb_client): + """A table keyed on string pk and sk, holding the item a/x.""" + with _keyed_table(dynamodb_client, "S") as name: + dynamodb_client.put_item( + TableName=name, + Item={"pk": {"S": "a"}, "sk": {"S": "x"}, "a": {"S": "x"}}, + ) + yield name + + +@pytest.fixture(scope="module") +def ntable(dynamodb_client): + """A table keyed on a string pk and a number sk, holding the item a/1.""" + with _keyed_table(dynamodb_client, "N") as name: + dynamodb_client.put_item(TableName=name, Item={"pk": {"S": "a"}, "sk": {"N": "1"}}) + yield name + + +@pytest.fixture(scope="module") +def btable(dynamodb_client): + """A table keyed on a string pk and a binary sk, holding the item a/x.""" + with _keyed_table(dynamodb_client, "B") as name: + dynamodb_client.put_item(TableName=name, Item={"pk": {"S": "a"}, "sk": {"B": b"x"}}) + yield name + + +@pytest.fixture(scope="module") +def gtable(dynamodb_client): + """A table with an index keyed on a number g and a string gs.""" + with scoped_table( + dynamodb_client, + attribute_definitions=[ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "g", "AttributeType": "N"}, + {"AttributeName": "gs", "AttributeType": "S"}, + ], + GlobalSecondaryIndexes=[ + { + "IndexName": "gix", + "KeySchema": [ + {"AttributeName": "g", "KeyType": "HASH"}, + {"AttributeName": "gs", "KeyType": "RANGE"}, + ], + "Projection": {"ProjectionType": "ALL"}, + } + ], + ) as name: + yield name + + +def _message(client, statement, **kwargs): + with pytest.raises(ClientError) as e: + client.execute_statement(Statement=statement, **kwargs) + err = e.value.response["Error"] + assert err["Code"] == "ValidationException", err + return err["Message"] + + +def _items(client, statement, **kwargs): + return client.execute_statement(Statement=statement, **kwargs)["Items"] + + +def _sort_keys(client, table, condition): + items = _items(client, f'SELECT * FROM "{table}" WHERE {condition}') + return sorted(i["sk"]["S"] for i in items) + + +def _batch_entries(client, statement): + return client.batch_execute_statement(Statements=[{"Statement": statement}])["Responses"] + + +# ---- which disjuncts have their key values checked --------------------------- + + +@pytest.mark.parametrize( + ("condition", "rows"), + [ + ("sk = 1", []), + ("sk = 1 AND a = 1", []), + ("sk IN [1, 2]", []), + ("sk > 1", []), + ("sk BETWEEN 1 AND 2", []), + ("1 = sk", []), + ("pk = 'a' OR sk = 1", ["x"]), + ("sk = 1 OR pk = 1", []), + ("a = 1 OR pk = 1", []), + ("pk = 'a' OR a = 1 OR pk = 1", ["x"]), + ("pk = 'a' AND sk = 'x' OR sk = 1", ["x"]), + ("pk = 'a' AND a = 1 OR sk = 1", []), + ("pk = 'a' AND (pk = 1 OR a = 1)", []), + ("pk = 'a' AND sk = 1 AND pk = 1", []), + ("pk = 1 AND pk = 'a' AND sk = 1", []), + ("NOT (pk = 1 AND sk = 1)", ["x"]), + ], +) +def test_a_value_outside_a_disjunct_read_by_key_is_not_checked( + dynamodb_client, ctable, condition, rows +): + assert _sort_keys(dynamodb_client, ctable, condition) == rows + + +@pytest.mark.parametrize( + "condition", + [ + "1 = pk", + "1 = pk AND a = 1", + "1 = pk OR a = 1", + "1 = pk OR pk = 'a'", + "pk = 'a' OR (a = 1 AND pk = 1)", + "(a = 1 AND pk = 1) OR pk = 'a'", + "pk = 'a' AND a = 1 OR pk = 1", + "pk = 'a' OR pk = 1 OR a = 1", + ], +) +def test_a_partition_key_value_of_the_wrong_type_in_a_keyed_disjunct( + dynamodb_client, htable, condition +): + statement = f'SELECT * FROM "{htable}" WHERE {condition}' + assert _message(dynamodb_client, statement) == KEY_TYPE.format("pk") + + +@pytest.mark.parametrize( + ("condition", "key"), + [ + ("(pk = 1 AND sk = 'x') OR (pk = 'b' AND sk = 1)", "pk"), + ("(pk = 'b' AND sk = 1) OR (pk = 1 AND sk = 'x')", "sk"), + ("1 = pk AND sk = 'x'", "pk"), + ("pk = 'a' AND sk = 'x' OR pk = 1", "pk"), + ("pk = 'a' AND 1 = sk", "sk"), + ("1 = sk AND pk = 'a'", "sk"), + ("pk = 'a' AND 1 < sk", "sk"), + ("'a' = pk AND 1 = sk", "sk"), + ("pk IN [1, 'b'] AND sk = 1", "sk"), + ("pk IN ['b', 1] AND sk = 1", "sk"), + ("pk = 'a' AND pk = 'a' AND sk = 1", "sk"), + ("pk = 'a' AND sk = 1 AND sk = 1", "sk"), + ("pk = 'a' AND sk = 1 AND sk = a", "sk"), + ("pk = 'a' AND (sk = 1 OR sk = 'y')", "sk"), + ("pk = 'a' AND (sk = 'x' OR sk = 1)", "sk"), + # Key values are checked before the overlap refusal. + ("pk = 'a' AND (sk = 1 OR a = 1)", "sk"), + ("pk = 'a' AND (a = 1 OR sk = 1)", "sk"), + ], +) +def test_a_key_value_of_the_wrong_type_in_a_keyed_disjunct( + dynamodb_client, ctable, condition, key +): + statement = f'SELECT * FROM "{ctable}" WHERE {condition}' + assert _message(dynamodb_client, statement) == KEY_TYPE.format(key) + + +def test_overlapping_disjuncts_of_the_right_type_are_refused(dynamodb_client, ctable): + statement = f"SELECT * FROM \"{ctable}\" WHERE pk = 'a' AND (sk = 'x' OR a = 1)" + assert _message(dynamodb_client, statement) == OVERLAP + + +@pytest.mark.parametrize( + "template", + [ + "SELECT * FROM \"{t}\" WHERE pk = 1 AND sk = 1", + "SELECT * FROM \"{t}\" WHERE sk = 1 AND pk = 1", + "SELECT * FROM \"{t}\" WHERE pk = 1 AND sk > 1", + "SELECT * FROM \"{t}\" WHERE pk = 1 AND sk BETWEEN 1 AND 2", + "SELECT * FROM \"{t}\" WHERE pk = 1 AND sk IN [1]", + "SELECT * FROM \"{t}\" WHERE a = 1 AND pk = 1 AND sk = 1", + "SELECT * FROM \"{t}\" WHERE 1 = pk AND 1 = sk", + "UPDATE \"{t}\" SET q = 1 WHERE pk = 1 AND sk = 1", + "UPDATE \"{t}\" SET q = 1 WHERE 1 = pk AND sk = 1", + "DELETE FROM \"{t}\" WHERE sk = 1 AND pk = 1", + "DELETE FROM \"{t}\" WHERE 'a' = pk AND 1 = sk", + ], +) +def test_the_sort_key_value_is_checked_before_the_partition_key_value( + dynamodb_client, ctable, template +): + assert _message(dynamodb_client, template.format(t=ctable)) == KEY_TYPE.format("sk") + + +def test_parameters_of_the_wrong_type_name_the_sort_key(dynamodb_client, ctable): + statement = f'SELECT * FROM "{ctable}" WHERE pk = ? AND sk = ?' + params = [{"N": "1"}, {"N": "1"}] + assert _message(dynamodb_client, statement, Parameters=params) == KEY_TYPE.format("sk") + + +def test_a_batch_read_names_the_sort_key(dynamodb_client, ctable): + statement = f'SELECT * FROM "{ctable}" WHERE pk = 1 AND sk = 1' + assert _batch_entries(dynamodb_client, statement) == [ + {"Error": {"Code": "ValidationError", "Message": KEY_TYPE.format("sk")}} + ] + + +# ---- a sort key with more than one condition --------------------------------- + + +@pytest.mark.parametrize( + "condition", + [ + "pk = 'a' AND sk = 'x' AND sk = 1", + "pk = 'a' AND sk < 1 AND sk > 0", + "pk = 'a' AND sk > 'a' AND sk < 1", + "pk = 'a' AND sk > 'a' AND sk = 1", + "pk = 'a' AND begins_with(sk, 'x') AND sk = 1", + "pk = 'a' AND sk = 1 AND NOT sk = 'y'", + "pk = 'a' AND sk = 1 AND contains(sk, 'x')", + "pk = 'a' AND sk = 1 AND attribute_exists(sk)", + "pk = 'a' AND sk = 1 AND sk <> 'y'", + "pk = 'a' AND sk = 1 AND sk IS NOT MISSING", + "pk = 'a' AND (sk = 'x' OR a = 1) AND sk = 1", + ], +) +def test_a_second_condition_on_the_sort_key_leaves_its_values_unchecked( + dynamodb_client, ctable, condition +): + assert _sort_keys(dynamodb_client, ctable, condition) == [] + + +def test_two_begins_with_on_a_number_sort_key_run(dynamodb_client, ntable): + statement = ( + f"SELECT * FROM \"{ntable}\" WHERE pk = 'a' AND begins_with(sk, 'x') " + "AND begins_with(sk, 'y')" + ) + assert _items(dynamodb_client, statement) == [] + + +@pytest.mark.parametrize("condition", ["pk = 1 AND pk > 'a'", "pk = 1 AND NOT pk = 'a'"]) +def test_a_second_condition_on_the_partition_key_leaves_its_value_unchecked( + dynamodb_client, htable, condition +): + assert _items(dynamodb_client, f'SELECT * FROM "{htable}" WHERE {condition}') == [] + + +# ---- begins_with on the sort key --------------------------------------------- + + +@pytest.mark.parametrize( + ("table", "condition"), + [ + ("ntable", "pk = 'a' AND begins_with(sk, 'x')"), + ("ntable", "pk = 1 AND begins_with(sk, 'x')"), + ("ntable", "begins_with(sk, 'x') AND pk = 1"), + ("ntable", "pk = 'a' AND begins_with(sk, 'x') AND a = 1"), + ("ntable", "pk = 'a' AND (begins_with(sk, 'x') OR a = 1)"), + ("ntable", "pk = 'a' AND begins_with(sk, 'x') OR pk = 'b'"), + ("btable", "pk = 'a' AND begins_with(sk, 'x')"), + ], +) +def test_a_string_prefix_on_a_number_or_binary_sort_key( + dynamodb_client, request, table, condition +): + name = request.getfixturevalue(table) + statement = f'SELECT * FROM "{name}" WHERE {condition}' + assert _message(dynamodb_client, statement) == KEY_TYPE.format("sk") + + +@pytest.mark.parametrize( + ("table", "prefix", "message"), + [ + ("ntable", {"S": "x"}, KEY_TYPE.format("sk")), + ("ntable", {"B": b"x"}, KEY_TYPE.format("sk")), + ("btable", {"S": "x"}, KEY_TYPE.format("sk")), + ("ntable", {"N": "1"}, BEGINS_WITH_NUMBER), + ("ctable", {"N": "1"}, BEGINS_WITH_NUMBER), + ], + ids=["number-string", "number-binary", "binary-string", "number-number", "string-number"], +) +def test_a_begins_with_parameter_of_another_type( + dynamodb_client, request, table, prefix, message +): + name = request.getfixturevalue(table) + statement = f"SELECT * FROM \"{name}\" WHERE pk = 'a' AND begins_with(sk, ?)" + assert _message(dynamodb_client, statement, Parameters=[prefix]) == message + + +def test_a_binary_prefix_on_a_binary_sort_key_reads(dynamodb_client, btable): + statement = f"SELECT * FROM \"{btable}\" WHERE pk = 'a' AND begins_with(sk, ?)" + items = _items(dynamodb_client, statement, Parameters=[{"B": b"x"}]) + assert items == [{"pk": {"S": "a"}, "sk": {"B": b"x"}}] + + +# ---- index keys -------------------------------------------------------------- + + +def test_an_index_sort_key_alone_is_not_checked(dynamodb_client, gtable): + assert _items(dynamodb_client, f'SELECT * FROM "{gtable}"."gix" WHERE gs = 1') == [] + + +@pytest.mark.parametrize( + ("condition", "key"), [("'x' = g", "g"), ("g = 'x'", "g"), ("g = 1 AND gs = 1", "gs")] +) +def test_an_index_key_value_of_the_wrong_type(dynamodb_client, gtable, condition, key): + statement = f'SELECT * FROM "{gtable}"."gix" WHERE {condition}' + assert _message(dynamodb_client, statement) == KEY_TYPE.format(key) + + +# ---- a value on the left ----------------------------------------------------- + + +def test_a_value_on_the_left_reads_by_key(dynamodb_client, htable, ctable): + assert _items(dynamodb_client, f"SELECT * FROM \"{htable}\" WHERE 'a' = pk") == [ + {"pk": {"S": "a"}, "a": {"N": "1"}} + ] + for statement in [ + f"SELECT * FROM \"{htable}\" WHERE 'a' = pk", + f"SELECT * FROM \"{ctable}\" WHERE 'a' = pk AND 'x' = sk", + ]: + batch = _batch_entries(dynamodb_client, statement) + assert [entry["Item"]["pk"] for entry in batch] == [{"S": "a"}] + txn = dynamodb_client.execute_transaction(TransactStatements=[{"Statement": statement}]) + assert [entry["Item"]["pk"] for entry in txn["Responses"]] == [{"S": "a"}] + + +def test_a_value_on_the_left_updates_by_key(dynamodb_client, htable, ctable): + dynamodb_client.put_item(TableName=htable, Item={"pk": {"S": "w1"}}) + statement = f"UPDATE \"{htable}\" SET q = 1 WHERE 'w1' = pk RETURNING ALL NEW *" + assert _items(dynamodb_client, statement) == [{"pk": {"S": "w1"}, "q": {"N": "1"}}] + dynamodb_client.put_item(TableName=htable, Item={"pk": {"S": "w2"}}) + dynamodb_client.execute_transaction( + TransactStatements=[{"Statement": f"UPDATE \"{htable}\" SET q = 2 WHERE 'w2' = pk"}] + ) + item = dynamodb_client.get_item( + TableName=htable, Key={"pk": {"S": "w2"}}, ConsistentRead=True + )["Item"] + assert item == {"pk": {"S": "w2"}, "q": {"N": "2"}} + statement = f"UPDATE \"{ctable}\" SET q = 1 WHERE 'a' = pk AND 'x' = sk" + assert _items(dynamodb_client, statement) == [] + + +@pytest.mark.parametrize( + "template", + ["UPDATE \"{t}\" SET q = 1 WHERE 1 = pk", "DELETE FROM \"{t}\" WHERE 1 = pk"], + ids=["update", "delete"], +) +def test_a_write_value_on_the_left_of_the_wrong_type(dynamodb_client, htable, template): + assert _message(dynamodb_client, template.format(t=htable)) == KEY_TYPE.format("pk") + + +@pytest.mark.parametrize( + "template", + [ + "UPDATE \"{t}\" SET q = 1 WHERE pk = 'a' AND 1 = pk", + "DELETE FROM \"{t}\" WHERE pk = 'a' AND 1 = pk", + ], + ids=["update", "delete"], +) +def test_a_write_with_two_values_for_a_key(dynamodb_client, htable, template): + assert _message(dynamodb_client, template.format(t=htable)) == MULTIPLE.format("pk") + + +def test_a_batch_read_with_a_value_of_the_wrong_type_on_the_left(dynamodb_client, htable): + statement = f'SELECT * FROM "{htable}" WHERE 1 = pk' + assert _batch_entries(dynamodb_client, statement) == [ + {"Error": {"Code": "ValidationError", "Message": KEY_TYPE.format("pk")}} + ] + + +def test_a_transaction_read_with_a_value_of_the_wrong_type_on_the_left(dynamodb_client, htable): + statement = f'SELECT * FROM "{htable}" WHERE 1 = pk' + with pytest.raises(ClientError) as e: + dynamodb_client.execute_transaction(TransactStatements=[{"Statement": statement}]) + response = e.value.response + assert response["Error"] == { + "Code": "TransactionCanceledException", + "Message": f"{CANCELLED} [ValidationError]", + } + assert response["CancellationReasons"] == [KEY_SCHEMA_MISMATCH] + + +# ---- key attributes in the filter of a read by key --------------------------- + + +@pytest.mark.parametrize( + ("table", "condition", "message"), + [ + ("ctable", "pk = 'a' AND sk = a", FILTER_KEY.format("sk")), + ("ctable", "pk = 'a' AND a = sk", FILTER_KEY.format("sk")), + ("ctable", "pk = 'a' AND sk = a AND a = 'x'", FILTER_KEY.format("sk")), + ("ctable", "pk = 'a' AND sk = 'x' AND sk = a", FILTER_KEY.format("sk")), + ("ctable", "pk = 'a' AND a = 'x' AND sk = a", FILTER_KEY.format("sk")), + ("ctable", "pk = 'a' AND sk = pk", FILTER_KEY.format("sk")), + ("ctable", "pk = 'a' AND pk = a", FILTER_KEY.format("pk")), + ("ctable", "pk = 'a' AND sk = 'x' AND pk = a", FILTER_KEY.format("pk")), + ("htable", "pk = 'a' AND pk = a", FILTER_KEY.format("pk")), + ("htable", "pk = 'a' AND a = pk", FILTER_KEY.format("pk")), + ("ctable", "pk = 'a' AND sk.x = 1", SCALAR_KEY.format("sk")), + ("ctable", "pk = 'a' AND sk[0] = 1", SCALAR_KEY.format("sk")), + ("htable", "pk = 'a' AND pk.x = 1", SCALAR_KEY.format("pk")), + # Under a NOT over an AND or an OR, a path into a key attribute gets + # the filter refusal; naming both keys, it names the sort key. + ("ctable", "pk = 'a' AND NOT (sk.q = 1 AND n = 1)", FILTER_KEY.format("sk")), + ("ctable", "pk = 'a' AND NOT (sk.q = 1 AND pk = b)", FILTER_KEY.format("sk")), + ("ctable", "pk = 'a' AND NOT (pk = b AND sk.q = 1)", FILTER_KEY.format("sk")), + ], +) +def test_a_keyed_filter_naming_a_key_attribute_is_refused( + dynamodb_client, request, table, condition, message +): + name = request.getfixturevalue(table) + assert _message(dynamodb_client, f'SELECT * FROM "{name}" WHERE {condition}') == message + + +@pytest.mark.parametrize( + ("condition", "rows"), + [ + ("sk = a", ["x"]), + ("pk = a", []), + ("pk = 'a' AND size(sk) = 1", ["x"]), + ("pk = 'a' AND attribute_exists(sk)", ["x"]), + ("pk = 'a' AND sk IS NOT MISSING", ["x"]), + ("pk = 'a' AND contains(sk, 'x')", ["x"]), + ("pk = 'a' AND sk <> 'y'", ["x"]), + ("pk = 'a' AND sk IN [a]", ["x"]), + ], +) +def test_a_scan_filter_may_name_a_key_attribute(dynamodb_client, ctable, condition, rows): + assert _sort_keys(dynamodb_client, ctable, condition) == rows + + +def test_overlapping_disjuncts_are_refused_before_the_filter(dynamodb_client, ctable): + statement = f"SELECT * FROM \"{ctable}\" WHERE pk = 'a' AND (sk = a OR a = 'x')" + assert _message(dynamodb_client, statement) == OVERLAP + + +@pytest.mark.parametrize( + "condition", + [ + "(pk = 'a' AND sk.x = 1) OR pk = 'b'", + # Ahead of the overlap refusal. + "(pk = 'a' AND sk.x = 1) OR (pk = 'a' AND sk = 'y')", + "(pk = 'a' AND sk = 'y') OR (pk = 'a' AND sk.x = 1)", + # Ahead of a key value of the wrong type in another disjunct. + "(pk = 'a' AND sk.x = 1) OR pk = 1", + "pk = 1 OR (pk = 'a' AND sk.x = 1)", + # When a later disjunct makes the statement a scan. + "(pk = 'a' AND sk.x = 1) OR x = 1", + ], + ids=[ + "beside-a-partition", + "overlap", + "overlap-reversed", + "key-type", + "key-type-reversed", + "scan", + ], +) +def test_a_path_into_a_key_attribute_in_one_disjunct_is_refused_at_once( + dynamodb_client, ctable, condition +): + statement = f'SELECT * FROM "{ctable}" WHERE {condition}' + assert _message(dynamodb_client, statement) == SCALAR_KEY.format("sk") + + +@pytest.mark.parametrize( + "condition", + ["sk.x = 1", "x = 1 OR (pk = 'a' AND sk.x = 1)"], + ids=["alone", "after-a-scan"], +) +def test_a_path_into_a_key_attribute_is_refused_anywhere_in_the_where( + dynamodb_client, ctable, condition +): + statement = f'SELECT * FROM "{ctable}" WHERE {condition}' + assert _message(dynamodb_client, statement) == SCALAR_KEY.format("sk") + + +def test_a_not_over_a_compound_condition_keeps_its_disjunct_read_by_key( + dynamodb_client, ctable +): + statement = ( + f"SELECT * FROM \"{ctable}\" WHERE pk = 'a' AND NOT (sk = 'x' AND a = 1) " + "OR pk = 'a' AND sk = 'y'" + ) + assert _message(dynamodb_client, statement) == OVERLAP + + +@pytest.mark.parametrize( + ("written", "condition", "extra"), + [ + ("w3", "pk = 'a' AND pk = 'b' AND a = 1", {}), + ("w4", "pk = 'a' AND pk = 'b'", {}), + # Item 'a' exists, and the reason still holds no item. + ( + "w5", + "pk = 'a' AND pk = 'b' AND a = 1", + {"ReturnValuesOnConditionCheckFailure": "ALL_OLD"}, + ), + ], + ids=["further-condition", "no-further-condition", "all-old"], +) +def test_an_exists_check_with_two_values_for_a_key_fails( + dynamodb_client, htable, written, condition, extra +): + dynamodb_client.put_item(TableName=htable, Item={"pk": {"S": written}}) + check = dict({"Statement": f'EXISTS(SELECT * FROM "{htable}" WHERE {condition})'}, **extra) + update = f"UPDATE \"{htable}\" SET q = 3 WHERE pk = '{written}'" + with pytest.raises(ClientError) as e: + dynamodb_client.execute_transaction(TransactStatements=[check, {"Statement": update}]) + response = e.value.response + assert response["Error"] == { + "Code": "TransactionCanceledException", + "Message": f"{CANCELLED} [ConditionalCheckFailed, None]", + } + assert response["CancellationReasons"] == [ + {"Code": "ConditionalCheckFailed", "Message": "The conditional request failed"}, + {"Code": "None"}, + ] + item = dynamodb_client.get_item( + TableName=htable, Key={"pk": {"S": written}}, ConsistentRead=True + )["Item"] + assert item == {"pk": {"S": written}} + + +@pytest.mark.parametrize( + ("written", "condition"), + [("w6", "pk = 'a' AND a = 2"), ("w7", "pk = 'a' AND pk = 'a' AND a = 2")], + ids=["one-value", "one-value-twice"], +) +def test_a_failed_exists_check_returns_no_item_for_all_old( + dynamodb_client, htable, written, condition +): + dynamodb_client.put_item(TableName=htable, Item={"pk": {"S": written}}) + check = { + "Statement": f'EXISTS(SELECT * FROM "{htable}" WHERE {condition})', + "ReturnValuesOnConditionCheckFailure": "ALL_OLD", + } + update = f"UPDATE \"{htable}\" SET q = 3 WHERE pk = '{written}'" + with pytest.raises(ClientError) as e: + dynamodb_client.execute_transaction(TransactStatements=[check, {"Statement": update}]) + response = e.value.response + assert response["Error"] == { + "Code": "TransactionCanceledException", + "Message": f"{CANCELLED} [ConditionalCheckFailed, None]", + } + # Item 'a' exists, and the reason holds no item. + assert response["CancellationReasons"] == [ + {"Code": "ConditionalCheckFailed", "Message": "The conditional request failed"}, + {"Code": "None"}, + ] + item = dynamodb_client.get_item( + TableName=htable, Key={"pk": {"S": written}}, ConsistentRead=True + )["Item"] + assert item == {"pk": {"S": written}} + + +# ---- key value sizes --------------------------------------------------------- + +BIG_HASH = "a" * 2049 +BIG_RANGE = "a" * 1025 + + +@pytest.mark.parametrize( + "condition", + [f"pk = '{BIG_HASH}'", f"pk = '{chr(0xE9) * 1025}'"], + ids=["ascii", "two-byte"], +) +def test_a_partition_key_value_past_2048_bytes(dynamodb_client, htable, condition): + statement = f'SELECT * FROM "{htable}" WHERE {condition}' + assert _message(dynamodb_client, statement) == HASH_SIZE + + +def test_a_partition_key_parameter_past_2048_bytes(dynamodb_client, htable): + statement = f'SELECT * FROM "{htable}" WHERE pk = ?' + assert _message(dynamodb_client, statement, Parameters=[{"S": BIG_HASH}]) == HASH_SIZE + + +@pytest.mark.parametrize( + ("condition", "message"), + [ + (f"pk = 'a' AND sk = '{BIG_RANGE}'", RANGE_SIZE), + (f"pk = 'a' AND sk > '{BIG_RANGE}'", RANGE_SIZE), + (f"pk = 'a' AND sk BETWEEN 'a' AND '{BIG_RANGE}'", RANGE_SIZE), + (f"pk = 'a' AND begins_with(sk, '{BIG_RANGE}')", RANGE_SIZE), + (f"pk = '{BIG_HASH}' AND sk = '{BIG_RANGE}'", HASH_SIZE), + (f"pk = '{BIG_HASH}'", HASH_SIZE), + ], + ids=["equal", "greater", "between", "begins-with", "both", "partition"], +) +def test_a_sort_key_value_past_1024_bytes(dynamodb_client, ctable, condition, message): + statement = f'SELECT * FROM "{ctable}" WHERE {condition}' + assert _message(dynamodb_client, statement) == message + + +@pytest.mark.parametrize( + ("table", "condition"), + [ + ("htable", f"pk = '{'a' * 2048}'"), + ("ctable", f"pk = 'a' AND sk = '{'a' * 1024}'"), + ("ctable", f"sk = '{BIG_RANGE}'"), + ], + ids=["hash-limit", "range-limit", "scan"], +) +def test_key_values_at_the_limit_or_outside_a_key_read_run( + dynamodb_client, request, table, condition +): + name = request.getfixturevalue(table) + assert _items(dynamodb_client, f'SELECT * FROM "{name}" WHERE {condition}') == [] + + +def test_a_long_value_in_a_scan_condition_on_the_partition_key_runs(dynamodb_client, htable): + items = _items(dynamodb_client, f"SELECT * FROM \"{htable}\" WHERE pk > '{BIG_HASH}'") + assert all(i["pk"]["S"] > BIG_HASH for i in items) + + +@pytest.mark.parametrize( + "condition", + [f"pk IN ['a', '{BIG_HASH}']", f"pk = 'a' OR pk = '{BIG_HASH}'"], + ids=["in", "or"], +) +def test_several_disjuncts_return_the_other_items_before_the_size_check( + dynamodb_client, htable, condition +): + response = dynamodb_client.execute_statement( + Statement=f'SELECT * FROM "{htable}" WHERE {condition}' + ) + assert [i["pk"] for i in response["Items"]] == [{"S": "a"}] + assert "NextToken" in response + + +@pytest.mark.parametrize( + "template", + [ + "SELECT * FROM \"{t}\" WHERE pk = '{v}'", + "UPDATE \"{t}\" SET q = 1 WHERE pk = '{v}'", + "DELETE FROM \"{t}\" WHERE pk = '{v}'", + "INSERT INTO \"{t}\" VALUE {{'pk': '{v}'}}", + ], + ids=["select", "update", "delete", "insert"], +) +def test_a_batch_key_value_past_its_size_limit_names_the_table( + dynamodb_client, htable, template +): + statement = template.format(t=htable, v=BIG_HASH) + assert _batch_entries(dynamodb_client, statement) == [ + {"Error": {"Code": "ValidationError", "Message": HASH_SIZE}, "TableName": htable} + ] + + +# ---- a projected call through a missing index -------------------------------- + + +@pytest.mark.parametrize( + ("statement", "position"), + [ + ("SELECT COUNT(*) FROM \"{t}\".\"nope\"", "1:8:5"), + ("SELECT COUNT(*) FROM \"{t}\".\"nope\" WHERE pk = 'a'", "1:8:5"), + ("SELECT COUNT(*) FROM \"{t}\".\"nope\" WHERE pk = 1e126", "1:8:5"), + ("SELECT a, COUNT(*) FROM \"{t}\".\"nope\"", "1:11:5"), + ("SELECT size(a) FROM \"{t}\".\"nope\"", "1:8:4"), + ("SELECT COUNT(a) FROM \"{t}\".\"nope\"", "1:8:5"), + ("SELECT count(*) FROM \"{t}\".\"nope\"", "1:8:5"), + ("SELECT\nCOUNT(*) FROM \"{t}\".\"nope\"", "2:1:5"), + ], +) +def test_a_projected_call_is_refused_before_the_index_lookup( + dynamodb_client, htable, statement, position +): + message = _message(dynamodb_client, statement.format(t=htable)) + assert message == f"Unexpected path component at {position}" + + +# ---- a path into a key attribute in the WHERE of a write --------------------- +# +# Measured 2026-11-05. A condition that tests a path into a key attribute +# against values (a comparison either way round, BETWEEN, IN, begins_with, +# contains, attribute_type, an IS type test, one of these compared with a +# boolean) is refused naming the key, in statement order with the refusal +# of two values for one key, ahead of the full key and the key type. Any +# other condition with a path into a key attribute (OR, NOT, a function of +# the path, IS MISSING, the path compared with a path) is refused later, +# after the key type and the SET of a key attribute, with "Key:" before the +# name, which is the sort key when a path into it is among the conditions. + +KEY_PATH_CONDITION = ( + "Key attributes must be scalars; list random access '[]' and map lookup '.' are not " + "allowed: Key: {}" +) +FULL_KEY = "Where clause does not contain a mandatory equality on all key attributes" +KEY_UPDATE = ( + "One or more parameter values were invalid: Cannot update attribute pk. This attribute " + "is part of the key" +) + + +def _write_item(client, table): + """The partition key of a fresh item k/b with x = 0.""" + pk = f"w-{uuid.uuid4().hex[:8]}" + client.put_item( + TableName=table, Item={"pk": {"S": pk}, "sk": {"S": "b"}, "x": {"N": "0"}} + ) + return pk + + +def _stored_x(client, table, pk): + item = client.get_item( + TableName=table, Key={"pk": {"S": pk}, "sk": {"S": "b"}}, ConsistentRead=True + )["Item"] + return item["x"]["N"] + + +@pytest.mark.parametrize( + ("template", "key"), + [ + ("DELETE FROM {t} WHERE pk = '{k}' AND sk = 'b' AND pk.c = 1", "pk"), + ("UPDATE {t} SET x = 1 WHERE pk = '{k}' AND sk = 'b' AND sk[0] = 'z'", "sk"), + ("UPDATE {t} SET x = 1 WHERE pk = '{k}' AND sk = 'b' AND 1 = pk.c", "pk"), + ("UPDATE {t} SET x = 1 WHERE pk = '{k}' AND sk = 'b' AND pk.c <> 1", "pk"), + ("UPDATE {t} SET x = 1 WHERE pk = '{k}' AND sk = 'b' AND pk.c BETWEEN 1 AND 2", "pk"), + ("UPDATE {t} SET x = 1 WHERE pk = '{k}' AND sk = 'b' AND pk.c IN [1]", "pk"), + ("UPDATE {t} SET x = 1 WHERE pk = '{k}' AND sk = 'b' AND begins_with(pk.c, 'x')", "pk"), + ("UPDATE {t} SET x = 1 WHERE pk = '{k}' AND sk = 'b' AND contains(pk.c, 'x')", "pk"), + ("UPDATE {t} SET x = 1 WHERE pk = '{k}' AND sk = 'b' AND pk.c IS NULL", "pk"), + ("UPDATE {t} SET x = 1 WHERE pk = '{k}' AND sk = 'b' AND pk.c = 1 = false", "pk"), + ("UPDATE {t} SET x = 1 WHERE pk = '{k}' AND sk = 'b' AND (pk.c = 1)", "pk"), + ("UPDATE {t} SET x = 1 WHERE pk = '{k}' AND sk = 'b' AND pk['c'] = 1", "pk"), + ("UPDATE {t} SET x = 1 WHERE pk = '{k}' AND pk.c = 1", "pk"), + ("UPDATE {t} SET x = 1 WHERE pk = 1 AND sk = 'b' AND pk.c = 1", "pk"), + ("UPDATE {t} SET x = 1 WHERE pk = '{k}' AND sk = 1 AND sk.c = 1", "sk"), + ("UPDATE {t} SET x = 1 WHERE pk = '{k}' AND sk = 'b' AND sk.c = 1 AND pk.d = 1", "sk"), + ("UPDATE {t} SET x = 1 WHERE pk = '{k}' AND sk = 'b' AND NOT pk.c = 1 AND sk.d = 1", "sk"), + ("UPDATE {t} SET x = 1 WHERE pk = '{k}' AND sk = 'b' AND pk.c = 1 AND pk = 'zz'", "pk"), + ], + ids=[ + "delete-map-lookup", + "update-list-index", + "value-on-the-left", + "not-equal", + "between", + "in", + "begins-with", + "contains", + "is-null", + "compared-with-false", + "in-parentheses", + "subscript", + "ahead-of-the-full-key", + "ahead-of-the-key-type", + "ahead-of-the-sort-key-type", + "first-in-statement-order", + "ahead-of-a-later-refusal", + "ahead-of-a-later-second-value", + ], +) +def test_a_write_refuses_a_path_into_a_key_tested_against_values( + dynamodb_client, ctable, template, key +): + pk = _write_item(dynamodb_client, ctable) + statement = template.format(t=f'"{ctable}"', k=pk) + assert _message(dynamodb_client, statement) == SCALAR_KEY.format(key) + assert _stored_x(dynamodb_client, ctable, pk) == "0" + + +def test_a_second_value_for_a_key_ahead_of_a_path_into_one(dynamodb_client, ctable): + pk = _write_item(dynamodb_client, ctable) + statement = ( + f"UPDATE \"{ctable}\" SET x = 1 WHERE pk = '{pk}' AND pk = 'zz' AND sk = 'b' " + "AND sk.c = 1" + ) + assert _message(dynamodb_client, statement) == MULTIPLE.format("pk") + assert _stored_x(dynamodb_client, ctable, pk) == "0" + + +@pytest.mark.parametrize( + ("template", "key"), + [ + ("UPDATE {t} SET x = 1 WHERE pk = '{k}' AND sk = 'b' AND (a = 1 OR pk.c = 1)", "pk"), + ("UPDATE {t} SET x = 1 WHERE pk = '{k}' AND sk = 'b' AND NOT pk.c = 1", "pk"), + ("UPDATE {t} SET x = 1 WHERE pk = '{k}' AND sk = 'b' AND attribute_exists(pk.c)", "pk"), + ("UPDATE {t} SET x = 1 WHERE pk = '{k}' AND sk = 'b' AND size(sk.c) > 1", "sk"), + ("UPDATE {t} SET x = 1 WHERE pk = '{k}' AND sk = 'b' AND pk.c IS MISSING", "pk"), + ("UPDATE {t} SET x = 1 WHERE pk = '{k}' AND sk = 'b' AND pk.c IS NOT NULL", "pk"), + ("UPDATE {t} SET x = 1 WHERE pk = '{k}' AND sk = 'b' AND pk.c NOT IN [1]", "pk"), + ("UPDATE {t} SET x = 1 WHERE pk = '{k}' AND sk = 'b' AND pk.c IN [a]", "pk"), + ("UPDATE {t} SET x = 1 WHERE pk = '{k}' AND sk = 'b' AND pk.c = a", "pk"), + ("DELETE FROM {t} WHERE pk = '{k}' AND sk = 'b' AND NOT pk.c = 1", "pk"), + ( + "UPDATE {t} SET x = 1 WHERE pk = '{k}' AND sk = 'b' AND NOT pk.c = 1 AND NOT sk.d = 1", + "sk", + ), + ("UPDATE {t} SET x = 1 WHERE pk = '{k}' AND sk = 'b' AND pk.c = sk.d", "sk"), + ("UPDATE {t} SET x = 1 WHERE pk = '{k}' AND sk = 'b' AND (pk.c = 1 OR sk = 'b')", "pk"), + ], + ids=[ + "or", + "not", + "attribute-exists", + "size", + "is-missing", + "is-not-null", + "not-in", + "in-with-a-path", + "compared-with-a-path", + "delete", + "the-sort-key-anywhere", + "the-sort-key-in-one-condition", + "a-bare-sort-key", + ], +) +def test_a_write_refuses_a_path_into_a_key_in_its_condition( + dynamodb_client, ctable, template, key +): + pk = _write_item(dynamodb_client, ctable) + statement = template.format(t=f'"{ctable}"', k=pk) + assert _message(dynamodb_client, statement) == KEY_PATH_CONDITION.format(key) + assert _stored_x(dynamodb_client, ctable, pk) == "0" + + +@pytest.mark.parametrize( + ("template", "message"), + [ + ("UPDATE {t} SET x = 1 WHERE pk = '{k}' AND NOT pk.c = 1", FULL_KEY), + ("UPDATE {t} SET x = 1 WHERE pk = 1 AND sk = 'b' AND NOT pk.c = 1", KEY_TYPE.format("pk")), + ("UPDATE {t} SET pk = 'z' WHERE pk = '{k}' AND sk = 'b' AND NOT pk.c = 1", KEY_UPDATE), + ( + "UPDATE {t} SET x = 1 WHERE pk = '{k}' AND sk = 'b' AND NOT pk.c = 1 AND sk = 'zz'", + MULTIPLE.format("sk"), + ), + ], + ids=["full-key", "key-type", "set-of-a-key", "second-value"], +) +def test_the_refusals_ahead_of_a_path_into_a_key_in_a_condition( + dynamodb_client, ctable, template, message +): + pk = _write_item(dynamodb_client, ctable) + statement = template.format(t=f'"{ctable}"', k=pk) + assert _message(dynamodb_client, statement) == message + assert _stored_x(dynamodb_client, ctable, pk) == "0" + + +@pytest.mark.parametrize( + "template", + [ + "DELETE FROM {t} WHERE pk = '{k}' AND sk = 'b' AND NOT pk.c = 1", + "UPDATE {t} SET x = 1 WHERE pk = '{k}' AND sk = 'b' AND NOT pk.c = 1", + ], + ids=["delete", "update"], +) +def test_a_path_into_a_key_in_a_condition_is_refused_for_a_missing_item( + dynamodb_client, ctable, template +): + statement = template.format(t=f'"{ctable}"', k=f"w-absent-{uuid.uuid4().hex[:8]}") + assert _message(dynamodb_client, statement) == KEY_PATH_CONDITION.format("pk") + + +def test_a_batch_names_the_table_only_for_a_path_in_the_condition(dynamodb_client, ctable): + first = _write_item(dynamodb_client, ctable) + second = _write_item(dynamodb_client, ctable) + t = f'"{ctable}"' + responses = dynamodb_client.batch_execute_statement( + Statements=[ + {"Statement": f"UPDATE {t} SET x = 1 WHERE pk = '{first}' AND sk = 'b' AND pk.c = 1"}, + { + "Statement": ( + f"UPDATE {t} SET x = 1 WHERE pk = '{second}' AND sk = 'b' AND NOT pk.c = 1" + ) + }, + ] + )["Responses"] + assert responses == [ + {"Error": {"Code": "ValidationError", "Message": SCALAR_KEY.format("pk")}}, + { + "Error": {"Code": "ValidationError", "Message": KEY_PATH_CONDITION.format("pk")}, + "TableName": ctable, + }, + ] + assert _stored_x(dynamodb_client, ctable, first) == "0" + assert _stored_x(dynamodb_client, ctable, second) == "0" + + +# Measured 2026-11-05. In a batch, a write that tests a path into a key +# attribute against values counts neither as a write nor in the check for +# two statements on one item. A write with a path into a key attribute among +# its other conditions counts as a write, and not in that check. + +MIXED = "Read and write requests together in the same batch is not supported." + + +def test_a_batch_leaves_a_path_into_a_key_tested_against_values_out_of_its_checks( + dynamodb_client, ctable +): + first = _write_item(dynamodb_client, ctable) + second = _write_item(dynamodb_client, ctable) + t = f'"{ctable}"' + read = {"Statement": f"SELECT x FROM {t} WHERE pk = '{first}' AND sk = 'b'"} + refused = { + "Statement": f"UPDATE {t} SET x = 1 WHERE pk = '{second}' AND sk = 'b' AND pk.c = 1" + } + again = {"Statement": f"UPDATE {t} SET x = 2 WHERE pk = '{second}' AND sk = 'b'"} + refusal = {"Error": {"Code": "ValidationError", "Message": SCALAR_KEY.format("pk")}} + responses = dynamodb_client.batch_execute_statement(Statements=[read, refused])["Responses"] + assert responses == [{"TableName": ctable, "Item": {"x": {"N": "0"}}}, refusal] + responses = dynamodb_client.batch_execute_statement(Statements=[refused, again])["Responses"] + assert responses == [refusal, {"TableName": ctable}] + assert _stored_x(dynamodb_client, ctable, second) == "2" + + +@pytest.mark.parametrize( + "template", + [ + "UPDATE {t} SET x = 1 WHERE pk = '{k}' AND sk = 'b' AND NOT pk.c = 1", + "DELETE FROM {t} WHERE pk = '{k}' AND sk = 'b' AND NOT pk.c = 1", + ], + ids=["update", "delete"], +) +def test_a_batch_counts_a_path_into_a_key_in_a_condition_as_a_write_of_no_item( + dynamodb_client, ctable, template +): + first = _write_item(dynamodb_client, ctable) + second = _write_item(dynamodb_client, ctable) + t = f'"{ctable}"' + read = {"Statement": f"SELECT * FROM {t} WHERE pk = '{first}' AND sk = 'b'"} + refused = {"Statement": template.format(t=t, k=second)} + with pytest.raises(ClientError) as e: + dynamodb_client.batch_execute_statement(Statements=[read, refused]) + assert e.value.response["Error"] == {"Code": "ValidationException", "Message": MIXED} + again = {"Statement": f"UPDATE {t} SET x = 2 WHERE pk = '{second}' AND sk = 'b'"} + refusal = { + "Error": {"Code": "ValidationError", "Message": KEY_PATH_CONDITION.format("pk")}, + "TableName": ctable, + } + for statements, answers in ( + ([refused, again], [refusal, {"TableName": ctable}]), + ([again, refused], [{"TableName": ctable}, refusal]), + ): + responses = dynamodb_client.batch_execute_statement(Statements=statements)["Responses"] + assert responses == answers + assert _stored_x(dynamodb_client, ctable, second) == "2" + + +@pytest.mark.parametrize( + ("template", "reason"), + [ + ("UPDATE {t} SET x = 1 WHERE pk = '{k}' AND sk = 'b' AND pk.c = 1", KEY_SCHEMA_MISMATCH), + ("DELETE FROM {t} WHERE pk = '{k}' AND sk = 'b' AND sk[0] = 'z'", KEY_SCHEMA_MISMATCH), + ( + "UPDATE {t} SET x = 1 WHERE pk = '{k}' AND sk = 'b' AND NOT pk.c = 1", + {"Code": "ValidationError", "Message": KEY_PATH_CONDITION.format("pk")}, + ), + ], + ids=["update-tested-against-a-value", "delete-tested-against-a-value", "in-the-condition"], +) +def test_a_transaction_write_with_a_path_into_a_key_is_cancelled( + dynamodb_client, ctable, template, reason +): + pk = _write_item(dynamodb_client, ctable) + statement = template.format(t=f'"{ctable}"', k=pk) + with pytest.raises(ClientError) as e: + dynamodb_client.execute_transaction(TransactStatements=[{"Statement": statement}]) + response = e.value.response + assert response["Error"] == { + "Code": "TransactionCanceledException", + "Message": f"{CANCELLED} [ValidationError]", + } + assert response["CancellationReasons"] == [reason] + assert _stored_x(dynamodb_client, ctable, pk) == "0" + + +def test_an_exists_check_with_a_path_into_a_key(dynamodb_client, ctable): + pk = _write_item(dynamodb_client, ctable) + t = f'"{ctable}"' + update = {"Statement": f"UPDATE {t} SET x = 1 WHERE pk = '{pk}' AND sk = 'b'"} + against_a_value = { + "Statement": f"EXISTS(SELECT * FROM {t} WHERE pk = '{pk}' AND sk = 'b' AND pk.c = 1)" + } + assert _transaction_message(dynamodb_client, [update, against_a_value]) == ( + "Validation failed in TransactStatements[1]: " + SCALAR_KEY.format("pk") + ) + in_the_condition = { + "Statement": f"EXISTS(SELECT * FROM {t} WHERE pk = '{pk}' AND sk = 'b' AND NOT pk.c = 1)" + } + with pytest.raises(ClientError) as e: + dynamodb_client.execute_transaction(TransactStatements=[update, in_the_condition]) + response = e.value.response + assert response["Error"]["Message"] == f"{CANCELLED} [None, ValidationError]" + assert response["CancellationReasons"] == [ + {"Code": "None"}, + {"Code": "ValidationError", "Message": KEY_PATH_CONDITION.format("pk")}, + ] + assert _stored_x(dynamodb_client, ctable, pk) == "0" + + +def _transaction_message(client, statements): + with pytest.raises(ClientError) as e: + client.execute_transaction(TransactStatements=statements) + err = e.value.response["Error"] + assert err["Code"] == "ValidationException", err + return err["Message"] + + +# ---- a path into a key attribute beside the refusals of an UPDATE ------------ +# +# Measured 2026-11-05 on a table with the index gsi on the string attributes +# g and gs. An UPDATE whose WHERE reads a path into a key attribute among its +# other conditions is first refused as its UpdateItem request is: for a SET +# or REMOVE of a key attribute, then for a value of the wrong type given to an +# index key, whether or not the item exists. An index key given the value of +# another attribute is checked on the written item, so the path into the key +# is refused first. In ExecuteTransaction a member has the first of these as +# its reason, and each member has its own reason. + +INDEX_KEY_TYPE = ( + "One or more parameter values were invalid: Type mismatch for Index Key {} Expected: S " + "Actual: N IndexName: gsi" +) +UPDATE_OF_A_KEY = ( + "One or more parameter values were invalid: Cannot update attribute {}. This attribute " + "is part of the key" +) +_IN_THE_CONDITION = " AND NOT pk.c = 1" + + +@pytest.fixture(scope="module") +def itable(dynamodb_client): + """A table keyed on string pk and sk, with the index gsi on string g and gs.""" + with gsi.gsi_table(dynamodb_client, "extenddb-test-keypath-gsi") as name: + yield name + + +@pytest.mark.parametrize( + ("template", "parameters", "message"), + [ + ( + "UPDATE {t} SET g = 1 WHERE pk = '{k}' AND sk = 'b'" + _IN_THE_CONDITION, + None, + INDEX_KEY_TYPE.format("g"), + ), + ( + "UPDATE {t} SET g = 1 WHERE pk = '{k}' AND sk = 'b' AND NOT sk.c = 1", + None, + INDEX_KEY_TYPE.format("g"), + ), + ( + "UPDATE {t} SET gs = 1 WHERE pk = '{k}' AND sk = 'b'" + _IN_THE_CONDITION, + None, + INDEX_KEY_TYPE.format("gs"), + ), + ( + "UPDATE {t} SET g = ? WHERE pk = '{k}' AND sk = 'b'" + _IN_THE_CONDITION, + [{"N": "1"}], + INDEX_KEY_TYPE.format("g"), + ), + ( + "UPDATE {t} SET g = 1 WHERE pk = '{n}' AND sk = 'b'" + _IN_THE_CONDITION, + None, + INDEX_KEY_TYPE.format("g"), + ), + ( + "UPDATE {t} SET g = 1 WHERE pk = '{k}' AND sk = 'b'" + _IN_THE_CONDITION + + " RETURNING ALL NEW *", + None, + INDEX_KEY_TYPE.format("g"), + ), + ( + "UPDATE {t} SET pk = 'z' WHERE pk = '{k}' AND sk = 'b'" + _IN_THE_CONDITION, + None, + UPDATE_OF_A_KEY.format("pk"), + ), + ( + "UPDATE {t} REMOVE sk WHERE pk = '{k}' AND sk = 'b'" + _IN_THE_CONDITION, + None, + UPDATE_OF_A_KEY.format("sk"), + ), + ( + "UPDATE {t} SET g = 1, pk = 'z' WHERE pk = '{k}' AND sk = 'b'" + _IN_THE_CONDITION, + None, + UPDATE_OF_A_KEY.format("pk"), + ), + ( + "UPDATE {t} SET g = 1 WHERE pk = '{k}' AND sk = 'b'", + None, + INDEX_KEY_TYPE.format("g"), + ), + ( + "UPDATE {t} SET g = 'v' WHERE pk = '{k}' AND sk = 'b'" + _IN_THE_CONDITION, + None, + KEY_PATH_CONDITION.format("pk"), + ), + ( + "UPDATE {t} SET g = a WHERE pk = '{k}' AND sk = 'b'" + _IN_THE_CONDITION, + None, + KEY_PATH_CONDITION.format("pk"), + ), + ( + "UPDATE {t} SET x = 1 WHERE pk = '{k}' AND sk = 'b'" + _IN_THE_CONDITION, + None, + KEY_PATH_CONDITION.format("pk"), + ), + ( + "UPDATE {t} SET g = 1 WHERE pk = '{k}' AND sk = 'b' AND pk.c = 1", + None, + SCALAR_KEY.format("pk"), + ), + ], + ids=[ + "index-key-type", + "index-key-type-and-a-path-into-the-sort-key", + "index-sort-key-type", + "index-key-type-from-a-parameter", + "index-key-type-on-a-missing-item", + "index-key-type-with-returning", + "set-of-a-key", + "remove-of-the-sort-key", + "set-of-a-key-ahead-of-an-index-key-type", + "index-key-type-alone", + "index-key-of-the-right-type", + "index-key-from-another-attribute", + "a-path-into-a-key-alone", + "a-path-into-a-key-tested-against-a-value", + ], +) +def test_the_update_refusals_ahead_of_a_path_into_a_key( + dynamodb_client, itable, template, parameters, message +): + pk = gsi.put(dynamodb_client, itable, f"kp-{uuid.uuid4().hex[:6]}") + absent = f"{pk}-absent" + statement = template.format(t=f'"{itable}"', k=pk, n=absent) + kwargs = {"Parameters": parameters} if parameters else {} + assert _message(dynamodb_client, statement, **kwargs) == message + assert gsi.get(dynamodb_client, itable, pk) == gsi.item(pk) + assert gsi.get(dynamodb_client, itable, absent) is None + + +def _validation_error(message): + return {"Code": "ValidationError", "Message": message} + + +_SET_INDEX_KEY = "UPDATE {t} SET g = 1 WHERE pk = '{k}' AND sk = 'b'" +_SET_INDEX_KEY_IN_THE_CONDITION = _SET_INDEX_KEY + _IN_THE_CONDITION +_INTO_A_KEY = "UPDATE {t} SET x = 1 WHERE pk = '{j}' AND sk = 'b'" + _IN_THE_CONDITION +_INDEX_REASON = _validation_error(INDEX_KEY_TYPE.format("g")) +_INTO_A_KEY_REASON = _validation_error(KEY_PATH_CONDITION.format("pk")) + + +@pytest.mark.parametrize( + ("templates", "reasons"), + [ + ([_SET_INDEX_KEY_IN_THE_CONDITION], [_INDEX_REASON]), + ([_SET_INDEX_KEY_IN_THE_CONDITION.replace("{k}", "{n}")], [_INDEX_REASON]), + ( + ["UPDATE {t} SET pk = 'z' WHERE pk = '{k}' AND sk = 'b'" + _IN_THE_CONDITION], + [_validation_error(UPDATE_OF_A_KEY.format("pk"))], + ), + ( + ["UPDATE {t} SET g = 1, pk = 'z' WHERE pk = '{k}' AND sk = 'b'" + _IN_THE_CONDITION], + [_validation_error(UPDATE_OF_A_KEY.format("pk"))], + ), + ( + ["UPDATE {t} SET g = a WHERE pk = '{k}' AND sk = 'b'" + _IN_THE_CONDITION], + [_INTO_A_KEY_REASON], + ), + ([_SET_INDEX_KEY], [_INDEX_REASON]), + ([_INTO_A_KEY], [_INTO_A_KEY_REASON]), + ([_SET_INDEX_KEY, _INTO_A_KEY], [_INDEX_REASON, _INTO_A_KEY_REASON]), + ([_INTO_A_KEY, _SET_INDEX_KEY], [_INTO_A_KEY_REASON, _INDEX_REASON]), + ( + ["UPDATE {t} SET x = 2 WHERE pk = '{j}' AND sk = 'b'", + _SET_INDEX_KEY_IN_THE_CONDITION], + [{"Code": "None"}, _INDEX_REASON], + ), + ( + [_SET_INDEX_KEY_IN_THE_CONDITION, + _SET_INDEX_KEY_IN_THE_CONDITION.replace("{k}", "{j}")], + [_INDEX_REASON, _INDEX_REASON], + ), + ( + [_SET_INDEX_KEY, + "DELETE FROM {t} WHERE pk = '{j}' AND sk = 'b'" + _IN_THE_CONDITION], + [_INDEX_REASON, _INTO_A_KEY_REASON], + ), + ( + [_SET_INDEX_KEY, + "EXISTS(SELECT * FROM {t} WHERE pk = '{j}' AND sk = 'b'" + _IN_THE_CONDITION + ")"], + [_INDEX_REASON, _INTO_A_KEY_REASON], + ), + ( + ["INSERT INTO {t} VALUE {{'pk': '{n}', 'sk': 'b', 'g': 1}}", _INTO_A_KEY], + [_INDEX_REASON, _INTO_A_KEY_REASON], + ), + ], + ids=[ + "index-key-type", + "index-key-type-on-a-missing-item", + "set-of-a-key", + "set-of-a-key-ahead-of-an-index-key-type", + "index-key-from-another-attribute", + "index-key-type-alone", + "a-path-into-a-key-alone", + "index-key-type-then-a-path-into-a-key", + "a-path-into-a-key-then-index-key-type", + "update-then-index-key-type", + "index-key-type-in-two-members", + "index-key-type-then-a-delete", + "index-key-type-then-an-exists-check", + "insert-index-key-type-then-a-path-into-a-key", + ], +) +def test_a_transaction_member_has_the_update_refusal_ahead_of_a_path_into_a_key( + dynamodb_client, itable, templates, reasons +): + run = uuid.uuid4().hex[:6] + first = gsi.put(dynamodb_client, itable, f"kpt-{run}-k") + second = gsi.put(dynamodb_client, itable, f"kpt-{run}-j") + absent = f"{first}-absent" + statements = [ + {"Statement": t.format(t=f'"{itable}"', k=first, j=second, n=absent)} for t in templates + ] + with pytest.raises(ClientError) as e: + dynamodb_client.execute_transaction(TransactStatements=statements) + response = e.value.response + codes = ", ".join(reason["Code"] for reason in reasons) + assert response["Error"] == { + "Code": "TransactionCanceledException", + "Message": f"{CANCELLED} [{codes}]", + } + assert response["CancellationReasons"] == reasons + assert gsi.get(dynamodb_client, itable, first) == gsi.item(first) + assert gsi.get(dynamodb_client, itable, second) == gsi.item(second) + assert gsi.get(dynamodb_client, itable, absent) is None diff --git a/tests/test_partiql_key_pages.py b/tests/test_partiql_key_pages.py new file mode 100644 index 000000000..2d140ab73 --- /dev/null +++ b/tests/test_partiql_key_pages.py @@ -0,0 +1,544 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""Dual-target wire tests for the pages of a PartiQL SELECT over several +disjuncts read by key. + +When every disjunct gives the full primary key of the table, the service +reads the items in key order, `Limit` keys a page (a key with no item counts +as one), and returns no NextToken once the last key is read, so a read of one +item never has one. Disjuncts that read a whole partition page as a Query and +Scan do. + +A statement mixing reads of whole items and reads of partitions takes its +disjuncts in key order, one after another: a whole item takes one unit of +`Limit`, a partition is a Query with the units left. A page of such a read +that holds `Limit` items has a LastEvaluatedKey, the table key of its last +item, with or without a NextToken; any other page has none. A page of whole +items stops before the item that would take it past 1 MB. + +A disjunct that fails a check made when it is read (a key value past its size +limit, or a key attribute compared with a path in its filter) does not fail +the statement while another disjunct is fine: the others are read first, and +the request with the NextToken of their last page gets the refusal of the +failing disjunct last in key order. When every disjunct fails, the first +request is refused. + +An ORDER BY of a read of more than one partition key value that does not +name the partition key is refused. ORDER BY takes a list of the keys, and +reads in key order with it. + +Dual-target: EXTENDDB_TEST_ENDPOINT selects an extenddb server; unset, the +tests run against the live service with the default credential chain. The +assertions are identical on both. Every expected value was measured on the +service (2026-11-02). +""" + +from __future__ import annotations + +import pytest +from botocore.exceptions import ClientError + +from conftest import scoped_table + +FILTER_KEY = ( + "Filter Expression can only contain non-primary key attributes: Primary key attribute: {}" +) +HASH_SIZE = ( + "One or more parameter values were invalid: Size of hashkey has exceeded the maximum " + "size limit of2048 bytes" +) +RANGE_SIZE = ( + "One or more parameter values were invalid: Aggregated size of all range keys has " + "exceeded the size limit of 1024 bytes" +) +BIG_HASH = "z" * 2049 +BIG_RANGE = "z" * 1025 +ORDER_BY_HASH = ( + "Must have hash key in ORDER BY clause when more than one hash key condition specified in " + "WHERE clause." +) + + +@pytest.fixture(scope="module") +def htable(dynamodb_client): + """A table keyed on a string pk, holding a, b and c, each with n = 1.""" + with scoped_table(dynamodb_client) as name: + for key in ["a", "b", "c"]: + dynamodb_client.put_item(TableName=name, Item={"pk": {"S": key}, "n": {"N": "1"}}) + yield name + + +@pytest.fixture(scope="module") +def ctable(dynamodb_client): + """A table keyed on string pk and sk, holding a/x, a/y, b/x and c/x.""" + with scoped_table( + dynamodb_client, + attribute_definitions=[ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "sk", "AttributeType": "S"}, + ], + key_schema=[ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"}, + ], + ) as name: + for pk, sk in [("a", "x"), ("a", "y"), ("b", "x"), ("c", "x")]: + dynamodb_client.put_item( + TableName=name, Item={"pk": {"S": pk}, "sk": {"S": sk}, "n": {"N": "1"}} + ) + yield name + + +@pytest.fixture(scope="module") +def qtable(dynamodb_client): + """A table keyed on string pk and sk, holding a/x, a/y, a/z and b/x.""" + with scoped_table( + dynamodb_client, + attribute_definitions=[ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "sk", "AttributeType": "S"}, + ], + key_schema=[ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"}, + ], + ) as name: + for pk, sk in [("a", "x"), ("a", "y"), ("a", "z"), ("b", "x")]: + dynamodb_client.put_item( + TableName=name, Item={"pk": {"S": pk}, "sk": {"S": sk}, "n": {"N": "1"}} + ) + yield name + + +@pytest.fixture(scope="module") +def bigtable(dynamodb_client): + """A table keyed on a string pk, holding a, b and c of about 380 KB each, + and a small item d.""" + with scoped_table(dynamodb_client) as name: + for key in ["a", "b", "c"]: + dynamodb_client.put_item( + TableName=name, Item={"pk": {"S": key}, "blob": {"S": "q" * 380_000}} + ) + dynamodb_client.put_item(TableName=name, Item={"pk": {"S": "d"}}) + yield name + + +@pytest.fixture(scope="module") +def ntable(dynamodb_client): + """A table keyed on a number pk, holding 9, 10 and 100.""" + with scoped_table( + dynamodb_client, + attribute_definitions=[{"AttributeName": "pk", "AttributeType": "N"}], + key_schema=[{"AttributeName": "pk", "KeyType": "HASH"}], + ) as name: + for key in ["9", "10", "100"]: + dynamodb_client.put_item(TableName=name, Item={"pk": {"N": key}}) + yield name + + +def _key(item): + return "/".join(item[k]["S"] for k in ("pk", "sk") if k in item) + + +def _last_keys(client, statement, **kwargs): + """Every page of `statement`, following NextToken: its items, its + LastEvaluatedKey (None when it has none) and whether it has a NextToken.""" + pages, token = [], None + for _ in range(12): + args = dict(kwargs) + if token: + args["NextToken"] = token + response = client.execute_statement(Statement=statement, **args) + token = response.get("NextToken") + pages.append((response["Items"], response.get("LastEvaluatedKey"), token is not None)) + if not token: + break + return pages + + +def _s(**key): + return {name: {"S": value} for name, value in key.items()} + + +class _KeyOrNone: + """The LastEvaluatedKey of a page that ends on the last item of a + partition, read by a Query that stopped at Limit. The service returns the + item's key for some partitions and none for others, by where the partition + sits in the table's storage (measured 2026-11-02), so either is accepted.""" + + def __init__(self, key): + self.key = key + + def __eq__(self, other): + return other is None or other == self.key + + def __repr__(self): + return f"{self.key!r} or None" + + +def _pages(client, table, condition, ordered=True, **kwargs): + """Every page of a SELECT on `table`, following NextToken: the keys of each + page (sorted unless `ordered`), and the refusal that ends the paging when + there is one. Every page but the last carries a NextToken.""" + statement = f'SELECT * FROM "{table}" WHERE {condition}' + pages, token = [], None + for _ in range(12): + args = dict(kwargs) + if token: + args["NextToken"] = token + try: + response = client.execute_statement(Statement=statement, **args) + except ClientError as e: + assert e.response["Error"]["Code"] == "ValidationException", e.response + pages.append(e.response["Error"]["Message"]) + break + keys = [_key(item) for item in response["Items"]] + pages.append(keys if ordered else sorted(keys)) + token = response.get("NextToken") + if not token: + break + return pages + + +# ---- whole items, in key order ----------------------------------------------- + + +@pytest.mark.parametrize( + ("condition", "limit", "pages"), + [ + ("pk IN ['c', 'a', 'b']", 1, [["a"], ["b"], ["c"]]), + ("pk IN ['c', 'a', 'b']", 2, [["a", "b"], ["c"]]), + ("pk IN ['c', 'a', 'b']", None, [["a", "b", "c"]]), + ("pk = 'c' OR pk = 'a' OR pk = 'b'", 1, [["a"], ["b"], ["c"]]), + ("pk IN ['a', 'b']", 1, [["a"], ["b"]]), + ("pk IN ['a', 'zz']", 1, [["a"], []]), + ("pk IN ['zz', 'a']", 1, [["a"], []]), + ("pk = 'a'", 1, [["a"]]), + ], +) +def test_whole_items_are_read_in_key_order_limit_keys_a_page( + dynamodb_client, htable, condition, limit, pages +): + kwargs = {"Limit": limit} if limit else {} + assert _pages(dynamodb_client, htable, condition, **kwargs) == pages + + +@pytest.mark.parametrize( + ("condition", "limit", "pages"), + [ + ("(pk = 'b' AND sk = 'x') OR (pk = 'a' AND sk = 'x')", 1, [["a/x"], ["b/x"]]), + ("(pk = 'b' AND sk = 'x') OR (pk = 'a' AND sk = 'x')", None, [["a/x", "b/x"]]), + ("(pk = 'a' AND sk = 'x') OR (pk = 'c' AND sk = 'x')", 1, [["a/x"], ["c/x"]]), + ("pk = 'a' AND sk = 'x'", 1, [["a/x"]]), + ], +) +def test_whole_items_by_partition_and_sort_key_are_read_in_key_order( + dynamodb_client, ctable, condition, limit, pages +): + kwargs = {"Limit": limit} if limit else {} + assert _pages(dynamodb_client, ctable, condition, **kwargs) == pages + + +def test_a_partition_read_pages_as_a_query(dynamodb_client, ctable): + # The Query stops at Limit on the partition's last item: whether an empty + # page follows depends on where the partition sits in the table's storage. + pages = _pages(dynamodb_client, ctable, "pk = 'a'", Limit=2) + assert pages in ([["a/x", "a/y"]], [["a/x", "a/y"], []]) + + +@pytest.mark.parametrize( + ("condition", "limit", "pages"), + [ + ( + "(pk = 'a' AND sk = 'y') OR pk = 'b' OR (pk = 'c' AND sk = 'x')", + 1, + [["a/y"], ["b/x"], ["c/x"]], + ), + ( + "(pk = 'a' AND sk = 'y') OR pk = 'b' OR (pk = 'c' AND sk = 'x')", + None, + [["a/y", "b/x", "c/x"]], + ), + ("(pk = 'b' AND sk = 'x') OR (pk = 'a' AND sk > 'a')", 1, [["a/x"], ["a/y"], ["b/x"]]), + ("(pk = 'b' AND sk = 'x') OR (pk = 'a' AND sk > 'a')", None, [["a/x", "a/y", "b/x"]]), + ("(pk = 'a' AND sk = 'x') OR pk = 'b'", None, [["a/x", "b/x"]]), + ("pk = 'b' OR (pk = 'a' AND sk = 'x')", None, [["a/x", "b/x"]]), + ], +) +def test_reads_of_items_and_partitions_are_taken_in_key_order( + dynamodb_client, ctable, condition, limit, pages +): + kwargs = {"Limit": limit} if limit else {} + assert _pages(dynamodb_client, ctable, condition, **kwargs) == pages + + +def test_reads_of_several_partitions_page_one_item_at_a_time(dynamodb_client, ctable): + pages = _pages(dynamodb_client, ctable, "pk IN ['c', 'a', 'b']", Limit=1) + # Whether an empty page follows the last item depends on where partition + # 'c' sits in the table's storage. + assert [len(page) for page in pages] in ([1, 1, 1, 1], [1, 1, 1, 1, 0]) + assert sorted(key for page in pages for key in page) == ["a/x", "a/y", "b/x", "c/x"] + assert _pages(dynamodb_client, ctable, "pk IN ['c', 'a', 'b']", ordered=False) == [ + ["a/x", "a/y", "b/x", "c/x"] + ] + + +# ---- the LastEvaluatedKey of a page ------------------------------------------ + + +@pytest.mark.parametrize( + ("condition", "limit", "pages"), + [ + ("pk IN ['a', 'b']", 1, [(["a"], _s(pk="a"), True), (["b"], _s(pk="b"), False)]), + ("pk IN ['a', 'zz']", 1, [(["a"], _s(pk="a"), True), ([], None, False)]), + ("pk IN ['zz', 'a']", 1, [(["a"], _s(pk="a"), True), ([], None, False)]), + ("pk = 'a'", 1, [(["a"], _s(pk="a"), False)]), + ( + "pk IN ['a', 'b', 'c']", + 2, + [(["a", "b"], _s(pk="b"), True), (["c"], None, False)], + ), + ("pk IN ['a', 'b']", None, [(["a", "b"], None, False)]), + ("pk IN ['a', 'b'] AND n = 2", 1, [([], None, True), ([], None, False)]), + ], + ids=["two", "missing-second", "missing-first", "one", "limit-two", "no-limit", "filtered"], +) +def test_a_page_of_whole_items_holding_limit_items_has_the_last_key( + dynamodb_client, htable, condition, limit, pages +): + kwargs = {"Limit": limit} if limit else {} + statement = f'SELECT * FROM "{htable}" WHERE {condition}' + got = _last_keys(dynamodb_client, statement, **kwargs) + assert [([_key(i) for i in items], lek, more) for items, lek, more in got] == pages + + +@pytest.mark.parametrize( + ("condition", "pages"), + [ + ( + "(pk = 'b' AND sk = 'x') OR (pk = 'a' AND sk > 'a')", + [ + (["a/x"], _s(pk="a", sk="x"), True), + (["a/y"], _KeyOrNone(_s(pk="a", sk="y")), True), + (["b/x"], _s(pk="b", sk="x"), False), + ], + ), + ( + "(pk = 'a' AND sk = 'x') OR (pk = 'b' AND sk = 'x')", + [(["a/x"], _s(pk="a", sk="x"), True), (["b/x"], _s(pk="b", sk="x"), False)], + ), + ( + "pk = 'a' AND sk IN ['x', 'y'] ORDER BY sk DESC", + [(["a/y"], _s(pk="a", sk="y"), True), (["a/x"], _s(pk="a", sk="x"), False)], + ), + ], + ids=["item-and-partition", "two-items", "descending"], +) +def test_a_page_of_items_and_partitions_holding_limit_items_has_the_last_key( + dynamodb_client, ctable, condition, pages +): + statement = f'SELECT * FROM "{ctable}" WHERE {condition}' + got = _last_keys(dynamodb_client, statement, Limit=1) + assert [([_key(i) for i in items], lek, more) for items, lek, more in got] == pages + + +def test_a_page_that_ends_in_a_partition_read_takes_its_last_key(dynamodb_client, ctable): + statement = f"SELECT * FROM \"{ctable}\" WHERE (pk = 'a' AND sk = 'x') OR pk = 'b'" + got = _last_keys(dynamodb_client, statement, Limit=1) + # The second page ends on the last item of partition 'b': its key, and + # whether an empty page follows, depend on where the partition sits in the + # table's storage, so only the first two pages are compared. + assert [([_key(i) for i in items], lek) for items, lek, _ in got[:2]] == [ + (["a/x"], _s(pk="a", sk="x")), + (["b/x"], _KeyOrNone(_s(pk="b", sk="x"))), + ] + assert got[0][2] is True + assert all(not items for items, _, _ in got[2:]) + + +def test_the_last_key_of_a_page_is_the_key_without_the_projection(dynamodb_client, htable): + statement = f"SELECT n FROM \"{htable}\" WHERE pk IN ['a', 'b']" + assert _last_keys(dynamodb_client, statement, Limit=1) == [ + ([{"n": {"N": "1"}}], _s(pk="a"), True), + ([{"n": {"N": "1"}}], _s(pk="b"), False), + ] + + +def test_the_last_key_of_a_page_holds_a_number_key(dynamodb_client, ntable): + statement = f'SELECT * FROM "{ntable}" WHERE pk IN [10, 9, 100]' + got = _last_keys(dynamodb_client, statement, Limit=1) + assert [(lek, more) for _, lek, more in got] == [ + ({"pk": {"N": "9"}}, True), + ({"pk": {"N": "10"}}, True), + ({"pk": {"N": "100"}}, False), + ] + + +# ---- the size of a page ------------------------------------------------------ + + +def test_a_page_of_whole_items_stops_before_it_passes_1_mb(dynamodb_client, bigtable): + statement = f"SELECT * FROM \"{bigtable}\" WHERE pk IN ['a', 'b', 'c', 'd']" + got = _last_keys(dynamodb_client, statement) + assert [([_key(i) for i in items], lek, more) for items, lek, more in got] == [ + (["a", "b"], None, True), + (["c", "d"], None, False), + ] + + +# ---- ORDER BY over several partitions ---------------------------------------- + + +@pytest.mark.parametrize( + "condition", + ["pk IN ['a', 'b']", "(pk = 'a' AND sk = 'x') OR pk = 'b'"], + ids=["in", "item-and-partition"], +) +def test_an_order_by_over_two_partitions_must_name_the_partition_key( + dynamodb_client, ctable, condition +): + assert _pages(dynamodb_client, ctable, f"{condition} ORDER BY sk DESC", Limit=1) == [ + ORDER_BY_HASH + ] + + +@pytest.mark.parametrize( + ("condition", "pages"), + [ + ( + "pk IN ['a', 'b']", + [ + (["a/x"], _s(pk="a", sk="x")), + (["a/y"], _s(pk="a", sk="y")), + (["a/z"], _KeyOrNone(_s(pk="a", sk="z"))), + (["b/x"], _KeyOrNone(_s(pk="b", sk="x"))), + ], + ), + ( + "(pk = 'a' AND sk = 'x') OR pk = 'b'", + [(["a/x"], _s(pk="a", sk="x")), (["b/x"], _KeyOrNone(_s(pk="b", sk="x")))], + ), + ], + ids=["in", "item-and-partition"], +) +def test_an_order_by_list_of_the_keys_reads_in_key_order( + dynamodb_client, qtable, condition, pages +): + statement = f'SELECT * FROM "{qtable}" WHERE {condition} ORDER BY pk, sk' + got = _last_keys(dynamodb_client, statement, Limit=1) + # Whether an empty page follows the last item depends on where partition + # 'b' sits in the table's storage, so a page after it is only checked to + # hold nothing. + assert [([_key(i) for i in items], lek) for items, lek, _ in got[: len(pages)]] == pages + assert all(more for _, _, more in got[: len(pages) - 1]) + assert all(not items for items, _, _ in got[len(pages) :]) + + +# ---- a disjunct that fails when it is read ----------------------------------- + + +@pytest.mark.parametrize( + ("condition", "limit", "pages"), + [ + (f"pk IN ['a', '{BIG_HASH}']", None, [["a"], HASH_SIZE]), + (f"pk IN ['{BIG_HASH}', 'a']", None, [["a"], HASH_SIZE]), + (f"pk = 'a' OR pk = '{BIG_HASH}' OR pk = 'b'", None, [["a", "b"], HASH_SIZE]), + (f"pk IN ['a', '{BIG_HASH}', 'b']", 1, [["a"], ["b"], HASH_SIZE]), + ], + ids=["in", "in-reversed", "or", "limit"], +) +def test_an_oversized_partition_value_is_refused_after_the_other_items( + dynamodb_client, htable, condition, limit, pages +): + kwargs = {"Limit": limit} if limit else {} + assert _pages(dynamodb_client, htable, condition, **kwargs) == pages + + +@pytest.mark.parametrize( + ("condition", "pages"), + [ + (f"pk IN ['a', '{BIG_HASH}']", [["a/x", "a/y"], HASH_SIZE]), + ( + f"(pk = 'a' AND sk = 'x') OR (pk = 'b' AND sk = '{BIG_RANGE}')", + [["a/x"], RANGE_SIZE], + ), + (f"pk = 'a' OR (pk = 'b' AND sk = '{BIG_RANGE}')", [["a/x", "a/y"], RANGE_SIZE]), + (f"pk = 'a' AND sk IN ['x', '{BIG_RANGE}']", [["a/x"], RANGE_SIZE]), + ], + ids=["partition", "point", "partition-and-point", "sort-in"], +) +def test_an_oversized_key_value_is_refused_after_the_other_disjuncts( + dynamodb_client, ctable, condition, pages +): + assert _pages(dynamodb_client, ctable, condition) == pages + + +@pytest.mark.parametrize( + ("condition", "pages"), + [ + ("(pk = 'a' AND sk = a) OR pk = 'b'", [["b/x"], FILTER_KEY.format("sk")]), + ("pk = 'b' OR (pk = 'a' AND sk = a)", [["b/x"], FILTER_KEY.format("sk")]), + ("(pk = 'c' AND sk = a) OR pk = 'b'", [["b/x"], FILTER_KEY.format("sk")]), + ( + "(pk = 'a' AND sk = a) OR (pk = 'b' AND sk = 'x')", + [["b/x"], FILTER_KEY.format("sk")], + ), + ( + "(pk = 'a' AND sk = a) OR pk = 'b' OR pk = 'c'", + [["b/x", "c/x"], FILTER_KEY.format("sk")], + ), + ("(pk = 'b' AND sk = a) OR pk = 'a'", [["a/x", "a/y"], FILTER_KEY.format("sk")]), + ("(pk = 'a' AND sk = a) OR pk = 'zz'", [[], FILTER_KEY.format("sk")]), + (f"(pk = 'a' AND sk = a) OR pk = '{BIG_HASH}' OR pk = 'b'", [["b/x"], HASH_SIZE]), + ], + ids=[ + "first", + "last", + "other-partition", + "beside-a-point", + "two-others", + "valid-partition-sorts-first", + "nothing-read", + "with-a-size-refusal", + ], +) +def test_a_key_attribute_in_one_disjunct_filter_is_refused_after_the_others( + dynamodb_client, ctable, condition, pages +): + assert _pages(dynamodb_client, ctable, condition, ordered=False) == pages + + +def test_a_filter_every_disjunct_shares_is_refused_at_once(dynamodb_client, ctable): + assert _pages(dynamodb_client, ctable, "pk IN ['a', 'b'] AND sk = a") == [ + FILTER_KEY.format("sk") + ] + + +@pytest.mark.parametrize( + ("condition", "pages"), + [ + (f"pk = '{BIG_HASH}' OR (pk = 'a' AND sk = a) OR pk = 'b'", [["b/x"], HASH_SIZE]), + ( + f"(pk = 'b' AND sk = '{BIG_RANGE}') OR pk = '{BIG_HASH}' OR pk = 'a'", + [["a/x", "a/y"], HASH_SIZE], + ), + ( + f"pk = '{BIG_HASH}' OR (pk = 'b' AND sk = '{BIG_RANGE}') OR pk = 'a'", + [["a/x", "a/y"], HASH_SIZE], + ), + ( + "(pk = 'a' AND pk = b) OR (pk = 'b' AND sk = a) OR pk = 'c'", + [["c/x"], FILTER_KEY.format("sk")], + ), + ( + "(pk = 'b' AND sk = a) OR (pk = 'a' AND pk = b) OR pk = 'c'", + [["c/x"], FILTER_KEY.format("sk")], + ), + ], + ids=["size-last", "two-sizes", "two-sizes-reversed", "two-filters", "two-filters-reversed"], +) +def test_of_several_failing_disjuncts_the_last_in_key_order_is_reported( + dynamodb_client, ctable, condition, pages +): + assert _pages(dynamodb_client, ctable, condition) == pages diff --git a/tests/test_partiql_key_types.py b/tests/test_partiql_key_types.py new file mode 100644 index 000000000..be5be0ec4 --- /dev/null +++ b/tests/test_partiql_key_types.py @@ -0,0 +1,125 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""Dual-target wire tests for a key value of the wrong type in a PartiQL read. + +A key attribute compared with a value of a type the table does not define for +it is refused with the key type message wherever the comparison could be a key +condition: an equality in an OR branch, an IN on the partition key, a sort-key +range. A begins_with key condition whose prefix is a number gets the +operand-type message a filter gets. BatchExecuteStatement carries the key type +message as the statement's error, and a read in ExecuteTransaction cancels the +transaction with a reason saying the key does not match the schema. + +Dual-target: EXTENDDB_TEST_ENDPOINT selects an extenddb server; unset, the +tests run against the live service with the default credential chain. The +assertions are identical on both. Every expected value was measured on the +service (2026-11-02). +""" + +from __future__ import annotations + +import pytest +from botocore.exceptions import ClientError + +from conftest import scoped_table + +KEY_TYPE = "Key attribute's data type should match its data type in table's schema: Key {}" +BEGINS_WITH_NUMBER = ( + "Incorrect operand type for operator or function; operator or function: begins_with, " + "operand type: N" +) +CANCELLED = "Transaction cancelled, please refer cancellation reasons for specific reasons" +KEY_SCHEMA_MISMATCH = { + "Code": "ValidationError", + "Message": "The provided key element does not match the schema", +} + + +@pytest.fixture(scope="module") +def stable(dynamodb_client): + """A table keyed on a string pk.""" + with scoped_table(dynamodb_client) as name: + yield name + + +@pytest.fixture(scope="module") +def ctable(dynamodb_client): + """A table keyed on a string pk and a string sk.""" + with scoped_table( + dynamodb_client, + attribute_definitions=[ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "sk", "AttributeType": "S"}, + ], + key_schema=[ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"}, + ], + ) as name: + yield name + + +def _message(client, statement): + with pytest.raises(ClientError) as e: + client.execute_statement(Statement=statement) + err = e.value.response["Error"] + assert err["Code"] == "ValidationException", err + return err["Message"] + + +def _batch_errors(client, statement): + resp = client.batch_execute_statement(Statements=[{"Statement": statement}]) + return [entry.get("Error") for entry in resp["Responses"]] + + +def test_an_in_list_of_the_wrong_type_on_the_partition_key(dynamodb_client, stable): + statement = f"SELECT * FROM \"{stable}\" WHERE pk IN [1, 2]" + assert _message(dynamodb_client, statement) == KEY_TYPE.format("pk") + + +@pytest.mark.parametrize( + "condition", ["sk > 1", "sk BETWEEN 1 AND 2"], ids=["greater", "between"] +) +def test_a_number_in_a_string_sort_key_range(dynamodb_client, ctable, condition): + statement = f"SELECT * FROM \"{ctable}\" WHERE pk = 'ov' AND {condition}" + assert _message(dynamodb_client, statement) == KEY_TYPE.format("sk") + + +def test_a_number_prefix_in_a_begins_with_key_condition(dynamodb_client, ctable): + statement = f"SELECT * FROM \"{ctable}\" WHERE pk = 'ov' AND begins_with(sk, 1)" + assert _message(dynamodb_client, statement) == BEGINS_WITH_NUMBER + + +@pytest.mark.parametrize("number", ["1", "1.e3"]) +def test_a_single_read_with_a_key_of_the_wrong_type(dynamodb_client, stable, number): + statement = f"SELECT * FROM \"{stable}\" WHERE pk = {number}" + assert _message(dynamodb_client, statement) == KEY_TYPE.format("pk") + + +@pytest.mark.parametrize("number", ["1", "1.e3"]) +def test_a_batch_read_with_a_key_of_the_wrong_type(dynamodb_client, stable, number): + statement = f"SELECT * FROM \"{stable}\" WHERE pk = {number}" + assert _batch_errors(dynamodb_client, statement) == [ + {"Code": "ValidationError", "Message": KEY_TYPE.format("pk")} + ] + + +def test_a_batch_write_with_a_key_of_the_wrong_type(dynamodb_client, stable): + statement = f"UPDATE \"{stable}\" SET a = 1 WHERE pk = 1" + assert _batch_errors(dynamodb_client, statement) == [ + {"Code": "ValidationError", "Message": KEY_TYPE.format("pk")} + ] + + +@pytest.mark.parametrize("number", ["1", "1.e3"]) +def test_a_transaction_read_with_a_key_of_the_wrong_type(dynamodb_client, stable, number): + statement = f"SELECT * FROM \"{stable}\" WHERE pk = {number}" + with pytest.raises(ClientError) as e: + dynamodb_client.execute_transaction(TransactStatements=[{"Statement": statement}]) + response = e.value.response + assert response["Error"] == { + "Code": "TransactionCanceledException", + "Message": f"{CANCELLED} [ValidationError]", + } + assert response["CancellationReasons"] == [KEY_SCHEMA_MISMATCH] diff --git a/tests/test_partiql_limits.py b/tests/test_partiql_limits.py new file mode 100644 index 000000000..7c30b6055 --- /dev/null +++ b/tests/test_partiql_limits.py @@ -0,0 +1,463 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""Dual-target wire tests for the limits on a PartiQL statement: its length, +the operators its WHERE condition may hold, how deep a literal value may nest, +signs written before an operand, and statements nested deep enough that the +server must refuse them rather than run out of stack. + +Dual-target: EXTENDDB_TEST_ENDPOINT selects an extenddb server; unset, the +tests run against the live service with the default credential chain. The +assertions are identical on both. +""" + +from __future__ import annotations + +import os + +import boto3 +import pytest +from botocore.config import Config +from botocore.exceptions import ClientError + +from conftest import scoped_table + +WF = "Statement wasn't well formed, can't be processed: " +TOO_MANY_OPERATORS = "The expression contains too many operators; operator count: 301" +MISSING_RHS = WF + "Missing right-hand side expression of infix operator" +MAX_LENGTH = 8192 + + +@pytest.fixture(scope="module") +def table(dynamodb_client): + """A table keyed on a string pk.""" + with scoped_table(dynamodb_client) as name: + yield name + + +@pytest.fixture(scope="module") +def once(endpoint_url): + """A client that sends each request once and does not check parameters. + + The service answers some of the statements below with an internal error, + which a client would otherwise retry. + """ + kwargs: dict = { + "service_name": "dynamodb", + "region_name": os.environ.get("AWS_DEFAULT_REGION", "us-east-1"), + "config": Config(parameter_validation=False, retries={"total_max_attempts": 1}), + } + if endpoint_url: + kwargs["endpoint_url"] = endpoint_url + if endpoint_url.startswith("https://"): + kwargs["verify"] = False + return boto3.client(**kwargs) + + +def _where(table: str) -> str: + return f'SELECT * FROM "{table}" WHERE ' + + +def _items(client, statement: str) -> list: + return client.execute_statement(Statement=statement).get("Items", []) + + +def _message(client, statement: str) -> str: + with pytest.raises(ClientError) as e: + client.execute_statement(Statement=statement) + err = e.value.response["Error"] + assert err["Code"] == "ValidationException", err + return err["Message"] + + +def _nested(levels: int) -> str: + """The path of the value inside `levels` nested lists.""" + return "root" + "[0]" * levels + + +# --- length ----------------------------------------------------------------- + + +def test_a_statement_of_8192_characters_runs(once, table): + head = _where(table) + "pk = 'a'" + assert _items(once, head + " " * (MAX_LENGTH - len(head))) == [] + + +def test_a_statement_one_character_longer_is_refused_echoing_it(once, table): + head = _where(table) + "pk = 'a'" + statement = head + " " * (MAX_LENGTH + 1 - len(head)) + assert _message(once, statement) == ( + f"1 validation error detected: Value '{statement}' at 'statement' failed to " + "satisfy constraint: Member must have length less than or equal to 8192" + ) + + +def test_the_next_page_of_a_long_statement_of_four_byte_characters_runs(dynamodb_client): + with scoped_table(dynamodb_client) as name: + for key in ["a", "b"]: + dynamodb_client.put_item(TableName=name, Item={"pk": {"S": key}, "note": {"S": "n"}}) + statement = f"SELECT * FROM \"{name}\" WHERE note <> '{chr(0x1F600) * 6200}'" + keys, token = [], None + for _ in range(4): + args = {"NextToken": token} if token else {} + response = dynamodb_client.execute_statement(Statement=statement, Limit=1, **args) + keys += [item["pk"]["S"] for item in response["Items"]] + token = response.get("NextToken") + if not token: + break + assert sorted(keys) == ["a", "b"] + + +# --- operators -------------------------------------------------------------- + + +def test_299_nots_before_a_comparison_run(dynamodb_client, table): + assert _items(dynamodb_client, _where(table) + "NOT " * 299 + "a = 1") == [] + + +def test_300_nots_before_a_comparison_are_too_many_operators(dynamodb_client, table): + statement = _where(table) + "NOT " * 300 + "a = 1" + assert _message(dynamodb_client, statement) == TOO_MANY_OPERATORS + + +@pytest.mark.parametrize("terms", [151, 299, 300, 301]) +def test_an_and_chain_past_300_operators_is_refused(dynamodb_client, table, terms): + # Each comparison and each AND count: 151 terms are 301 operators. + condition = " AND ".join(f"a{i} = 1" for i in range(terms)) + assert _message(dynamodb_client, _where(table) + condition) == TOO_MANY_OPERATORS + + +@pytest.mark.parametrize("joiner", [" AND ", " OR "]) +def test_150_comparisons_are_299_operators_and_run(dynamodb_client, table, joiner): + condition = joiner.join(f"a{i} = 1" for i in range(150)) + assert _items(dynamodb_client, _where(table) + condition) == [] + + +def test_a_run_of_nots_in_a_write_condition_keeps_its_parity(dynamodb_client, table): + dynamodb_client.execute_statement( + Statement=f"INSERT INTO \"{table}\" VALUE {{'pk': 'nots', 'a': 2}}" + ) + update = f"UPDATE \"{table}\" SET b = 1 WHERE pk = 'nots' AND " + # 297 NOTs: the key equality, AND, the NOTs and `a = 1` are 300 operators. + dynamodb_client.execute_statement(Statement=update + "NOT " * 297 + "a = 1") + with pytest.raises(ClientError) as e: + dynamodb_client.execute_statement(Statement=update + "NOT " * 296 + "a = 1") + assert e.value.response["Error"]["Code"] == "ConditionalCheckFailedException" + item = _items(dynamodb_client, _where(table) + "pk = 'nots'") + assert item == [{"pk": {"S": "nots"}, "a": {"N": "2"}, "b": {"N": "1"}}] + + +# --- what comes before the operator count ---------------------------------- + + +NOTS = "NOT " * 300 + + +@pytest.mark.parametrize( + "tail, message", + [ + ( + "a = 1 AND b = 1e126", + "Number overflow. Attempting to store a number with magnitude larger than " + "supported range under root", + ), + ( + "a = 1 AND b = 1e-131", + "Number underflow. Attempting to store a number with magnitude smaller than " + "supported range under root", + ), + ( + "a = 1 AND b = 1" + "1" * 40, + "Attempting to store more than 38 significant digits in a Number under root", + ), + ("a = 1 AND b = ?", "Number of parameters in request and statement don't match."), + ], +) +def test_a_number_or_parameter_problem_comes_before_the_operator_count( + dynamodb_client, table, tail, message +): + assert _message(dynamodb_client, _where(table) + NOTS + tail) == message + + +def test_a_number_problem_before_the_operators_comes_before_a_missing_table( + dynamodb_client, table +): + statement = f'SELECT * FROM "{table}-missing" WHERE b = 1e126 AND {NOTS}a = 1' + assert _message(dynamodb_client, statement) == ( + "Number overflow. Attempting to store a number with magnitude larger than " + "supported range under root" + ) + + +@pytest.mark.parametrize( + "tail, message", + [ + ( + "a = 1 AND b = [" + "[" * 32 + "1" + "]" * 33, + "Nesting Levels have exceeded supported limits under " + _nested(32), + ), + ("a = 1 AND b = 1 + 1", "Unsupported operator in Condition Expression. Operator: +"), + ("a = 1 AND b = 'x' || 'y'", "Unsupported operator in Condition Expression. Operator: ||"), + ("a = 1 AND", MISSING_RHS), + ("a = 1 AND b = MISSING", "MISSING must be compared using IS operator"), + # A call with too many arguments is refused after the operator count. + ("a = 1 AND size(b, c) = 1", TOO_MANY_OPERATORS), + ], +) +def test_the_other_refusals_keep_their_place_around_the_operator_count( + dynamodb_client, table, tail, message +): + assert _message(dynamodb_client, _where(table) + NOTS + tail) == message + + +def test_an_unknown_function_comes_before_the_operator_count(dynamodb_client, table): + statement = _where(table) + NOTS + "a = 1 AND foo(b)" + column = statement.index("foo(") + 1 + assert _message(dynamodb_client, statement) == f"Unrecognized function: foo at 1:{column}:3" + + +# --- long conditions within the operator limit ------------------------------- + + +def test_150_begins_with_calls_anded_run(dynamodb_client, table): + dynamodb_client.execute_statement( + Statement=f"INSERT INTO \"{table}\" VALUE {{'pk': 'bw', 'bw': 'xy'}}" + ) + condition = " AND ".join(["begins_with(bw, 'x')"] * 150) + assert _items(dynamodb_client, _where(table) + condition) == [ + {"pk": {"S": "bw"}, "bw": {"S": "xy"}} + ] + + +def test_35_in_lists_of_100_members_ored_run(dynamodb_client, table): + dynamodb_client.execute_statement( + Statement=f"INSERT INTO \"{table}\" VALUE {{'pk': 'ors', 'ors': 1}}" + ) + condition = " OR ".join("ors IN [" + ",".join(["1"] * 100) + "]" for _ in range(35)) + statement = _where(table) + condition + assert len(statement) <= MAX_LENGTH + assert _items(dynamodb_client, statement) == [{"pk": {"S": "ors"}, "ors": {"N": "1"}}] + + +def test_a_projection_of_1000_paths_runs(dynamodb_client, table): + dynamodb_client.execute_statement( + Statement=f"INSERT INTO \"{table}\" VALUE {{'pk': 'paths', 'x5': 5}}" + ) + paths = ", ".join(f"x{i}" for i in range(1000)) + statement = f"SELECT {paths} FROM \"{table}\" WHERE pk = 'paths'" + assert _items(dynamodb_client, statement) == [{"x5": {"N": "5"}}] + + +def test_a_set_clause_of_400_assignments_runs(dynamodb_client, table): + dynamodb_client.execute_statement( + Statement=f"INSERT INTO \"{table}\" VALUE {{'pk': 'sets'}}" + ) + sets = ", ".join(f"x{i} = {i}" for i in range(400)) + dynamodb_client.execute_statement( + Statement=f"UPDATE \"{table}\" SET {sets} WHERE pk = 'sets'" + ) + [item] = _items(dynamodb_client, _where(table) + "pk = 'sets'") + assert item == {"pk": {"S": "sets"}, **{f"x{i}": {"N": str(i)} for i in range(400)}} + + +@pytest.mark.parametrize("members", [101, 1000]) +def test_an_in_list_of_more_than_100_members_is_refused(dynamodb_client, table, members): + statement = _where(table) + "a IN [" + ", ".join(str(i) for i in range(members)) + "]" + assert _message(dynamodb_client, statement) == ( + f"The IN operator is provided with too many operands; number of operands: {members}" + ) + + +# --- conditions nested within the operator limit ----------------------------- + + +def _alternated(joiner: str, depth: int) -> str: + """`NOT (a = 1 NOT (a = 1 ...))`, `depth` levels.""" + return f"NOT (a = 1 {joiner} " * depth + "a = 1" + ")" * depth + + +@pytest.mark.parametrize( + "joiner, depth", [("AND", 99), ("OR", 74), ("OR", 99)] +) +def test_not_alternated_with_and_or_or_to_depth_99_runs(dynamodb_client, table, joiner, depth): + dynamodb_client.execute_statement(Statement=_where(table) + _alternated(joiner, depth)) + + +def test_not_alternated_with_and_to_depth_100_is_too_many_operators(dynamodb_client, table): + statement = _where(table) + _alternated("AND", 100) + assert _message(dynamodb_client, statement) == TOO_MANY_OPERATORS + + +@pytest.mark.parametrize("depth, runs", [(149, True), (150, False)]) +def test_and_nested_to_depth_149_runs(dynamodb_client, table, depth, runs): + statement = _where(table) + "(a = 1 AND " * depth + "a = 1" + ")" * depth + if runs: + dynamodb_client.execute_statement(Statement=statement) + else: + assert _message(dynamodb_client, statement) == TOO_MANY_OPERATORS + + +@pytest.mark.parametrize("depth, runs", [(99, True), (140, False)]) +def test_nested_and_of_size_calls(dynamodb_client, table, depth, runs): + statement = _where(table) + "(size(a) = 1 AND " * depth + "a = 1" + ")" * depth + if runs: + dynamodb_client.execute_statement(Statement=statement) + else: + assert _message(dynamodb_client, statement) == TOO_MANY_OPERATORS + + +@pytest.mark.parametrize("depth", [99, 148]) +def test_an_update_condition_nested_to_depth_148_runs(dynamodb_client, table, depth): + dynamodb_client.execute_statement( + Statement=f"INSERT INTO \"{table}\" VALUE {{'pk': 'nested{depth}', 'n': 1}}" + ) + dynamodb_client.execute_statement( + Statement=f"UPDATE \"{table}\" SET q = 1 WHERE pk = 'nested{depth}' AND " + + "(n = 1 AND " * depth + + "n = 1" + + ")" * depth + ) + + +# --- groups ----------------------------------------------------------------- + + +@pytest.mark.parametrize("after_key", [False, True]) +def test_1000_nested_groups_run(dynamodb_client, table, after_key): + head = _where(table) + ("pk = 'a' AND " if after_key else "") + assert _items(dynamodb_client, head + "(" * 1000 + "a = 1" + ")" * 1000) == [] + + +# --- literal nesting -------------------------------------------------------- + + +@pytest.mark.parametrize("levels", [32, 33, 34, 40, 100, 1000]) +def test_a_list_literal_nested_32_levels_or_more_names_the_32nd( + dynamodb_client, table, levels +): + statement = ( + f"INSERT INTO \"{table}\" VALUE {{'pk': 'l{levels}', 'x': " + + "[" * levels + + "1" + + "]" * levels + + "}" + ) + assert _message(dynamodb_client, statement) == ( + "Nesting Levels have exceeded supported limits under " + _nested(32) + ) + + +@pytest.mark.parametrize("levels", [32, 33, 34, 40, 100, 1000]) +def test_a_map_literal_nested_32_levels_or_more_names_the_32nd( + dynamodb_client, table, levels +): + statement = ( + f"INSERT INTO \"{table}\" VALUE {{'pk': 'm{levels}', 'x': " + + "{'a': " * levels + + "1" + + "}" * levels + + "}" + ) + assert _message(dynamodb_client, statement) == ( + "Nesting Levels have exceeded supported limits under root" + ".a" * 32 + ) + + +@pytest.mark.parametrize("levels", [33, 1000]) +def test_a_deep_list_literal_in_a_condition_names_the_32nd(dynamodb_client, table, levels): + statement = _where(table) + "x = " + "[" * levels + "1" + "]" * levels + assert _message(dynamodb_client, statement) == ( + "Nesting Levels have exceeded supported limits under " + _nested(32) + ) + + +def test_a_deep_list_literal_as_a_set_value_names_the_32nd(dynamodb_client, table): + statement = f"UPDATE \"{table}\" SET x = " + "[" * 33 + "1" + "]" * 33 + " WHERE pk = 'u'" + assert _message(dynamodb_client, statement) == ( + "Nesting Levels have exceeded supported limits under " + _nested(32) + ) + + +def test_a_list_literal_of_31_levels_is_stored(dynamodb_client, table): + statement = ( + f"INSERT INTO \"{table}\" VALUE {{'pk': 'l31', 'x': " + "[" * 31 + "1" + "]" * 31 + "}" + ) + dynamodb_client.execute_statement(Statement=statement) + [item] = _items(dynamodb_client, _where(table) + "pk = 'l31'") + value, levels = item["x"], 0 + while "L" in value: + value, levels = value["L"][0], levels + 1 + assert (levels, value) == (31, {"N": "1"}) + + +def test_a_deep_literal_that_breaks_off_reports_the_syntax_error(dynamodb_client, table): + statement = ( + f"INSERT INTO \"{table}\" VALUE {{'pk':'mix','x':{{'a':[{{'b':" + + "[" * 30 + + "1" + + "]" * 30 + + "}]}}}}" + ) + assert _message(dynamodb_client, statement) == WF + "Unexpected token after expression" + + +# --- signs ------------------------------------------------------------------ + + +@pytest.mark.parametrize("signs", [2, 10, 100]) +@pytest.mark.parametrize("operand", ["1", "b"]) +def test_two_minus_signs_start_a_comment(dynamodb_client, table, signs, operand): + statement = _where(table) + "a = " + "-" * signs + operand + assert _message(dynamodb_client, statement) == MISSING_RHS + + +def test_1000_plus_signs_before_a_name_are_refused_naming_the_operator(dynamodb_client, table): + # The service gives this answer for 1000 signs on every try. From about + # 1500 signs it sometimes answers InternalFailure instead, so the test + # stays at 1000. + statement = _where(table) + "a = " + "+" * 1000 + "a" + assert _message(dynamodb_client, statement) == ( + "Unsupported operator in Condition Expression. Operator: +" + ) + + +# --- statements nested to the length limit ---------------------------------- + + +def _fill(head: str, open_: str, inner: str, close: str = "", tail: str = "") -> str: + """`head`, as many `open_` as fit in the length limit, `inner`, as many + `close`, then `tail`.""" + times = (MAX_LENGTH - len(head) - len(inner) - len(tail)) // (len(open_) + len(close)) + return head + open_ * times + inner + close * times + tail + + +def _shapes(table: str) -> dict[str, str]: + where = _where(table) + insert = f"INSERT INTO \"{table}\" VALUE {{'pk': 'deep', 'x': " + return { + "groups": _fill(where, "(", "a = 1", ")"), + "groups after a key": _fill(where + "pk = 'a' AND ", "(", "a = 1", ")"), + "nots": _fill(where, "NOT ", "a = 1"), + "plus signs": _fill(where + "a = ", "+", "a"), + "spaced minus signs": _fill(where + "a = ", "- ", "b"), + "bags": _fill(where + "a = ", "<<", "1", ">>"), + "lists": _fill(insert, "[", "1", "]", "}"), + "maps": _fill(insert, "{'a': ", "1", "}", "}"), + "and chain": where + " AND ".join(f"a{i} = 1" for i in range(600)), + "or chain": where + " OR ".join(f"a{i} = 1" for i in range(600)), + "calls": _fill(f"UPDATE \"{table}\" SET a = ", "f(", "x", ")", " WHERE pk = 'p'"), + } + + +SHAPES = list(_shapes("t")) + + +@pytest.mark.parametrize("shape", SHAPES) +def test_a_statement_nested_to_the_length_limit_is_refused_and_the_server_keeps_serving( + once, table, shape +): + statement = _shapes(table)[shape] + assert len(statement) <= MAX_LENGTH + with pytest.raises(ClientError): + once.execute_statement(Statement=statement) + assert _items(once, _where(table) + "pk = 'a'") == [] diff --git a/tests/test_partiql_not_forms.py b/tests/test_partiql_not_forms.py new file mode 100644 index 000000000..9695265bd --- /dev/null +++ b/tests/test_partiql_not_forms.py @@ -0,0 +1,363 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""Dual-target wire tests for `NOT IN`, `NOT BETWEEN`, the members `IN` +takes, and `LIKE`. + +`a NOT IN [...]`, `a NOT IN (...)` and `a NOT BETWEEN x AND y` run wherever a +condition does: a read's WHERE, an UPDATE or DELETE condition, on key +attributes and with parameters. Each means the negation of the predicate after +the NOT and counts two operators towards the limit of 300. A malformed form +gets the message of the predicate it spells, and a NOT before anything else +after an operand is a token left over. `IN` takes its members in brackets, in +parentheses or in a bag, which is not checked as a set literal there and may +hold names, and `LIKE` is refused with or without NOT and ESCAPE, also inside +the argument of a call. A predicate may be compared with a boolean, and a +comparison with a number, which holds for no item; a partition key equality +compared with false selects no item. An IS after a predicate or an IS test +is refused for its left side. An operand is not a condition, alone or as an +argument of AND. + +Dual-target: EXTENDDB_TEST_ENDPOINT selects an extenddb server; unset, the +tests run against the live service with the default credential chain. The +assertions are identical on both. Every expected value was measured on the +service (2026-11-02). +""" + +from __future__ import annotations + +import pytest +from botocore.exceptions import ClientError + +from conftest import scoped_table + +WF = "Statement wasn't well formed, can't be processed: " +IN_OPERANDS = ( + "IN operator must have a left hand argument of type Variable Reference and right hand " + "argument of type Seq with at least one member" +) +TOO_MANY_OPERATORS = "The expression contains too many operators; operator count: 301" +IS_LEFT_SIDE = "The left hand side argument to IS must be a Variable Reference, Literal, or Path" +ITEM = {"pk": {"S": "a"}, "a": {"N": "1"}} +FIVE = {"pk": {"S": "b"}, "a": {"N": "5"}} + + +@pytest.fixture(scope="module") +def readable(dynamodb_client): + """A table keyed on a string pk, holding one item with a = 1.""" + with scoped_table(dynamodb_client) as name: + dynamodb_client.put_item(TableName=name, Item=ITEM) + yield name + + +@pytest.fixture(scope="module") +def sorted_table(dynamodb_client): + """A table keyed on a string pk and a string sk, holding one item.""" + with scoped_table( + dynamodb_client, + attribute_definitions=[ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "sk", "AttributeType": "S"}, + ], + key_schema=[ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"}, + ], + ) as name: + dynamodb_client.put_item( + TableName=name, Item={"pk": {"S": "a"}, "sk": {"S": "x"}, "a": {"S": "x"}} + ) + yield name + + +@pytest.fixture(scope="module") +def pair(dynamodb_client): + """A table keyed on a string pk, holding items with a = 1 and a = 5.""" + with scoped_table(dynamodb_client) as name: + dynamodb_client.put_item(TableName=name, Item=ITEM) + dynamodb_client.put_item(TableName=name, Item=FIVE) + yield name + + +@pytest.fixture +def writable(dynamodb_client): + """A table keyed on a string pk, holding one item with a = 1, per test.""" + with scoped_table(dynamodb_client) as name: + dynamodb_client.put_item(TableName=name, Item=ITEM) + yield name + + +def _rows(client, statement: str, **kwargs) -> list: + return client.execute_statement(Statement=statement, **kwargs)["Items"] + + +def _message(client, statement: str) -> str: + with pytest.raises(ClientError) as e: + client.execute_statement(Statement=statement) + err = e.value.response["Error"] + assert err["Code"] == "ValidationException", err + return err["Message"] + + +@pytest.mark.parametrize( + "condition, matches", + [ + ("a NOT IN [1, 2]", False), + ("a NOT IN [2]", True), + ("a NOT IN (1, 2)", False), + ("a NOT BETWEEN 1 AND 2", False), + ("a NOT BETWEEN 2 AND 3", True), + ("NOT a NOT IN [1]", True), + ("a NOT IN [1] AND pk = 'a'", False), + ("pk = 'a' AND a NOT IN [1]", False), + ("a NOT IN [1, 'x', [1], {'k': 1}]", False), + ("pk NOT IN ['a']", False), + ("pk NOT IN [1]", True), + ("pk NOT BETWEEN 'a' AND 'b'", False), + ], +) +def test_a_not_form_selects_what_its_predicate_does_not( + dynamodb_client, readable, condition, matches +): + rows = _rows(dynamodb_client, f'SELECT * FROM "{readable}" WHERE {condition}') + assert rows == ([ITEM] if matches else []) + + +def test_in_takes_its_members_in_parentheses(dynamodb_client, readable): + rows = _rows(dynamodb_client, f'SELECT * FROM "{readable}" WHERE a IN (1, 2)') + assert rows == [ITEM] + + +def test_not_in_takes_parameters(dynamodb_client, readable): + rows = _rows( + dynamodb_client, + f'SELECT * FROM "{readable}" WHERE a NOT IN [?, ?]', + Parameters=[{"N": "1"}, {"N": "2"}], + ) + assert rows == [] + + +@pytest.mark.parametrize( + "condition", ["pk = 'a' AND sk NOT IN ['y']", "pk = 'a' AND sk NOT BETWEEN 'y' AND 'z'"] +) +def test_a_not_form_on_a_sort_key_filters_the_query(dynamodb_client, sorted_table, condition): + rows = _rows(dynamodb_client, f'SELECT * FROM "{sorted_table}" WHERE {condition}') + assert rows == [{"pk": {"S": "a"}, "sk": {"S": "x"}, "a": {"S": "x"}}] + + +def test_an_update_condition_with_not_in_that_fails(dynamodb_client, writable): + with pytest.raises(ClientError) as e: + dynamodb_client.execute_statement( + Statement=f"UPDATE \"{writable}\" SET q = 1 WHERE pk = 'a' AND a NOT IN [1]" + ) + assert e.value.response["Error"]["Code"] == "ConditionalCheckFailedException" + + +def test_an_update_condition_with_not_between_that_holds(dynamodb_client, writable): + rows = _rows( + dynamodb_client, + f"UPDATE \"{writable}\" SET q = 2 WHERE pk = 'a' AND a NOT BETWEEN 5 AND 6 " + "RETURNING ALL NEW *", + ) + assert rows == [{"pk": {"S": "a"}, "a": {"N": "1"}, "q": {"N": "2"}}] + + +def test_a_delete_of_a_missing_item_with_not_in(dynamodb_client, writable): + rows = _rows(dynamodb_client, f"DELETE FROM \"{writable}\" WHERE pk = 'zz' AND a NOT IN [1]") + assert rows == [] + + +@pytest.mark.parametrize("term", ["a NOT IN [1]", "a NOT BETWEEN 1 AND 2"]) +def test_a_not_form_counts_two_operators(dynamodb_client, readable, term): + head = f'SELECT * FROM "{readable}" WHERE ' + assert _rows(dynamodb_client, head + "NOT " * 298 + f"({term})") == [] + assert _message(dynamodb_client, head + "NOT " * 299 + f"({term})") == TOO_MANY_OPERATORS + + +@pytest.mark.parametrize( + "condition, message", + [ + ("a NOT IN 1", IN_OPERANDS), + ("a NOT IN []", IN_OPERANDS), + ("a NOT BETWEEN 1", WF + "Expected AND keyword"), + ("a NOT = 1", WF + "Unexpected token after expression"), + ("a NOT IS MISSING", WF + "Unexpected token after expression"), + ("a IS NOT IN [1]", WF + "Expected type name"), + ("a NOT LIKE", WF + "Missing right-hand side expression of infix operator"), + ], +) +def test_a_malformed_not_form_is_refused(dynamodb_client, readable, condition, message): + statement = f'SELECT * FROM "{readable}" WHERE {condition}' + assert _message(dynamodb_client, statement) == message + + +@pytest.mark.parametrize("condition", ["a LIKE 'x'", "a NOT LIKE 'x'", "a LIKE 'x' ESCAPE 'y'"]) +def test_like_is_refused_as_an_operator_a_condition_does_not_take( + dynamodb_client, readable, condition +): + statement = f'SELECT * FROM "{readable}" WHERE {condition}' + assert _message(dynamodb_client, statement) == ( + "Unsupported operator in Condition Expression. Operator: like" + ) + + +@pytest.mark.parametrize( + "condition, keys", + [ + # A member in parentheses of its own is a list when a comma separates + # what they hold, and what they hold otherwise. + ("a IN ((1, 2))", []), + ("a IN ((5))", ["b"]), + ("a IN ([5])", []), + ("a IN ([1], 2)", []), + ("a NOT IN ([5])", ["a", "b"]), + ("a IN (1)", ["a"]), + ("a IN (pk, 1)", ["a"]), + ("a IN <<1, 5>>", ["a", "b"]), + # A bag there is not checked as a set literal. + ("a IN <<1, 1>>", ["a"]), + ("a IN <<'x', 1>>", ["a"]), + # An IN, with or without NOT, may be compared with a boolean, and so + # may a comparison. + ("a NOT IN [1] = true", ["b"]), + ("(a NOT IN [1]) = true", ["b"]), + ("a IN [1] AND pk = 'a' = true", ["a"]), + ], +) +def test_the_members_an_in_takes(dynamodb_client, pair, condition, keys): + rows = _rows(dynamodb_client, f'SELECT * FROM "{pair}" WHERE {condition}') + assert sorted(row["pk"]["S"] for row in rows) == keys + + +@pytest.mark.parametrize( + "condition, message", + [ + ("a IN ()", WF + "Unexpected term"), + ("a NOT IN ()", WF + "Unexpected term"), + ("a IN (1,)", WF + "Unexpected term"), + ("a IN (1, 2", WF + "Expected RIGHT_PAREN"), + ("a IN 5", IN_OPERANDS), + ("a IN (1, 2) IS MISSING", IS_LEFT_SIDE), + ("(a NOT = 1)", WF + "Expected RIGHT_PAREN"), + ("a NOT", WF + "Unexpected token after expression"), + ("a NOT AND pk = 'a'", WF + "Unexpected token after expression"), + ("a NOT NOT IN [1]", WF + "Unexpected token after expression"), + ("a NOT NOT BETWEEN 1 AND 2", WF + "Unexpected token after expression"), + # A group that is not closed. + ("(a = 1 b)", WF + "Expected RIGHT_PAREN"), + ("(a = 1", WF + "Expected RIGHT_PAREN"), + ("((a = 1)", WF + "Expected RIGHT_PAREN"), + # An IS after any predicate is refused for its left side. + ("a BETWEEN 1 AND 2 IS MISSING", IS_LEFT_SIDE), + ("a = 1 IS MISSING", IS_LEFT_SIDE), + ], +) +def test_a_malformed_in_or_not_is_refused(dynamodb_client, pair, condition, message): + statement = f'SELECT * FROM "{pair}" WHERE {condition}' + assert _message(dynamodb_client, statement) == message + + +def test_a_name_inside_a_list_in_parentheses_is_refused_at_its_position(dynamodb_client, pair): + statement = f'SELECT * FROM "{pair}" WHERE a IN ((pk, 1))' + column = statement.index("pk,") + 1 + assert _message(dynamodb_client, statement) == ( + f"Unsupported data type: Id under key root[0] at 1:{column}:2" + ) + + +def test_like_inside_the_argument_of_a_call_is_refused(dynamodb_client, readable): + statement = f"SELECT * FROM \"{readable}\" WHERE foo(a LIKE 'x')" + assert _message(dynamodb_client, statement) == ( + "Unsupported operator in Condition Expression. Operator: like" + ) + + +def test_escape_with_nothing_after_it(dynamodb_client, readable): + statement = f"SELECT * FROM \"{readable}\" WHERE a LIKE 'x' ESCAPE" + assert _message(dynamodb_client, statement) == WF + "Expected expression after ESCAPE" + + +@pytest.mark.parametrize( + "condition, message", + [ + ( + "1 IS MISSING", + "Operator or function requires a document path; operator or function: " + "attribute_not_exists", + ), + ( + "'x' IS NOT MISSING", + "Operator or function requires a document path; operator or function: " + "attribute_not_exists", + ), + ("a IS NULL IS NULL", IS_LEFT_SIDE), + ], +) +def test_an_is_test_on_a_literal_or_after_an_is_test(dynamodb_client, pair, condition, message): + assert _message(dynamodb_client, f'SELECT * FROM "{pair}" WHERE {condition}') == message + + +@pytest.mark.parametrize( + "member, token, kind, length", + [('"pk"', '"pk"', "Id", 4), ("m.b", "m.b", "Path", 1), ("foo(b)", "foo", "Call", 3)], + ids=["quoted-name", "path", "call"], +) +def test_a_term_inside_a_list_in_parentheses_is_refused_by_what_it_is( + dynamodb_client, pair, member, token, kind, length +): + statement = f'SELECT * FROM "{pair}" WHERE a IN (({member}, 1))' + column = statement.index(token) + 1 + assert _message(dynamodb_client, statement) == ( + f"Unsupported data type: {kind} under key root[0] at 1:{column}:{length}" + ) + + +def test_a_name_in_a_bag_after_in_is_an_operand(dynamodb_client, pair): + assert _rows(dynamodb_client, f'SELECT * FROM "{pair}" WHERE a IN <>') == [] + + +@pytest.mark.parametrize( + "condition, keys", + [ + ("a = 1 = true = true", ["a"]), + ("a = 1 = 1", []), + # The partition key equality is still read as the key condition, so + # the item it reads fails the comparison with false. + ("pk = 'a' = false", []), + ], +) +def test_a_comparison_compared_with_a_boolean_or_a_number(dynamodb_client, pair, condition, keys): + rows = _rows(dynamodb_client, f'SELECT * FROM "{pair}" WHERE {condition}') + assert sorted(row["pk"]["S"] for row in rows) == keys + + +@pytest.mark.parametrize( + "condition, parameters", + [ + ("a", []), + ("1", []), + ("'x'", []), + ("?", [{"N": "1"}]), + ("[1]", []), + ("m.b", []), + ("true", []), + ], +) +def test_an_operand_that_is_the_whole_condition_is_refused( + dynamodb_client, pair, condition, parameters +): + kwargs = {"Parameters": parameters} if parameters else {} + with pytest.raises(ClientError) as e: + dynamodb_client.execute_statement( + Statement=f'SELECT * FROM "{pair}" WHERE {condition}', **kwargs + ) + err = e.value.response["Error"] + assert (err["Code"], err["Message"]) == ( + "ValidationException", + "WHERE clause must contain an expression that resolves to a boolean", + ) + + +def test_an_operand_that_and_takes_is_refused(dynamodb_client, pair): + statement = f"SELECT * FROM \"{pair}\" WHERE a AND pk = 'a'" + assert _message(dynamodb_client, statement) == "The arguments to AND/OR must be Expressions" diff --git a/tests/test_partiql_numbers.py b/tests/test_partiql_numbers.py new file mode 100644 index 000000000..0ab6e7035 --- /dev/null +++ b/tests/test_partiql_numbers.py @@ -0,0 +1,699 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""Dual-target wire tests for PartiQL number conversion and the checks that +answer before a statement's table is resolved. + +Covered: how a number in ``Parameters`` or in the statement text is read and +which message a bad one gets, the order between nesting depth and number +conversion in parameters, the operand-type check on ordering comparisons, and +where these answers fall relative to parse errors, table resolution, the +multi-attribute key refusal and the batch and transaction envelopes. + +Runs against real DynamoDB (EXTENDDB_TEST_ENDPOINT unset, AWS credentials) and +against an extenddb server (EXTENDDB_TEST_ENDPOINT set) with identical +assertions. Every expected value was measured on the service (2026-11-02). +Tables use the ``extenddb-measure-pqn-`` prefix and are deleted at the end. +""" + +from __future__ import annotations + +import os +import uuid + +import pytest +from botocore.exceptions import ClientError + +from conftest import wait_for_active + +# ExtendDB walks the members of a parameter map in the order the parsed request +# holds them. That is the order the request sent, as the service walks them +# (measured 2026-11-02), when the server is built together with the MongoDB +# backend crate, whose JSON dependency keeps map order: any build of the whole +# workspace, as CI's are. A build of the server package alone with another +# backend sorts map members by name, and answers differently when a map's first +# member in request order is not its first by name. Not strict, because which +# order a server has depends on how it was built. +MAP_MEMBERS_IN_NAME_ORDER = pytest.mark.xfail( + bool(os.environ.get("EXTENDDB_TEST_ENDPOINT", "").strip()), + reason="a server built without the MongoDB crate checks parameter map members in name order", + raises=AssertionError, + strict=False, +) + +NOT_WELL_FORMED = "Statement wasn't well formed, can't be processed: " +UNCONVERTIBLE = "The parameter cannot be converted to a numeric value" +OVERFLOW = ( + "Number overflow. Attempting to store a number with magnitude larger than " + "supported range" +) +UNDERFLOW = ( + "Number underflow. Attempting to store a number with magnitude smaller than " + "supported range" +) +DIGITS = "Attempting to store more than 38 significant digits in a Number" +NESTED = ( + "Nesting Levels have exceeded supported limits: Attributes in the item have " + "nested levels beyond supported limit" +) +REFUSED = ( + "PartiQL statements cannot be executed on tables with multi-attribute HASH " + "keys or multi-attribute RANGE keys." +) +DIGITS_39 = "123456789012345678901234567890123456789" +PROJECTED_CALL = "Unexpected path component at 1:8:5" + + +def operand_type(op: str, type_code: str) -> str: + return ( + "Incorrect operand type for operator or function; operator or function: " + f"{op}, operand type: {type_code}" + ) + + +def deep(levels: int, leaf: dict | None = None) -> dict: + """``levels`` maps wrapped around ``leaf``.""" + value = leaf if leaf is not None else {"S": "leaf"} + for _ in range(levels): + value = {"M": {"n": value}} + return value + + +# ---- tables ------------------------------------------------------------------ + + +@pytest.fixture(scope="module") +def tables(dynamodb_client): + """A string-keyed composite table, a number-keyed table and an absent name.""" + prefix = f"extenddb-measure-pqn-{uuid.uuid4().hex[:12]}" + names = {"str": f"{prefix}-s", "num": f"{prefix}-n"} + dynamodb_client.create_table( + TableName=names["str"], + BillingMode="PAY_PER_REQUEST", + AttributeDefinitions=[ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "sk", "AttributeType": "S"}, + ], + KeySchema=[ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"}, + ], + ) + dynamodb_client.create_table( + TableName=names["num"], + BillingMode="PAY_PER_REQUEST", + AttributeDefinitions=[{"AttributeName": "pk", "AttributeType": "N"}], + KeySchema=[{"AttributeName": "pk", "KeyType": "HASH"}], + ) + try: + for name in names.values(): + wait_for_active(dynamodb_client, name) + dynamodb_client.put_item( + TableName=names["str"], + Item={"pk": {"S": "p"}, "sk": {"S": "s"}, "a": {"N": "1"}, "val": {"S": "v"}}, + ) + names["absent"] = f"{prefix}-absent" + yield names + finally: + for name in ("str", "num"): + try: + dynamodb_client.delete_table(TableName=names[name]) + except ClientError: + pass + + +@pytest.fixture(scope="module") +def multi_table(dynamodb_client): + """A table whose GSI key has two HASH attributes. A backend that cannot store + a multi-attribute GSI skips the tests that need it.""" + name = f"extenddb-measure-pqn-{uuid.uuid4().hex[:12]}-m" + try: + dynamodb_client.create_table( + TableName=name, + BillingMode="PAY_PER_REQUEST", + AttributeDefinitions=[ + {"AttributeName": n, "AttributeType": "S"} for n in ("pk", "h1", "h2", "r1") + ], + KeySchema=[{"AttributeName": "pk", "KeyType": "HASH"}], + GlobalSecondaryIndexes=[ + { + "IndexName": "gsi", + "KeySchema": [ + {"AttributeName": "h1", "KeyType": "HASH"}, + {"AttributeName": "h2", "KeyType": "HASH"}, + {"AttributeName": "r1", "KeyType": "RANGE"}, + ], + "Projection": {"ProjectionType": "ALL"}, + } + ], + ) + except ClientError as exc: + pytest.skip(f"multi-attribute GSI not supported here: {exc}") + try: + wait_for_active(dynamodb_client, name) + yield name + finally: + try: + dynamodb_client.delete_table(TableName=name) + except ClientError: + pass + + +# ---- helpers ----------------------------------------------------------------- + + +def error(call) -> dict: + with pytest.raises(ClientError) as excinfo: + call() + return excinfo.value.response["Error"] + + +def message(client, statement: str, parameters: list | None = None) -> str: + kwargs: dict = {"Statement": statement} + if parameters is not None: + kwargs["Parameters"] = parameters + err = error(lambda: client.execute_statement(**kwargs)) + assert err["Code"] == "ValidationException", err + return err["Message"] + + +def transaction_message(client, statements: list[dict]) -> str: + err = error(lambda: client.execute_transaction(TransactStatements=statements)) + assert err["Code"] == "ValidationException", err + return err["Message"] + + +def batch_errors(client, statements: list[dict]) -> list: + resp = client.batch_execute_statement(Statements=statements) + return [entry.get("Error") for entry in resp["Responses"]] + + +def positions(tables) -> dict: + """Every position a parameter can take, as statements with one ``?``.""" + s = tables["str"] + return { + "where": f"SELECT * FROM \"{s}\" WHERE pk = 'p' AND n = ?", + "insert": f"INSERT INTO \"{s}\" VALUE {{'pk': 'p-insert', 'sk': 's', 'n': ?}}", + "update": f"UPDATE \"{s}\" SET n = ? WHERE pk = 'p' AND sk = 's'", + "key": f"SELECT * FROM \"{s}\" WHERE pk = ?", + } + + +POSITIONS = ["where", "insert", "update", "key"] + + +# ---- parameters ---------------------------------------------------------------- + + +class TestParameterNumbers: + @pytest.mark.parametrize("position", POSITIONS) + @pytest.mark.parametrize( + "value", + ["1e99999999999", "-1e99999999999", "1e-99999999999", "1e2147483648", "1e-2147483648"], + ) + def test_an_exponent_or_scale_outside_32_bits_cannot_be_converted( + self, dynamodb_client, tables, value, position + ): + statement = positions(tables)[position] + assert message(dynamodb_client, statement, [{"N": value}]) == f"{UNCONVERTIBLE}: {value}" + + @pytest.mark.parametrize("position", POSITIONS) + def test_the_largest_exponent_wraps_to_an_underflow(self, dynamodb_client, tables, position): + statement = positions(tables)[position] + assert message(dynamodb_client, statement, [{"N": "1e2147483647"}]) == UNDERFLOW + + @pytest.mark.parametrize("position", POSITIONS) + def test_an_empty_number_has_nothing_after_the_message( + self, dynamodb_client, tables, position + ): + statement = positions(tables)[position] + assert message(dynamodb_client, statement, [{"N": ""}]) == UNCONVERTIBLE + + @pytest.mark.parametrize( + "parameter", + [ + {"L": [{"N": "1"}, {"N": "1e99999999999"}]}, + {"M": {"x": {"N": "1e99999999999"}}}, + {"NS": ["1", "1e99999999999"]}, + ], + ids=["list", "map", "number-set"], + ) + def test_a_number_inside_a_collection_is_converted(self, dynamodb_client, tables, parameter): + statement = positions(tables)["where"] + assert ( + message(dynamodb_client, statement, [parameter]) + == f"{UNCONVERTIBLE}: 1e99999999999" + ) + + @pytest.mark.parametrize( + ("value", "expected"), + [("1e126", OVERFLOW), ("1e-131", UNDERFLOW), (DIGITS_39, DIGITS)], + ids=["overflow", "underflow", "digits"], + ) + def test_a_number_rule_gets_the_bare_message(self, dynamodb_client, tables, value, expected): + statement = positions(tables)["where"] + assert message(dynamodb_client, statement, [{"N": value}]) == expected + + def test_forms_the_decimal_reader_accepts_are_stored_normalized( + self, dynamodb_client, tables + ): + s = tables["str"] + dynamodb_client.execute_statement( + Statement=( + f"INSERT INTO \"{s}\" VALUE {{'pk': 'p-forms', 'sk': 's', " + "'a': ?, 'b': ?, 'c': ?, 'd': ?, 'e': ?, 'f': ?}" + ), + Parameters=[ + {"N": "+1"}, + {"N": ".5"}, + {"N": "5."}, + {"N": "1.50"}, + {"N": "1e+2"}, + {"N": "1e00000000001"}, + ], + ) + item = dynamodb_client.get_item( + TableName=s, Key={"pk": {"S": "p-forms"}, "sk": {"S": "s"}}, ConsistentRead=True + )["Item"] + assert {k: item[k]["N"] for k in "abcdef"} == { + "a": "1", + "b": "0.5", + "c": "5", + "d": "1.5", + "e": "100", + "f": "10", + } + + @pytest.mark.parametrize( + "value", [" 1", "1 ", "1e", "1.2.3", "0x10", "Infinity", "1e12345678901"] + ) + def test_forms_the_decimal_reader_refuses(self, dynamodb_client, tables, value): + statement = positions(tables)["where"] + assert message(dynamodb_client, statement, [{"N": value}]) == f"{UNCONVERTIBLE}: {value}" + + def test_a_parameter_is_converted_ahead_of_a_parse_error(self, dynamodb_client, tables): + statement = f"SELECT * FROM \"{tables['str']}\" WHERE pk = ? AND" + assert message(dynamodb_client, statement, [{"N": "abc"}]) == f"{UNCONVERTIBLE}: abc" + + def test_a_parameter_is_converted_ahead_of_table_resolution(self, dynamodb_client, tables): + statement = f"SELECT * FROM \"{tables['absent']}\" WHERE pk = ?" + assert message(dynamodb_client, statement, [{"N": "abc"}]) == f"{UNCONVERTIBLE}: abc" + + def test_a_parameter_without_a_placeholder_is_converted(self, dynamodb_client, tables): + statement = f"SELECT * FROM \"{tables['str']}\" WHERE pk = 'p'" + assert message(dynamodb_client, statement, [{"N": "abc"}]) == f"{UNCONVERTIBLE}: abc" + + def test_execute_transaction_reports_no_statement_position(self, dynamodb_client, tables): + s = tables["str"] + statements = [ + { + "Statement": f"UPDATE \"{s}\" SET n = ? WHERE pk = 'p' AND sk = 's'", + "Parameters": [{"N": "1e99999999999"}], + } + ] + assert ( + transaction_message(dynamodb_client, statements) + == f"{UNCONVERTIBLE}: 1e99999999999" + ) + + def test_a_statement_check_ahead_of_a_later_parameter_in_a_transaction( + self, dynamodb_client, tables + ): + s = tables["str"] + statements = [ + { + "Statement": f"UPDATE \"{s}\" SET a = 1 WHERE pk = 'p' AND sk = 's' AND val < ?", + "Parameters": [{"BOOL": True}], + }, + { + "Statement": f"UPDATE \"{s}\" SET a = ? WHERE pk = 'q' AND sk = 's'", + "Parameters": [{"N": "abc"}], + }, + ] + assert transaction_message(dynamodb_client, statements) == ( + "Validation failed in TransactStatements[0]: " + operand_type("<", "BOOL") + ) + + def test_batch_execute_statement_fails_only_that_statement(self, dynamodb_client, tables): + s = tables["str"] + errors = batch_errors( + dynamodb_client, + [ + {"Statement": f"SELECT * FROM \"{s}\" WHERE pk = 'p' AND sk = 's'"}, + { + "Statement": f"SELECT * FROM \"{s}\" WHERE pk = 'q' AND sk = 's' AND n = ?", + "Parameters": [{"N": "1e99999999999"}], + }, + ], + ) + assert errors == [ + None, + {"Code": "ValidationError", "Message": f"{UNCONVERTIBLE}: 1e99999999999"}, + ] + + +class TestParameterNesting: + def pair(self, tables) -> str: + return f"SELECT * FROM \"{tables['str']}\" WHERE pk = 'p' AND x = ? AND y = ?" + + def single(self, tables) -> str: + return f"SELECT * FROM \"{tables['str']}\" WHERE pk = 'p' AND x = ?" + + def test_the_first_problem_in_parameter_order_is_reported(self, dynamodb_client, tables): + bad = {"N": "abc"} + assert message(dynamodb_client, self.pair(tables), [deep(40), bad]) == NESTED + assert ( + message(dynamodb_client, self.pair(tables), [bad, deep(40)]) + == f"{UNCONVERTIBLE}: abc" + ) + + @MAP_MEMBERS_IN_NAME_ORDER + def test_the_first_problem_in_map_member_order_is_reported(self, dynamodb_client, tables): + deep_first = {"M": {"d": deep(40), "bad": {"N": "abc"}}} + assert message(dynamodb_client, self.single(tables), [deep_first]) == NESTED + + def test_a_bad_member_ahead_of_a_deep_one_is_reported(self, dynamodb_client, tables): + bad_first = {"M": {"bad": {"N": "abc"}, "d": deep(40)}} + assert ( + message(dynamodb_client, self.single(tables), [bad_first]) + == f"{UNCONVERTIBLE}: abc" + ) + + def test_a_bad_number_below_the_limit_is_never_reached(self, dynamodb_client, tables): + parameter = deep(40, {"N": "abc"}) + assert message(dynamodb_client, self.single(tables), [parameter]) == NESTED + + def test_depth_is_reported_ahead_of_a_later_overflow(self, dynamodb_client, tables): + parameters = [deep(40), {"N": "1e126"}] + assert message(dynamodb_client, self.pair(tables), parameters) == NESTED + + def test_a_parameter_holds_32_levels_and_refuses_33(self, dynamodb_client, tables): + resp = dynamodb_client.execute_statement( + Statement=self.single(tables), Parameters=[deep(32)] + ) + assert resp["Items"] == [] + assert message(dynamodb_client, self.single(tables), [deep(33)]) == NESTED + + def test_execute_transaction_reports_depth_without_a_position( + self, dynamodb_client, tables + ): + statements = [ + { + "Statement": ( + f"UPDATE \"{tables['str']}\" SET x = ?, y = ? WHERE pk = 'p' AND sk = 's'" + ), + "Parameters": [deep(40), {"N": "abc"}], + } + ] + assert transaction_message(dynamodb_client, statements) == NESTED + + def test_batch_execute_statement_reports_depth_per_statement(self, dynamodb_client, tables): + errors = batch_errors( + dynamodb_client, + [{"Statement": self.pair(tables), "Parameters": [deep(40), {"N": "abc"}]}], + ) + assert errors == [{"Code": "ValidationError", "Message": NESTED}] + + +# ---- literals ------------------------------------------------------------------ + + +class TestLiteralNumbers: + @pytest.mark.parametrize("literal", ["1e2147483648", "1e99999999999", "1.5e-2147483647"]) + def test_a_literal_that_cannot_be_converted_has_nothing_after_the_colon( + self, dynamodb_client, tables, literal + ): + statement = f"SELECT * FROM \"{tables['num']}\" WHERE pk = {literal}" + assert message(dynamodb_client, statement) == NOT_WELL_FORMED + + def test_a_literal_that_cannot_be_converted_wins_over_an_earlier_parse_error( + self, dynamodb_client, tables + ): + statement = f"FOO * FROM \"{tables['num']}\" WHERE pk = 1e99999999999" + assert message(dynamodb_client, statement) == NOT_WELL_FORMED + + # `1.2.3` reads as two numbers; tests/test_partiql_grammar.py covers it. + @pytest.mark.parametrize("literal", ["1e"]) + def test_a_malformed_number_literal_has_nothing_after_the_colon( + self, dynamodb_client, tables, literal + ): + statement = f"SELECT * FROM \"{tables['num']}\" WHERE pk = {literal}" + assert message(dynamodb_client, statement) == NOT_WELL_FORMED + + @pytest.mark.parametrize( + ("clause", "expected"), + [ + ("WHERE pk = 1e126", f"{OVERFLOW} under root"), + ("WHERE pk = 1e-131", f"{UNDERFLOW} under root"), + (f"WHERE pk = {DIGITS_39}", f"{DIGITS} under root"), + ("WHERE pk = 1e2147483647", f"{UNDERFLOW} under root"), + ("WHERE pk IN [1, 1e126]", f"{OVERFLOW} under root"), + ], + ids=["overflow", "underflow", "digits", "wrap", "in-list"], + ) + def test_a_where_literal_names_root(self, dynamodb_client, tables, clause, expected): + statement = f"SELECT * FROM \"{tables['num']}\" {clause}" + assert message(dynamodb_client, statement) == expected + + def test_write_literals_name_root(self, dynamodb_client, tables): + n = tables["num"] + assert ( + message(dynamodb_client, f"INSERT INTO \"{n}\" VALUE {{'pk': 1e126}}") + == f"{OVERFLOW} under root" + ) + assert ( + message(dynamodb_client, f"UPDATE \"{n}\" SET a = 1e-131 WHERE pk = 1") + == f"{UNDERFLOW} under root" + ) + + @pytest.mark.parametrize( + ("value", "position"), + [ + ("{'x': 1e126}", "root.x"), + ("[1, 1e126]", "root[1]"), + ("{'a': {'b': 1e126}}", "root.a.b"), + ("[[0, 1e126]]", "root[0][1]"), + ("{'a': [1, 1e126]}", "root.a[1]"), + ("[{'a': 1e126}]", "root[0].a"), + ("<<1, 1e126>>", "root"), + ], + ids=["map", "list", "map-map", "list-list", "map-list", "list-map", "set"], + ) + def test_a_literal_inside_a_value_names_its_position( + self, dynamodb_client, tables, value, position + ): + statement = f"INSERT INTO \"{tables['num']}\" VALUE {{'pk': 1, 'v': {value}}}" + assert message(dynamodb_client, statement) == f"{OVERFLOW} under {position}" + + def test_a_literal_rule_answers_ahead_of_table_resolution(self, dynamodb_client, tables): + a = tables["absent"] + assert ( + message(dynamodb_client, f"SELECT * FROM \"{a}\" WHERE pk = 1e126") + == f"{OVERFLOW} under root" + ) + assert ( + message(dynamodb_client, f"INSERT INTO \"{a}\" VALUE {{'pk': 1e126}}") + == f"{OVERFLOW} under root" + ) + + def test_a_literal_rule_answers_ahead_of_the_key_type(self, dynamodb_client, tables): + statement = f"SELECT * FROM \"{tables['str']}\" WHERE pk = 1e126" + assert message(dynamodb_client, statement) == f"{OVERFLOW} under root" + + def test_a_literal_rule_in_a_transaction_names_the_statement(self, dynamodb_client, tables): + statements = [{"Statement": f"UPDATE \"{tables['num']}\" SET a = 1e126 WHERE pk = 1"}] + assert transaction_message(dynamodb_client, statements) == ( + f"Validation failed in TransactStatements[0]: {OVERFLOW} under root" + ) + + def test_a_literal_rule_in_a_batch_fails_that_statement(self, dynamodb_client, tables): + errors = batch_errors( + dynamodb_client, + [{"Statement": f"UPDATE \"{tables['num']}\" SET a = 1e126 WHERE pk = 1"}], + ) + assert errors == [{"Code": "ValidationError", "Message": f"{OVERFLOW} under root"}] + + +# ---- ordering operand types ------------------------------------------------------ + + +class TestOrderingOperandTypes: + @pytest.mark.parametrize("op", ["<", "<=", ">", ">="]) + def test_each_operator_is_named_as_written(self, dynamodb_client, tables, op): + statement = f"SELECT pk FROM \"{tables['str']}\" WHERE pk = ? AND val {op} ?" + parameters = [{"S": "p"}, {"BOOL": True}] + assert message(dynamodb_client, statement, parameters) == operand_type(op, "BOOL") + + def test_between_is_named(self, dynamodb_client, tables): + statement = f"SELECT pk FROM \"{tables['str']}\" WHERE pk = 'p' AND val BETWEEN ? AND ?" + parameters = [{"BOOL": True}, {"BOOL": False}] + assert message(dynamodb_client, statement, parameters) == operand_type("BETWEEN", "BOOL") + + def test_the_left_operand_is_reported_first(self, dynamodb_client, tables): + statement = f"SELECT pk FROM \"{tables['str']}\" WHERE pk = 'p' AND ? < ?" + parameters = [{"NULL": True}, {"BOOL": True}] + assert message(dynamodb_client, statement, parameters) == operand_type("<", "NULL") + + @pytest.mark.parametrize( + "statement", + [ + "SELECT pk FROM \"{t}\" WHERE pk = 'p' AND val < ?", + "UPDATE \"{t}\" SET a = 1 WHERE pk = 'p' AND sk = 's' AND val < ?", + "DELETE FROM \"{t}\" WHERE pk = 'p' AND sk = 's' AND val < ?", + ], + ids=["select", "update", "delete"], + ) + def test_the_check_answers_ahead_of_table_resolution( + self, dynamodb_client, tables, statement + ): + text = statement.format(t=tables["absent"]) + assert message(dynamodb_client, text, [{"BOOL": True}]) == operand_type("<", "BOOL") + + def test_a_literal_operand_is_checked(self, dynamodb_client, tables): + statement = f"SELECT pk FROM \"{tables['absent']}\" WHERE pk = 'p' AND val < true" + assert message(dynamodb_client, statement) == operand_type("<", "BOOL") + + def test_a_batch_statement_gets_the_bare_message(self, dynamodb_client, tables): + s = tables["str"] + errors = batch_errors( + dynamodb_client, + [ + { + "Statement": f"SELECT * FROM \"{s}\" WHERE pk = 'p' AND sk = 's' AND val < ?", + "Parameters": [{"BOOL": True}], + } + ], + ) + assert errors == [{"Code": "ValidationError", "Message": operand_type("<", "BOOL")}] + + def test_a_transaction_statement_is_named(self, dynamodb_client, tables): + s = tables["str"] + statements = [ + { + "Statement": f"UPDATE \"{s}\" SET a = 1 WHERE pk = 'p' AND sk = 's' AND val < ?", + "Parameters": [{"BOOL": True}], + } + ] + assert transaction_message(dynamodb_client, statements) == ( + "Validation failed in TransactStatements[0]: " + operand_type("<", "BOOL") + ) + + def test_a_function_operand_type_message_is_bare(self, dynamodb_client, tables): + statement = f"SELECT * FROM \"{tables['str']}\" WHERE pk = 'p' AND begins_with(val, ?)" + assert message(dynamodb_client, statement, [{"N": "1"}]) == operand_type( + "begins_with", "N" + ) + + +# ---- multi-attribute key refusal ----------------------------------------------- + + +class TestMultiAttributeOrder: + def test_an_unconvertible_literal_is_reported_ahead_of_the_refusal( + self, dynamodb_client, multi_table + ): + statement = f"SELECT * FROM \"{multi_table}\".\"gsi\" WHERE h1 = 1e99999999999" + assert message(dynamodb_client, statement) == NOT_WELL_FORMED + + def test_an_unconvertible_parameter_is_reported_ahead_of_the_refusal( + self, dynamodb_client, multi_table + ): + statement = f"SELECT * FROM \"{multi_table}\".\"gsi\" WHERE h1 = ?" + assert ( + message(dynamodb_client, statement, [{"N": "1e99999999999"}]) + == f"{UNCONVERTIBLE}: 1e99999999999" + ) + + def test_the_refusal_is_reported_ahead_of_a_projected_call( + self, dynamodb_client, multi_table + ): + statement = f"SELECT COUNT(*) FROM \"{multi_table}\".\"gsi\"" + assert message(dynamodb_client, statement) == REFUSED + + def test_a_batch_reports_the_refusal_ahead_of_a_projected_call( + self, dynamodb_client, multi_table + ): + statement = f"SELECT COUNT(*) FROM \"{multi_table}\".\"gsi\" WHERE h1 = 'a'" + assert batch_errors(dynamodb_client, [{"Statement": statement}]) == [ + {"Code": "ValidationError", "Message": REFUSED} + ] + + +class TestProjectedCall: + """The three operations check a projected call after the table name rule + and ahead of number literals and operand types.""" + + TABLE_NAME_RULE = ( + "Table name should be within the length [3, 255] and only contain 'a-z', 'A-Z', " + "'0-9', '-', '_', '.'." + ) + + def answers(self, client, statement: str) -> tuple: + """The message from ExecuteStatement, the member error from + BatchExecuteStatement and the message from ExecuteTransaction.""" + members = [{"Statement": statement}] + return ( + message(client, statement), + batch_errors(client, members), + transaction_message(client, members), + ) + + def test_the_table_name_rule_is_reported_ahead_of_a_projected_call(self, dynamodb_client): + rule = self.TABLE_NAME_RULE + assert self.answers(dynamodb_client, "SELECT COUNT(*) FROM \"t\" WHERE pk = 'a'") == ( + rule, + [{"Code": "ValidationError", "Message": rule}], + f"Validation failed in TransactStatements[0]: {rule}", + ) + + @pytest.mark.parametrize( + "where", ["pk = 1e126", "pk = 'p' AND v < true"], ids=["number", "operand-type"] + ) + def test_a_projected_call_is_reported_ahead_of_the_literal_checks( + self, dynamodb_client, tables, where + ): + statement = f"SELECT COUNT(*) FROM \"{tables['str']}\" WHERE {where}" + assert self.answers(dynamodb_client, statement) == ( + PROJECTED_CALL, + [{"Code": "ValidationError", "Message": PROJECTED_CALL}], + f"Validation failed in TransactStatements[0]: {PROJECTED_CALL}", + ) + + @pytest.mark.parametrize("where", ["", " WHERE pk = 'p'"], ids=["no-where", "where"]) + def test_a_projected_call_is_reported_ahead_of_a_missing_table( + self, dynamodb_client, tables, where + ): + statement = f"SELECT COUNT(*) FROM \"{tables['absent']}\"{where}" + assert message(dynamodb_client, statement) == PROJECTED_CALL + + @pytest.mark.parametrize("table", ["str", "absent"]) + def test_a_batch_fails_only_the_statement_with_a_projected_call( + self, dynamodb_client, tables, table + ): + s = tables["str"] + errors = batch_errors( + dynamodb_client, + [ + {"Statement": f"SELECT * FROM \"{s}\" WHERE pk = 'p' AND sk = 's'"}, + { + "Statement": ( + f"SELECT COUNT(*) FROM \"{tables[table]}\" WHERE pk = 'q' AND sk = 's'" + ) + }, + ], + ) + assert errors == [None, {"Code": "ValidationError", "Message": PROJECTED_CALL}] + + @pytest.mark.parametrize("position", [0, 1]) + @pytest.mark.parametrize("table", ["str", "absent"]) + def test_a_transaction_names_the_member_with_a_projected_call( + self, dynamodb_client, tables, table, position + ): + statements = [ + {"Statement": f"SELECT * FROM \"{tables['str']}\" WHERE pk = 'p' AND sk = 's'"} + ] + count = f"SELECT COUNT(*) FROM \"{tables[table]}\" WHERE pk = 'q' AND sk = 's'" + statements.insert(position, {"Statement": count}) + assert transaction_message(dynamodb_client, statements) == ( + f"Validation failed in TransactStatements[{position}]: {PROJECTED_CALL}" + ) diff --git a/tests/test_partiql_order_by.py b/tests/test_partiql_order_by.py new file mode 100644 index 000000000..242954c45 --- /dev/null +++ b/tests/test_partiql_order_by.py @@ -0,0 +1,473 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""Dual-target wire tests for ORDER BY in a PartiQL SELECT. + +Every path in the list must be a key attribute of what is read, the table's +key or the index's; the statement must have a WHERE, and the WHERE must read +by key. Over more than one partition key value the list must name the +partition key somewhere. Each path takes ASC or DESC. + +The partitions ascend unless the path that names the partition key says +DESC. Within a partition the items follow the path that names the sort key, +and descend when no path names it: `ORDER BY pk` reads the items of a +partition in descending order. + +Among the checks of the WHERE, a path into a key attribute comes first, then +a path in the list that is not a key attribute, then the key values, then +more than one partition key value without the partition key in the list, +then overlapping disjuncts and the number of disjuncts. + +Dual-target: EXTENDDB_TEST_ENDPOINT selects an extenddb server; unset, the +tests run against the live service with the default credential chain. The +assertions are identical on both. Every expected value was measured on the +service (2026-11-03). +""" + +from __future__ import annotations + +import time + +import pytest +from botocore.exceptions import ClientError + +from conftest import scoped_table + +WF = "Statement wasn't well formed, can't be processed: " +NO_WHERE = "Must have WHERE clause in the statement when using ORDER BY clause." +NOT_KEYED = ( + "Must have at least one non-optional hash key condition in WHERE clause when using " + "ORDER BY clause." +) +HASH_KEY = ( + "Must have hash key in ORDER BY clause when more than one hash key condition specified in " + "WHERE clause." +) +OVERLAP = "Overlapping conditions with range keys are not supported in where clause" +SCALAR = ( + "Key attributes must be scalars; list random access '[]' and map lookup '.' are not " + "allowed: Key pk" +) +VALUES_51 = ", ".join(f"'v{i:03}'" for i in range(51)) + + +def _key_type(name): + return f"Key attribute's data type should match its data type in table's schema: Key {name}" + + +def _attribute(name): + return f"Attribute {name} in ORDER BY clause must be part of the primary key" + + +def _keyed_table(client, sort_type="S"): + """A table keyed on a string pk and a sort key sk of type `sort_type`.""" + return scoped_table( + client, + attribute_definitions=[ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "sk", "AttributeType": sort_type}, + ], + key_schema=[ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"}, + ], + ) + + +@pytest.fixture(scope="module") +def five(dynamodb_client): + """Items a/x, a/y, a/z, b/x and b/y, each with n = 1 and m = 'x'.""" + with _keyed_table(dynamodb_client) as name: + for pk, sk in [("a", "x"), ("a", "y"), ("a", "z"), ("b", "x"), ("b", "y")]: + dynamodb_client.put_item( + TableName=name, + Item={"pk": {"S": pk}, "sk": {"S": sk}, "n": {"N": "1"}, "m": {"S": "x"}}, + ) + yield name + + +@pytest.fixture(scope="module") +def four(dynamodb_client): + """Items a/x, a/y, b/x and c/x, each with n = 1.""" + with _keyed_table(dynamodb_client) as name: + for pk, sk in [("a", "x"), ("a", "y"), ("b", "x"), ("c", "x")]: + dynamodb_client.put_item( + TableName=name, Item={"pk": {"S": pk}, "sk": {"S": sk}, "n": {"N": "1"}} + ) + yield name + + +@pytest.fixture(scope="module") +def numbers(dynamodb_client): + """Partition a with number sort keys 1, 2 and 10.""" + with _keyed_table(dynamodb_client, sort_type="N") as name: + for sk in ["1", "2", "10"]: + dynamodb_client.put_item(TableName=name, Item={"pk": {"S": "a"}, "sk": {"N": sk}}) + yield name + + +@pytest.fixture(scope="module") +def hashed(dynamodb_client): + """A table keyed on pk alone holding a, b and c, each with n = 1.""" + with scoped_table(dynamodb_client) as name: + for pk in ["a", "b", "c"]: + dynamodb_client.put_item(TableName=name, Item={"pk": {"S": pk}, "n": {"N": "1"}}) + yield name + + +@pytest.fixture(scope="module") +def indexed(dynamodb_client): + """A table keyed on pk with a GSI gix keyed on g and gs, holding p1 (a/x), + p2 (a/y) and p3 (a/z).""" + with scoped_table( + dynamodb_client, + attribute_definitions=[ + {"AttributeName": n, "AttributeType": "S"} for n in ("pk", "g", "gs") + ], + GlobalSecondaryIndexes=[ + { + "IndexName": "gix", + "KeySchema": [ + {"AttributeName": "g", "KeyType": "HASH"}, + {"AttributeName": "gs", "KeyType": "RANGE"}, + ], + "Projection": {"ProjectionType": "ALL"}, + } + ], + ) as name: + for pk, gs in [("p1", "x"), ("p2", "y"), ("p3", "z")]: + dynamodb_client.put_item( + TableName=name, Item={"pk": {"S": pk}, "g": {"S": "a"}, "gs": {"S": gs}} + ) + for _ in range(240): + count = dynamodb_client.scan(TableName=name, IndexName="gix", Select="COUNT")["Count"] + if count == 3: + break + time.sleep(0.5) + yield name + + +def _key(item): + if "gs" in item: + return item["pk"]["S"] + sort = item.get("sk") + return item["pk"]["S"] + ("" if sort is None else "/" + (sort.get("S") or sort.get("N"))) + + +def _read(client, statement, **kwargs): + """The keys every page of `statement` holds, in order, following + NextToken, or the refusal's message.""" + keys, token = [], None + for _ in range(12): + args = dict(kwargs) + if token: + args["NextToken"] = token + try: + response = client.execute_statement(Statement=statement, **args) + except ClientError as e: + assert e.response["Error"]["Code"] == "ValidationException", e.response + return e.response["Error"]["Message"] + keys += [_key(item) for item in response["Items"]] + token = response.get("NextToken") + if not token: + break + return keys + + +DOWN = ["a/z", "a/y", "a/x"] +UP = ["a/x", "a/y", "a/z"] + + +@pytest.mark.parametrize("where", ["pk = 'a'", "pk = 'a' AND sk > 'a'", "pk = 'a' AND n = 1"]) +@pytest.mark.parametrize( + ("order", "keys"), + [ + ("pk", DOWN), + ("pk ASC", DOWN), + ("pk DESC", DOWN), + ("sk", UP), + ("sk DESC", DOWN), + ("pk, sk", UP), + ("pk, pk", DOWN), + ("sk, sk", UP), + ('"pk"', DOWN), + ("n", _attribute("n")), + ("PK", _attribute("PK")), + ], +) +def test_the_items_of_a_partition_follow_the_sort_key_or_descend( + dynamodb_client, five, where, order, keys +): + statement = f'SELECT * FROM "{five}" WHERE {where} ORDER BY {order}' + assert _read(dynamodb_client, statement) == keys + + +@pytest.mark.parametrize( + ("where", "order", "keys"), + [ + ("pk = 'a'", "pk", ["a/10", "a/2", "a/1"]), + ("pk = 'a'", "sk", ["a/1", "a/2", "a/10"]), + ("pk = 'a'", "sk DESC", ["a/10", "a/2", "a/1"]), + ("pk = 'a' AND sk > 0", "pk", ["a/10", "a/2", "a/1"]), + ("pk = 'a' AND sk > 0", "pk, sk", ["a/1", "a/2", "a/10"]), + ("pk = 'a' AND sk <> 5", "pk", NOT_KEYED), + ("pk = 'a' AND sk <> 5", "n", _attribute("n")), + ], +) +def test_a_number_sort_key_orders_by_value(dynamodb_client, numbers, where, order, keys): + statement = f'SELECT * FROM "{numbers}" WHERE {where} ORDER BY {order}' + assert _read(dynamodb_client, statement) == keys + + +@pytest.mark.parametrize( + ("order", "keys"), + [ + ("pk", ["a/z", "a/y", "a/x", "b/y", "b/x"]), + ("pk ASC", ["a/z", "a/y", "a/x", "b/y", "b/x"]), + ("sk, pk", ["a/x", "a/y", "a/z", "b/x", "b/y"]), + ("sk DESC, pk", ["a/z", "a/y", "a/x", "b/y", "b/x"]), + ("pk DESC, sk DESC", ["b/y", "b/x", "a/z", "a/y", "a/x"]), + ], +) +def test_partitions_follow_the_partition_key(dynamodb_client, five, order, keys): + statement = f"SELECT * FROM \"{five}\" WHERE pk IN ['a', 'b'] ORDER BY {order}" + assert _read(dynamodb_client, statement) == keys + + +@pytest.mark.parametrize( + ("order", "keys"), + [ + ("sk, pk", ["a/x", "a/y", "b/x"]), + ("pk DESC, sk", ["b/x", "a/x", "a/y"]), + ("pk ASC, sk", ["a/x", "a/y", "b/x"]), + ("pk, sk DESC", ["a/y", "a/x", "b/x"]), + ("pk, sk ASC", ["a/x", "a/y", "b/x"]), + ("pk", ["a/y", "a/x", "b/x"]), + ("pk, sk", ["a/x", "a/y", "b/x"]), + ("sk", HASH_KEY), + ("pk, n", _attribute("n")), + ], +) +def test_each_path_takes_a_direction_one_item_a_page(dynamodb_client, four, order, keys): + statement = f"SELECT * FROM \"{four}\" WHERE pk IN ['a', 'b'] ORDER BY {order}" + assert _read(dynamodb_client, statement, Limit=1) == keys + + +@pytest.mark.parametrize( + ("where", "by_n", "by_sk"), + [ + ("pk.x = 1", SCALAR, SCALAR), + ("pk = 1", _attribute("n"), _key_type("pk")), + ("(pk = 'a' OR pk = 'a')", _attribute("n"), OVERLAP), + ("(pk = 'a' OR pk = 'b' OR pk = 'a')", _attribute("n"), HASH_KEY), + (f"pk IN [{VALUES_51}]", _attribute("n"), HASH_KEY), + ("pk IN ['a', 1]", _attribute("n"), _key_type("pk")), + ("pk = 'a' AND sk = 1", _attribute("n"), _key_type("sk")), + ("pk = 'a' AND (sk = 'x' OR sk > 'a')", _attribute("n"), OVERLAP), + ], + ids=[ + "key-path", + "key-type", + "overlap", + "overlap-two-keys", + "51-disjuncts", + "key-type-in-an-in", + "sort-key-type", + "sort-key-overlap", + ], +) +def test_the_order_by_refusals_sit_among_the_key_checks(dynamodb_client, five, where, by_n, by_sk): + # A path into a key comes first, then a term that is not a key + # attribute, then the key values, then more than one partition key value + # without the partition key in the list, then overlapping disjuncts and + # the number of disjuncts. + for order, result in (("n", by_n), ("sk", by_sk)): + statement = f'SELECT * FROM "{five}" WHERE {where} ORDER BY {order}' + assert _read(dynamodb_client, statement) == result, order + + +@pytest.mark.parametrize( + ("clause", "result"), + [ + ("ORDER BY pk.x", NO_WHERE), + ("WHERE n = 1 ORDER BY n", _attribute("n")), + ("WHERE pk = 1 OR pk = 1", _key_type("pk")), + ("WHERE pk.x = 1 AND pk = 1", SCALAR), + ], +) +def test_the_key_checks_keep_their_order_with_and_without_an_order_by( + dynamodb_client, five, clause, result +): + assert _read(dynamodb_client, f'SELECT * FROM "{five}" {clause}') == result + + +def test_a_list_that_ends_in_a_comma_is_an_unexpected_term(dynamodb_client, four): + statement = f"SELECT * FROM \"{four}\" WHERE pk = 'a' ORDER BY pk," + assert _read(dynamodb_client, statement) == WF + "Unexpected term" + + +@pytest.mark.parametrize( + ("where", "order", "result"), + [ + (None, "pk", NO_WHERE), + ("n = 1", "pk", NOT_KEYED), + ("n = 1", "sk", NOT_KEYED), + ("pk = 'a' OR n = 1", "pk", NOT_KEYED), + ("pk > 'a'", "pk", NOT_KEYED), + ("pk = 'a' AND sk <> 'x'", "sk", NOT_KEYED), + ("pk = 'a' AND NOT sk = 'x'", "sk", NOT_KEYED), + ("pk = 'a' AND contains(sk, 'x')", "sk", NOT_KEYED), + ("pk = 'a' AND sk IS NOT MISSING", "sk", NOT_KEYED), + ("pk = 'a' AND size(sk) = 1", "sk", NOT_KEYED), + ("pk = 'a' AND sk = 'x' AND sk = 'y'", "sk", NOT_KEYED), + ("pk = 'a' AND sk > 'a' AND sk < 'z'", "sk", NOT_KEYED), + ("pk IN ['a', 'b'] AND sk <> 'x'", "pk", NOT_KEYED), + ("pk = 'a' AND sk <> 'x'", "pk", NOT_KEYED), + ("pk = 'a' AND sk IN ['x', 'y']", "sk", ["a/x", "a/y"]), + ("pk = 'a' AND (sk = 'x' OR sk = 'y')", "sk", ["a/x", "a/y"]), + ("pk = 'a' AND m = 'x'", "sk", UP), + ("pk = 'a' AND NOT m = 'y'", "sk", UP), + ("pk = 'a' AND (sk = 'x' OR m = 'x')", "sk", OVERLAP), + ], +) +def test_an_order_by_needs_a_where_that_reads_by_key(dynamodb_client, five, where, order, result): + clause = f" WHERE {where}" if where else "" + statement = f'SELECT * FROM "{five}"{clause} ORDER BY {order}' + assert _read(dynamodb_client, statement) == result + + +@pytest.mark.parametrize( + ("where", "order", "result"), + [ + ("pk = 'a'", "pk", ["a"]), + ("pk IN ['a', 'b']", "pk", ["a", "b"]), + ("pk IN ['a', 'b']", "pk DESC", ["b", "a"]), + ("pk IN ['c', 'a']", "pk", ["a", "c"]), + ("n = 1", "pk", NOT_KEYED), + ("pk = 'a' AND n = 1", "pk", ["a"]), + ("pk = 'a'", "n", _attribute("n")), + ], +) +def test_an_order_by_on_a_table_keyed_on_its_partition_key( + dynamodb_client, hashed, where, order, result +): + statement = f'SELECT * FROM "{hashed}" WHERE {where} ORDER BY {order}' + assert _read(dynamodb_client, statement) == result + + +@pytest.mark.parametrize( + ("order", "result"), + [ + ("g", ["p3", "p2", "p1"]), + ("g DESC", ["p3", "p2", "p1"]), + ("gs", ["p1", "p2", "p3"]), + ("g, gs", ["p1", "p2", "p3"]), + ("pk", _attribute("pk")), + ], +) +def test_an_order_by_on_an_index_takes_the_index_keys(dynamodb_client, indexed, order, result): + statement = f"SELECT * FROM \"{indexed}\".\"gix\" WHERE g = 'a' ORDER BY {order}" + assert _read(dynamodb_client, statement) == result + + +# ---- ORDER BY on an index, and a path in the list ------------------------------- +# +# Measured 2026-11-05. An index read takes the refusals of the WHERE an ORDER +# BY reads that a table read takes. A statement with no WHERE is refused for +# it ahead of the list, on a table and on an index. A path with steps in the +# list is refused at its first token, in list order with a name that is not a +# key attribute, after a path into a key attribute in the WHERE and ahead of +# the key values. + +PATH_TERM = "Argument to ORDER BY must be of type VariableReference. Type: PATH at 1:{}:{}" + + +@pytest.mark.parametrize( + ("clause", "result"), + [ + ("ORDER BY g", NO_WHERE), + ("ORDER BY n", NO_WHERE), + ("ORDER BY m.x", NO_WHERE), + ("WHERE g > 'a' ORDER BY g", NOT_KEYED), + ("WHERE g = 'a' OR n = 1 ORDER BY gs", NOT_KEYED), + ("WHERE gs = 'x' ORDER BY gs", NOT_KEYED), + ("WHERE g = 'a' AND gs <> 'x' ORDER BY gs", NOT_KEYED), + ("WHERE g IN ['a', 'b'] ORDER BY gs", HASH_KEY), + ("WHERE g > 'a' ORDER BY n", _attribute("n")), + ], +) +def test_an_order_by_on_an_index_needs_a_where_that_reads_it_by_key( + dynamodb_client, indexed, clause, result +): + statement = f'SELECT * FROM "{indexed}"."gix" {clause}' + assert _read(dynamodb_client, statement) == result + + +def test_an_order_by_on_an_index_over_two_partitions_names_its_partition_key( + dynamodb_client, indexed +): + statement = f"SELECT * FROM \"{indexed}\".\"gix\" WHERE g IN ['a', 'b'] ORDER BY g, gs" + assert _read(dynamodb_client, statement) == ["p1", "p2", "p3"] + + +def test_an_order_by_with_no_where_is_refused_ahead_of_its_list(dynamodb_client, five): + for order in ("n", "m.x", "pk"): + assert _read(dynamodb_client, f'SELECT * FROM "{five}" ORDER BY {order}') == NO_WHERE + + +@pytest.mark.parametrize( + ("clause", "term"), + [ + ("WHERE pk = 'a' ORDER BY m.x", "m"), + ("WHERE pk = 'a' ORDER BY m[0]", "m"), + ("WHERE pk = 'a' ORDER BY m['k']", "m"), + ("WHERE pk = 'a' ORDER BY pk.x", "pk"), + ("WHERE pk = 'a' ORDER BY \"m\".\"x\"", '"m"'), + ("WHERE pk = 'a' ORDER BY m.x DESC", "m"), + ("WHERE pk = 'a' ORDER BY m.x, n", "m"), + ("WHERE pk = 'a' ORDER BY sk, m.x", "m"), + ("WHERE pk = 1 ORDER BY m.x", "m"), + ("WHERE pk = 'a' AND sk = 1 ORDER BY m.x", "m"), + ("WHERE pk > 'a' ORDER BY m.x", "m"), + ], + ids=[ + "map-member", + "list-element", + "member-in-brackets", + "into-a-key", + "quoted", + "with-a-direction", + "ahead-of-a-name", + "after-a-key", + "ahead-of-the-key-type", + "ahead-of-the-sort-key-type", + "ahead-of-a-scan", + ], +) +def test_a_path_in_the_order_by_list_is_refused_at_its_first_token( + dynamodb_client, five, clause, term +): + statement = f'SELECT * FROM "{five}" {clause}' + column = statement.index(term, statement.index("ORDER BY") + len("ORDER BY")) + 1 + assert _read(dynamodb_client, statement) == PATH_TERM.format(column, len(term)) + + +@pytest.mark.parametrize( + ("clause", "result"), + [ + ("WHERE pk = 'a' ORDER BY n, m.x", _attribute("n")), + ("WHERE pk = 'a' AND sk.x = 1 ORDER BY m.x", SCALAR.replace("Key pk", "Key sk")), + ], +) +def test_a_path_in_the_order_by_list_after_other_refusals(dynamodb_client, five, clause, result): + assert _read(dynamodb_client, f'SELECT * FROM "{five}" {clause}') == result + + +@pytest.mark.parametrize( + "clause", + ["WHERE g = 'a' ORDER BY m.x", "WHERE g > 'a' ORDER BY m.x", "WHERE g = 'a' ORDER BY g, m.x"], +) +def test_a_path_in_the_order_by_list_of_an_index_read(dynamodb_client, indexed, clause): + statement = f'SELECT * FROM "{indexed}"."gix" {clause}' + column = statement.index("m.x") + 1 + assert _read(dynamodb_client, statement) == PATH_TERM.format(column, 1) diff --git a/tests/test_partiql_page_bytes.py b/tests/test_partiql_page_bytes.py new file mode 100644 index 000000000..6bc923f1b --- /dev/null +++ b/tests/test_partiql_page_bytes.py @@ -0,0 +1,152 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""Dual-target wire tests for the size of a page of a PartiQL SELECT that +reads several partitions by key. + +The disjuncts of a page share one 1 MB budget of the item bytes they read. A +Query of a partition reads until the item that takes the page past 1 MB, +keeps that item, and ends the page there with that item's key as the +LastEvaluatedKey; the next page goes on inside that Query. Whether an empty +page follows a page that ends on the last item of a partition depends on +where the partition sits in the table's storage, so these tests do not assert +what follows such a page. + +Dual-target: EXTENDDB_TEST_ENDPOINT selects an extenddb server; unset, the +tests run against the live service with the default credential chain. The +assertions are identical on both. The expected values were measured on the +service (2026-11-03), except the read of a partition past 1 MB in +`test_a_partition_past_1_mb_ends_the_page_on_the_item_that_passes_it`, which +follows the rule the other shapes measured. +""" + +from __future__ import annotations + +import contextlib + +import pytest + +from conftest import scoped_table + +BIG = "x" * (380 * 1024) + + +@pytest.fixture(scope="module") +def big_table(dynamodb_client): + """Yield a function that fills a fresh table keyed on string pk and sk + with items of about 380 KB, `{partition: [sort keys]}`, plus small items + `small`, and returns its name.""" + with contextlib.ExitStack() as stack: + + def make(big, small=()): + name = stack.enter_context( + scoped_table( + dynamodb_client, + attribute_definitions=[ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "sk", "AttributeType": "S"}, + ], + key_schema=[ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"}, + ], + ) + ) + for pk, sorts in big.items(): + for sk in sorts: + dynamodb_client.put_item( + TableName=name, + Item={"pk": {"S": pk}, "sk": {"S": sk}, "v": {"S": BIG}}, + ) + for pk, sk in small: + dynamodb_client.put_item(TableName=name, Item={"pk": {"S": pk}, "sk": {"S": sk}}) + return name + + yield make + + +@pytest.fixture(scope="module") +def one_each(big_table): + """Partitions a, b and c, each holding one item of about 380 KB.""" + return big_table({"a": ["1"], "b": ["1"], "c": ["1"]}) + + +@pytest.fixture(scope="module") +def two_each(big_table): + """Partitions a and b, each holding two items of about 380 KB.""" + return big_table({"a": ["1", "2"], "b": ["1", "2"]}) + + +@pytest.fixture(scope="module") +def four_in_a(big_table): + """Partition a holding four items of about 380 KB, partition b one small item.""" + return big_table({"a": ["1", "2", "3", "4"]}, small=[("b", "1")]) + + +def _key(item): + return f"{item['pk']['S']}/{item['sk']['S']}" + + +def _pages(client, statement): + """Every page of `statement`, following NextToken: its keys, its + LastEvaluatedKey (None when it has none) and whether it has a NextToken.""" + pages, token = [], None + for _ in range(8): + args = {"NextToken": token} if token else {} + response = client.execute_statement(Statement=statement, **args) + token = response.get("NextToken") + keys = [_key(item) for item in response["Items"]] + pages.append((keys, response.get("LastEvaluatedKey"), token is not None)) + if not token: + break + return pages + + +def _sk(pk, sk): + return {"pk": {"S": pk}, "sk": {"S": sk}} + + +@pytest.mark.parametrize("projection", ["*", "pk, sk"]) +def test_a_page_ends_inside_the_query_that_takes_it_past_1_mb( + dynamodb_client, two_each, projection +): + statement = f"SELECT {projection} FROM \"{two_each}\" WHERE pk IN ['a', 'b']" + assert _pages(dynamodb_client, statement) == [ + (["a/1", "a/2", "b/1"], _sk("b", "1"), True), + (["b/2"], None, False), + ] + + +def test_a_page_within_1_mb_reads_every_partition(dynamodb_client, one_each): + statement = f"SELECT * FROM \"{one_each}\" WHERE pk IN ['a', 'b']" + assert _pages(dynamodb_client, statement) == [(["a/1", "b/1"], None, False)] + + +@pytest.mark.parametrize( + "condition", + ["pk IN ['a', 'b', 'c']", "pk = 'a' OR pk = 'b' OR pk = 'c'"], + ids=["in", "or"], +) +def test_a_page_keeps_the_item_that_takes_it_past_1_mb(dynamodb_client, one_each, condition): + pages = _pages(dynamodb_client, f'SELECT * FROM "{one_each}" WHERE {condition}') + # The page ends on the last item of partition c: whether it has a key and + # whether an empty page follows depend on the table's storage. + assert pages[0][0] == ["a/1", "b/1", "c/1"] + assert all(not keys for keys, _, _ in pages[1:]) + + +def test_a_page_of_an_item_and_a_partition_shares_the_budget(dynamodb_client, two_each): + statement = f"SELECT * FROM \"{two_each}\" WHERE (pk = 'a' AND sk = '1') OR pk = 'b'" + pages = _pages(dynamodb_client, statement) + assert pages[0][0] == ["a/1", "b/1", "b/2"] + assert all(not keys for keys, _, _ in pages[1:]) + + +def test_a_partition_past_1_mb_ends_the_page_on_the_item_that_passes_it( + dynamodb_client, four_in_a +): + statement = f"SELECT * FROM \"{four_in_a}\" WHERE pk IN ['a', 'b']" + assert _pages(dynamodb_client, statement) == [ + (["a/1", "a/2", "a/3"], _sk("a", "3"), True), + (["a/4", "b/1"], None, False), + ] diff --git a/tests/test_partiql_page_reads.py b/tests/test_partiql_page_reads.py new file mode 100644 index 000000000..3823f66eb --- /dev/null +++ b/tests/test_partiql_page_reads.py @@ -0,0 +1,479 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""Dual-target wire tests for the pages of a PartiQL SELECT read by key over +many disjuncts. + +A page of a SELECT whose every disjunct reads by key reads at most 10 of its +disjuncts, whatever its Limit: the first page in read order (key order, or +the order an ORDER BY gives), later pages in the order that +tests/test_partiql_visit_order.py describes. Within that cap the rules of a +page hold: a disjunct +that names one item takes one unit of Limit whether the item exists or not, +any other disjunct is a Query with the units left. A page has a +LastEvaluatedKey only when it returned Limit items, so a page the cap ends +with fewer has none. There is no NextToken after the page that reads the +last disjunct, and the capacity a page reports is that of its reads. + +A disjunct held back for the refusal it gets when it is read takes its +place among the 10 and reads nothing; a request whose page would visit only +held-back disjuncts gets the refusal, so the request after the page that +reads the last of the others does. + +An index read takes one Query for each disjunct, also when several name one +partition of the index: each Query is one of the 10 of a page and reports +its own capacity. Whether a page that ends in one of these Queries has a +LastEvaluatedKey, and whether an empty page follows the last one, is the +answer a Query gives at the end of a partition, which differs on the service +by where the partition sits in its storage, so the tests assert neither. + +Dual-target: EXTENDDB_TEST_ENDPOINT selects an extenddb server; unset, the +tests run against the live service with the default credential chain. The +assertions are identical on both. Every expected value was measured on the +service (2026-11-03). +""" + +from __future__ import annotations + +import time + +import pytest +from botocore.exceptions import ClientError + +from conftest import scoped_table + +HASH_SIZE = ( + "One or more parameter values were invalid: Size of hashkey has exceeded the maximum " + "size limit of2048 bytes" +) +FILTER_KEY = ( + "Filter Expression can only contain non-primary key attributes: Primary key attribute: sk" +) +BIG = "z" * 2049 + + +def _values(n, prefix="v", start=0): + """`n` string values `000`, `001`, ... from `start`.""" + return ", ".join(f"'{prefix}{i:03}'" for i in range(start, start + n)) + + +def _keyed_table(client): + """A table keyed on string pk and sk.""" + return scoped_table( + client, + attribute_definitions=[ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "sk", "AttributeType": "S"}, + ], + key_schema=[ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"}, + ], + ) + + +@pytest.fixture(scope="module") +def htable(dynamodb_client): + """A table keyed on a string pk holding v000 to v029.""" + with scoped_table(dynamodb_client) as name: + for i in range(30): + dynamodb_client.put_item(TableName=name, Item={"pk": {"S": f"v{i:03}"}}) + yield name + + +@pytest.fixture(scope="module") +def ctable(dynamodb_client): + """A table keyed on string pk and sk holding v000/s to v029/s, each with + a = 's'.""" + with _keyed_table(dynamodb_client) as name: + for i in range(30): + dynamodb_client.put_item( + TableName=name, + Item={"pk": {"S": f"v{i:03}"}, "sk": {"S": "s"}, "a": {"S": "s"}}, + ) + yield name + + +@pytest.fixture(scope="module") +def c3table(dynamodb_client): + """A table keyed on string pk and sk holding partitions v000 to v010 of + three items each, s0, s1 and s2.""" + with _keyed_table(dynamodb_client) as name: + for i in range(11): + for j in range(3): + dynamodb_client.put_item( + TableName=name, Item={"pk": {"S": f"v{i:03}"}, "sk": {"S": f"s{j}"}} + ) + yield name + + +def _indexed_table(client, *sort): + """A table keyed on a string pk with a GSI gix keyed on a string g and, + when `sort` names it, a string sort key.""" + schema = [{"AttributeName": "g", "KeyType": "HASH"}] + schema += [{"AttributeName": name, "KeyType": "RANGE"} for name in sort] + return scoped_table( + client, + attribute_definitions=[ + {"AttributeName": name, "AttributeType": "S"} for name in ("pk", "g", *sort) + ], + GlobalSecondaryIndexes=[ + {"IndexName": "gix", "KeySchema": schema, "Projection": {"ProjectionType": "ALL"}} + ], + ) + + +def _wait_for_index(client, table, count): + """Wait until the index gix shows every item written (it fills + asynchronously on the service).""" + for _ in range(240): + scan = client.scan(TableName=table, IndexName="gix", Select="COUNT") + if scan["Count"] == count: + return + time.sleep(0.5) + raise TimeoutError(f"gix of {table} did not reach {count} items") + + +@pytest.fixture(scope="module") +def gtable(dynamodb_client): + """A table with a GSI gix keyed on g, holding p000 to p029 with g v000 to + v029.""" + with _indexed_table(dynamodb_client) as name: + for i in range(30): + dynamodb_client.put_item( + TableName=name, Item={"pk": {"S": f"p{i:03}"}, "g": {"S": f"v{i:03}"}} + ) + _wait_for_index(dynamodb_client, name, 30) + yield name + + +@pytest.fixture(scope="module") +def g2table(dynamodb_client): + """A table with a GSI gix keyed on g and gs, holding p000 to p014 in index + partition a with gs w000 to w014.""" + with _indexed_table(dynamodb_client, "gs") as name: + for i in range(15): + dynamodb_client.put_item( + TableName=name, + Item={"pk": {"S": f"p{i:03}"}, "g": {"S": "a"}, "gs": {"S": f"w{i:03}"}}, + ) + _wait_for_index(dynamodb_client, name, 15) + yield name + + +@pytest.fixture(scope="module") +def xyz(dynamodb_client): + """A table with a GSI gix keyed on g and gs, holding p1 (a/x), p2 (a/y) + and p3 (a/z).""" + with _indexed_table(dynamodb_client, "gs") as name: + for pk, gs in [("p1", "x"), ("p2", "y"), ("p3", "z")]: + dynamodb_client.put_item( + TableName=name, Item={"pk": {"S": pk}, "g": {"S": "a"}, "gs": {"S": gs}} + ) + _wait_for_index(dynamodb_client, name, 3) + yield name + + +def _key(item): + return "/".join(item[k]["S"] for k in ("pk", "sk") if k in item) + + +def _gs(item): + return item["gs"]["S"] + + +def _read(client, statement, key=_key, **kwargs): + """Every page of `statement`, following NextToken: the `key` of each of + its items in order, its LastEvaluatedKey (None when it has none), whether + it has a NextToken and the capacity it reports; and the message of the + ValidationException that ends the paging when there is one.""" + pages, token = [], None + for _ in range(12): + args = dict(kwargs) + if token: + args["NextToken"] = token + try: + response = client.execute_statement(Statement=statement, **args) + except ClientError as e: + assert e.response["Error"]["Code"] == "ValidationException", e.response + pages.append(e.response["Error"]["Message"]) + break + token = response.get("NextToken") + capacity = response.get("ConsumedCapacity", {}).get("CapacityUnits") + keys = [key(item) for item in response["Items"]] + pages.append((keys, response.get("LastEvaluatedKey"), token is not None, capacity)) + if not token: + break + return pages + + +def _sizes(client, statement, **kwargs): + """How many items each page of `statement` holds, and the refusal that + ends the paging when there is one.""" + return [ + page if isinstance(page, str) else len(page[0]) + for page in _read(client, statement, **kwargs) + ] + + +def _sorted_keys(client, statement, **kwargs): + """The sorted keys of each page of `statement`, and the refusal that ends + the paging when there is one.""" + return [ + page if isinstance(page, str) else sorted(page[0]) + for page in _read(client, statement, **kwargs) + ] + + +# ---- the cap of 10 disjuncts a page ------------------------------------------ + + +@pytest.mark.parametrize( + ("count", "sizes"), + [ + (4, [4]), + (9, [9]), + (10, [10]), + (11, [10, 1]), + (12, [10, 2]), + (15, [10, 5]), + (20, [10, 10]), + (24, [10, 10, 4]), + (25, [10, 10, 5]), + (30, [10, 10, 10]), + (50, [10, 10, 10, 0, 0]), + ], +) +def test_a_page_reads_at_most_10_items_by_key(dynamodb_client, htable, count, sizes): + statement = f'SELECT * FROM "{htable}" WHERE pk IN [{_values(count)}]' + assert _sizes(dynamodb_client, statement) == sizes + + +@pytest.mark.parametrize(("count", "sizes"), [(10, [0]), (11, [0, 0]), (25, [0, 0, 0])]) +def test_a_key_with_no_item_takes_its_place_in_a_page(dynamodb_client, htable, count, sizes): + statement = f"SELECT * FROM \"{htable}\" WHERE pk IN [{_values(count, 'zz')}]" + assert _sizes(dynamodb_client, statement) == sizes + + +@pytest.mark.parametrize(("count", "sizes"), [(11, [10, 1]), (25, [10, 10, 5])]) +def test_an_or_of_keys_pages_as_an_in_list(dynamodb_client, htable, count, sizes): + condition = " OR ".join(f"pk = 'v{i:03}'" for i in range(count)) + assert _sizes(dynamodb_client, f'SELECT * FROM "{htable}" WHERE {condition}') == sizes + + +@pytest.mark.parametrize( + ("count", "limit", "sizes"), + [ + (25, 100, [10, 10, 5]), + (25, 15, [10, 10, 5]), + (25, 11, [10, 10, 5]), + (25, 10, [10, 10, 5]), + (25, 7, [7, 7, 7, 4]), + (21, 10, [10, 10, 1]), + ], +) +def test_the_cap_holds_whatever_the_limit(dynamodb_client, htable, count, limit, sizes): + statement = f'SELECT * FROM "{htable}" WHERE pk IN [{_values(count)}]' + assert _sizes(dynamodb_client, statement, Limit=limit) == sizes + + +@pytest.mark.parametrize( + ("limit", "last_keys"), + [ + (None, [None, None]), + (15, [None, None, None]), + (11, [None, None, None]), + (10, [{"pk": {"S": "v009"}}, {"pk": {"S": "v019"}}, None]), + ], + ids=["no-limit", "limit-15", "limit-11", "limit-10"], +) +def test_a_page_has_a_last_key_only_when_it_holds_limit_items( + dynamodb_client, htable, limit, last_keys +): + count = 11 if limit is None else 25 + statement = f'SELECT * FROM "{htable}" WHERE pk IN [{_values(count)}]' + kwargs = {"Limit": limit} if limit else {} + pages = _read(dynamodb_client, statement, **kwargs) + assert [lek for _, lek, _, _ in pages] == last_keys + assert [more for _, _, more, _ in pages] == [True] * (len(last_keys) - 1) + [False] + + +def test_a_page_reports_the_capacity_of_its_reads(dynamodb_client, htable): + statement = f'SELECT * FROM "{htable}" WHERE pk IN [{_values(11)}]' + pages = _read(dynamodb_client, statement, ReturnConsumedCapacity="TOTAL") + assert [(len(keys), capacity) for keys, _, _, capacity in pages] == [(10, 5.0), (1, 0.5)] + + +def test_an_order_by_pages_in_its_order(dynamodb_client, htable): + statement = f'SELECT * FROM "{htable}" WHERE pk IN [{_values(25)}] ORDER BY pk DESC' + pages = _read(dynamodb_client, statement) + assert [keys for keys, _, _, _ in pages] == [ + [f"v{i:03}" for i in range(24, 14, -1)], + [f"v{i:03}" for i in range(14, 4, -1)], + [f"v{i:03}" for i in range(4, -1, -1)], + ] + + +@pytest.mark.parametrize( + ("condition", "sizes"), + [ + (f"pk IN [{_values(11)}]", [10, 1]), + (f"pk IN [{_values(11)}] AND sk = 's'", [10, 1]), + (f"pk IN [{_values(25)}]", [10, 10, 5]), + (f"pk IN [{_values(25)}] AND sk = 's'", [10, 10, 5]), + ], + ids=["partitions-11", "items-11", "partitions-25", "items-25"], +) +def test_a_partition_read_is_one_of_the_10(dynamodb_client, ctable, condition, sizes): + assert _sizes(dynamodb_client, f'SELECT * FROM "{ctable}" WHERE {condition}') == sizes + + +def test_items_and_partitions_share_the_10(dynamodb_client, ctable): + condition = f"(pk = 'v000' AND sk = 's') OR pk IN [{_values(10, start=1)}]" + pages = _read(dynamodb_client, f'SELECT * FROM "{ctable}" WHERE {condition}') + assert [keys for keys, _, _, _ in pages] == [ + [f"v{i:03}/s" for i in range(10)], + ["v010/s"], + ] + + +@pytest.mark.parametrize( + ("limit", "sizes"), [(None, [30, 3]), (5, [5, 5, 5, 5, 5, 5, 3])], ids=["no-limit", "limit-5"] +) +def test_a_page_reads_10_partitions_of_several_items(dynamodb_client, c3table, limit, sizes): + statement = f'SELECT * FROM "{c3table}" WHERE pk IN [{_values(11)}]' + kwargs = {"Limit": limit} if limit else {} + assert _sizes(dynamodb_client, statement, **kwargs) == sizes + + +def test_sort_key_values_of_one_partition_take_a_place_each(dynamodb_client, c3table): + statement = f"SELECT * FROM \"{c3table}\" WHERE pk = 'v000' AND sk IN [{_values(11, 's')}]" + assert _sizes(dynamodb_client, statement) == [0, 0] + + +# ---- a disjunct held back for a refusal -------------------------------------- + + +@pytest.mark.parametrize( + ("held_back", "limit"), + [(f"v005{BIG}", None), (f"v005{BIG}", 15), (f"v000{BIG}", None)], + ids=["after-v005", "after-v005-limit-15", "after-v000"], +) +def test_a_held_back_key_takes_its_place_in_a_page(dynamodb_client, htable, held_back, limit): + statement = f"SELECT * FROM \"{htable}\" WHERE pk IN [{_values(11)}, '{held_back}']" + kwargs = {"Limit": limit} if limit else {} + assert _sorted_keys(dynamodb_client, statement, **kwargs) == [ + [f"v{i:03}" for i in range(9)], + ["v009", "v010"], + HASH_SIZE, + ] + + +def test_a_held_back_key_read_last_is_refused_after_the_last_page(dynamodb_client, htable): + statement = f"SELECT * FROM \"{htable}\" WHERE pk IN [{_values(49)}, '{BIG}']" + assert _sizes(dynamodb_client, statement) == [10, 10, 10, 0, 0, HASH_SIZE] + + +def test_a_held_back_filter_takes_its_place_in_a_page(dynamodb_client, ctable): + others = ", ".join(f"'v{i:03}'" for i in range(12) if i != 3) + condition = f"(pk = 'v003' AND sk = a) OR pk IN [{others}]" + pages = _sorted_keys(dynamodb_client, f'SELECT * FROM "{ctable}" WHERE {condition}') + assert pages == [ + [f"v{i:03}/s" for i in range(10) if i != 3], + ["v010/s", "v011/s"], + FILTER_KEY, + ] + + +# ---- an index read by several disjuncts -------------------------------------- + + +@pytest.mark.parametrize(("count", "sizes"), [(11, [10, 1]), (25, [10, 10, 5])]) +def test_an_index_page_reads_at_most_10_partitions(dynamodb_client, gtable, count, sizes): + statement = f'SELECT * FROM "{gtable}"."gix" WHERE g IN [{_values(count)}]' + assert _sizes(dynamodb_client, statement) == sizes + + +@pytest.mark.parametrize( + ("condition", "keys", "capacity"), + [ + ("g = 'a' AND gs IN ['w000', 'qqq']", ["w000"], 1.0), + ("g = 'a' AND gs IN ['w000', 'w001', 'w002']", ["w000", "w001", "w002"], 1.5), + ("(g = 'a' AND gs = 'w000') OR (g = 'a' AND gs = 'w001')", ["w000", "w001"], 1.0), + ("g = 'a' AND (gs = 'w000' OR gs > 'w012')", ["w000", "w013", "w014"], 1.0), + ("g = 'a' AND gs IN ['w000', 'w001'] AND pk = 'zz'", [], 1.0), + # One key condition is one Query. + ("g = 'a'", [f"w{i:03}" for i in range(15)], 0.5), + ("g = 'a' AND gs > 'w005'", [f"w{i:03}" for i in range(6, 15)], 0.5), + ], + ids=["in-two", "in-three", "or", "equality-or-range", "filtered", "partition", "range"], +) +def test_each_disjunct_of_an_index_partition_is_a_query_of_its_own( + dynamodb_client, g2table, condition, keys, capacity +): + statement = f'SELECT * FROM "{g2table}"."gix" WHERE {condition}' + pages = _read(dynamodb_client, statement, key=_gs, ReturnConsumedCapacity="TOTAL") + assert [(found, more, units) for found, _, more, units in pages] == [(keys, False, capacity)] + + +def test_each_query_of_an_index_partition_is_one_of_the_10(dynamodb_client, g2table): + statement = ( + f"SELECT * FROM \"{g2table}\".\"gix\" WHERE g = 'a' AND gs IN [{_values(11, 'w')}]" + ) + pages = _read(dynamodb_client, statement, key=_gs, ReturnConsumedCapacity="TOTAL") + assert [(found, more, units) for found, _, more, units in pages] == [ + ([f"w{i:03}" for i in range(10)], True, 5.0), + (["w010"], False, 0.5), + ] + + +def test_a_held_back_value_takes_its_place_on_an_index(dynamodb_client, gtable): + statement = ( + f"SELECT pk FROM \"{gtable}\".\"gix\" WHERE g IN [{_values(11)}, 'v005{BIG}']" + ) + pages = _read(dynamodb_client, statement, key=lambda item: item["pk"]["S"]) + assert [page if isinstance(page, str) else sorted(page[0]) for page in pages] == [ + [f"p{i:03}" for i in range(9)], + ["p009", "p010"], + HASH_SIZE, + ] + + +@pytest.mark.parametrize( + ("condition", "first_pages"), + [ + ("g = 'a' AND gs IN ['x', 'z']", [["x"], ["z"]]), + ("(g = 'a' AND gs = 'x') OR (g = 'a' AND gs > 'y')", [["x"], ["z"]]), + ("g = 'a' AND gs IN ['x', 'z'] ORDER BY gs DESC", [["z"], ["x"]]), + ("g = 'a' AND gs IN ['x', 'z'] AND pk = 'p3'", [[], ["z"]]), + ], + ids=["in", "or", "descending", "filtered"], +) +def test_queries_of_one_index_partition_take_the_limit_in_turn( + dynamodb_client, xyz, condition, first_pages +): + statement = f'SELECT * FROM "{xyz}"."gix" WHERE {condition}' + pages = _read(dynamodb_client, statement, key=_gs, Limit=1) + # The second page ends in the Query of the last disjunct: whether an + # empty page follows depends on where the partition sits in storage. + assert [found for found, _, _, _ in pages[:2]] == first_pages + assert pages[0][2] is True + assert all(not found for found, _, _, _ in pages[2:]) + + +@pytest.mark.parametrize( + ("condition", "keys"), + [ + ("g = 'a' AND gs IN ['x', 'z']", ["x", "z"]), + ("(g = 'a' AND gs = 'x') OR (g = 'a' AND gs > 'y')", ["x", "z"]), + ("g = 'a' AND gs IN ['x', 'z'] ORDER BY gs DESC", ["z", "x"]), + ("g = 'a' AND gs IN ['x', 'z'] AND pk = 'p3'", ["z"]), + ], + ids=["in", "or", "descending", "filtered"], +) +def test_queries_of_one_index_partition_add_up_their_capacity( + dynamodb_client, xyz, condition, keys +): + statement = f'SELECT * FROM "{xyz}"."gix" WHERE {condition}' + pages = _read(dynamodb_client, statement, key=_gs, ReturnConsumedCapacity="TOTAL") + assert [(found, more, units) for found, _, more, units in pages] == [(keys, False, 1.0)] diff --git a/tests/test_partiql_phase3.py b/tests/test_partiql_phase3.py new file mode 100644 index 000000000..baf7fb604 --- /dev/null +++ b/tests/test_partiql_phase3.py @@ -0,0 +1,201 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""PartiQL phase-3 behaviours, measured on the service and asserted on both +targets: the LIMIT-clause error, the IN-on-sort-key result, a projected batch +read, and a vector-index SELECT refusal. + +Dual-target: EXTENDDB_TEST_ENDPOINT selects a fix server, EXTENDDB_TEST_PROFILE +selects the live-service profile. Every expected value was measured on the +service; see conf-run/partiql/probe_gaps.py. +""" + +from __future__ import annotations + +import os +import uuid + +import boto3 +import pytest +from botocore.config import Config +from botocore.exceptions import ClientError + +from conftest import wait_for_active + + +@pytest.fixture +def dynamodb_client(endpoint_url): + region = os.environ.get("AWS_DEFAULT_REGION", "us-east-1") + profile = os.environ.get("EXTENDDB_TEST_PROFILE") or None + session = boto3.Session(profile_name=profile, region_name=region) + kwargs: dict = {"service_name": "dynamodb"} + if endpoint_url: + kwargs["endpoint_url"] = endpoint_url + if endpoint_url.startswith("https://"): + kwargs["verify"] = False + kwargs["config"] = Config(parameter_validation=True) + return session.client(**kwargs) + + +@pytest.fixture +def table(dynamodb_client): + name = f"extenddb-measure-p3-{uuid.uuid4().hex[:12]}" + dynamodb_client.create_table( + TableName=name, + AttributeDefinitions=[ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "sk", "AttributeType": "S"}, + ], + KeySchema=[ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"}, + ], + BillingMode="PAY_PER_REQUEST", + ) + wait_for_active(dynamodb_client, name) + try: + yield name + finally: + try: + dynamodb_client.delete_table(TableName=name) + except ClientError: + pass + + +class TestPartiqlPhase3: + def test_limit_clause_is_unsupported(self, dynamodb_client, table): + with pytest.raises(ClientError) as ei: + dynamodb_client.execute_statement( + Statement=f"SELECT * FROM \"{table}\" WHERE pk = 'p1' LIMIT 2" + ) + err = ei.value.response["Error"] + assert err["Code"] == "ValidationException" + assert err["Message"].startswith("Unsupported clause: LIMIT at 1:") + assert err["Message"].endswith(":1") + + def test_in_on_sort_key_returns_the_requested_items(self, dynamodb_client, table): + for sk in ("s1", "s2", "s3"): + dynamodb_client.put_item(TableName=table, Item={"pk": {"S": "p1"}, "sk": {"S": sk}}) + r = dynamodb_client.execute_statement( + Statement=f"SELECT sk FROM \"{table}\" WHERE pk = 'p1' AND sk IN ['s1','s3']" + ) + assert sorted(i["sk"]["S"] for i in r["Items"]) == ["s1", "s3"] + + def test_batch_select_projects(self, dynamodb_client, table): + dynamodb_client.put_item( + TableName=table, + Item={"pk": {"S": "p1"}, "sk": {"S": "s1"}, "name": {"S": "Alice"}, "age": {"N": "30"}}, + ) + r = dynamodb_client.batch_execute_statement( + Statements=[{"Statement": f"SELECT name FROM \"{table}\" WHERE pk = 'p1' AND sk = 's1'"}] + ) + assert r["Responses"][0]["Item"] == {"name": {"S": "Alice"}} + + def test_duplicate_leaf_projection_is_rejected(self, dynamodb_client, table): + dynamodb_client.put_item( + TableName=table, + Item={"pk": {"S": "p1"}, "sk": {"S": "s1"}, + "a1": {"M": {"x": {"S": "A1X"}}}, "a2": {"M": {"x": {"S": "A2X"}}}}, + ) + with pytest.raises(ClientError) as ei: + dynamodb_client.execute_statement( + Statement=f"SELECT a1.x, a2.x FROM \"{table}\" WHERE pk = 'p1' AND sk = 's1'" + ) + err = ei.value.response["Error"] + assert err["Code"] == "ValidationException" + assert err["Message"] == "Duplicate identifiers in select clause: x" + + def test_multi_dot_projection_is_rejected(self, dynamodb_client, table): + dynamodb_client.put_item( + TableName=table, + Item={"pk": {"S": "p1"}, "sk": {"S": "s1"}, + "meta": {"M": {"inner": {"M": {"leaf": {"S": "deep"}}}}}}, + ) + with pytest.raises(ClientError) as ei: + dynamodb_client.execute_statement( + Statement=f"SELECT meta.inner.leaf FROM \"{table}\" WHERE pk = 'p1' AND sk = 's1'" + ) + err = ei.value.response["Error"] + assert err["Code"] == "ValidationException" + assert err["Message"] == ( + "Statement wasn't well formed, can't be processed: Invalid path dot component" + ) + + def test_count_star_message(self, dynamodb_client, table): + with pytest.raises(ClientError) as ei: + dynamodb_client.execute_statement( + Statement=f"SELECT COUNT(*) FROM \"{table}\" WHERE pk = 'p1'" + ) + err = ei.value.response["Error"] + assert err["Code"] == "ValidationException" + assert err["Message"] == "Unexpected path component at 1:8:5" + + def test_duplicate_bag_member_message(self, dynamodb_client, table): + with pytest.raises(ClientError) as ei: + dynamodb_client.execute_statement( + Statement=f"INSERT INTO \"{table}\" VALUE {{'pk':'p1','sk':'s1','db':<<'a','a'>>}}" + ) + err = ei.value.response["Error"] + assert err["Code"] == "ValidationException" + assert err["Message"] == ( + "One or more parameter values were invalid: Input collection [a, a] contains duplicates." + ) + + def test_arithmetic_string_literal_operand_message(self, dynamodb_client, table): + dynamodb_client.put_item( + TableName=table, Item={"pk": {"S": "p1"}, "sk": {"S": "s1"}, "a": {"N": "1"}} + ) + with pytest.raises(ClientError) as ei: + dynamodb_client.execute_statement( + Statement=f"UPDATE \"{table}\" SET a = a + 'x' WHERE pk = 'p1' AND sk = 's1'" + ) + err = ei.value.response["Error"] + assert err["Code"] == "ValidationException" + assert err["Message"] == ( + "Incorrect operand type for operator or function; operator or function: +, operand type: S" + ) + + +@pytest.fixture +def vector_table(dynamodb_client): + name = f"extenddb-measure-p3v-{uuid.uuid4().hex[:12]}" + try: + dynamodb_client.create_table( + TableName=name, + BillingMode="PAY_PER_REQUEST", + AttributeDefinitions=[{"AttributeName": "pk", "AttributeType": "S"}], + KeySchema=[{"AttributeName": "pk", "KeyType": "HASH"}], + VectorIndexes=[ + { + "IndexName": "vix", + "VectorAttribute": {"AttributeName": "embedding"}, + "Dimensions": 3, + "DistanceFunction": "COSINE", + "Projection": {"ProjectionType": "ALL"}, + } + ], + ) + except ClientError as exc: + # A backend without vector support rejects the index; the SELECT refusal + # is only reachable where vector indexes exist (the service, PostgreSQL). + msg = exc.response["Error"]["Message"].lower() + if "vector" in msg or exc.response["Error"]["Code"] == "ValidationException": + pytest.skip("vector indexes not supported by this backend") + raise + try: + wait_for_active(dynamodb_client, name) + yield name + finally: + try: + dynamodb_client.delete_table(TableName=name) + except ClientError: + pass + + +class TestPartiqlVectorIndex: + def test_select_from_vector_index_is_refused(self, dynamodb_client, vector_table): + with pytest.raises(ClientError) as ei: + dynamodb_client.execute_statement(Statement=f'SELECT * FROM "{vector_table}"."vix"') + err = ei.value.response["Error"] + assert err["Code"] == "ValidationException" + assert err["Message"] == "Scan operation not supported on this index type" diff --git a/tests/test_partiql_read_capacity.py b/tests/test_partiql_read_capacity.py new file mode 100644 index 000000000..7887b48b0 --- /dev/null +++ b/tests/test_partiql_read_capacity.py @@ -0,0 +1,63 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""Dual-target wire tests for the consumed capacity a PartiQL SELECT reports +when it reads several sort-key values or partitions by key. + +Each sort-key value of an IN is a read of one item, billed whether the item +exists or not; each partition is a Query. + +Dual-target: EXTENDDB_TEST_ENDPOINT selects an extenddb server; unset, the +tests run against the live service with the default credential chain. The +assertions are identical on both. Every expected value was measured on the +service (2026-11-03). +""" + +from __future__ import annotations + +import pytest + +from conftest import scoped_table + + +@pytest.fixture(scope="module") +def xyz(dynamodb_client): + """A table keyed on string pk and sk holding a/x, a/y and a/z.""" + with scoped_table( + dynamodb_client, + attribute_definitions=[ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "sk", "AttributeType": "S"}, + ], + key_schema=[ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"}, + ], + ) as name: + for sk in ["x", "y", "z"]: + dynamodb_client.put_item(TableName=name, Item={"pk": {"S": "a"}, "sk": {"S": sk}}) + yield name + + +@pytest.mark.parametrize( + ("condition", "consistent", "units"), + [ + ("pk = 'a' AND sk IN ['x', 'y']", False, 1.0), + ("pk = 'a' AND sk IN ['x', 'y']", True, 2.0), + ("pk = 'a' AND sk IN ['x', 'q']", False, 1.0), + ("pk = 'a' AND sk IN ['x', 'y', 'z']", False, 1.5), + ("pk = 'a'", False, 0.5), + ("pk = 'a' AND sk = 'x'", False, 0.5), + ("(pk = 'a' AND sk = 'x') OR (pk = 'a' AND sk = 'y')", False, 1.0), + ("pk IN ['a', 'b']", False, 1.0), + ], +) +def test_a_read_by_key_reports_the_capacity_of_its_reads( + dynamodb_client, xyz, condition, consistent, units +): + response = dynamodb_client.execute_statement( + Statement=f'SELECT * FROM "{xyz}" WHERE {condition}', + ReturnConsumedCapacity="TOTAL", + ConsistentRead=consistent, + ) + assert response["ConsumedCapacity"]["CapacityUnits"] == units diff --git a/tests/test_partiql_repeated_names.py b/tests/test_partiql_repeated_names.py new file mode 100644 index 000000000..b9d5f3513 --- /dev/null +++ b/tests/test_partiql_repeated_names.py @@ -0,0 +1,140 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""Dual-target wire tests for a map literal that names a key twice. + +The statement is refused with "Duplicate keys in tuple. Key: under + at ::", the path being where the map sits in the +value that holds it ("root", "root[0]", "root.y"). At the top level of an +INSERT VALUE the position is the second occurrence of the name, its length +the quoted name's; in any other map (one nested in a value, a map in a list, +a SET value, a WHERE literal) it is the map's opening `{`, length 1. Names +differing only in case are different names, and a syntax error later in the +statement wins. BatchExecuteStatement gives the refusal as the statement's +ValidationError entry, ExecuteTransaction names the statement. + +Dual-target: EXTENDDB_TEST_ENDPOINT selects an extenddb server; unset, the +tests run against the live service with the default credential chain. The +assertions are identical on both. Every expected value was measured on the +service (2026-11-03). +""" + +from __future__ import annotations + +import pytest +from botocore.exceptions import ClientError + +from conftest import scoped_table + +WF = "Statement wasn't well formed, can't be processed: " + + +@pytest.fixture(scope="module") +def table(dynamodb_client): + """A table keyed on a string pk and a string sk.""" + with scoped_table( + dynamodb_client, + attribute_definitions=[ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "sk", "AttributeType": "S"}, + ], + key_schema=[ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"}, + ], + ) as name: + yield name + + +def _message(client, statement: str) -> str: + with pytest.raises(ClientError) as e: + client.execute_statement(Statement=statement) + err = e.value.response["Error"] + assert err["Code"] == "ValidationException", err + return err["Message"] + + +def _refusal(name: str, path: str, statement: str, at: str, length: int) -> str: + """The refusal of `name` given twice in the map at `path`, at the column + where `at` starts in `statement`.""" + column = statement.index(at) + 1 + return f"Duplicate keys in tuple. Key: {name} under {path} at 1:{column}:{length}" + + +def _second(statement: str, quoted: str) -> int: + """The index of the second occurrence of `quoted` in `statement`.""" + return statement.index(quoted, statement.index(quoted) + 1) + + +@pytest.mark.parametrize( + ("value", "name"), + [ + ("{'pk': 'd1', 'pk': 'd2', 'sk': 's'}", "pk"), + ("{'pk': 'd3', 'sk': 's', 'v': '1', 'v': '2'}", "v"), + ("{'pk': 'd4', 'sk': 's1', 'sk': 's2'}", "sk"), + ], +) +def test_a_name_given_twice_at_the_top_of_a_value_is_refused_at_its_second_occurrence( + dynamodb_client, table, value, name +): + statement = f'INSERT INTO "{table}" VALUE {value}' + quoted = f"'{name}'" + column = _second(statement, quoted) + 1 + assert _message(dynamodb_client, statement) == ( + f"Duplicate keys in tuple. Key: {name} under root at 1:{column}:{len(quoted)}" + ) + + +@pytest.mark.parametrize( + ("template", "path"), + [ + ("INSERT INTO {t} VALUE {{'pk': 'n1', 'sk': 's', 'm': {{'x': 1, 'x': 2}}}}", "root"), + ("INSERT INTO {t} VALUE {{'pk': 'n2', 'sk': 's', 'l': [{{'x': 1, 'x': 2}}]}}", "root[0]"), + ( + "INSERT INTO {t} VALUE {{'pk': 'n4', 'sk': 's', 'm': {{'y': {{'x': 1, 'x': 2}}}}}}", + "root.y", + ), + ("UPDATE {t} SET m = {{'x': 1, 'x': 2}} WHERE pk = 'a' AND sk = 'x'", "root"), + ("SELECT * FROM {t} WHERE pk = 'a' AND m = {{'x': 1, 'x': 2}}", "root"), + ], + ids=["nested-map", "map-in-a-list", "two-levels-down", "set-value", "where-literal"], +) +def test_a_name_given_twice_in_any_other_map_is_refused_at_its_opening_brace( + dynamodb_client, table, template, path +): + statement = template.format(t=f'"{table}"') + assert _message(dynamodb_client, statement) == _refusal("x", path, statement, "{'x'", 1) + + +def test_names_that_differ_in_case_are_different_names(dynamodb_client, table): + statement = f"INSERT INTO \"{table}\" VALUE {{'pk': 'n3', 'sk': 's', 'v': 1, 'V': 2}}" + assert dynamodb_client.execute_statement(Statement=statement)["Items"] == [] + item = dynamodb_client.get_item( + TableName=table, Key={"pk": {"S": "n3"}, "sk": {"S": "s"}}, ConsistentRead=True + )["Item"] + assert item["v"] == {"N": "1"} and item["V"] == {"N": "2"} + + +def test_a_syntax_error_after_the_value_wins(dynamodb_client, table): + statement = f"INSERT INTO \"{table}\" VALUE {{'pk': 'n7', 'pk': 'n8', 'sk': 's'}} WHERE" + assert _message(dynamodb_client, statement) == WF + "Unexpected token after expression" + + +def test_a_batch_and_a_transaction_give_the_refusal_of_their_statement(dynamodb_client, table): + statement = f"INSERT INTO \"{table}\" VALUE {{'pk': 'n5', 'pk': 'n6', 'sk': 's'}}" + refusal = _refusal("pk", "root", statement, "'pk': 'n6'", 4) + responses = dynamodb_client.batch_execute_statement( + Statements=[{"Statement": statement}] + )["Responses"] + assert responses == [{"Error": {"Code": "ValidationError", "Message": refusal}}] + with pytest.raises(ClientError) as e: + dynamodb_client.execute_transaction(TransactStatements=[{"Statement": statement}]) + assert e.value.response["Error"] == { + "Code": "ValidationException", + "Message": f"Validation failed in TransactStatements[0]: {refusal}", + } + for pk in ("n5", "n6"): + found = dynamodb_client.get_item( + TableName=table, Key={"pk": {"S": pk}, "sk": {"S": "s"}}, ConsistentRead=True + ) + assert "Item" not in found, pk diff --git a/tests/test_partiql_requests.py b/tests/test_partiql_requests.py new file mode 100644 index 000000000..e2e15a740 --- /dev/null +++ b/tests/test_partiql_requests.py @@ -0,0 +1,701 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""Dual-target wire tests for the constraints the request model puts on the +fields of a PartiQL request: the length of each statement, the number of +statements, the client request token, NextToken, Limit, Parameters, the two +enum fields, and the members a request must carry. + +The service checks these before it reads a statement, and reports every one +that fails in a single ValidationException: "1 validation error detected: +" for one, "N validation errors detected: ; " for more, in +the order it walks the request, at most ten parts. A statement list past its +limit is echoed with every field of every statement, or in some answers with +each statement's class name and a hash. + +Runs against real DynamoDB (EXTENDDB_TEST_ENDPOINT unset, AWS credentials) and +against an extenddb server (EXTENDDB_TEST_ENDPOINT set) with identical +assertions. Every expected message was measured on the service (2026-11-02). +The client skips its own parameter checks, which would refuse some of these +requests before they are sent. +""" + +from __future__ import annotations + +import re + +import pytest +from botocore.exceptions import ClientError + +from conftest import scoped_table + +MAX_LENGTH = 8192 +TOO_LONG = f"Member must have length less than or equal to {MAX_LENGTH}" +TOO_SHORT = "Member must have length greater than or equal to 1" +AT_LEAST_ONE = "Member must have value greater than or equal to 1" +ON_FAILURE_SET = "Member must satisfy enum value set: [ALL_OLD, NONE]" +CAPACITY_SET = "Member must satisfy enum value set: [INDEXES, TOTAL, NONE]" +TOKEN_TOO_LONG = "Member must have length less than or equal to 36" + +# The members of a parameter's value class, as the service echoes them. +VALUE_FIELDS = ( + "s", "n", "b", "iNT", "dECIMAL", "dOUBLE", "sS", "nS", "bS", "iNTSET", + "dECIMALSET", "dOUBLESET", "m", "l", "nULL", "bOOL", +) +BUFFER = "java.nio.HeapByteBuffer[pos=0 lim=1 cap=1]" + +# The service echoes a statement list past its limit in one of two forms. +# Most answers write every field of every request object; the others write +# each object as its class name and a hash, for example +# "com.amazonaws.dynamodb.v20120810.BatchStatementRequest@8e2ac1ba" (8 of 40 +# batch answers and 14 of 40 transaction answers, measured 2026-11-02). +# ExtendDB always writes the fields; a test accepts either form. +HASHED_CLASS = { + "BatchStatementRequest": "com.amazonaws.dynamodb.v20120810.BatchStatementRequest", + "ParameterizedStatement": "com.amazonaws.dynamodb.v20120810.ParameterizedStatement", +} + + +@pytest.fixture(scope="module") +def table(dynamodb_client): + """A table keyed on a string pk, holding the item pk = 'a'.""" + with scoped_table(dynamodb_client) as name: + dynamodb_client.put_item(TableName=name, Item={"pk": {"S": "a"}, "a": {"N": "1"}}) + yield name + + +@pytest.fixture(scope="module") +def client(dynamodb_client_no_validation): + return dynamodb_client_no_validation + + +def _refusal(call, **kwargs) -> str: + with pytest.raises(ClientError) as e: + call(**kwargs) + err = e.value.response["Error"] + assert err["Code"] == "ValidationException", err + return err["Message"] + + +def _part(value: str, path: str, constraint: str) -> str: + return f"Value '{value}' at '{path}' failed to satisfy constraint: {constraint}" + + +def _null(path: str) -> str: + return f"Value null at '{path}' failed to satisfy constraint: Member must not be null" + + +class _Expected: + """A refusal that holds an echoed statement list: equal to the text with + every field written out, or to the same text with the list in its class + and hash form.""" + + def __init__(self, exact: str, pattern: str): + self.exact, self.pattern = exact, pattern + + def __eq__(self, other: object) -> bool: + if not isinstance(other, str): + return NotImplemented + return other == self.exact or re.fullmatch(self.pattern, other) is not None + + def __repr__(self) -> str: + return repr(self.exact) + + +def _list_part(kind: str, members: list[str], path: str, constraint: str) -> tuple[str, str]: + """The part for a statement list past its limit, as its exact text and as + a pattern for the class and hash form.""" + exact = _part(f"[{', '.join(members)}]", path, constraint) + one = re.escape(HASHED_CLASS[kind]) + "@[0-9a-f]{1,8}" + hashed = r"\[" + ", ".join([one] * len(members)) + r"\]" + tail = f"' at '{path}' failed to satisfy constraint: {constraint}" + return exact, re.escape("Value '") + hashed + re.escape(tail) + + +def _errors(*parts): + head = ( + "1 validation error detected: " + if len(parts) == 1 + else f"{len(parts)} validation errors detected: " + ) + exact = head + "; ".join(p if isinstance(p, str) else p[0] for p in parts) + if all(isinstance(p, str) for p in parts): + return exact + pattern = re.escape(head) + "; ".join( + re.escape(p) if isinstance(p, str) else p[1] for p in parts + ) + return _Expected(exact, pattern) + + +def _select(table: str) -> str: + return f"SELECT * FROM \"{table}\" WHERE pk = 'a'" + + +def _long(table: str) -> str: + """A statement one character past the limit.""" + head = _select(table) + return head + " " * (MAX_LENGTH + 1 - len(head)) + + +def _value(**fields: str) -> str: + """A parameter as echoed, the given members set and the rest null.""" + shown = ", ".join(f"{name}={fields.get(name, 'null')}" for name in VALUE_FIELDS) + return f"AttributeValue({shown})" + + +def _batch_member(statement: str, parameters="null", consistent_read="null", on_failure="null"): + return ( + f"BatchStatementRequest(statement={statement}, parameters={parameters}, " + f"consistentRead={consistent_read}, returnValuesOnConditionCheckFailure={on_failure})" + ) + + +def _transaction_member(statement: str, parameters="null", on_failure="null"): + return ( + f"ParameterizedStatement(statement={statement}, parameters={parameters}, " + f"returnValuesOnConditionCheckFailure={on_failure})" + ) + + +# --- one statement too long or too short ------------------------------------ + + +def test_a_long_statement_in_a_batch_fails_the_request(client, table): + long = _long(table) + message = _refusal(client.batch_execute_statement, Statements=[{"Statement": long}]) + assert message == _errors(_part(long, "statements.1.member.statement", TOO_LONG)) + + +def test_a_long_statement_in_a_transaction_fails_the_request(client, table): + long = _long(table) + message = _refusal(client.execute_transaction, TransactStatements=[{"Statement": long}]) + assert message == _errors(_part(long, "transactStatements.1.member.statement", TOO_LONG)) + + +def test_an_empty_statement_in_a_batch_fails_the_request(client, table): + statements = [{"Statement": ""}, {"Statement": _select(table)}] + message = _refusal(client.batch_execute_statement, Statements=statements) + assert message == _errors(_part("", "statements.1.member.statement", TOO_SHORT)) + + +def test_an_empty_statement_in_a_transaction_fails_the_request(client, table): + statements = [{"Statement": _select(table)}, {"Statement": ""}] + message = _refusal(client.execute_transaction, TransactStatements=statements) + assert message == _errors(_part("", "transactStatements.2.member.statement", TOO_SHORT)) + + +def test_an_empty_statement_is_too_short(client): + message = _refusal(client.execute_statement, Statement="") + assert message == _errors(_part("", "statement", TOO_SHORT)) + + +def test_the_length_counts_code_points(client, table): + head = f"SELECT * FROM \"{table}\" WHERE pk = 'a' AND a = '" + emoji = "\U0001F600" + at_limit = head + emoji * (MAX_LENGTH - len(head) - 1) + "'" + assert client.execute_statement(Statement=at_limit)["Items"] == [] + over = head + emoji * (MAX_LENGTH - len(head)) + "'" + assert _refusal(client.execute_statement, Statement=over) == _errors( + _part(over, "statement", TOO_LONG) + ) + + +# --- several failures in one request ----------------------------------------- + + +def test_each_long_statement_in_a_batch_is_reported(client, table): + long = _long(table) + statements = [{"Statement": long}, {"Statement": long}] + assert _refusal(client.batch_execute_statement, Statements=statements) == _errors( + _part(long, "statements.1.member.statement", TOO_LONG), + _part(long, "statements.2.member.statement", TOO_LONG), + ) + + +def test_an_empty_and_a_long_statement_in_a_batch(client, table): + long = _long(table) + statements = [{"Statement": ""}, {"Statement": long}] + assert _refusal(client.batch_execute_statement, Statements=statements) == _errors( + _part("", "statements.1.member.statement", TOO_SHORT), + _part(long, "statements.2.member.statement", TOO_LONG), + ) + + +def test_a_batch_statement_before_its_parameters(client, table): + long = _long(table) + statements = [{"Statement": long, "Parameters": []}] + assert _refusal(client.batch_execute_statement, Statements=statements) == _errors( + _part(long, "statements.1.member.statement", TOO_LONG), + _part("[]", "statements.1.member.parameters", TOO_SHORT), + ) + + +def test_a_transaction_statement_with_empty_parameters(client, table): + statements = [{"Statement": _select(table), "Parameters": []}] + assert _refusal(client.execute_transaction, TransactStatements=statements) == _errors( + _part("[]", "transactStatements.1.member.parameters", TOO_SHORT), + ) + + +def test_each_long_statement_in_a_transaction_is_reported(client, table): + long = _long(table) + statements = [{"Statement": long}, {"Statement": long}] + assert _refusal(client.execute_transaction, TransactStatements=statements) == _errors( + _part(long, "transactStatements.1.member.statement", TOO_LONG), + _part(long, "transactStatements.2.member.statement", TOO_LONG), + ) + + +def test_a_long_token_comes_before_the_statements(client, table): + long, token = _long(table), "t" * 37 + message = _refusal( + client.execute_transaction, TransactStatements=[{"Statement": long}], ClientRequestToken=token + ) + assert message == _errors( + _part(token, "clientRequestToken", "Member must have length less than or equal to 36"), + _part(long, "transactStatements.1.member.statement", TOO_LONG), + ) + + +def test_execute_statement_reports_the_statement_before_its_parameters(client, table): + long = _long(table) + assert _refusal(client.execute_statement, Statement=long, Parameters=[]) == _errors( + _part(long, "statement", TOO_LONG), + _part("[]", "parameters", TOO_SHORT), + ) + + +def test_execute_statement_reports_the_limit_before_the_statement(client, table): + long = _long(table) + assert _refusal(client.execute_statement, Statement=long, Limit=0) == _errors( + _part("0", "limit", "Member must have value greater than or equal to 1"), + _part(long, "statement", TOO_LONG), + ) + + +def test_empty_parameters_are_refused(client, table): + assert _refusal(client.execute_statement, Statement=_select(table), Parameters=[]) == ( + _errors(_part("[]", "parameters", TOO_SHORT)) + ) + + +# --- a statement list past its limit ----------------------------------------- + + +def test_a_batch_past_25_statements_is_echoed_before_a_long_statement(client, table): + plain, long = _select(table), _long(table) + statements = [{"Statement": plain}] * 25 + [{"Statement": long}] + members = [_batch_member(plain)] * 25 + [_batch_member(long)] + assert _refusal(client.batch_execute_statement, Statements=statements) == _errors( + _list_part("BatchStatementRequest", members, "statements", "Member must have length less than or equal to 25"), + _part(long, "statements.26.member.statement", TOO_LONG), + ) + + +def test_a_transaction_past_100_statements_is_echoed_before_a_long_statement(client, table): + plain, long = _select(table), _long(table) + statements = [{"Statement": plain}] * 100 + [{"Statement": long}] + members = [_transaction_member(plain)] * 100 + [_transaction_member(long)] + assert _refusal(client.execute_transaction, TransactStatements=statements) == _errors( + _list_part("ParameterizedStatement", members, "transactStatements", "Member must have length less than or equal to 100"), + _part(long, "transactStatements.101.member.statement", TOO_LONG), + ) + + +def test_a_long_token_comes_before_the_statement_count(client, table): + plain, token = _select(table), "t" * 37 + message = _refusal( + client.execute_transaction, + TransactStatements=[{"Statement": plain}] * 101, + ClientRequestToken=token, + ) + members = [_transaction_member(plain)] * 101 + assert message == _errors( + _part(token, "clientRequestToken", "Member must have length less than or equal to 36"), + _list_part("ParameterizedStatement", members, "transactStatements", "Member must have length less than or equal to 100"), + ) + + +PARAMETERS = [ + {"S": "a"}, {"N": "1"}, {"B": b"x"}, {"BOOL": True}, {"NULL": True}, + {"L": [{"S": "x"}]}, {"M": {"k": {"S": "v"}}}, {"SS": ["a", "b"]}, + {"NS": ["1"]}, {"BS": [b"x"]}, +] +ECHOED_PARAMETERS = [ + _value(s="a"), _value(n="1"), _value(b=BUFFER), _value(bOOL="true"), _value(nULL="true"), + _value(l=f"[{_value(s='x')}]"), _value(m=f"{{k={_value(s='v')}}}"), _value(sS="[a, b]"), + _value(nS="[1]"), _value(bS=f"[{BUFFER}]"), +] + + +def test_a_batch_past_its_limit_echoes_every_field_of_a_statement(client, table): + plain = _select(table) + rich = f'SELECT * FROM "{table}" WHERE pk = ?' + first = { + "Statement": rich, + "Parameters": PARAMETERS, + "ConsistentRead": True, + "ReturnValuesOnConditionCheckFailure": "ALL_OLD", + } + message = _refusal( + client.batch_execute_statement, Statements=[first] + [{"Statement": plain}] * 25 + ) + members = ( + [_batch_member(rich, f"[{', '.join(ECHOED_PARAMETERS)}]", "true", "ALL_OLD")] + + [_batch_member(plain)] * 25 + ) + assert message == _errors( + _list_part("BatchStatementRequest", members, "statements", "Member must have length less than or equal to 25") + ) + + +def test_a_batch_past_its_limit_echoes_consistent_read_false(client, table): + plain = _select(table) + statements = [{"Statement": plain, "ConsistentRead": False}] * 26 + members = [_batch_member(plain, consistent_read="false")] * 26 + assert _refusal(client.batch_execute_statement, Statements=statements) == _errors( + _list_part("BatchStatementRequest", members, "statements", "Member must have length less than or equal to 25") + ) + + +def test_a_transaction_past_its_limit_echoes_every_field_of_a_statement(client, table): + plain = _select(table) + rich = f'SELECT * FROM "{table}" WHERE pk = ?' + first = { + "Statement": rich, + "Parameters": PARAMETERS[:2], + "ReturnValuesOnConditionCheckFailure": "NONE", + } + message = _refusal( + client.execute_transaction, TransactStatements=[first] + [{"Statement": plain}] * 100 + ) + members = ( + [_transaction_member(rich, f"[{', '.join(ECHOED_PARAMETERS[:2])}]", "NONE")] + + [_transaction_member(plain)] * 100 + ) + assert message == _errors( + _list_part("ParameterizedStatement", members, "transactStatements", "Member must have length less than or equal to 100") + ) + + +# --- every field of a request in one refusal --------------------------------- + + +@pytest.mark.parametrize( + "token, constraint", + [("", TOO_SHORT), ("n" * 32769, "Member must have length less than or equal to 32768")], + ids=["empty token", "long token"], +) +def test_execute_statement_reports_every_field_in_order(client, table, token, constraint): + long = _long(table) + message = _refusal( + client.execute_statement, + Statement=long, + Parameters=[], + Limit=0, + NextToken=token, + ReturnConsumedCapacity="X", + ReturnValuesOnConditionCheckFailure="X", + ConsistentRead=True, + ) + assert message == _errors( + _part("X", "returnValuesOnConditionCheckFailure", ON_FAILURE_SET), + _part(token, "nextToken", constraint), + _part("0", "limit", AT_LEAST_ONE), + _part(long, "statement", TOO_LONG), + _part("X", "returnConsumedCapacity", CAPACITY_SET), + _part("[]", "parameters", TOO_SHORT), + ) + + +def test_a_batch_reports_its_capacity_then_its_count_then_each_member(client, table): + plain, long = _select(table), _long(table) + first = { + "Statement": long, + "Parameters": [], + "ConsistentRead": True, + "ReturnValuesOnConditionCheckFailure": "X", + } + message = _refusal( + client.batch_execute_statement, + Statements=[first] + [{"Statement": plain}] * 25, + ReturnConsumedCapacity="X", + ) + members = [_batch_member(long, "[]", "true", "X")] + [_batch_member(plain)] * 25 + assert message == _errors( + _part("X", "returnConsumedCapacity", CAPACITY_SET), + _list_part("BatchStatementRequest", members, "statements", "Member must have length less than or equal to 25"), + _part("X", "statements.1.member.returnValuesOnConditionCheckFailure", ON_FAILURE_SET), + _part(long, "statements.1.member.statement", TOO_LONG), + _part("[]", "statements.1.member.parameters", TOO_SHORT), + ) + + +def test_a_transaction_reports_its_capacity_after_its_members(client, table): + plain, long, token = _select(table), _long(table), "t" * 37 + first = {"Statement": long, "Parameters": [], "ReturnValuesOnConditionCheckFailure": "X"} + message = _refusal( + client.execute_transaction, + TransactStatements=[first] + [{"Statement": plain}] * 100, + ClientRequestToken=token, + ReturnConsumedCapacity="X", + ) + members = [_transaction_member(long, "[]", "X")] + [_transaction_member(plain)] * 100 + assert message == _errors( + _part(token, "clientRequestToken", TOKEN_TOO_LONG), + _list_part("ParameterizedStatement", members, "transactStatements", "Member must have length less than or equal to 100"), + _part("X", "transactStatements.1.member.returnValuesOnConditionCheckFailure", ON_FAILURE_SET), + _part(long, "transactStatements.1.member.statement", TOO_LONG), + _part("[]", "transactStatements.1.member.parameters", TOO_SHORT), + _part("X", "returnConsumedCapacity", CAPACITY_SET), + ) + + +def test_a_refusal_lists_the_first_ten_failed_constraints(client): + statements = [{"Statement": "", "Parameters": []}] * 25 + message = _refusal(client.batch_execute_statement, Statements=statements) + parts = [] + for i in range(1, 6): + parts.append(_part("", f"statements.{i}.member.statement", TOO_SHORT)) + parts.append(_part("[]", f"statements.{i}.member.parameters", TOO_SHORT)) + assert message == _errors(*parts) + assert message.startswith("10 validation errors detected: ") + + +# --- a member the request does not carry ------------------------------------ + + +def test_execute_statement_without_a_statement(client): + assert _refusal(client.execute_statement) == _errors(_null("statement")) + + +def test_a_missing_statement_joins_the_other_parts(client): + assert _refusal(client.execute_statement, Limit=0, Parameters=[]) == _errors( + _part("0", "limit", AT_LEAST_ONE), + _null("statement"), + _part("[]", "parameters", TOO_SHORT), + ) + + +def test_a_batch_without_statements(client): + assert _refusal(client.batch_execute_statement) == _errors(_null("statements")) + assert _refusal(client.batch_execute_statement, ReturnConsumedCapacity="X") == _errors( + _part("X", "returnConsumedCapacity", CAPACITY_SET), + _null("statements"), + ) + + +def test_a_batch_member_without_a_statement(client): + message = _refusal(client.batch_execute_statement, Statements=[{"Parameters": [{"S": "a"}]}]) + assert message == _errors(_null("statements.1.member.statement")) + message = _refusal(client.batch_execute_statement, Statements=[{"Parameters": []}]) + assert message == _errors( + _null("statements.1.member.statement"), + _part("[]", "statements.1.member.parameters", TOO_SHORT), + ) + + +def test_a_transaction_without_statements(client): + assert _refusal(client.execute_transaction) == _errors(_null("transactStatements")) + token = "t" * 37 + assert _refusal(client.execute_transaction, ClientRequestToken=token) == _errors( + _part(token, "clientRequestToken", TOKEN_TOO_LONG), + _null("transactStatements"), + ) + + +def test_a_transaction_member_without_a_statement(client): + message = _refusal( + client.execute_transaction, TransactStatements=[{"Parameters": [{"S": "a"}]}] + ) + assert message == _errors(_null("transactStatements.1.member.statement")) + + +# --- the enum fields --------------------------------------------------------- + + +def test_an_unknown_return_values_on_failure_is_refused(client, table): + plain = _select(table) + message = _refusal( + client.execute_statement, Statement=plain, ReturnValuesOnConditionCheckFailure="X" + ) + assert message == _errors(_part("X", "returnValuesOnConditionCheckFailure", ON_FAILURE_SET)) + member = {"Statement": plain, "ReturnValuesOnConditionCheckFailure": "X"} + message = _refusal(client.batch_execute_statement, Statements=[member]) + assert message == _errors( + _part("X", "statements.1.member.returnValuesOnConditionCheckFailure", ON_FAILURE_SET) + ) + message = _refusal(client.execute_transaction, TransactStatements=[member]) + assert message == _errors( + _part("X", "transactStatements.1.member.returnValuesOnConditionCheckFailure", ON_FAILURE_SET) + ) + + +def test_an_unknown_consumed_capacity_fails_the_request(client, table): + plain = _select(table) + expected = _errors(_part("X", "returnConsumedCapacity", CAPACITY_SET)) + assert _refusal( + client.execute_statement, Statement=plain, ReturnConsumedCapacity="X" + ) == expected + assert _refusal( + client.batch_execute_statement, Statements=[{"Statement": plain}], ReturnConsumedCapacity="X" + ) == expected + assert _refusal( + client.execute_transaction, + TransactStatements=[{"Statement": plain}], + ReturnConsumedCapacity="X", + ) == expected + + +# --- the two tokens ---------------------------------------------------------- + + +def test_an_empty_next_token_is_too_short(client, table): + message = _refusal(client.execute_statement, Statement=_select(table), NextToken="") + assert message == _errors(_part("", "nextToken", TOO_SHORT)) + + +def test_a_next_token_past_32768_characters_is_too_long(client, table): + token = "n" * 32769 + message = _refusal(client.execute_statement, Statement=_select(table), NextToken=token) + assert message == _errors( + _part(token, "nextToken", "Member must have length less than or equal to 32768") + ) + + +@pytest.mark.parametrize( + "token", + ["n" * 32768, "x", "a b!", "bm90LWEtdG9rZW4="], + ids=["32768 characters", "one character", "a space and a bang", "base64 text"], +) +def test_a_next_token_that_does_not_decode_is_invalid(client, table, token): + message = _refusal(client.execute_statement, Statement=_select(table), NextToken=token) + assert message == "Invalid NextToken" + + +def test_an_empty_client_request_token_is_too_short(client, table): + message = _refusal( + client.execute_transaction, + TransactStatements=[{"Statement": _select(table)}], + ClientRequestToken="", + ) + assert message == _errors(_part("", "clientRequestToken", TOO_SHORT)) + + +# --- the order of a map's members in the echo --------------------------------- +# +# The service prints a map's members in the order of the buckets a Java +# HashMap sorts them into. These maps have no two keys in one bucket, so the +# order does not depend on the order the request lists them in. + + +def _map(*names: str) -> dict: + return {"M": {name: {"S": name} for name in names}} + + +def _echoed_map(*names: str) -> str: + members = ", ".join(f"{name}={_value(s=name)}" for name in names) + return f"[{_value(m='{' + members + '}')}]" + + +@pytest.mark.parametrize( + "sent, printed", + [(("c", "a", "p"), ("p", "a", "c")), + (("zeta", "alpha", "mid", "beta"), ("zeta", "mid", "beta", "alpha")), + (("m0", "m1", "m2"), ("m1", "m2", "m0")), + (("k0", "k1", "k2", "k3", "k4", "k5"), ("k3", "k4", "k5", "k0", "k1", "k2"))], + ids=["three keys", "four keys", "three keys in four buckets", "six keys in eight buckets"], +) +def test_a_batch_past_its_limit_echoes_a_map_in_the_service_order(client, table, sent, printed): + plain = _select(table) + first = {"Statement": plain, "Parameters": [_map(*sent)]} + message = _refusal( + client.batch_execute_statement, Statements=[first] + [{"Statement": plain}] * 25 + ) + members = [_batch_member(plain, _echoed_map(*printed))] + [_batch_member(plain)] * 25 + assert message == _errors( + _list_part("BatchStatementRequest", members, "statements", "Member must have length less than or equal to 25") + ) + + +def test_a_transaction_past_its_limit_echoes_a_map_in_the_service_order(client, table): + plain = _select(table) + first = {"Statement": plain, "Parameters": [_map("c", "a", "p")]} + message = _refusal( + client.execute_transaction, TransactStatements=[first] + [{"Statement": plain}] * 100 + ) + members = ( + [_transaction_member(plain, _echoed_map("p", "a", "c"))] + [_transaction_member(plain)] * 100 + ) + assert message == _errors( + _list_part("ParameterizedStatement", members, "transactStatements", "Member must have length less than or equal to 100") + ) + + +# ---- a statement of only whitespace ------------------------------------------- +# +# Measured 2026-11-05. A statement that passes the length constraint and holds +# only whitespace is refused as empty by BatchExecuteStatement, for that +# member, and by ExecuteTransaction, for the whole request without the +# member's position, ahead of the member's parameters and of the members +# after it, and after the members before it. ExecuteStatement reads it as a +# statement with nothing in it. A comment is not whitespace here. + +EMPTY_IN_A_BATCH = "The statement was empty." +EMPTY_IN_A_TRANSACTION = "ExecuteTransaction can not have empty statement." +WF = "Statement wasn't well formed, can't be processed: " + +BLANK = [" ", "\t\n ", "\r", "\x0b", "\x0c", "\x1f", "\u2028", "\u3000"] + + +@pytest.mark.parametrize("blank", BLANK, ids=[repr(b) for b in BLANK]) +def test_a_blank_statement_in_a_batch_fails_that_statement(client, table, blank): + responses = client.batch_execute_statement( + Statements=[{"Statement": _select(table)}, {"Statement": blank}] + )["Responses"] + assert responses[1] == {"Error": {"Code": "ValidationError", "Message": EMPTY_IN_A_BATCH}} + assert responses[0]["TableName"] == table + + +@pytest.mark.parametrize("blank", BLANK, ids=[repr(b) for b in BLANK]) +def test_a_blank_statement_in_a_transaction_fails_the_request(client, table, blank): + message = _refusal( + client.execute_transaction, + TransactStatements=[{"Statement": _select(table)}, {"Statement": blank}], + ) + assert message == EMPTY_IN_A_TRANSACTION + + +@pytest.mark.parametrize( + ("statement", "reason"), + [("-- a note", "Unexpected term"), ("/* a note */", "Unexpected term"), ("\x01", "")], + ids=["line-comment", "block-comment", "control-character"], +) +def test_a_statement_that_is_not_blank(client, table, statement, reason): + responses = client.batch_execute_statement(Statements=[{"Statement": statement}])["Responses"] + assert responses == [{"Error": {"Code": "ValidationError", "Message": WF + reason}}] + message = _refusal(client.execute_transaction, TransactStatements=[{"Statement": statement}]) + assert message == f"Validation failed in TransactStatements[0]: {WF}{reason}" + + +def test_a_blank_statement_comes_before_its_parameters(client, table): + member = {"Statement": " ", "Parameters": [{"N": "x"}]} + responses = client.batch_execute_statement(Statements=[member])["Responses"] + assert responses == [{"Error": {"Code": "ValidationError", "Message": EMPTY_IN_A_BATCH}}] + message = _refusal(client.execute_transaction, TransactStatements=[member]) + assert message == EMPTY_IN_A_TRANSACTION + + +def test_a_blank_transaction_member_comes_after_the_members_before_it(client, table): + broken = {"Statement": f'SELECT * FROM "{table}" WHERE'} + message = _refusal(client.execute_transaction, TransactStatements=[broken, {"Statement": " "}]) + assert message == f"Validation failed in TransactStatements[0]: {WF}Unexpected term" + missing = {"Statement": f"SELECT * FROM \"{table}-absent\" WHERE pk = 'a'"} + for statements in ([{"Statement": " "}, missing], [missing, {"Statement": " "}]): + message = _refusal(client.execute_transaction, TransactStatements=statements) + assert message == EMPTY_IN_A_TRANSACTION + + +def test_execute_statement_reads_a_blank_statement_as_an_empty_one(client): + for blank in [" ", "\t\n"]: + assert _refusal(client.execute_statement, Statement=blank) == WF + "Unexpected term" diff --git a/tests/test_partiql_select.py b/tests/test_partiql_select.py new file mode 100644 index 000000000..f8a901a82 --- /dev/null +++ b/tests/test_partiql_select.py @@ -0,0 +1,362 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""Dual-target wire tests for PartiQL SELECT (ExecuteStatement). + +Runs against real DynamoDB (EXTENDDB_TEST_ENDPOINT unset, AWS creds) and against +an extenddb server (EXTENDDB_TEST_ENDPOINT set). Identical assertions on both, +so the service is the oracle for every case. Every expected value and error +string was measured on the live service; see conf-run/partiql/char-select.md. + +The table is created with a unique ``extenddb-measure-2a-`` prefix and deleted in +a finally so nothing is left in the live account. +""" + +from __future__ import annotations + +import time +import uuid + +import boto3 +import pytest +from botocore.exceptions import ClientError + + +# ---- local polling helpers (independent of conftest internals) ------------- + +def _is_service() -> bool: + import os + return not os.environ.get("EXTENDDB_TEST_ENDPOINT", "").strip() + + +def _interval() -> float: + return 0.3 if _is_service() else 0.03 + + +def _wait_active(client, name: str, timeout: float = 180.0) -> None: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + d = client.describe_table(TableName=name)["Table"] + if d["TableStatus"] == "ACTIVE": + gsis = d.get("GlobalSecondaryIndexes", []) + if all(g.get("IndexStatus") == "ACTIVE" for g in gsis): + return + time.sleep(_interval()) + raise TimeoutError(f"{name} not ACTIVE within {timeout}s") + + +def _wait_deleted(client, name: str, timeout: float = 180.0) -> None: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + client.describe_table(TableName=name) + except client.exceptions.ResourceNotFoundException: + return + time.sleep(_interval()) + + +def _items(resp) -> list: + return resp.get("Items", []) + + +def _poll_statement(client, timeout: float = 20.0, want: int = 0, **kwargs) -> list: + """Run an ExecuteStatement, retrying until it yields >= ``want`` items. + + A GSI is eventually consistent (on the service and on extenddb), so a query + right after the seed can be short until propagation completes. + """ + deadline = time.monotonic() + timeout + items = _items(client.execute_statement(**kwargs)) + while len(items) < want and time.monotonic() < deadline: + time.sleep(_interval()) + items = _items(client.execute_statement(**kwargs)) + return items + + +# ---- seeded table fixture -------------------------------------------------- + +@pytest.fixture(scope="module") +def table(dynamodb_client): + name = f"extenddb-measure-2a-{uuid.uuid4().hex[:12]}" + dynamodb_client.create_table( + TableName=name, + AttributeDefinitions=[ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "sk", "AttributeType": "S"}, + {"AttributeName": "gsi_pk", "AttributeType": "S"}, + {"AttributeName": "gsi_sk", "AttributeType": "N"}, + ], + KeySchema=[ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"}, + ], + GlobalSecondaryIndexes=[ + { + "IndexName": "gsi1", + "KeySchema": [ + {"AttributeName": "gsi_pk", "KeyType": "HASH"}, + {"AttributeName": "gsi_sk", "KeyType": "RANGE"}, + ], + "Projection": {"ProjectionType": "ALL"}, + } + ], + BillingMode="PAY_PER_REQUEST", + ) + try: + _wait_active(dynamodb_client, name) + items = [ + { + "pk": {"S": "user1"}, "sk": {"S": "profile"}, + "name": {"S": "Alice"}, "age": {"N": "30"}, + "addr": {"M": {"city": {"S": "NYC"}, "zip": {"S": "10001"}}}, + "tags": {"L": [{"S": "a"}, {"S": "b"}]}, + "Comment": {"S": "hello"}, + "gsi_pk": {"S": "g1"}, "gsi_sk": {"N": "1"}, + }, + { + "pk": {"S": "user1"}, "sk": {"S": "post#1"}, + "name": {"S": "Post One"}, "age": {"N": "5"}, + "gsi_pk": {"S": "g1"}, "gsi_sk": {"N": "2"}, + }, + { + "pk": {"S": "user1"}, "sk": {"S": "post#2"}, + "name": {"S": "Post Two"}, "age": {"N": "25"}, + "gsi_pk": {"S": "g1"}, "gsi_sk": {"N": "3"}, + }, + { + "pk": {"S": "user1"}, "sk": {"S": "post#3"}, + "name": {"S": "Post Three"}, "age": {"N": "50"}, + }, + {"pk": {"S": "user2"}, "sk": {"S": "profile"}, "name": {"S": "Bob"}, "age": {"N": "40"}}, + {"pk": {"S": "user3"}, "sk": {"S": "profile"}, "name": {"S": "Carol"}, "age": {"N": "10"}}, + ] + for it in items: + dynamodb_client.put_item(TableName=name, Item=it) + yield name + finally: + try: + dynamodb_client.delete_table(TableName=name) + _wait_deleted(dynamodb_client, name) + except ClientError: + pass + + +def q(client, table, where="", proj="*", **kwargs): + stmt = f'SELECT {proj} FROM "{table}"' + if where: + stmt += f" WHERE {where}" + return client.execute_statement(Statement=stmt, **kwargs) + + +def sks(items) -> list: + return [i["sk"]["S"] for i in items] + + +# ---- planning: point read / Query / Scan ----------------------------------- + +def test_full_key_point_read(dynamodb_client, table): + items = _items(q(dynamodb_client, table, "pk='user1' AND sk='profile'")) + assert len(items) == 1 + assert items[0]["sk"]["S"] == "profile" + + +def test_point_read_miss_is_empty(dynamodb_client, table): + assert _items(q(dynamodb_client, table, "pk='nope' AND sk='nope'")) == [] + + +def test_partition_only_query_ascending(dynamodb_client, table): + assert sks(_items(q(dynamodb_client, table, "pk='user1'"))) == [ + "post#1", "post#2", "post#3", "profile", + ] + + +def test_sort_range_query(dynamodb_client, table): + assert sks(_items(q(dynamodb_client, table, "pk='user1' AND sk > 'post#0'"))) == [ + "post#1", "post#2", "post#3", "profile", + ] + + +def test_begins_with_sort_key(dynamodb_client, table): + assert sks(_items(q(dynamodb_client, table, "pk='user1' AND begins_with(\"sk\", 'post#')"))) == [ + "post#1", "post#2", "post#3", + ] + + +def test_between_sort_key(dynamodb_client, table): + assert sks(_items(q(dynamodb_client, table, "pk='user1' AND sk BETWEEN 'post#1' AND 'post#3'"))) == [ + "post#1", "post#2", "post#3", + ] + + +def test_no_where_scans_all(dynamodb_client, table): + assert len(_items(q(dynamodb_client, table))) == 6 + + +def test_sort_only_is_scan_filter(dynamodb_client, table): + pks = sorted(i["pk"]["S"] for i in _items(q(dynamodb_client, table, "sk='profile'"))) + assert pks == ["user1", "user2", "user3"] + + +def test_non_key_filter_scan(dynamodb_client, table): + assert len(_items(q(dynamodb_client, table, "age > 20"))) == 4 + + +def test_pk_eq_plus_non_key_filter_is_query(dynamodb_client, table): + assert sorted(sks(_items(q(dynamodb_client, table, "pk='user1' AND age > 20")))) == [ + "post#2", "post#3", "profile", + ] + + +def test_in_on_sort_key(dynamodb_client, table): + assert sorted(sks(_items(q(dynamodb_client, table, "pk='user1' AND sk IN ['post#1','post#2']")))) == [ + "post#1", "post#2", + ] + + +def test_order_by_desc(dynamodb_client, table): + stmt = f'SELECT * FROM "{table}" WHERE pk=\'user1\' ORDER BY sk DESC' + assert sks(_items(dynamodb_client.execute_statement(Statement=stmt))) == [ + "profile", "post#3", "post#2", "post#1", + ] + + +# ---- projection ------------------------------------------------------------ + +def test_projection_subset(dynamodb_client, table): + items = _items(q(dynamodb_client, table, "pk='user1' AND sk='profile'", proj="pk, sk, name")) + assert set(items[0].keys()) == {"pk", "sk", "name"} + + +def test_projection_nested_path_keys_on_leaf(dynamodb_client, table): + items = _items(q(dynamodb_client, table, "pk='user1' AND sk='profile'", proj="addr.city")) + assert items[0] == {"city": {"S": "NYC"}} + + +def test_projection_list_index_keys_on_literal_text(dynamodb_client, table): + items = _items(q(dynamodb_client, table, "pk='user1' AND sk='profile'", proj="tags[0]")) + assert items[0] == {"tags[0]": {"S": "a"}} + + +def test_projection_reserved_word_attribute(dynamodb_client, table): + items = _items(q(dynamodb_client, table, "pk='user1' AND sk='profile'", proj="Comment")) + assert items[0] == {"Comment": {"S": "hello"}} + + +# ---- parameters ------------------------------------------------------------ + +def test_positional_parameters(dynamodb_client, table): + stmt = f'SELECT * FROM "{table}" WHERE pk=? AND sk=?' + items = _items(dynamodb_client.execute_statement( + Statement=stmt, Parameters=[{"S": "user1"}, {"S": "profile"}])) + assert len(items) == 1 + + +def test_too_few_parameters(dynamodb_client, table): + stmt = f'SELECT * FROM "{table}" WHERE pk=? AND sk=?' + with pytest.raises(ClientError) as e: + dynamodb_client.execute_statement(Statement=stmt, Parameters=[{"S": "user1"}]) + assert e.value.response["Error"]["Code"] == "ValidationException" + assert e.value.response["Error"]["Message"] == ( + "Number of parameters in request and statement don't match." + ) + + +# ---- pagination: Limit + NextToken ----------------------------------------- + +def test_limit_and_next_token_round_trip(dynamodb_client, table): + stmt = f'SELECT * FROM "{table}" WHERE pk=\'user1\'' + collected: list = [] + token = None + for _ in range(6): + kwargs = {"Statement": stmt, "Limit": 2} + if token: + kwargs["NextToken"] = token + resp = dynamodb_client.execute_statement(**kwargs) + collected.extend(sks(_items(resp))) + token = resp.get("NextToken") + if not token: + break + assert collected == ["post#1", "post#2", "post#3", "profile"] + + +def test_next_token_bound_to_statement(dynamodb_client, table): + first = dynamodb_client.execute_statement( + Statement=f'SELECT * FROM "{table}" WHERE pk=\'user1\'', Limit=2) + token = first["NextToken"] + with pytest.raises(ClientError) as e: + dynamodb_client.execute_statement( + Statement=f'SELECT * FROM "{table}" WHERE pk=\'user1\' AND sk > \'post#0\'', + Limit=2, NextToken=token) + assert e.value.response["Error"]["Code"] == "ValidationException" + assert e.value.response["Error"]["Message"] == "NextToken does not match request" + + +def test_next_token_bound_to_parameter_values(dynamodb_client, table): + stmt = f'SELECT * FROM "{table}" WHERE pk = ?' + first = dynamodb_client.execute_statement( + Statement=stmt, Parameters=[{"S": "user1"}], Limit=1) + assert sks(_items(first)) == ["post#1"] + token = first["NextToken"] + with pytest.raises(ClientError) as e: + dynamodb_client.execute_statement( + Statement=stmt, Parameters=[{"S": "user2"}], Limit=1, NextToken=token) + assert e.value.response["Error"]["Code"] == "ValidationException" + assert e.value.response["Error"]["Message"] == "NextToken does not match request" + second = dynamodb_client.execute_statement( + Statement=stmt, Parameters=[{"S": "user1"}], Limit=1, NextToken=token) + assert sks(_items(second)) == ["post#2"] + + +# ---- index targeting ------------------------------------------------------- + +def test_gsi_query(dynamodb_client, table): + stmt = f'SELECT * FROM "{table}"."gsi1" WHERE gsi_pk=\'g1\'' + items = _poll_statement(dynamodb_client, want=3, Statement=stmt) + assert [i["gsi_sk"]["N"] for i in items] == ["1", "2", "3"] + + +def test_gsi_strong_read_rejected(dynamodb_client, table): + stmt = f'SELECT * FROM "{table}"."gsi1" WHERE gsi_pk=\'g1\'' + with pytest.raises(ClientError) as e: + dynamodb_client.execute_statement(Statement=stmt, ConsistentRead=True) + assert e.value.response["Error"]["Code"] == "ValidationException" + assert e.value.response["Error"]["Message"] == ( + "Strongly consistent read is not supported on Global Secondary Indexes" + ) + + +def test_missing_index(dynamodb_client, table): + stmt = f'SELECT * FROM "{table}"."nope" WHERE gsi_pk=\'g1\'' + with pytest.raises(ClientError) as e: + dynamodb_client.execute_statement(Statement=stmt) + assert e.value.response["Error"]["Code"] == "ValidationException" + assert e.value.response["Error"]["Message"] == "The table does not have the specified index" + + +# ---- errors ---------------------------------------------------------------- + +def test_missing_table(dynamodb_client): + with pytest.raises(ClientError) as e: + dynamodb_client.execute_statement( + Statement='SELECT * FROM "extenddb-measure-2a-does-not-exist-xyz" WHERE pk=\'x\'') + assert e.value.response["Error"]["Code"] == "ResourceNotFoundException" + assert e.value.response["Error"]["Message"] == "Requested resource not found" + + +def test_aliasing_rejected(dynamodb_client, table): + with pytest.raises(ClientError) as e: + dynamodb_client.execute_statement( + Statement=f'SELECT name AS n FROM "{table}" WHERE pk=\'user1\' AND sk=\'profile\'') + assert e.value.response["Error"]["Code"] == "ValidationException" + assert e.value.response["Error"]["Message"] == "Aliasing is not supported" + + +def test_empty_in_list_rejected(dynamodb_client, table): + with pytest.raises(ClientError) as e: + dynamodb_client.execute_statement( + Statement=f'SELECT * FROM "{table}" WHERE pk=\'user1\' AND sk IN []') + assert e.value.response["Error"]["Code"] == "ValidationException" + assert e.value.response["Error"]["Message"] == ( + "IN operator must have a left hand argument of type Variable Reference " + "and right hand argument of type Seq with at least one member" + ) diff --git a/tests/test_partiql_set_members.py b/tests/test_partiql_set_members.py new file mode 100644 index 000000000..91d16c80f --- /dev/null +++ b/tests/test_partiql_set_members.py @@ -0,0 +1,173 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""Dual-target wire tests for set_add and set_delete in a PartiQL UPDATE: what +a RETURNING clause gives back, and how a number literal in the bag is checked. + +RETURNING MODIFIED NEW * and RETURNING MODIFIED OLD * give the attribute a +set_add or set_delete names, beside any plain SET target, as it is after or +before the update. An attribute the update empties, or one the item does not +have yet, gives no row. ALL NEW * and ALL OLD * give the whole item. A number +literal in the bag is checked like any literal value: a rule's message ends +with "under root", and a literal that cannot be converted is a parse error with +nothing after the colon. + +Runs against real DynamoDB (EXTENDDB_TEST_ENDPOINT unset, AWS credentials) and +against an extenddb server (EXTENDDB_TEST_ENDPOINT set) with identical +assertions. Every expected value was measured on the service (2026-11-02). The +service returns the members of a set in no fixed order, so sets compare as +sets. Tables use the ``extenddb-measure-pqs-`` prefix and are deleted at the +end. +""" + +from __future__ import annotations + +import uuid +from decimal import Decimal + +import pytest +from botocore.exceptions import ClientError + +from conftest import wait_for_active + +NOT_WELL_FORMED = "Statement wasn't well formed, can't be processed: " +OVERFLOW = ( + "Number overflow. Attempting to store a number with magnitude larger than " + "supported range" +) +UNDERFLOW = ( + "Number underflow. Attempting to store a number with magnitude smaller than " + "supported range" +) +DIGITS = "Attempting to store more than 38 significant digits in a Number" +DIGITS_39 = "123456789012345678901234567890123456789" + +SET_ADD = "SET ss = set_add(ss, <<'b'>>)" +SET_DELETE = "SET ns = set_delete(ns, <<1>>)" + + +@pytest.fixture(scope="module") +def table(dynamodb_client): + name = f"extenddb-measure-pqs-{uuid.uuid4().hex[:12]}" + dynamodb_client.create_table( + TableName=name, + BillingMode="PAY_PER_REQUEST", + AttributeDefinitions=[{"AttributeName": "pk", "AttributeType": "S"}], + KeySchema=[{"AttributeName": "pk", "KeyType": "HASH"}], + ) + try: + wait_for_active(dynamodb_client, name) + yield name + finally: + try: + dynamodb_client.delete_table(TableName=name) + except ClientError: + pass + + +def seeded(key: str) -> dict: + """The item every test starts from.""" + return { + "pk": {"S": key}, + "ss": {"SS": ["a"]}, + "ns": {"NS": ["1", "2"]}, + "other": {"N": "5"}, + } + + +@pytest.fixture +def key(dynamodb_client, table) -> str: + """The key of a fresh copy of the seeded item.""" + key = f"s-{uuid.uuid4().hex[:8]}" + dynamodb_client.put_item(TableName=table, Item=seeded(key)) + return key + + +def as_sets(item: dict) -> dict: + """``item`` with each string set and number set held as a Python set.""" + out = {} + for name, value in item.items(): + if "SS" in value: + out[name] = ("SS", frozenset(value["SS"])) + elif "NS" in value: + out[name] = ("NS", frozenset(Decimal(n) for n in value["NS"])) + else: + out[name] = value + return out + + +def update(client, table: str, key: str, assignment: str, returning: str) -> list: + statement = f"UPDATE \"{table}\" {assignment} WHERE pk = '{key}' RETURNING {returning} *" + return client.execute_statement(Statement=statement)["Items"] + + +@pytest.mark.parametrize( + ("assignment", "returning", "expected"), + [ + (SET_ADD, "MODIFIED NEW", [{"ss": {"SS": ["a", "b"]}}]), + (SET_ADD, "MODIFIED OLD", [{"ss": {"SS": ["a"]}}]), + (SET_DELETE, "MODIFIED NEW", [{"ns": {"NS": ["2"]}}]), + (SET_DELETE, "MODIFIED OLD", [{"ns": {"NS": ["1", "2"]}}]), + ( + f"{SET_ADD} SET other = 6", + "MODIFIED NEW", + [{"other": {"N": "6"}, "ss": {"SS": ["a", "b"]}}], + ), + ("SET ss = set_delete(ss, <<'a'>>)", "MODIFIED NEW", []), + ("SET ss = set_delete(ss, <<'a'>>)", "MODIFIED OLD", [{"ss": {"SS": ["a"]}}]), + ("SET zz = set_add(zz, <<'q'>>)", "MODIFIED OLD", []), + ], + ids=[ + "add-new", + "add-old", + "delete-new", + "delete-old", + "add-and-set-new", + "emptied-new", + "emptied-old", + "absent-old", + ], +) +def test_modified_gives_the_set_the_function_names( + dynamodb_client, table, key, assignment, returning, expected +): + items = update(dynamodb_client, table, key, assignment, returning) + assert [as_sets(item) for item in items] == [as_sets(item) for item in expected] + + +@pytest.mark.parametrize( + ("assignment", "returning", "changed"), + [ + (SET_ADD, "ALL NEW", {"ss": {"SS": ["a", "b"]}}), + (SET_DELETE, "ALL NEW", {"ns": {"NS": ["2"]}}), + (SET_ADD, "ALL OLD", {}), + (SET_DELETE, "ALL OLD", {}), + ], + ids=["add-new", "delete-new", "add-old", "delete-old"], +) +def test_all_gives_the_whole_item(dynamodb_client, table, key, assignment, returning, changed): + items = update(dynamodb_client, table, key, assignment, returning) + assert [as_sets(item) for item in items] == [as_sets({**seeded(key), **changed})] + + +@pytest.mark.parametrize("function", ["set_add", "set_delete"]) +@pytest.mark.parametrize( + ("literal", "expected"), + [ + (DIGITS_39, f"{DIGITS} under root"), + ("1e126", f"{OVERFLOW} under root"), + ("1e-131", f"{UNDERFLOW} under root"), + ("1e99999999999", NOT_WELL_FORMED), + ], + ids=["digits", "overflow", "underflow", "unconvertible"], +) +def test_a_number_literal_in_the_bag_is_checked( + dynamodb_client, table, key, function, literal, expected +): + statement = f"UPDATE \"{table}\" SET ns = {function}(ns, <<{literal}>>) WHERE pk = '{key}'" + with pytest.raises(ClientError) as excinfo: + dynamodb_client.execute_statement(Statement=statement) + err = excinfo.value.response["Error"] + assert (err["Code"], err["Message"]) == ("ValidationException", expected) + item = dynamodb_client.get_item(TableName=table, Key={"pk": {"S": key}}, ConsistentRead=True) + assert as_sets(item["Item"]) == as_sets(seeded(key)) diff --git a/tests/test_partiql_transaction.py b/tests/test_partiql_transaction.py new file mode 100644 index 000000000..de987ff66 --- /dev/null +++ b/tests/test_partiql_transaction.py @@ -0,0 +1,1262 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""PartiQL ExecuteTransaction over the wire. + +Dual-target: identical assertions run against real DynamoDB and against an +ExtendDB fix server (selected by EXTENDDB_TEST_ENDPOINT, see conftest). Every +expected response and error Code+Message was measured on the live service; the +records are in conf-run/partiql/char-batch-txn.md (Part 2). + +A transaction is all reads or all writes, all-or-nothing. Reads behave like +TransactGetItems (Responses[i] = {Item}, no TableName); any write failure +cancels with a positional CancellationReasons list. ClientRequestToken gives +idempotency. The one shared composite pk(S)/sk(S) table is reused with distinct +keys per test for isolation. +""" + +from __future__ import annotations + +import os +import uuid +from string import Template + +import boto3 +import pytest +from botocore.config import Config +from botocore.exceptions import ClientError + +from conftest import wait_for_active +from partiql_gsi_table import get as _gget +from partiql_gsi_table import gsi_table, item, key, put + + +@pytest.fixture(scope="session") +def dynamodb_client(endpoint_url): + """DynamoDB client honoring EXTENDDB_TEST_PROFILE for the live-service run.""" + region = os.environ.get("AWS_DEFAULT_REGION", "us-east-1") + profile = os.environ.get("EXTENDDB_TEST_PROFILE") or None + session = boto3.Session(profile_name=profile, region_name=region) + kwargs: dict = {"service_name": "dynamodb"} + if endpoint_url: + kwargs["endpoint_url"] = endpoint_url + if endpoint_url.startswith("https://"): + kwargs["verify"] = False + kwargs["config"] = Config(parameter_validation=True) + return session.client(**kwargs) + + +@pytest.fixture(scope="module") +def ttable(dynamodb_client): + """A composite pk(S)/sk(S) table, PAY_PER_REQUEST.""" + name = f"extenddb-measure-2c-txn-{uuid.uuid4().hex[:12]}" + dynamodb_client.create_table( + TableName=name, + AttributeDefinitions=[ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "sk", "AttributeType": "S"}, + ], + KeySchema=[ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"}, + ], + BillingMode="PAY_PER_REQUEST", + ) + wait_for_active(dynamodb_client, name) + try: + yield name + finally: + try: + dynamodb_client.delete_table(TableName=name) + except dynamodb_client.exceptions.ResourceNotFoundException: + pass + + +def _txn(client, statements, **kwargs): + return client.execute_transaction(TransactStatements=statements, **kwargs) + + +def _error(excinfo): + err = excinfo.value.response["Error"] + return err["Code"], err["Message"] + + +def _seed(client, table, pk, sk="s1", a=1): + client.put_item( + TableName=table, + Item={"pk": {"S": pk}, "sk": {"S": sk}, "a": {"N": str(a)}}, + ) + + +def _get(client, table, pk, sk="s1"): + return client.get_item( + TableName=table, Key={"pk": {"S": pk}, "sk": {"S": sk}} + ).get("Item") + + +# --- T1 / T2 / T2b: success shapes ---------------------------------------- + + +def test_all_writes_returns_empty_responses(dynamodb_client, ttable): + _seed(dynamodb_client, ttable, "t1u", a=1) + _seed(dynamodb_client, ttable, "t1d", a=1) + resp = _txn( + dynamodb_client, + [ + {"Statement": f"INSERT INTO \"{ttable}\" VALUE {{'pk':'t1i','sk':'s1','a':1}}"}, + {"Statement": f"UPDATE \"{ttable}\" SET a=2 WHERE pk='t1u' AND sk='s1'"}, + {"Statement": f"DELETE FROM \"{ttable}\" WHERE pk='t1d' AND sk='s1'"}, + ], + ) + assert resp["Responses"] == [] + assert _get(dynamodb_client, ttable, "t1i")["a"] == {"N": "1"} + assert _get(dynamodb_client, ttable, "t1u")["a"] == {"N": "2"} + assert _get(dynamodb_client, ttable, "t1d") is None + + +def test_all_reads_items_no_table_name(dynamodb_client, ttable): + _seed(dynamodb_client, ttable, "t2a", a=99) + _seed(dynamodb_client, ttable, "t2b", a=2) + resp = _txn( + dynamodb_client, + [ + {"Statement": f"SELECT * FROM \"{ttable}\" WHERE pk='t2a' AND sk='s1'"}, + {"Statement": f"SELECT * FROM \"{ttable}\" WHERE pk='t2b' AND sk='s1'"}, + ], + ) + assert resp["Responses"] == [ + {"Item": {"pk": {"S": "t2a"}, "sk": {"S": "s1"}, "a": {"N": "99"}}}, + {"Item": {"pk": {"S": "t2b"}, "sk": {"S": "s1"}, "a": {"N": "2"}}}, + ] + + +def test_read_absent_is_empty_object(dynamodb_client, ttable): + resp = _txn( + dynamodb_client, + [{"Statement": f"SELECT * FROM \"{ttable}\" WHERE pk='ghost' AND sk='s1'"}], + ) + assert resp["Responses"] == [{}] + + +# --- T2c: partial-key read (request-level) -------------------------------- + + +def test_read_partial_key_rejects_whole_request(dynamodb_client, ttable): + _seed(dynamodb_client, ttable, "t2c") + with pytest.raises(ClientError) as excinfo: + _txn( + dynamodb_client, + [{"Statement": f"SELECT * FROM \"{ttable}\" WHERE pk='t2c'"}], + ) + code, message = _error(excinfo) + assert code == "ValidationException" + assert message == ( + "Validation failed in TransactStatements[0]: Select statements within " + "ExecuteTransaction must specify the primary key in the where clause." + ) + + +# --- T3: read/write mix (request-level) ----------------------------------- + + +def test_mixed_read_and_write_rejects(dynamodb_client, ttable): + _seed(dynamodb_client, ttable, "t3") + with pytest.raises(ClientError) as excinfo: + _txn( + dynamodb_client, + [ + {"Statement": f"SELECT * FROM \"{ttable}\" WHERE pk='t3' AND sk='s1'"}, + {"Statement": f"UPDATE \"{ttable}\" SET a=1 WHERE pk='t3' AND sk='s1'"}, + ], + ) + code, message = _error(excinfo) + assert code == "ValidationException" + assert message == ( + "ExecuteTransaction API does not support both read and write operations in the same request." + ) + + +# --- T4 / T4b: all-or-nothing via a failing condition --------------------- + + +def test_cancel_on_failed_condition_is_positional(dynamodb_client, ttable): + _seed(dynamodb_client, ttable, "t4a", a=99) + _seed(dynamodb_client, ttable, "t4b", a=2) + with pytest.raises(ClientError) as excinfo: + _txn( + dynamodb_client, + [ + {"Statement": f"UPDATE \"{ttable}\" SET a=5 WHERE pk='t4a' AND sk='s1'"}, + { + "Statement": f"UPDATE \"{ttable}\" SET a=5 WHERE pk='t4b' AND sk='s1' AND attribute_exists(zzz)" + }, + ], + ) + response = excinfo.value.response + assert response["Error"]["Code"] == "TransactionCanceledException" + assert response["Error"]["Message"] == ( + "Transaction cancelled, please refer cancellation reasons for specific reasons " + "[None, ConditionalCheckFailed]" + ) + assert response["CancellationReasons"] == [ + {"Code": "None"}, + {"Code": "ConditionalCheckFailed", "Message": "The conditional request failed"}, + ] + # Nothing committed: the good UPDATE on t4a did not apply. + assert _get(dynamodb_client, ttable, "t4a")["a"] == {"N": "99"} + + +# --- T4c: cancel via duplicate-key INSERT --------------------------------- + + +def test_cancel_on_duplicate_insert(dynamodb_client, ttable): + _seed(dynamodb_client, ttable, "t4exist", a=1) + with pytest.raises(ClientError) as excinfo: + _txn( + dynamodb_client, + [ + {"Statement": f"INSERT INTO \"{ttable}\" VALUE {{'pk':'t4new','sk':'s1','a':1}}"}, + {"Statement": f"INSERT INTO \"{ttable}\" VALUE {{'pk':'t4exist','sk':'s1','a':2}}"}, + ], + ) + response = excinfo.value.response + assert response["Error"]["Code"] == "TransactionCanceledException" + assert response["Error"]["Message"] == ( + "Transaction cancelled, please refer cancellation reasons for specific reasons " + "[None, DuplicateItem]" + ) + assert response["CancellationReasons"] == [ + {"Code": "None"}, + {"Code": "DuplicateItem", "Message": "Duplicate primary key exists in table"}, + ] + # The good INSERT did not commit (all-or-nothing). + assert _get(dynamodb_client, ttable, "t4new") is None + + +# --- T4d: ReturnValuesOnConditionCheckFailure on a cancelled statement ----- + + +def test_cancel_returns_item_on_all_old(dynamodb_client, ttable): + _seed(dynamodb_client, ttable, "t4d", a=2) + with pytest.raises(ClientError) as excinfo: + _txn( + dynamodb_client, + [ + { + "Statement": f"UPDATE \"{ttable}\" SET a=5 WHERE pk='t4d' AND sk='s1' AND a=12345", + "ReturnValuesOnConditionCheckFailure": "ALL_OLD", + } + ], + ) + reasons = excinfo.value.response["CancellationReasons"] + assert reasons == [ + { + "Item": {"pk": {"S": "t4d"}, "sk": {"S": "s1"}, "a": {"N": "2"}}, + "Code": "ConditionalCheckFailed", + "Message": "The conditional request failed", + } + ] + + +# --- T5: ClientRequestToken idempotency ----------------------------------- + + +def test_idempotent_replay_and_mismatch(dynamodb_client, ttable): + token = uuid.uuid4().hex + statements = [ + {"Statement": f"INSERT INTO \"{ttable}\" VALUE {{'pk':'t5','sk':'s1','a':1}}"} + ] + first = _txn(dynamodb_client, statements, ClientRequestToken=token) + assert first["Responses"] == [] + # Same token + same body: idempotent replay, no DuplicateItem though the key + # now exists. + replay = _txn(dynamodb_client, statements, ClientRequestToken=token) + assert replay["Responses"] == [] + # Same token + different body: mismatch. + with pytest.raises(ClientError) as excinfo: + _txn( + dynamodb_client, + [{"Statement": f"INSERT INTO \"{ttable}\" VALUE {{'pk':'t5x','sk':'s1','a':9}}"}], + ClientRequestToken=token, + ) + code, message = _error(excinfo) + assert code == "IdempotentParameterMismatchException" + assert message == ( + "Specified idempotent token was used with different request parameters within the idempotency window" + ) + + +# --- T6: conflicting keys (request-level) --------------------------------- + + +def test_conflicting_keys_reject(dynamodb_client, ttable): + _seed(dynamodb_client, ttable, "t6") + with pytest.raises(ClientError) as excinfo: + _txn( + dynamodb_client, + [ + {"Statement": f"UPDATE \"{ttable}\" SET a=1 WHERE pk='t6' AND sk='s1'"}, + {"Statement": f"UPDATE \"{ttable}\" SET a=2 WHERE pk='t6' AND sk='s1'"}, + ], + ) + code, message = _error(excinfo) + assert code == "ValidationException" + assert message == "Transaction request cannot include multiple operations on one item" + + +# --- T7: statement-count limit -------------------------------------------- + + +def test_one_hundred_statements_ok(dynamodb_client, ttable): + statements = [ + {"Statement": f"SELECT * FROM \"{ttable}\" WHERE pk='n{i}' AND sk='s1'"} + for i in range(100) + ] + resp = _txn(dynamodb_client, statements) + assert len(resp["Responses"]) == 100 + + +def test_one_hundred_one_statements_rejected(dynamodb_client, ttable): + statements = [ + {"Statement": f"SELECT * FROM \"{ttable}\" WHERE pk='n{i}' AND sk='s1'"} + for i in range(101) + ] + with pytest.raises(ClientError) as excinfo: + _txn(dynamodb_client, statements) + code, message = _error(excinfo) + assert code == "ValidationException" + assert message.endswith("Member must have length less than or equal to 100") + + +# --- T8: capacity (a list, 2.0 RCU transactional read) -------------------- + + +def test_transactional_read_capacity_is_two_rcu_list(dynamodb_client, ttable): + _seed(dynamodb_client, ttable, "t8", a=1) + resp = _txn( + dynamodb_client, + [{"Statement": f"SELECT * FROM \"{ttable}\" WHERE pk='t8' AND sk='s1'"}], + ReturnConsumedCapacity="TOTAL", + ) + assert resp["ConsumedCapacity"] == [ + {"TableName": ttable, "CapacityUnits": 2.0, "ReadCapacityUnits": 2.0} + ] + + +# --- RETURNING inside a transaction (request-level) ----------------------- +# A member with a RETURNING clause fails the whole request up front with a +# ValidationException naming its position, and no member applies (measured +# 2026-11-02). + +_RETURNING_REFUSAL = "RETURNING clause is not supported in ExecuteTransaction." + + +@pytest.mark.parametrize("mode", ["ALL OLD", "MODIFIED OLD", "ALL NEW", "MODIFIED NEW"]) +def test_update_returning_member_rejects_whole_request(dynamodb_client, ttable, mode): + pk = "tr-" + mode.replace(" ", "-").lower() + _seed(dynamodb_client, ttable, pk, a=1) + with pytest.raises(ClientError) as excinfo: + _txn( + dynamodb_client, + [ + { + "Statement": f"UPDATE \"{ttable}\" SET a=2 WHERE pk='{pk}' AND sk='s1' RETURNING {mode} *" + } + ], + ) + code, message = _error(excinfo) + assert code == "ValidationException" + assert message == f"Validation failed in TransactStatements[0]: {_RETURNING_REFUSAL}" + assert _get(dynamodb_client, ttable, pk)["a"] == {"N": "1"} + + +def test_returning_member_is_named_by_its_position(dynamodb_client, ttable): + _seed(dynamodb_client, ttable, "tr-pos-a", a=1) + _seed(dynamodb_client, ttable, "tr-pos-b", a=1) + with pytest.raises(ClientError) as excinfo: + _txn( + dynamodb_client, + [ + {"Statement": f"UPDATE \"{ttable}\" SET a=2 WHERE pk='tr-pos-a' AND sk='s1'"}, + { + "Statement": f"DELETE FROM \"{ttable}\" WHERE pk='tr-pos-b' AND sk='s1' RETURNING ALL OLD *" + }, + ], + ) + code, message = _error(excinfo) + assert code == "ValidationException" + assert message == f"Validation failed in TransactStatements[1]: {_RETURNING_REFUSAL}" + # Nothing applied: neither the plain UPDATE nor the DELETE ran. + assert _get(dynamodb_client, ttable, "tr-pos-a")["a"] == {"N": "1"} + assert _get(dynamodb_client, ttable, "tr-pos-b") is not None + + +def test_returning_member_after_a_read_reports_the_returning_clause(dynamodb_client, ttable): + _seed(dynamodb_client, ttable, "tr-mix-r", a=1) + _seed(dynamodb_client, ttable, "tr-mix-w", a=1) + with pytest.raises(ClientError) as excinfo: + _txn( + dynamodb_client, + [ + {"Statement": f"SELECT * FROM \"{ttable}\" WHERE pk='tr-mix-r' AND sk='s1'"}, + { + "Statement": f"UPDATE \"{ttable}\" SET a=2 WHERE pk='tr-mix-w' AND sk='s1' RETURNING ALL NEW *" + }, + ], + ) + code, message = _error(excinfo) + assert code == "ValidationException" + assert message == f"Validation failed in TransactStatements[1]: {_RETURNING_REFUSAL}" + + +_MISSING_TABLE = "extenddb-measure-2c-no-such-table" +_NOT_FOUND_CANCEL = ( + "Transaction cancelled, please refer cancellation reasons for specific reasons [ResourceNotFound]" +) + + +@pytest.mark.parametrize( + "returning", + ["UPDATE-RETURNING ALL NEW *", "DELETE-RETURNING ALL OLD *", "DELETE-RETURNING MODIFIED OLD *"], +) +def test_returning_on_a_missing_table_is_a_not_found_cancellation(dynamodb_client, ttable, returning): + verb, clause = returning.split("-") + if verb == "UPDATE": + statement = f"UPDATE \"{_MISSING_TABLE}\" SET a=2 WHERE pk='x' AND sk='s1' {clause}" + else: + statement = f"DELETE FROM \"{_MISSING_TABLE}\" WHERE pk='x' AND sk='s1' {clause}" + with pytest.raises(ClientError) as excinfo: + _txn(dynamodb_client, [{"Statement": statement}]) + code, message = _error(excinfo) + assert code == "TransactionCanceledException" + assert message == _NOT_FOUND_CANCEL + reasons = excinfo.value.response["CancellationReasons"] + assert [r["Code"] for r in reasons] == ["ResourceNotFound"] + + +def test_returning_on_an_existing_table_is_refused_beside_a_missing_one(dynamodb_client, ttable): + returning = f"UPDATE \"{ttable}\" SET a=2 WHERE pk='tr-miss' AND sk='s1' RETURNING ALL NEW *" + plain_missing = f"UPDATE \"{_MISSING_TABLE}\" SET a=2 WHERE pk='x' AND sk='s1'" + returning_missing = returning.replace(ttable, _MISSING_TABLE) + for statements, index in ( + ([returning, plain_missing], 0), + ([plain_missing, returning], 1), + ([returning_missing, returning], 1), + ): + with pytest.raises(ClientError) as excinfo: + _txn(dynamodb_client, [{"Statement": s} for s in statements]) + code, message = _error(excinfo) + assert code == "ValidationException" + assert message == f"Validation failed in TransactStatements[{index}]: {_RETURNING_REFUSAL}" + + +def test_returning_on_a_missing_table_beside_a_read_reports_the_mixed_request(dynamodb_client, ttable): + with pytest.raises(ClientError) as excinfo: + _txn( + dynamodb_client, + [ + {"Statement": f"SELECT * FROM \"{ttable}\" WHERE pk='x' AND sk='s1'"}, + { + "Statement": f"UPDATE \"{_MISSING_TABLE}\" SET a=2 " + "WHERE pk='x' AND sk='s1' RETURNING ALL NEW *" + }, + ], + ) + code, message = _error(excinfo) + assert code == "ValidationException" + assert message == ( + "ExecuteTransaction API does not support both read and write operations in the same request." + ) + + +def test_delete_invalid_returning_clause_beside_a_missing_table(dynamodb_client, ttable): + with pytest.raises(ClientError) as excinfo: + _txn( + dynamodb_client, + [ + {"Statement": f"UPDATE \"{_MISSING_TABLE}\" SET a=2 WHERE pk='x' AND sk='s1'"}, + { + "Statement": f"DELETE FROM \"{ttable}\" WHERE pk='tr-miss2' AND sk='s1' " + "RETURNING MODIFIED OLD *" + }, + ], + ) + code, message = _error(excinfo) + assert code == "ValidationException" + assert message == ( + "Validation failed in TransactStatements[1]: Invalid returning clause: " + "RETURNING MODIFIED OLD *. Only RETURNING ALL OLD * is allowed in DELETE statements." + ) + + +def test_delete_with_another_returning_clause_reports_the_invalid_clause(dynamodb_client, ttable): + _seed(dynamodb_client, ttable, "tr-delmod", a=1) + with pytest.raises(ClientError) as excinfo: + _txn( + dynamodb_client, + [ + { + "Statement": f"DELETE FROM \"{ttable}\" WHERE pk='tr-delmod' AND sk='s1' RETURNING MODIFIED OLD *" + } + ], + ) + code, message = _error(excinfo) + assert code == "ValidationException" + assert message == ( + "Validation failed in TransactStatements[0]: Invalid returning clause: " + "RETURNING MODIFIED OLD *. Only RETURNING ALL OLD * is allowed in DELETE statements." + ) + assert _get(dynamodb_client, ttable, "tr-delmod") is not None + + +# --- empty TransactStatements (request-level) ----------------------------- + + +def test_empty_transact_statements_rejected(dynamodb_client_no_validation): + with pytest.raises(ClientError) as excinfo: + dynamodb_client_no_validation.execute_transaction(TransactStatements=[]) + code, message = _error(excinfo) + assert code == "ValidationException" + assert message == ( + "1 validation error detected: Value '[]' at 'transactStatements' failed to satisfy " + "constraint: Member must have length greater than or equal to 1" + ) + + +# --- DELETE on a missing key with a non-key predicate ----------------------- +# The predicate is a condition on an item that exists, so a member naming a +# missing key does not cancel the transaction, as the statement does not fail +# on its own. + + +def test_delete_missing_key_with_predicate_does_not_cancel(dynamodb_client, ttable): + _seed(dynamodb_client, ttable, "tr-np-other", a=1) + resp = _txn( + dynamodb_client, + [ + {"Statement": f"DELETE FROM \"{ttable}\" WHERE pk='tr-np-ghost' AND sk='s1' AND a = 1"}, + {"Statement": f"UPDATE \"{ttable}\" SET a=2 WHERE pk='tr-np-other' AND sk='s1'"}, + ], + ) + assert resp["Responses"] == [] + assert _get(dynamodb_client, ttable, "tr-np-other")["a"] == {"N": "2"} + + +# --- request-stage refusals of a write member ------------------------------- +# A statement that does not parse, or that has more placeholders than the +# request gives parameters, fails the whole request with a ValidationException +# naming its position (measured 2026-11-02). + + +def test_parse_error_names_its_member(dynamodb_client, ttable): + with pytest.raises(ClientError) as excinfo: + _txn( + dynamodb_client, + [ + {"Statement": f"UPDATE \"{ttable}\" SET a=1 WHERE pk='tp-parse' AND sk='s1'"}, + {"Statement": "UPDATE WHERE"}, + ], + ) + code, message = _error(excinfo) + assert code == "ValidationException" + assert message == ( + "Validation failed in TransactStatements[1]: Statement wasn't well formed, can't be " + "processed: Expected identifier for simple path" + ) + + +def test_parameter_count_mismatch_names_its_member(dynamodb_client, ttable): + with pytest.raises(ClientError) as excinfo: + _txn( + dynamodb_client, + [{"Statement": f"UPDATE \"{ttable}\" SET a=? WHERE pk='tp-count' AND sk='s1'"}], + ) + code, message = _error(excinfo) + assert code == "ValidationException" + assert message == ( + "Validation failed in TransactStatements[0]: Number of parameters in request and " + "statement don't match." + ) + + +# --- member-stage cancellations ---------------------------------------------- +# Each member is translated against its table before anything runs. A member on +# a missing table, or one whose key does not match the table's schema, cancels +# the request with one reason per member, "None" for the others. Nothing runs: +# no condition is evaluated, no duplicate INSERT is found, and two members on +# one item are not refused (measured 2026-11-02). + +_NONE = {"Code": "None"} +_NOT_FOUND = {"Code": "ResourceNotFound", "Message": "Requested resource not found"} +_KEY_MISMATCH = { + "Code": "ValidationError", + "Message": "The provided key element does not match the schema", +} + + +def _invalid(message): + return { + "Code": "ValidationError", + "Message": f"One or more parameter values were invalid: {message}", + } + + +def _statements(templates, table): + """Fill $T (the shared table) and $M (a table that does not exist).""" + return [ + {"Statement": Template(t).substitute(T=table, M=_MISSING_TABLE)} for t in templates + ] + + +def _expect_cancellation(client, table, templates, reasons): + with pytest.raises(ClientError) as excinfo: + _txn(client, _statements(templates, table)) + response = excinfo.value.response + codes = ", ".join(reason["Code"] for reason in reasons) + assert response["Error"]["Code"] == "TransactionCanceledException" + assert response["Error"]["Message"] == ( + f"Transaction cancelled, please refer cancellation reasons for specific reasons [{codes}]" + ) + assert response["CancellationReasons"] == reasons + + +_ON_T = "UPDATE \"$T\" SET a=2 WHERE pk='tm-y' AND sk='s1'" +_ON_M = "UPDATE \"$M\" SET a=2 WHERE pk='tm-x' AND sk='s1'" +_PARTIAL = "UPDATE \"$T\" SET a=1 WHERE pk='tm-a'" +_SAME_KEY = "UPDATE \"$T\" SET a=1 WHERE pk='tm-b' AND sk='s1'" + + +@pytest.mark.parametrize( + ("templates", "reasons"), + [ + pytest.param([_ON_T, _ON_M], [_NONE, _NOT_FOUND], id="table-then-missing-table"), + pytest.param([_ON_M, _ON_T], [_NOT_FOUND, _NONE], id="missing-table-then-table"), + pytest.param([_ON_M], [_NOT_FOUND], id="update-on-missing-table"), + pytest.param( + ["INSERT INTO \"$M\" VALUE {'pk':'tm-a','sk':'b'}"], + [_NOT_FOUND], + id="insert-on-missing-table", + ), + pytest.param( + ["DELETE FROM \"$M\" WHERE pk='tm-a' AND sk='b'"], + [_NOT_FOUND], + id="delete-on-missing-table", + ), + pytest.param([_PARTIAL, _ON_M], [_KEY_MISMATCH, _NOT_FOUND], id="partial-key-then-missing-table"), + pytest.param([_ON_M, _PARTIAL], [_NOT_FOUND, _KEY_MISMATCH], id="missing-table-then-partial-key"), + pytest.param( + [_ON_M, "UPDATE \"$T\" SET a=1 WHERE pk='tm-a' AND sk=1"], + [_NOT_FOUND, _KEY_MISMATCH], + id="missing-table-then-key-of-the-wrong-type", + ), + pytest.param( + [_SAME_KEY, _SAME_KEY, _ON_M], + [_NONE, _NONE, _NOT_FOUND], + id="one-item-twice-then-missing-table", + ), + pytest.param([_PARTIAL], [_KEY_MISMATCH], id="partial-key"), + pytest.param( + ["UPDATE \"$T\" SET a=1 WHERE pk='tm-a' AND sk=1"], + [_KEY_MISMATCH], + id="key-of-the-wrong-type", + ), + pytest.param(["UPDATE \"$T\" SET a=1 WHERE a=2"], [_KEY_MISMATCH], id="no-key-in-where"), + pytest.param( + ["DELETE FROM \"$T\" WHERE pk='tm-a'"], [_KEY_MISMATCH], id="delete-with-a-partial-key" + ), + pytest.param( + ["UPDATE \"$T\" SET pk='tm-z' WHERE pk='tm-a' AND sk='s1'"], + [_invalid("Cannot update attribute pk. This attribute is part of the key")], + id="set-on-a-key-attribute", + ), + pytest.param( + ["INSERT INTO \"$T\" VALUE {'pk':'tm-a'}"], + [_invalid("Missing the key sk in the item")], + id="insert-missing-the-sort-key", + ), + pytest.param( + [_PARTIAL, _SAME_KEY, _SAME_KEY], + [_KEY_MISMATCH, _NONE, _NONE], + id="partial-key-then-one-item-twice", + ), + ], +) +def test_member_stage_failure_cancels_with_a_reason_per_member( + dynamodb_client, ttable, templates, reasons +): + _expect_cancellation(dynamodb_client, ttable, templates, reasons) + + +def test_member_stage_failure_skips_duplicate_insert_detection(dynamodb_client, ttable): + _seed(dynamodb_client, ttable, "tm-dup", a=1) + _expect_cancellation( + dynamodb_client, + ttable, + ["INSERT INTO \"$T\" VALUE {'pk':'tm-dup','sk':'s1'}", _PARTIAL], + [_NONE, _KEY_MISMATCH], + ) + assert _get(dynamodb_client, ttable, "tm-dup") == { + "pk": {"S": "tm-dup"}, + "sk": {"S": "s1"}, + "a": {"N": "1"}, + } + + +def test_member_stage_failure_skips_condition_evaluation(dynamodb_client, ttable): + _seed(dynamodb_client, ttable, "tm-cond", a=1) + _expect_cancellation( + dynamodb_client, + ttable, + ["UPDATE \"$T\" SET a=5 WHERE pk='tm-cond' AND sk='s1' AND a=99", _PARTIAL], + [_NONE, _KEY_MISMATCH], + ) + assert _get(dynamodb_client, ttable, "tm-cond")["a"] == {"N": "1"} + + +def test_member_stage_failure_applies_nothing(dynamodb_client, ttable): + _seed(dynamodb_client, ttable, "tm-keep", a=1) + _seed(dynamodb_client, ttable, "tm-gone", a=1) + _expect_cancellation( + dynamodb_client, + ttable, + [ + "UPDATE \"$T\" SET a=2 WHERE pk='tm-keep' AND sk='s1'", + "INSERT INTO \"$T\" VALUE {'pk':'tm-new','sk':'s1'}", + "DELETE FROM \"$T\" WHERE pk='tm-gone' AND sk='s1'", + _ON_M, + ], + [_NONE, _NONE, _NONE, _NOT_FOUND], + ) + assert _get(dynamodb_client, ttable, "tm-keep")["a"] == {"N": "1"} + assert _get(dynamodb_client, ttable, "tm-new") is None + assert _get(dynamodb_client, ttable, "tm-gone") is not None + + +# --- key and parameter variants --------------------------------------------- +# The same rules applied to an INSERT key of the wrong type or a missing +# partition key, a DELETE key of the wrong type, a REMOVE of a key attribute, +# and a member with a partial key that also sets a key attribute. + + +@pytest.mark.parametrize( + ("templates", "reasons"), + [ + pytest.param( + ["INSERT INTO \"$T\" VALUE {'pk':'tv-a','sk':1}"], + [_invalid("Type mismatch for key sk expected: S actual: N")], + id="insert-sort-key-of-the-wrong-type", + ), + pytest.param( + ["INSERT INTO \"$T\" VALUE {'sk':'s1'}"], + [_invalid("Missing the key pk in the item")], + id="insert-missing-the-partition-key", + ), + pytest.param( + ["INSERT INTO \"$T\" VALUE {'a':1}"], + [_invalid("Missing the key pk in the item")], + id="insert-missing-both-keys", + ), + pytest.param( + ["DELETE FROM \"$T\" WHERE pk='tv-a' AND sk=1"], + [_KEY_MISMATCH], + id="delete-key-of-the-wrong-type", + ), + pytest.param( + ["UPDATE \"$T\" REMOVE sk WHERE pk='tv-a' AND sk='s1'"], + [_invalid("Cannot update attribute sk. This attribute is part of the key")], + id="remove-a-key-attribute", + ), + pytest.param( + ["UPDATE \"$T\" SET pk='tv-z' WHERE pk='tv-a'"], + [_KEY_MISMATCH], + id="set-a-key-attribute-with-a-partial-key", + ), + ], +) +def test_key_problem_variants_cancel_with_a_reason(dynamodb_client, ttable, templates, reasons): + _expect_cancellation(dynamodb_client, ttable, templates, reasons) + + +@pytest.mark.parametrize( + ("templates", "index"), + [ + pytest.param(["UPDATE \"$M\" SET a=? WHERE pk='tv-x' AND sk='s1'"], 0, id="on-the-missing-table"), + pytest.param( + [_ON_M, "UPDATE \"$T\" SET a=? WHERE pk='tv-x' AND sk='s1'"], + 1, + id="beside-the-missing-table", + ), + ], +) +def test_parameter_count_mismatch_beside_a_missing_table(dynamodb_client, ttable, templates, index): + with pytest.raises(ClientError) as excinfo: + _txn(dynamodb_client, _statements(templates, ttable)) + code, message = _error(excinfo) + assert code == "ValidationException" + assert message == ( + f"Validation failed in TransactStatements[{index}]: Number of parameters in request " + "and statement don't match." + ) + +# --- the table with a global secondary index ------------------------------ + + +@pytest.fixture(scope="module") +def gtable(dynamodb_client): + """The table with the index `gsi`, holding the items of the reads.""" + with gsi_table(dynamodb_client, "extenddb-measure-2c-txng") as name: + put(dynamodb_client, name, "seed") + put(dynamodb_client, name, "seed2", gs="2") + put(dynamodb_client, name, "lst", l={"L": [{"N": "1"}, {"N": "2"}, {"N": "3"}]}) + yield name + + +def _sql(template, table, **names): + """`template` with `{t}` as the quoted table and each other field as the + key that name takes in this run.""" + return template.format(t=f'"{table}"', **{n: key(v) for n, v in names.items()}) + + +def _members(table, templates, **names): + return [{"Statement": _sql(t, table, **names)} for t in templates] + + +# --- the projection of a read member ------------------------------------------ +# A read member honours its projection list, and the item keeps the shape of +# the paths, as a ProjectionExpression gives it. An item that has none of the +# paths is returned empty, and a missing item returns no item. A projection +# list that names an identifier twice, or whose paths overlap, is refused with +# the member's position before its table is read (measured 2026-11-05). + + +@pytest.mark.parametrize( + "projection, name, expected", + [ + ("a, m.k", "seed", {"Item": {"a": {"N": "1"}, "m": {"M": {"k": {"N": "1"}}}}}), + ( + "m.k, m.\"my key\"", + "seed", + {"Item": {"m": {"M": {"k": {"N": "1"}, "my key": {"N": "1"}}}}}, + ), + ("sk, pk", "seed", {"Item": {"pk": {"S": key("seed")}, "sk": {"S": "b"}}}), + ("l[1], m.k", "lst", {"Item": {"l": {"L": [{"N": "2"}]}, "m": {"M": {"k": {"N": "1"}}}}}), + ("nope", "seed", {"Item": {}}), + ("m.nope, a", "seed", {"Item": {"a": {"N": "1"}}}), + ("a.b", "seed", {"Item": {}}), + ("l[7]", "lst", {"Item": {}}), + ("a", "no-such-item", {}), + ("m['k'], a", "seed", {"Item": {"a": {"N": "1"}, "m": {"M": {"k": {"N": "1"}}}}}), + ], + ids=[ + "path", + "quoted-name", + "key-attributes", + "list-element", + "absent-attribute", + "absent-member", + "path-into-a-number", + "element-past-the-list", + "missing-item", + "member-in-brackets", + ], +) +def test_a_read_member_honours_its_projection(dynamodb_client, gtable, projection, name, expected): + resp = _txn( + dynamodb_client, + _members(gtable, [f"SELECT {projection} FROM {{t}} WHERE pk = '{{k}}' AND sk = 'b'"], + k=name), + ) + assert resp["Responses"] == [expected] + + +def test_each_read_member_takes_its_own_projection(dynamodb_client, gtable): + resp = _txn( + dynamodb_client, + _members( + gtable, + [ + "SELECT pk, a FROM {t} WHERE pk = '{s}' AND sk = 'b'", + "SELECT * FROM {t} WHERE pk = '{s2}' AND sk = 'b'", + ], + s="seed", + s2="seed2", + ), + ) + assert resp["Responses"] == [ + {"Item": {"pk": {"S": key("seed")}, "a": {"N": "1"}}}, + {"Item": item(key("seed2"), gs="2")}, + ] + + +def test_a_projection_does_not_change_the_read_units(dynamodb_client, gtable): + resp = _txn( + dynamodb_client, + _members(gtable, ["SELECT a FROM {t} WHERE pk = '{s}' AND sk = 'b'"], s="seed"), + ReturnConsumedCapacity="TOTAL", + ) + assert resp["Responses"] == [{"Item": {"a": {"N": "1"}}}] + assert resp["ConsumedCapacity"] == [ + {"TableName": gtable, "CapacityUnits": 2.0, "ReadCapacityUnits": 2.0} + ] + + +_REPEATED = "Duplicate identifiers in select clause: a" + + +def _overlap(one, two): + return ( + "Two document paths overlap with each other; must remove or rewrite one of these " + f"paths; path one: {one}, path two: {two}" + ) + + +@pytest.mark.parametrize( + "templates, index, message", + [ + (["SELECT a, a FROM {t} WHERE pk = '{s}' AND sk = 'b'"], 0, _REPEATED), + (["SELECT m.k, k FROM {t} WHERE pk = '{s}' AND sk = 'b'"], 0, + "Duplicate identifiers in select clause: k"), + (["SELECT a, \"a\" FROM {t} WHERE pk = '{s}' AND sk = 'b'"], 0, _REPEATED), + (["SELECT m.k, m FROM {t} WHERE pk = '{s}' AND sk = 'b'"], 0, _overlap("[m]", "[m, k]")), + (["SELECT m, m.k FROM {t} WHERE pk = '{s}' AND sk = 'b'"], 0, _overlap("[m, k]", "[m]")), + (["SELECT l[1], l FROM {t} WHERE pk = '{l}' AND sk = 'b'"], 0, + _overlap("[l]", "[l, [1]]")), + (["SELECT a, a FROM nosuch_tbl WHERE pk = 'a' AND sk = 'b'"], 0, _REPEATED), + (["SELECT a, a FROM {t} WHERE pk = '{s}'"], 0, _REPEATED), + (["SELECT a, a FROM {t}.\"gsi\" WHERE g = 'a' AND gs = '1'"], 0, _REPEATED), + (["SELECT m.k, m FROM nosuch_tbl WHERE pk = 'a' AND sk = 'b'"], 0, + _overlap("[m]", "[m, k]")), + (["SELECT * FROM {t} WHERE pk = '{s}' AND sk = 'b'", + "SELECT a, a FROM {t} WHERE pk = '{s2}' AND sk = 'b'"], 1, _REPEATED), + (["INSERT INTO {t} VALUE {{'pk':'{n}','sk':'b','a':1}}", + "SELECT a, a FROM {t} WHERE pk = '{s2}' AND sk = 'b'"], 1, _REPEATED), + (["SELECT a, a FROM {t} WHERE pk = '{s}' AND sk = 'b'", "SELECT * FROM"], 0, _REPEATED), + (["SELECT m['k'], m.k FROM {t} WHERE pk = '{s}' AND sk = 'b'"], 0, + "Duplicate identifiers in select clause: k"), + (["SELECT m['k'], m FROM {t} WHERE pk = '{s}' AND sk = 'b'"], 0, + _overlap("[m]", "[m, k]")), + ], + ids=[ + "repeated", + "repeated-leaf", + "repeated-quoted", + "overlap", + "overlap-reversed", + "overlap-list", + "ahead-of-a-missing-table", + "ahead-of-a-partial-key", + "ahead-of-an-index", + "overlap-ahead-of-a-missing-table", + "second-member", + "ahead-of-the-mix", + "ahead-of-a-later-syntax-error", + "repeated-member-in-brackets", + "overlap-member-in-brackets", + ], +) +def test_a_projection_refusal_names_its_member(dynamodb_client, gtable, templates, index, message): + with pytest.raises(ClientError) as excinfo: + _txn(dynamodb_client, _members(gtable, templates, s="seed", s2="seed2", l="lst", + n=f"tq-{uuid.uuid4().hex[:6]}")) + assert _error(excinfo) == ( + "ValidationException", + f"Validation failed in TransactStatements[{index}]: {message}", + ) + + +# --- which refusal a transaction with several faults reports ----------------- +# The service checks each member in order and stops at the first member that +# fails: its statement, its parameters, its form against its table (an index +# read, a read without the full key, a key-only EXISTS, an EXISTS on an index +# ahead of a missing table, a RETURNING clause), then whether it is a read +# beside a write. A missing table and a key value of the wrong type cancel the +# request once every member passes (measured 2026-11-05). + +_MIXED = ( + "ExecuteTransaction API does not support both read and write operations in the same " + "request." +) +_SEED_READ = "SELECT * FROM {t} WHERE pk = '{s}' AND sk = 'b'" +_SYNTAX = "SELECT * FROM" +_INDEX_READ = "SELECT * FROM {t}.\"gsi\" WHERE g = 'a' AND gs = '1'" +_PARTIAL_READ = "SELECT * FROM {t} WHERE pk = '{s}'" +_KEY_ONLY_EXISTS = "EXISTS(SELECT * FROM {t} WHERE pk = '{s}' AND sk = 'b')" +_MISSING_READ = "SELECT * FROM nosuch_tbl WHERE pk = 'a' AND sk = 'b'" +_INSERT = "INSERT INTO {t} VALUE {{'pk':'{n}','sk':'b','a':1}}" +_UNBOUND_INSERT = "INSERT INTO {t} VALUE {{'pk':'{n}','sk':?}}" +_RETURNING_UPDATE = "UPDATE {t} SET x = 1 WHERE pk = '{r}' AND sk = 'b' RETURNING ALL NEW *" + + +def _positional(index, message): + return f"Validation failed in TransactStatements[{index}]: {message}" + + +_RETURNING = "RETURNING clause is not supported in ExecuteTransaction." +_INDEX = "Reads on indices are not supported within transactions." +_PARTIAL_KEY = ( + "Select statements within ExecuteTransaction must specify the primary key in the where " + "clause." +) +_EXISTS_FORM = "EXISTS() must contain a single item read with additional condition" +_EXISTS_INDEX = "Operations on indices are not supported in transactions" +_UNBOUND = "Number of parameters in request and statement don't match." +_UNEXPECTED_TERM = "Statement wasn't well formed, can't be processed: Unexpected term" +_BLANK = " " +_INTO_KEY_EXISTS = "EXISTS(SELECT * FROM {t} WHERE pk = '{s}' AND sk = 'b' AND pk.c = 1)" +_INTO_KEY = ( + "Key attributes must be scalars; list random access '[]' and map lookup '.' are not " + "allowed: Key pk" +) + + +@pytest.mark.parametrize( + "templates, message", + [ + ([_SEED_READ, _INSERT, _SYNTAX], _MIXED), + ([_RETURNING_UPDATE, _SYNTAX], _positional(0, _RETURNING)), + ([_INDEX_READ, _INSERT], _positional(0, _INDEX)), + ([_PARTIAL_READ, _MISSING_READ], _positional(0, _PARTIAL_KEY)), + ([_KEY_ONLY_EXISTS, _SEED_READ], _positional(0, _EXISTS_FORM)), + (["EXISTS(SELECT * FROM \"nosuch_tbl\".\"idx\" WHERE pk = 'a' AND x = 1)"], + _positional(0, _EXISTS_INDEX)), + ([_SEED_READ, _UNBOUND_INSERT], _positional(1, _UNBOUND)), + ([_MISSING_READ, _PARTIAL_READ], _positional(1, _PARTIAL_KEY)), + ([_MISSING_READ, _INDEX_READ], _positional(1, _INDEX)), + ([_SEED_READ, _RETURNING_UPDATE], _positional(1, _RETURNING)), + ([_INSERT, _INDEX_READ], _positional(1, _INDEX)), + ([_INSERT, _PARTIAL_READ], _positional(1, _PARTIAL_KEY)), + ([_SEED_READ, _KEY_ONLY_EXISTS], _positional(1, _EXISTS_FORM)), + ([_SEED_READ, + "EXISTS(SELECT * FROM {t}.\"gsi\" WHERE g = 'a' AND gs = '1' AND x = 0)"], + _positional(1, _EXISTS_INDEX)), + ([_SEED_READ, "UPDATE {t} SET a = ? WHERE pk = '{n}' AND sk = 'b'"], + _positional(1, _UNBOUND)), + ([_SEED_READ, _INDEX_READ, _INSERT], _positional(1, _INDEX)), + ([_UNBOUND_INSERT, _RETURNING_UPDATE], _positional(0, _UNBOUND)), + ([_KEY_ONLY_EXISTS, _UNBOUND_INSERT], _positional(0, _EXISTS_FORM)), + (["INSERT INTO nosuch_tbl VALUE {{'pk':'a','sk':'b'}}", _RETURNING_UPDATE], + _positional(1, _RETURNING)), + (["UPDATE {t} SET x = 1 WHERE pk = '{n}'", _RETURNING_UPDATE], + _positional(1, _RETURNING)), + (["INSERT INTO nosuch_tbl VALUE {{'pk':'a','sk':'b'}}", _KEY_ONLY_EXISTS], + _positional(1, _EXISTS_FORM)), + (["SELECT * FROM {t} WHERE pk = '{s}' AND sk = 1", _PARTIAL_READ], + _positional(1, _PARTIAL_KEY)), + ([_MISSING_READ, _SYNTAX], _positional(1, _UNEXPECTED_TERM)), + ([_SEED_READ, "INSERT INTO nosuch_tbl VALUE {{'pk':'a','sk':'b'}}"], _MIXED), + ([_INSERT, _MISSING_READ], _MIXED), + (["UPDATE nosuch_tbl SET x = 1 WHERE pk = 'a' AND sk = 'b' RETURNING ALL NEW *", + _SEED_READ], _MIXED), + ([_PARTIAL_READ, _BLANK], _positional(0, _PARTIAL_KEY)), + ([_RETURNING_UPDATE, _BLANK], _positional(0, _RETURNING)), + ([_SEED_READ, _INSERT, _BLANK], _MIXED), + ([_INDEX_READ, _BLANK], _positional(0, _INDEX)), + ([_INTO_KEY_EXISTS, _BLANK], _positional(0, _INTO_KEY)), + ([_INTO_KEY_EXISTS, _SYNTAX], _positional(0, _INTO_KEY)), + ([_RETURNING_UPDATE, "INSERT INTO {t} VALUE {{'pk':'{n}','sk':'b','x':-0.0}}"], + _positional(0, _RETURNING)), + (["UPDATE {t} SET x = 1 WHERE pk = '{r}' AND sk = 'b' AND pk.c = 1", _PARTIAL_READ], + _positional(1, _PARTIAL_KEY)), + ], + ids=[ + "read-write-then-syntax", + "returning-then-syntax", + "index-read-then-write", + "partial-key-then-missing-table", + "key-only-exists-then-read", + "exists-on-an-index-of-a-missing-table", + "read-then-unbound-insert", + "missing-table-then-partial-key", + "missing-table-then-index-read", + "read-then-returning", + "write-then-index-read", + "write-then-partial-key", + "read-then-key-only-exists", + "read-then-exists-on-an-index", + "read-then-unbound-set", + "read-then-index-read-then-write", + "unbound-insert-then-returning", + "key-only-exists-then-unbound-insert", + "write-on-a-missing-table-then-returning", + "partial-key-write-then-returning", + "write-on-a-missing-table-then-key-only-exists", + "key-type-then-partial-key", + "missing-table-then-syntax", + "read-then-write-on-a-missing-table", + "write-then-read-of-a-missing-table", + "returning-on-a-missing-table-then-read", + "partial-key-then-blank", + "returning-then-blank", + "read-write-then-blank", + "index-read-then-blank", + "exists-into-a-key-then-blank", + "exists-into-a-key-then-syntax", + "returning-then-negative-zero", + "write-into-a-key-then-partial-key", + ], +) +def test_the_first_member_that_fails_is_reported(dynamodb_client, gtable, templates, message): + run = uuid.uuid4().hex[:6] + with pytest.raises(ClientError) as excinfo: + _txn(dynamodb_client, _members(gtable, templates, s="seed", n=f"tq-{run}", + r=f"tq-r-{run}")) + assert _error(excinfo) == ("ValidationException", message) + + +_KEY_TYPE_READ = "SELECT * FROM {t} WHERE pk = '{s}' AND sk = 1" + + +@pytest.mark.parametrize( + "templates, reasons", + [ + ([_MISSING_READ, _KEY_TYPE_READ], [_NOT_FOUND, _KEY_MISMATCH]), + ([_KEY_TYPE_READ, _MISSING_READ], [_KEY_MISMATCH, _NOT_FOUND]), + ([_MISSING_READ, "SELECT * FROM nosuch_tbl2 WHERE pk = 'a' AND sk = 'b'", _SEED_READ], + [_NOT_FOUND, _NOT_FOUND, _NONE]), + (["SELECT * FROM \"nosuch_tbl\".\"idx\" WHERE pk = 'a'"], [_NOT_FOUND]), + ], + ids=["missing-table-then-key-type", "key-type-then-missing-table", "two-missing-tables", + "index-read-of-a-missing-table"], +) +def test_a_read_cancellation_gives_every_member_its_reason( + dynamodb_client, gtable, templates, reasons +): + with pytest.raises(ClientError) as excinfo: + _txn(dynamodb_client, _members(gtable, templates, s="seed")) + err = excinfo.value.response + assert err["Error"]["Code"] == "TransactionCanceledException" + assert err["CancellationReasons"] == reasons + + +# --- an index key of the wrong type ----------------------------------------------- +# A write member that gives an index key a value of the wrong type, in an +# UPDATE's SET or an INSERT's item, has that refusal as its reason, beside +# the reasons of the other members: each such member has it, a member on a +# missing table or without the full key has its own, and no condition is +# evaluated. A member without the full key has that reason instead. An empty +# value set on an index key by an UPDATE is no reason (measured 2026-11-05). + +_INDEX_KEY = _invalid("Type mismatch for Index Key g Expected: S Actual: N IndexName: gsi") +_SET_INDEX_KEY = "UPDATE {t} SET g = 1 WHERE pk = '{k}' AND sk = 'b'" +_INSERT_INDEX_KEY = "INSERT INTO {t} VALUE {{'pk':'{n}','sk':'b','g':1}}" + + +@pytest.mark.parametrize( + "templates, reasons", + [ + ([_SET_INDEX_KEY], [_INDEX_KEY]), + ([_INSERT_INDEX_KEY], [_INDEX_KEY]), + ([_SET_INDEX_KEY, "UPDATE {t} SET g = 1 WHERE pk = '{j}' AND sk = 'b'"], + [_INDEX_KEY, _INDEX_KEY]), + ([_SET_INDEX_KEY, _INSERT_INDEX_KEY], [_INDEX_KEY, _INDEX_KEY]), + ([_INSERT_INDEX_KEY, _SET_INDEX_KEY], [_INDEX_KEY, _INDEX_KEY]), + ([_SET_INDEX_KEY, "UPDATE {t} SET x = 1 WHERE pk = '{j}'"], [_INDEX_KEY, _KEY_MISMATCH]), + (["UPDATE {t} SET x = 1 WHERE pk = '{j}'", _SET_INDEX_KEY], [_KEY_MISMATCH, _INDEX_KEY]), + ([_SET_INDEX_KEY, "UPDATE {t} SET pk = 'z' WHERE pk = '{j}' AND sk = 'b'"], + [_INDEX_KEY, _invalid("Cannot update attribute pk. This attribute is part of the key")]), + ([_SET_INDEX_KEY, "UPDATE {t} SET x = 1 WHERE pk = '{j}' AND sk = 'b' AND a = 99"], + [_INDEX_KEY, _NONE]), + ([_SET_INDEX_KEY, "UPDATE nosuch_tbl SET x = 1 WHERE pk = 'a' AND sk = 'b'"], + [_INDEX_KEY, _NOT_FOUND]), + (["UPDATE {t} SET g = 1 WHERE pk = '{k}'"], [_KEY_MISMATCH]), + (["UPDATE {t} SET g = 1 WHERE pk = '{k}' AND sk = 1"], [_KEY_MISMATCH]), + ([_SET_INDEX_KEY, "UPDATE {t} SET g = '' WHERE pk = '{j}' AND sk = 'b'"], + [_INDEX_KEY, _NONE]), + ], + ids=[ + "update", + "insert", + "two-updates", + "update-then-insert", + "insert-then-update", + "update-then-partial-key", + "partial-key-then-update", + "update-then-set-of-a-key", + "update-then-failed-condition", + "update-then-missing-table", + "partial-key-in-the-member", + "key-type-in-the-member", + "update-then-empty-index-key", + ], +) +def test_each_member_that_gives_an_index_key_the_wrong_type_has_the_reason( + dynamodb_client, gtable, templates, reasons +): + run = uuid.uuid4().hex[:6] + first = put(dynamodb_client, gtable, f"ik-{run}-k") + second = put(dynamodb_client, gtable, f"ik-{run}-j") + absent = f"{first}-absent" + statements = [ + {"Statement": t.format(t=f'"{gtable}"', k=first, j=second, n=absent)} for t in templates + ] + with pytest.raises(ClientError) as excinfo: + _txn(dynamodb_client, statements) + err = excinfo.value.response + codes = ", ".join(reason["Code"] for reason in reasons) + assert err["Error"] == { + "Code": "TransactionCanceledException", + "Message": ( + "Transaction cancelled, please refer cancellation reasons for specific reasons " + f"[{codes}]" + ), + } + assert err["CancellationReasons"] == reasons + assert _gget(dynamodb_client, gtable, first) == item(first) + assert _gget(dynamodb_client, gtable, second) == item(second) + assert _gget(dynamodb_client, gtable, absent) is None + + +# --- ConsumedCapacity of an index ------------------------------------------------ +# A write transaction charges each write to an index at the units an +# UpdateItem charges it, beside the doubled units of the table (measured +# 2026-11-05). ExtendDB's TransactWriteItems has no old or new item of an +# Update or Delete member to rate an index write from, so it reports the +# table alone, for PartiQL and for the classic operation. + +TRANSACTION_INDEX_UNITS = pytest.mark.xfail( + bool(os.environ.get("EXTENDDB_TEST_ENDPOINT", "").strip()), + reason="TransactWriteItems reports no index units", + raises=AssertionError, + strict=True, +) + + +@TRANSACTION_INDEX_UNITS +def test_a_transaction_reports_its_index_units(dynamodb_client, gtable): + put(dynamodb_client, gtable, "ct17") + resp = _txn( + dynamodb_client, + _members(gtable, ["UPDATE {t} SET g = 'x' WHERE pk = '{k}' AND sk = 'b'"], k="ct17"), + ReturnConsumedCapacity="INDEXES", + ) + assert resp["ConsumedCapacity"] == [ + { + "TableName": gtable, + "CapacityUnits": 4.0, + "WriteCapacityUnits": 4.0, + "Table": {"CapacityUnits": 2.0, "WriteCapacityUnits": 2.0}, + "GlobalSecondaryIndexes": {"gsi": {"CapacityUnits": 2.0, "WriteCapacityUnits": 2.0}}, + } + ] + + +@TRANSACTION_INDEX_UNITS +def test_a_transaction_total_counts_its_index_units(dynamodb_client, gtable): + put(dynamodb_client, gtable, "twr") + put(dynamodb_client, gtable, "tws") + resp = _txn( + dynamodb_client, + _members( + gtable, + [ + "UPDATE {t} SET a = 9 WHERE pk = '{r}' AND sk = 'b'", + "UPDATE {t} SET a = 9 WHERE pk = '{s}' AND sk = 'b'", + ], + r="twr", + s="tws", + ), + ReturnConsumedCapacity="TOTAL", + ) + assert resp["ConsumedCapacity"] == [ + {"TableName": gtable, "CapacityUnits": 6.0, "WriteCapacityUnits": 6.0} + ] diff --git a/tests/test_partiql_update_paths.py b/tests/test_partiql_update_paths.py new file mode 100644 index 000000000..a39fa95cd --- /dev/null +++ b/tests/test_partiql_update_paths.py @@ -0,0 +1,190 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""Dual-target wire tests for the document paths a PartiQL UPDATE writes. + +An UPDATE whose SET, REMOVE, set_add and set_delete targets name one path +twice, a path and a path inside it, or one element both as a map member and +as a list element, is refused before anything is written, with or without +RETURNING. set_add and set_delete apply through a map path and through a list +element. + +Runs against real DynamoDB (EXTENDDB_TEST_ENDPOINT unset, AWS credentials) and +against an extenddb server (EXTENDDB_TEST_ENDPOINT set) with identical +assertions. Every expected value was measured on the service (2026-11-02). The +service returns the members of a set in no fixed order, so sets compare as +sets. Tables use the ``extenddb-measure-pqp-`` prefix and are deleted at the +end. +""" + +from __future__ import annotations + +import os +import uuid + +import pytest +from botocore.exceptions import ClientError + +from conftest import wait_for_active + +OVERLAP = ( + "Two document paths overlap with each other; must remove or rewrite one of these " + "paths; path one: {}, path two: {}" +) +CONFLICT = ( + "Two document paths conflict with each other; must remove or rewrite one of these " + "paths; path one: {}, path two: {}" +) + +# ExtendDB translates set_add and set_delete into the ADD and DELETE update +# actions, which it applies only to an attribute in a map. The service applies +# set_add to a list element (measured 2026-11-02). +SET_FUNCTION_ON_A_LIST_ELEMENT = pytest.mark.xfail( + bool(os.environ.get("EXTENDDB_TEST_ENDPOINT", "").strip()), + reason="ExtendDB applies ADD and DELETE only to an attribute in a map", + raises=ClientError, + strict=True, +) + + +@pytest.fixture(scope="module") +def table(dynamodb_client): + name = f"extenddb-measure-pqp-{uuid.uuid4().hex[:12]}" + dynamodb_client.create_table( + TableName=name, + BillingMode="PAY_PER_REQUEST", + AttributeDefinitions=[{"AttributeName": "pk", "AttributeType": "S"}], + KeySchema=[{"AttributeName": "pk", "KeyType": "HASH"}], + ) + try: + wait_for_active(dynamodb_client, name) + yield name + finally: + try: + dynamodb_client.delete_table(TableName=name) + except ClientError: + pass + + +def stored_item(client, table: str, key: str) -> dict | None: + return client.get_item(TableName=table, Key={"pk": {"S": key}}, ConsistentRead=True).get( + "Item" + ) + + +def as_sets(value): + """``value`` with every string set held as a Python set, at any depth.""" + if isinstance(value, dict): + if "SS" in value: + return ("SS", frozenset(value["SS"])) + return {name: as_sets(member) for name, member in value.items()} + if isinstance(value, list): + return [as_sets(member) for member in value] + return value + + +@pytest.mark.parametrize( + ("assignments", "one", "two"), + [ + ("SET q = 1, q = 2", "[q]", "[q]"), + ("SET q = 1 REMOVE q", "[q]", "[q]"), + ("SET m = {'a': 1}, m.a = 2", "[m, a]", "[m]"), + ("SET ss = set_add(ss, <<'b'>>), ss = set_add(ss, <<'c'>>)", "[ss]", "[ss]"), + ("SET ss = 'plain', ss = set_add(ss, <<'b'>>)", "[ss]", "[ss]"), + ("SET q = 1, q = 2 RETURNING MODIFIED NEW *", "[q]", "[q]"), + ], + ids=[ + "set-twice", + "set-then-remove", + "parent-and-child", + "set-add-twice", + "set-beside-set-add", + "returning-modified", + ], +) +def test_overlapping_paths_are_refused_before_anything_is_written( + dynamodb_client, table, assignments, one, two +): + key = f"o-{uuid.uuid4().hex[:8]}" + dynamodb_client.execute_statement( + Statement=f"INSERT INTO \"{table}\" VALUE {{'pk': '{key}', 'ss': <<'a'>>, 'm': {{'a': 0}}}}" + ) + head, _, tail = assignments.partition(" RETURNING ") + statement = f"UPDATE \"{table}\" {head} WHERE pk = '{key}'" + if tail: + statement += f" RETURNING {tail}" + with pytest.raises(ClientError) as excinfo: + dynamodb_client.execute_statement(Statement=statement) + err = excinfo.value.response["Error"] + assert (err["Code"], err["Message"]) == ("ValidationException", OVERLAP.format(one, two)) + assert stored_item(dynamodb_client, table, key) == { + "pk": {"S": key}, + "ss": {"SS": ["a"]}, + "m": {"M": {"a": {"N": "0"}}}, + } + + +@pytest.mark.parametrize( + "stored", + [{"M": {"b": {"N": "0"}}}, {"L": [{"N": "0"}]}, None, "missing"], + ids=["map", "list", "absent", "missing-item"], +) +@pytest.mark.parametrize( + ("assignments", "one", "two"), + [ + ("SET a.b = 1, a[0] = 2", "[a, [0]]", "[a, b]"), + ("SET a[0] = 2, a.b = 1", "[a, b]", "[a, [0]]"), + ], + ids=["member-then-element", "element-then-member"], +) +def test_one_element_named_as_a_member_and_an_element_conflicts( + dynamodb_client, table, stored, assignments, one, two +): + key = f"c-{uuid.uuid4().hex[:8]}" + item = {"pk": {"S": key}} + if isinstance(stored, dict): + item["a"] = stored + if stored != "missing": + dynamodb_client.put_item(TableName=table, Item=item) + with pytest.raises(ClientError) as excinfo: + dynamodb_client.execute_statement( + Statement=f"UPDATE \"{table}\" {assignments} WHERE pk = '{key}'" + ) + err = excinfo.value.response["Error"] + assert (err["Code"], err["Message"]) == ("ValidationException", CONFLICT.format(one, two)) + assert stored_item(dynamodb_client, table, key) == (None if stored == "missing" else item) + + +def insert_nested(client, table: str) -> str: + key = f"n-{uuid.uuid4().hex[:8]}" + client.execute_statement( + Statement=f"INSERT INTO \"{table}\" VALUE {{'pk': '{key}', 'm': {{'ss': <<'a'>>}}, " + "'l': [<<'a'>>]}" + ) + return key + + +def update(client, table: str, key: str, assignment: str) -> list: + statement = f"UPDATE \"{table}\" {assignment} WHERE pk = '{key}' RETURNING ALL NEW *" + return client.execute_statement(Statement=statement)["Items"] + + +def test_set_add_and_set_delete_apply_through_a_map_path(dynamodb_client, table): + key = insert_nested(dynamodb_client, table) + added = update(dynamodb_client, table, key, "SET m.ss = set_add(m.ss, <<'b'>>)") + assert as_sets(added) == as_sets( + [{"pk": {"S": key}, "m": {"M": {"ss": {"SS": ["a", "b"]}}}, "l": {"L": [{"SS": ["a"]}]}}] + ) + deleted = update(dynamodb_client, table, key, "SET m.ss = set_delete(m.ss, <<'a'>>)") + assert as_sets(deleted) == as_sets( + [{"pk": {"S": key}, "m": {"M": {"ss": {"SS": ["b"]}}}, "l": {"L": [{"SS": ["a"]}]}}] + ) + + +@SET_FUNCTION_ON_A_LIST_ELEMENT +def test_set_add_applies_to_a_list_element(dynamodb_client, table): + key = insert_nested(dynamodb_client, table) + added = update(dynamodb_client, table, key, "SET l[0] = set_add(l[0], <<'b'>>)") + assert as_sets(added) == as_sets( + [{"pk": {"S": key}, "m": {"M": {"ss": {"SS": ["a"]}}}, "l": {"L": [{"SS": ["a", "b"]}]}}] + ) diff --git a/tests/test_partiql_visit_order.py b/tests/test_partiql_visit_order.py new file mode 100644 index 000000000..4c2d55afc --- /dev/null +++ b/tests/test_partiql_visit_order.py @@ -0,0 +1,427 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""Dual-target wire tests for the order in which the pages of a PartiQL +SELECT read by key visit its disjuncts. + +Number the disjuncts 0 to N-1 in read order (key order, or the order an +ORDER BY gives), the ones held back for a refusal included. The first page +takes them in that order. Each later page takes the list the page before it +left and visits it by position modulo a bucket count, in list order within +one bucket; the bucket count is 16, doubled while the list holds more than +three quarters of it. A page visits at most 10 disjuncts and stops when its +Limit is spent. The list it leaves holds the disjuncts it visited and did +not read to their end, in the order it visited them (a disjunct held back +for a refusal, which takes a visit on every page that reaches it, and a +Query the page stopped inside, which goes on from where it stopped), then +the ones it did not visit. The pages return their items in visit order, and +a page with Limit items has the key of its last item as its +LastEvaluatedKey. A request whose page visits only disjuncts held back gets +their refusal instead of a page, also when other disjuncts are left. + +The tests write each page as the positions of the items it returns, in +order, and the hash key size refusal that ends a read as "R". + +Dual-target: EXTENDDB_TEST_ENDPOINT selects an extenddb server; unset, the +tests run against the live service with the default credential chain. The +assertions are identical on both. Every expected value was measured on the +service (2026-11-03). +""" + +from __future__ import annotations + +import time + +import pytest +from botocore.exceptions import ClientError + +from conftest import scoped_table + +HASH_SIZE = ( + "One or more parameter values were invalid: Size of hashkey has exceeded the maximum " + "size limit of2048 bytes" +) +FILTER_KEY = ( + "Filter Expression can only contain non-primary key attributes: Primary key attribute: sk" +) + + +def _values(n, prefix="v"): + """The `n` string values `000` to ``.""" + return [f"{prefix}{i:03}" for i in range(n)] + + +def _listed(values): + return ", ".join(f"'{value}'" for value in values) + + +def _expected(*pages): + """Pages written as positions, `a..b` for the positions a to b, and "R" + for the refusal.""" + out = [] + for page in pages: + if page == "R": + out.append("R") + continue + positions = [] + for token in page.split(): + low, _, high = token.partition("..") + positions.extend(range(int(low), int(high or low) + 1)) + out.append(positions) + return out + + +def _number(attribute): + """The position of an item whose `attribute` ends in three digits.""" + return lambda item: int(item[attribute]["S"][-3:]) + + +def _read(client, statement, position, **kwargs): + """Every page of `statement`, following NextToken: the positions of its + items in the order it returns them, and its LastEvaluatedKey (None when it + has none). The hash key size refusal ends the paging as "R".""" + pages, keys, token = [], [], None + for _ in range(40): + args = dict(kwargs) + if token: + args["NextToken"] = token + try: + response = client.execute_statement(Statement=statement, **args) + except ClientError as e: + assert e.response["Error"]["Code"] == "ValidationException", e.response + message = e.response["Error"]["Message"] + pages.append("R" if message == HASH_SIZE else message) + break + pages.append([position(item) for item in response["Items"]]) + keys.append(response.get("LastEvaluatedKey")) + token = response.get("NextToken") + if not token: + break + return pages, keys + + +def _wait_for_index(client, table, count): + """Wait until the index gix shows every item written (it fills + asynchronously on the service).""" + for _ in range(240): + scan = client.scan(TableName=table, IndexName="gix", Select="COUNT") + if scan["Count"] == count: + return + time.sleep(0.5) + raise TimeoutError(f"gix of {table} did not reach {count} items") + + +def _keyed_table(client): + """A table keyed on string pk and sk.""" + return scoped_table( + client, + attribute_definitions=[ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "sk", "AttributeType": "S"}, + ], + key_schema=[ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"}, + ], + ) + + +def _indexed_table(client, *sort): + """A table keyed on a string pk with a GSI gix keyed on a string g and, + when `sort` names it, a string sort key.""" + schema = [{"AttributeName": "g", "KeyType": "HASH"}] + schema += [{"AttributeName": name, "KeyType": "RANGE"} for name in sort] + return scoped_table( + client, + attribute_definitions=[ + {"AttributeName": name, "AttributeType": "S"} for name in ("pk", "g", *sort) + ], + GlobalSecondaryIndexes=[ + {"IndexName": "gix", "KeySchema": schema, "Projection": {"ProjectionType": "ALL"}} + ], + ) + + +@pytest.fixture(scope="module") +def htable(dynamodb_client): + """A table keyed on a string pk holding v000 to v049.""" + with scoped_table(dynamodb_client) as name: + for value in _values(50): + dynamodb_client.put_item(TableName=name, Item={"pk": {"S": value}}) + yield name + + +@pytest.fixture(scope="module") +def ctable(dynamodb_client): + """A table keyed on pk and sk holding the partitions v000 to v039 of one + item each (sk s) and the partition p of the items s000 to s039.""" + with _keyed_table(dynamodb_client) as name: + for value in _values(40): + dynamodb_client.put_item(TableName=name, Item={"pk": {"S": value}, "sk": {"S": "s"}}) + for value in _values(40, "s"): + dynamodb_client.put_item(TableName=name, Item={"pk": {"S": "p"}, "sk": {"S": value}}) + yield name + + +@pytest.fixture(scope="module") +def gtable(dynamodb_client): + """A table with a GSI gix keyed on g, holding p000 to p039 with g v000 to + v039.""" + with _indexed_table(dynamodb_client) as name: + for i, value in enumerate(_values(40)): + dynamodb_client.put_item( + TableName=name, Item={"pk": {"S": f"p{i:03}"}, "g": {"S": value}} + ) + _wait_for_index(dynamodb_client, name, 40) + yield name + + +@pytest.fixture(scope="module") +def g2table(dynamodb_client): + """A table with a GSI gix keyed on g and gs, holding q000 to q039 in index + partition a with gs w000 to w039.""" + with _indexed_table(dynamodb_client, "gs") as name: + for i, value in enumerate(_values(40, "w")): + dynamodb_client.put_item( + TableName=name, + Item={"pk": {"S": f"q{i:03}"}, "g": {"S": "a"}, "gs": {"S": value}}, + ) + _wait_for_index(dynamodb_client, name, 40) + yield name + + +@pytest.fixture(scope="module") +def qtable(dynamodb_client): + """A table keyed on pk and sk holding the partitions p00 to p18 of one + item each (sk s), except p10, which holds the items s00 to s20.""" + with _keyed_table(dynamodb_client) as name: + for i in range(19): + sorts = [f"s{j:02}" for j in range(21)] if i == 10 else ["s"] + for sort in sorts: + dynamodb_client.put_item( + TableName=name, Item={"pk": {"S": f"p{i:02}"}, "sk": {"S": sort}} + ) + yield name + + +# Pages of point reads, as measured on N disjuncts. +POINT_READS = { + 17: _expected("0..9", "16 10..15"), + 21: _expected("0..9", "16..20 10..14", "15"), + 22: _expected("0..9", "16..21 10..13", "14 15"), + 23: _expected("0..9", "10..19", "20..22"), + 33: _expected("0..9", "32 10..18", "19..28", "29..31"), + 37: _expected("0..9", "10..19", "32..36 20..24", "25..31"), + 41: _expected("0..9", "10..19", "32..40 20", "21..30", "31"), + 50: _expected("0..9", "10..19", "20..29", "32..41", "48 49 42..46 30 47 31"), +} + + +@pytest.mark.parametrize("count", sorted(POINT_READS)) +def test_later_pages_visit_the_disjuncts_left_by_position_modulo_the_buckets( + dynamodb_client, htable, count +): + statement = f'SELECT * FROM "{htable}" WHERE pk IN [{_listed(_values(count))}]' + pages, _ = _read(dynamodb_client, statement, _number("pk")) + assert pages == POINT_READS[count] + + +@pytest.mark.parametrize( + ("count", "limit", "pages"), + [ + (21, 3, _expected("0..2", "3..5", "6..8", "16..18", "19 20 9", "10..12", "13..15")), + ( + 33, + 4, + _expected( + "0..3", "4..7", "8..11", "32 12..14", "15..18", "19..22", "23..26", "27..30", "31" + ), + ), + ], +) +def test_pages_with_a_limit_visit_in_the_same_order( + dynamodb_client, htable, count, limit, pages +): + statement = f'SELECT * FROM "{htable}" WHERE pk IN [{_listed(_values(count))}]' + got, _ = _read(dynamodb_client, statement, _number("pk"), Limit=limit) + assert got == pages + + +@pytest.mark.parametrize( + ("count", "limit", "pages", "last_keys"), + [ + (21, 10, _expected("0..9", "16..20 10..14", "15"), ["v009", "v014", None]), + ( + 33, + 7, + _expected("0..6", "7..13", "32 14..19", "20..26", "27..31"), + ["v006", "v013", "v019", "v026", None], + ), + ], +) +def test_the_last_key_of_a_full_page_is_its_last_item_in_visit_order( + dynamodb_client, htable, count, limit, pages, last_keys +): + statement = f'SELECT * FROM "{htable}" WHERE pk IN [{_listed(_values(count))}]' + got, keys = _read(dynamodb_client, statement, _number("pk"), Limit=limit) + assert got == pages + assert keys == [None if key is None else {"pk": {"S": key}} for key in last_keys] + + +def test_an_order_by_numbers_the_disjuncts_in_its_order(dynamodb_client, htable): + statement = ( + f'SELECT * FROM "{htable}" WHERE pk IN [{_listed(_values(33))}] ORDER BY pk DESC' + ) + pages, _ = _read(dynamodb_client, statement, lambda item: 32 - _number("pk")(item)) + assert pages == POINT_READS[33] + + +@pytest.mark.parametrize("count", [21, 33]) +def test_an_index_read_visits_in_the_same_order(dynamodb_client, gtable, count): + statement = f'SELECT * FROM "{gtable}"."gix" WHERE g IN [{_listed(_values(count))}]' + pages, _ = _read(dynamodb_client, statement, _number("g")) + assert pages == POINT_READS[count] + + +@pytest.mark.parametrize("count", [21, 33]) +def test_queries_of_partitions_visit_in_the_same_order(dynamodb_client, ctable, count): + statement = f'SELECT * FROM "{ctable}" WHERE pk IN [{_listed(_values(count))}]' + pages, _ = _read(dynamodb_client, statement, _number("pk")) + assert pages == POINT_READS[count] + + +@pytest.mark.parametrize("count", [21, 33]) +def test_sort_key_values_of_one_partition_visit_in_the_same_order( + dynamodb_client, ctable, count +): + statement = ( + f"SELECT * FROM \"{ctable}\" WHERE pk = 'p' AND sk IN [{_listed(_values(count, 's'))}]" + ) + pages, _ = _read(dynamodb_client, statement, _number("sk")) + assert pages == POINT_READS[count] + + +@pytest.mark.parametrize("count", [21, 33]) +def test_sort_key_values_of_one_index_partition_visit_in_the_same_order( + dynamodb_client, g2table, count +): + values = _listed(_values(count, "w")) + statement = f"SELECT * FROM \"{g2table}\".\"gix\" WHERE g = 'a' AND gs IN [{values}]" + pages, _ = _read(dynamodb_client, statement, _number("gs")) + assert pages == POINT_READS[count] + + +def _with_oversized(values, oversized): + """The statement on `values` and the values past the key size limit + `oversized`, and the position of each value in key order.""" + ordered = sorted(values + oversized) + return _listed(values + oversized), {value: i for i, value in enumerate(ordered)} + + +@pytest.mark.parametrize( + ("oversized", "pages"), + [ + ("a" * 2049, _expected("1..9", "32 10..17", "18..26", "27..31", "R")), + ("v004" + "z" * 2049, _expected("0..4 6..9", "32 10..17", "18..26", "27..31", "R")), + ("v011" + "z" * 2049, _expected("0..9", "32 10 11 13..18", "19..27", "28..31", "R")), + ("v019" + "z" * 2049, _expected("0..9", "32 10..18", "19 21..28", "29..31", "R")), + ("v031" + "z" * 2049, _expected("0..9", "10..18", "19..27", "28..31", "R")), + ], + ids=["at-0", "at-5", "at-12", "at-20", "at-32"], +) +def test_a_held_back_value_takes_a_visit_on_every_page_that_reaches_it( + dynamodb_client, htable, oversized, pages +): + listed, position = _with_oversized(_values(32), [oversized]) + statement = f'SELECT * FROM "{htable}" WHERE pk IN [{listed}]' + got, _ = _read(dynamodb_client, statement, lambda item: position[item["pk"]["S"]]) + assert got == pages + + +def test_a_held_back_value_that_sorts_last_is_visited_again(dynamodb_client, htable): + listed, position = _with_oversized(_values(20), ["z" * 2049]) + statement = f'SELECT * FROM "{htable}" WHERE pk IN [{listed}]' + got, _ = _read(dynamodb_client, statement, lambda item: position[item["pk"]["S"]]) + assert got == _expected("0..9", "16..19 10..14", "15", "R") + + +def test_two_held_back_values_with_a_limit(dynamodb_client, htable): + listed, position = _with_oversized( + _values(21), ["v004" + "z" * 2049, "v015" + "z" * 2049] + ) + statement = f'SELECT * FROM "{htable}" WHERE pk IN [{listed}]' + got, _ = _read(dynamodb_client, statement, lambda item: position[item["pk"]["S"]], Limit=4) + assert got == _expected( + "0..3", "4 6 7 8", "9..12", "16 18 19 20", "21 22 13 14", "15", "R" + ) + + +def _partitions(first, last): + """The partitions v to v of ctable, listed.""" + return _listed(f"v{i:03}" for i in range(first, last + 1)) + + +@pytest.mark.parametrize("limit", [None, 3], ids=["no-limit", "limit-3"]) +def test_a_request_whose_page_visits_only_held_back_disjuncts_is_refused( + dynamodb_client, ctable, limit +): + # Positions 0 to 9 are held back for the filter refusal and sort first; + # positions 10 to 20 read partitions. + statement = ( + f'SELECT * FROM "{ctable}" WHERE (pk IN [{_partitions(0, 9)}] AND sk = a) ' + f"OR pk IN [{_partitions(10, 20)}]" + ) + kwargs = {"Limit": limit} if limit else {} + pages, _ = _read(dynamodb_client, statement, _number("pk"), **kwargs) + assert pages == [FILTER_KEY] + + +def test_a_later_request_whose_page_visits_only_held_back_disjuncts_is_refused( + dynamodb_client, ctable +): + # The second page's ten visits go to positions 10 to 19, all held back, + # while 20 to 24 are left unvisited. + statement = ( + f'SELECT * FROM "{ctable}" WHERE pk IN [{_partitions(0, 9)}] ' + f"OR (pk IN [{_partitions(10, 19)}] AND sk = a) OR pk IN [{_partitions(20, 24)}]" + ) + pages, _ = _read(dynamodb_client, statement, _number("pk")) + assert pages == _expected("0..9") + [FILTER_KEY] + + +def test_a_page_of_nine_held_back_visits_and_one_read_is_a_page(dynamodb_client, ctable): + statement = ( + f'SELECT * FROM "{ctable}" WHERE (pk IN [{_partitions(0, 8)}] AND sk = a) ' + f"OR pk IN [{_partitions(9, 20)}]" + ) + pages, _ = _read(dynamodb_client, statement, _number("pk")) + assert pages == _expected(*[str(p) for p in range(9, 18)], "18..20") + [FILTER_KEY] + + +def test_a_query_stopped_at_the_limit_goes_on_at_its_place(dynamodb_client, qtable): + conditions = [ + f"pk = 'p{i:02}'" if i == 10 else f"(pk = 'p{i:02}' AND sk = 's')" for i in range(19) + ] + statement = f'SELECT * FROM "{qtable}" WHERE ' + " OR ".join(conditions) + pages, _ = _read(dynamodb_client, statement, lambda item: int(item["pk"]["S"][1:]), Limit=6) + + def runs(page): + """The page with each run of items of partition p10 written as + 10x.""" + out = [] + for value in page: + if value == 10 and out and out[-1].startswith("10x"): + out[-1] = f"10x{int(out[-1][3:]) + 1}" + else: + out.append("10x1" if value == 10 else str(value)) + return " ".join(out) + + assert [runs(page) for page in pages] == [ + "0 1 2 3 4 5", + "6 7 8 9 10x2", + "16 17 18 10x3", + "10x6", + "10x6", + "10x4 11 12", + "13 14 15", + ] diff --git a/tests/test_partiql_words.py b/tests/test_partiql_words.py new file mode 100644 index 000000000..506de1342 --- /dev/null +++ b/tests/test_partiql_words.py @@ -0,0 +1,193 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""Dual-target wire tests for how a PartiQL statement answers a reserved word: +where a condition starts, as the argument of attribute_exists, as a SET target, +as a projection, as a table, and as a write target. + +The service reads a reserved word where a term starts as the start of a form +of its own when the word begins one (CASE, DATE, a FROM-led form, EXEC, ...), +so the answer is often the error that form meets at the next token. + +Dual-target: EXTENDDB_TEST_ENDPOINT selects an extenddb server; unset, the +tests run against the live service with the default credential chain. The +assertions are identical on both. +""" + +from __future__ import annotations + +import uuid + +import pytest +from botocore.exceptions import ClientError + +from conftest import scoped_table + +WF = "Statement wasn't well formed, can't be processed: " +NOT_FOUND = ("ResourceNotFoundException", "Requested resource not found") + +KEYWORD = ("ValidationException", f"{WF}Unexpected keyword") +OPERATOR = ("ValidationException", f"{WF}Unexpected operator") +TERM = ("ValidationException", f"{WF}Unexpected term") +SIMPLE_PATH = ("ValidationException", f"{WF}Expected identifier for simple path") +MANIPULATION = ("ValidationException", f"{WF}Expected data manipulation") + + +def _wf(reason): + return ("ValidationException", f"{WF}{reason}") + + +def _plain(message): + return ("ValidationException", message) + + +def _in_select_list(word): + return _wf( + f"Expected identifier or an expression but found unexpected keyword '{word}' in a select list." + ) + + +def _answer(client, statement): + try: + client.execute_statement(Statement=statement) + except ClientError as e: + return (e.response["Error"]["Code"], e.response["Error"]["Message"]) + return ("OK", "") + + +@pytest.fixture(scope="module") +def absent(): + """A valid table name no table has.""" + return f"extenddb-absent-{uuid.uuid4().hex[:12]}" + + +@pytest.fixture(scope="module") +def stable(dynamodb_client): + with scoped_table(dynamodb_client) as name: + yield name + + +DATE = _wf("Expected date string followed by the keyword DATE, found SYMBOL") +TIME = _wf("Expected time string followed by the keyword TIME OR TIME WITH TIME ZONE, found SYMBOL") +PATH_NEEDED = _plain( + "Operator or function requires a document path; operator or function: attribute_exists" +) +MISSING = _plain("MISSING must be compared using IS operator") + +# word -> answers to `WHERE = 1`, `WHERE attribute_exists()`, +# `SET = 1` and `SELECT FROM`, on a table that does not exist. +POSITIONS = { + "order": (KEYWORD, KEYWORD, SIMPLE_PATH, KEYWORD), + "inner": (KEYWORD, KEYWORD, SIMPLE_PATH, KEYWORD), + "count": (KEYWORD, KEYWORD, SIMPLE_PATH, KEYWORD), + "insert": (KEYWORD, KEYWORD, SIMPLE_PATH, KEYWORD), + "and": (OPERATOR, OPERATOR, SIMPLE_PATH, OPERATOR), + "like": (OPERATOR, OPERATOR, SIMPLE_PATH, OPERATOR), + "as": (TERM, TERM, SIMPLE_PATH, TERM), + "first": (TERM, TERM, SIMPLE_PATH, TERM), + "not": (OPERATOR, TERM, SIMPLE_PATH, MANIPULATION), + "case": (OPERATOR, TERM, SIMPLE_PATH, MANIPULATION), + "select": (OPERATOR, TERM, SIMPLE_PATH, _in_select_list("select")), + "from": (OPERATOR, TERM, SIMPLE_PATH, _in_select_list("from")), + "distinct": (KEYWORD, KEYWORD, SIMPLE_PATH, _in_select_list("from")), + "value": (KEYWORD, KEYWORD, SIMPLE_PATH, MANIPULATION), + "where": (KEYWORD, KEYWORD, SIMPLE_PATH, _in_select_list("where")), + "delete": ( + _wf("Expected FROM after DELETE"), + _wf("Expected FROM after DELETE"), + SIMPLE_PATH, + _in_select_list("delete"), + ), + "set": (SIMPLE_PATH, SIMPLE_PATH, SIMPLE_PATH, _in_select_list("set")), + "exec": ( + _wf("Keyword = only expected at the top level in the query"), + _wf("Expected RIGHT_PAREN"), + SIMPLE_PATH, + _wf("Expected FROM after SELECT list"), + ), + "date": (DATE, DATE, SIMPLE_PATH, DATE), + "time": (TIME, TIME, SIMPLE_PATH, _wf("Expected WITH keyword")), + "cast": ( + _wf("Missing left parenthesis after cast"), + _wf("Missing left parenthesis after cast"), + SIMPLE_PATH, + _wf("Missing left parenthesis after cast"), + ), + "substring": (_wf("Expected LEFT_PAREN"), _wf("Expected LEFT_PAREN"), SIMPLE_PATH, _wf("Expected LEFT_PAREN")), + "values": ( + _wf("Expected LEFT_PAREN for row value constructor"), + _wf("Expected LEFT_PAREN for row value constructor"), + SIMPLE_PATH, + _wf("Expected LEFT_PAREN for row value constructor"), + ), + "create": ( + _wf("Unexpected token following CREATE"), + _wf("Unexpected token following CREATE"), + SIMPLE_PATH, + _wf("Unexpected token following CREATE"), + ), + "missing": (MISSING, MISSING, SIMPLE_PATH, _plain("Unexpected path component at 1:8:7")), + "null": (NOT_FOUND, PATH_NEEDED, SIMPLE_PATH, _plain("Unexpected path component at 1:8:4")), + "true": (NOT_FOUND, PATH_NEEDED, SIMPLE_PATH, _plain("Unexpected path component at 1:8:4")), + "year": (NOT_FOUND, NOT_FOUND, NOT_FOUND, NOT_FOUND), + "partition": (NOT_FOUND, NOT_FOUND, NOT_FOUND, NOT_FOUND), +} + + +@pytest.mark.parametrize("word", sorted(POSITIONS)) +def test_a_word_in_the_four_positions(dynamodb_client, absent, word): + q = f'"{absent}"' + statements = [ + f"SELECT * FROM {q} WHERE {word} = 1", + f"SELECT * FROM {q} WHERE attribute_exists({word})", + f"UPDATE {q} SET {word} = 1 WHERE pk = 'a'", + f"SELECT {word} FROM {q}", + ] + answers = tuple(_answer(dynamodb_client, s) for s in statements) + assert answers == POSITIONS[word] + + +@pytest.mark.parametrize( + ("word", "expected"), + [ + ("where", KEYWORD), + ("value", KEYWORD), + ("as", TERM), + ("and", OPERATOR), + ("set", SIMPLE_PATH), + ("select", _in_select_list("where")), + ("null", _plain("Unexpected from source")), + ], +) +def test_a_word_where_the_table_of_a_select_starts(dynamodb_client, word, expected): + assert _answer(dynamodb_client, f"SELECT * FROM {word} WHERE pk = 'p'") == expected + + +@pytest.mark.parametrize("word", ["where", "select", "inner", "as", "and", "null", "count"]) +def test_a_word_where_the_table_of_a_delete_starts(dynamodb_client, word): + assert _answer(dynamodb_client, f"DELETE FROM {word} WHERE pk = 'p'") == SIMPLE_PATH + + +@pytest.mark.parametrize( + ("word", "expected"), + [ + ("case", MANIPULATION), + ("exec", _wf("Expected FROM after SELECT list")), + ("time", _wf("Expected WITH keyword")), + ("nulls", TERM), + ], +) +def test_a_projected_word_before_a_where(dynamodb_client, stable, word, expected): + assert _answer(dynamodb_client, f"SELECT {word} FROM \"{stable}\" WHERE pk = 'p'") == expected + + +def test_exec_after_a_condition(dynamodb_client, stable): + statement = f"SELECT * FROM \"{stable}\" WHERE pk = 'p' AND exec = 1" + assert _answer(dynamodb_client, statement) == _wf( + "Keyword = only expected at the top level in the query" + ) + + +def test_order_by_a_reserved_word(dynamodb_client, stable): + statement = f"SELECT * FROM \"{stable}\" WHERE pk = 'p' ORDER BY value" + assert _answer(dynamodb_client, statement) == KEYWORD diff --git a/tests/test_partiql_write.py b/tests/test_partiql_write.py new file mode 100644 index 000000000..0d98d09e8 --- /dev/null +++ b/tests/test_partiql_write.py @@ -0,0 +1,588 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""PartiQL write statements (INSERT / UPDATE / DELETE) over ExecuteStatement. + +Dual-target: identical assertions run against real DynamoDB and against an +ExtendDB fix server (selected by EXTENDDB_TEST_ENDPOINT, see conftest). Every +expected response and error Code+Message was measured on the live service; the +records are in conf-run/partiql/char-write.md. + +The table is composite pk(S)/sk(S) with a GSI gsi1 on gpk(S), matching the +oracle table. Each test uses a distinct partition key for isolation on the one +shared table. +""" + +from __future__ import annotations + +import os +import uuid + +import boto3 +import pytest +from botocore.config import Config +from botocore.exceptions import ClientError + +from conftest import wait_for_active + + +@pytest.fixture(scope="session") +def dynamodb_client(endpoint_url): + """DynamoDB client honoring EXTENDDB_TEST_PROFILE for the live-service run. + + Overrides conftest's fixture only for this module. When EXTENDDB_TEST_ENDPOINT + is set (fix-server run) this behaves exactly like the conftest client; when + targeting the live service, EXTENDDB_TEST_PROFILE selects the named profile + whose credentials boto3 resolves via the broker credential_process. + """ + region = os.environ.get("AWS_DEFAULT_REGION", "us-east-1") + profile = os.environ.get("EXTENDDB_TEST_PROFILE") or None + session = boto3.Session(profile_name=profile, region_name=region) + kwargs: dict = {"service_name": "dynamodb"} + if endpoint_url: + kwargs["endpoint_url"] = endpoint_url + if endpoint_url.startswith("https://"): + kwargs["verify"] = False + kwargs["config"] = Config(parameter_validation=True) + return session.client(**kwargs) + + +@pytest.fixture(scope="module") +def wtable(dynamodb_client): + """A composite pk(S)/sk(S) table with GSI gsi1 on gpk(S).""" + name = f"extenddb-measure-b-pqlw-{uuid.uuid4().hex[:12]}" + dynamodb_client.create_table( + TableName=name, + AttributeDefinitions=[ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "sk", "AttributeType": "S"}, + {"AttributeName": "gpk", "AttributeType": "S"}, + ], + KeySchema=[ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"}, + ], + GlobalSecondaryIndexes=[ + { + "IndexName": "gsi1", + "KeySchema": [{"AttributeName": "gpk", "KeyType": "HASH"}], + "Projection": {"ProjectionType": "ALL"}, + } + ], + BillingMode="PAY_PER_REQUEST", + ) + wait_for_active(dynamodb_client, name) + try: + yield name + finally: + try: + dynamodb_client.delete_table(TableName=name) + except dynamodb_client.exceptions.ResourceNotFoundException: + pass + + +def _exec(client, statement, **kwargs): + return client.execute_statement(Statement=statement, **kwargs) + + +def _error(excinfo): + err = excinfo.value.response["Error"] + return err["Code"], err["Message"] + + +def _get(client, table, pk, sk): + return client.get_item( + TableName=table, Key={"pk": {"S": pk}, "sk": {"S": sk}} + ).get("Item") + + +# --- INSERT --------------------------------------------------------------- + + +def test_insert_success_writes_full_item(dynamodb_client, wtable): + resp = _exec( + dynamodb_client, + f"INSERT INTO \"{wtable}\" VALUE {{'pk':'p1','sk':'s1','a':1,'gpk':'g1'}}", + ) + assert resp["Items"] == [] + assert _get(dynamodb_client, wtable, "p1", "s1") == { + "a": {"N": "1"}, + "gpk": {"S": "g1"}, + "pk": {"S": "p1"}, + "sk": {"S": "s1"}, + } + + +def test_insert_duplicate_key_is_duplicate_item_exception(dynamodb_client, wtable): + _exec(dynamodb_client, f"INSERT INTO \"{wtable}\" VALUE {{'pk':'dup','sk':'s1','a':1}}") + with pytest.raises(ClientError) as excinfo: + _exec(dynamodb_client, f"INSERT INTO \"{wtable}\" VALUE {{'pk':'dup','sk':'s1','a':2}}") + code, message = _error(excinfo) + assert code == "DuplicateItemException" + assert message == "Duplicate primary key exists in table" + # The original item is untouched. + assert _get(dynamodb_client, wtable, "dup", "s1")["a"] == {"N": "1"} + + +def test_insert_missing_key_attribute_names_the_key(dynamodb_client, wtable): + with pytest.raises(ClientError) as excinfo: + _exec(dynamodb_client, f"INSERT INTO \"{wtable}\" VALUE {{'pk':'p2','a':1}}") + code, message = _error(excinfo) + assert code == "ValidationException" + assert message == "Key attribute should be present in the item: Key sk" + + +def test_insert_wrong_key_type_names_the_key(dynamodb_client, wtable): + with pytest.raises(ClientError) as excinfo: + _exec(dynamodb_client, f"INSERT INTO \"{wtable}\" VALUE {{'pk':9,'sk':'s1'}}") + code, message = _error(excinfo) + assert code == "ValidationException" + assert ( + message + == "Key attribute's data type should match its data type in table's schema: Key pk" + ) + + +def test_insert_rich_literal_round_trips(dynamodb_client, wtable): + _exec( + dynamodb_client, + f"INSERT INTO \"{wtable}\" VALUE {{'pk':'p3','sk':'s1','m':{{'x':1,'y':'z'}}," + f"'l':[1,'two',true,null],'b':true,'n':null,'num':3.5}}", + ) + assert _get(dynamodb_client, wtable, "p3", "s1") == { + "b": {"BOOL": True}, + "num": {"N": "3.5"}, + "l": {"L": [{"N": "1"}, {"S": "two"}, {"BOOL": True}, {"NULL": True}]}, + "pk": {"S": "p3"}, + "m": {"M": {"x": {"N": "1"}, "y": {"S": "z"}}}, + "n": {"NULL": True}, + "sk": {"S": "s1"}, + } + + +def test_insert_string_set_via_parameter(dynamodb_client, wtable): + _exec( + dynamodb_client, + f"INSERT INTO \"{wtable}\" VALUE {{'pk':?,'sk':?,'ss':?}}", + Parameters=[{"S": "p4"}, {"S": "s1"}, {"SS": ["a", "b", "c"]}], + ) + assert _get(dynamodb_client, wtable, "p4", "s1")["ss"] == {"SS": ["a", "b", "c"]} + + +def test_insert_number_set_via_parameter(dynamodb_client, wtable): + _exec( + dynamodb_client, + f"INSERT INTO \"{wtable}\" VALUE {{'pk':?,'sk':?,'ns':?}}", + Parameters=[{"S": "p7"}, {"S": "s1"}, {"NS": ["1", "2", "3"]}], + ) + assert _get(dynamodb_client, wtable, "p7", "s1")["ns"] == {"NS": ["1", "2", "3"]} + + +def test_insert_binary_via_parameter(dynamodb_client, wtable): + _exec( + dynamodb_client, + f"INSERT INTO \"{wtable}\" VALUE {{'pk':?,'sk':?,'bin':?}}", + Parameters=[{"S": "p6"}, {"S": "s1"}, {"B": b"\x01\x02\x03"}], + ) + assert _get(dynamodb_client, wtable, "p6", "s1")["bin"] == {"B": b"\x01\x02\x03"} + + +def test_insert_string_set_literal_stores_ss(dynamodb_client, wtable): + _exec( + dynamodb_client, + f"INSERT INTO \"{wtable}\" VALUE {{'pk':'p5','sk':'s1','ss':<<'a','b'>>}}", + ) + assert _get(dynamodb_client, wtable, "p5", "s1")["ss"] == {"SS": ["a", "b"]} + + +def test_insert_return_consumed_capacity(dynamodb_client, wtable): + resp = _exec( + dynamodb_client, + f"INSERT INTO \"{wtable}\" VALUE {{'pk':'p11','sk':'s1'}}", + ReturnConsumedCapacity="TOTAL", + ) + assert resp["Items"] == [] + assert resp["ConsumedCapacity"]["TableName"] == wtable + assert resp["ConsumedCapacity"]["CapacityUnits"] == 1.0 + + +# --- UPDATE --------------------------------------------------------------- + + +def _seed(client, table, pk, extra=""): + body = f"'pk':'{pk}','sk':'s1','a':1,'l':[1,2]" + if extra: + body += "," + extra + _exec(client, f"INSERT INTO \"{table}\" VALUE {{{body}}}") + + +def test_update_set_literal(dynamodb_client, wtable): + _seed(dynamodb_client, wtable, "u1") + resp = _exec(dynamodb_client, f"UPDATE \"{wtable}\" SET a = 2 WHERE pk='u1' AND sk='s1'") + assert resp["Items"] == [] + assert _get(dynamodb_client, wtable, "u1", "s1")["a"] == {"N": "2"} + + +def test_update_increment(dynamodb_client, wtable): + _seed(dynamodb_client, wtable, "u2") + _exec(dynamodb_client, f"UPDATE \"{wtable}\" SET a = a + 1 WHERE pk='u2' AND sk='s1'") + assert _get(dynamodb_client, wtable, "u2", "s1")["a"] == {"N": "2"} + + +def test_update_list_append(dynamodb_client, wtable): + _seed(dynamodb_client, wtable, "u3") + _exec( + dynamodb_client, + f"UPDATE \"{wtable}\" SET l = list_append(l, [3]) WHERE pk='u3' AND sk='s1'", + ) + assert _get(dynamodb_client, wtable, "u3", "s1")["l"] == { + "L": [{"N": "1"}, {"N": "2"}, {"N": "3"}] + } + + +def test_update_remove(dynamodb_client, wtable): + _seed(dynamodb_client, wtable, "u4") + _exec(dynamodb_client, f"UPDATE \"{wtable}\" REMOVE a WHERE pk='u4' AND sk='s1'") + item = _get(dynamodb_client, wtable, "u4", "s1") + assert "a" not in item + assert "l" in item + + +def test_update_set_and_remove(dynamodb_client, wtable): + _seed(dynamodb_client, wtable, "u5") + _exec(dynamodb_client, f"UPDATE \"{wtable}\" SET a = 9 REMOVE l WHERE pk='u5' AND sk='s1'") + item = _get(dynamodb_client, wtable, "u5", "s1") + assert item["a"] == {"N": "9"} + assert "l" not in item + + +def test_update_missing_item_is_conditional_check_failed(dynamodb_client, wtable): + with pytest.raises(ClientError) as excinfo: + _exec(dynamodb_client, f"UPDATE \"{wtable}\" SET a = 5 WHERE pk='nope' AND sk='nope'") + code, message = _error(excinfo) + assert code == "ConditionalCheckFailedException" + assert message == "The conditional request failed" + assert _get(dynamodb_client, wtable, "nope", "nope") is None + + +def test_update_partial_key_is_full_key_error(dynamodb_client, wtable): + _seed(dynamodb_client, wtable, "u6") + with pytest.raises(ClientError) as excinfo: + _exec(dynamodb_client, f"UPDATE \"{wtable}\" SET a = 5 WHERE pk='u6'") + code, message = _error(excinfo) + assert code == "ValidationException" + assert message == "Where clause does not contain a mandatory equality on all key attributes" + + +def test_update_non_key_where_is_full_key_error(dynamodb_client, wtable): + with pytest.raises(ClientError) as excinfo: + _exec(dynamodb_client, f"UPDATE \"{wtable}\" SET a = 5 WHERE a = 2") + code, message = _error(excinfo) + assert code == "ValidationException" + assert message == "Where clause does not contain a mandatory equality on all key attributes" + + +def test_update_condition_met(dynamodb_client, wtable): + _seed(dynamodb_client, wtable, "u7") + _exec( + dynamodb_client, + f"UPDATE \"{wtable}\" SET a = 7 WHERE pk='u7' AND sk='s1' AND attribute_exists(a)", + ) + assert _get(dynamodb_client, wtable, "u7", "s1")["a"] == {"N": "7"} + + +def test_update_condition_not_met(dynamodb_client, wtable): + _seed(dynamodb_client, wtable, "u8") + with pytest.raises(ClientError) as excinfo: + _exec( + dynamodb_client, + f"UPDATE \"{wtable}\" SET a = 7 WHERE pk='u8' AND sk='s1' AND attribute_exists(zzz)", + ) + code, message = _error(excinfo) + assert code == "ConditionalCheckFailedException" + assert message == "The conditional request failed" + assert _get(dynamodb_client, wtable, "u8", "s1")["a"] == {"N": "1"} + + +def test_update_returning_all_old(dynamodb_client, wtable): + _seed(dynamodb_client, wtable, "r1") + resp = _exec( + dynamodb_client, + f"UPDATE \"{wtable}\" SET a = 100 WHERE pk='r1' AND sk='s1' RETURNING ALL OLD *", + ) + assert resp["Items"] == [ + {"a": {"N": "1"}, "l": {"L": [{"N": "1"}, {"N": "2"}]}, "pk": {"S": "r1"}, "sk": {"S": "s1"}} + ] + + +def test_update_returning_modified_old(dynamodb_client, wtable): + _seed(dynamodb_client, wtable, "r2") + resp = _exec( + dynamodb_client, + f"UPDATE \"{wtable}\" SET a = 100 WHERE pk='r2' AND sk='s1' RETURNING MODIFIED OLD *", + ) + assert resp["Items"] == [{"a": {"N": "1"}}] + + +def test_update_returning_all_new(dynamodb_client, wtable): + _seed(dynamodb_client, wtable, "r3") + resp = _exec( + dynamodb_client, + f"UPDATE \"{wtable}\" SET a = 100 WHERE pk='r3' AND sk='s1' RETURNING ALL NEW *", + ) + assert resp["Items"] == [ + {"a": {"N": "100"}, "l": {"L": [{"N": "1"}, {"N": "2"}]}, "pk": {"S": "r3"}, "sk": {"S": "s1"}} + ] + + +def test_update_returning_modified_new(dynamodb_client, wtable): + _seed(dynamodb_client, wtable, "r4") + resp = _exec( + dynamodb_client, + f"UPDATE \"{wtable}\" SET a = 100 WHERE pk='r4' AND sk='s1' RETURNING MODIFIED NEW *", + ) + assert resp["Items"] == [{"a": {"N": "100"}}] + + +def test_update_returning_missing_item_still_fails(dynamodb_client, wtable): + with pytest.raises(ClientError) as excinfo: + _exec( + dynamodb_client, + f"UPDATE \"{wtable}\" SET a = 100 WHERE pk='missr' AND sk='s1' RETURNING ALL OLD *", + ) + code, message = _error(excinfo) + assert code == "ConditionalCheckFailedException" + assert message == "The conditional request failed" + + +def test_update_set_key_attribute_rejected(dynamodb_client, wtable): + _seed(dynamodb_client, wtable, "kk") + with pytest.raises(ClientError) as excinfo: + _exec( + dynamodb_client, + f"UPDATE \"{wtable}\" SET pk = 'changed' WHERE pk='kk' AND sk='s1'", + ) + code, message = _error(excinfo) + assert code == "ValidationException" + assert ( + message + == "One or more parameter values were invalid: Cannot update attribute pk. " + "This attribute is part of the key" + ) + + +def test_update_return_consumed_capacity(dynamodb_client, wtable): + _seed(dynamodb_client, wtable, "ucc") + resp = _exec( + dynamodb_client, + f"UPDATE \"{wtable}\" SET a = 2 WHERE pk='ucc' AND sk='s1'", + ReturnConsumedCapacity="TOTAL", + ) + assert resp["Items"] == [] + assert resp["ConsumedCapacity"]["TableName"] == wtable + assert resp["ConsumedCapacity"]["CapacityUnits"] == 1.0 + + +# --- DELETE --------------------------------------------------------------- + + +def test_delete_success(dynamodb_client, wtable): + _seed(dynamodb_client, wtable, "d1") + resp = _exec(dynamodb_client, f"DELETE FROM \"{wtable}\" WHERE pk='d1' AND sk='s1'") + assert resp["Items"] == [] + assert _get(dynamodb_client, wtable, "d1", "s1") is None + + +def test_delete_missing_item_is_silent_success(dynamodb_client, wtable): + resp = _exec(dynamodb_client, f"DELETE FROM \"{wtable}\" WHERE pk='ghost' AND sk='s1'") + assert resp["Items"] == [] + + +def test_delete_returning_all_old(dynamodb_client, wtable): + _seed(dynamodb_client, wtable, "d2") + resp = _exec( + dynamodb_client, + f"DELETE FROM \"{wtable}\" WHERE pk='d2' AND sk='s1' RETURNING ALL OLD *", + ) + assert resp["Items"] == [ + {"a": {"N": "1"}, "l": {"L": [{"N": "1"}, {"N": "2"}]}, "pk": {"S": "d2"}, "sk": {"S": "s1"}} + ] + + +def test_delete_returning_missing_is_empty(dynamodb_client, wtable): + resp = _exec( + dynamodb_client, + f"DELETE FROM \"{wtable}\" WHERE pk='ghost2' AND sk='s1' RETURNING ALL OLD *", + ) + assert resp["Items"] == [] + + +def test_delete_returning_all_new_rejected(dynamodb_client, wtable): + with pytest.raises(ClientError) as excinfo: + _exec( + dynamodb_client, + f"DELETE FROM \"{wtable}\" WHERE pk='d3' AND sk='s1' RETURNING ALL NEW *", + ) + code, message = _error(excinfo) + assert code == "ValidationException" + assert ( + message + == "Invalid returning clause: RETURNING ALL NEW *. " + "Only RETURNING ALL OLD * is allowed in DELETE statements." + ) + + +def test_delete_condition_met(dynamodb_client, wtable): + _seed(dynamodb_client, wtable, "d4") + resp = _exec( + dynamodb_client, + f"DELETE FROM \"{wtable}\" WHERE pk='d4' AND sk='s1' AND attribute_exists(a)", + ) + assert resp["Items"] == [] + assert _get(dynamodb_client, wtable, "d4", "s1") is None + + +def test_delete_condition_not_met(dynamodb_client, wtable): + _seed(dynamodb_client, wtable, "d5") + with pytest.raises(ClientError) as excinfo: + _exec( + dynamodb_client, + f"DELETE FROM \"{wtable}\" WHERE pk='d5' AND sk='s1' AND attribute_exists(zzz)", + ) + code, message = _error(excinfo) + assert code == "ConditionalCheckFailedException" + assert message == "The conditional request failed" + assert _get(dynamodb_client, wtable, "d5", "s1") is not None + + +def test_delete_partial_key_is_full_key_error(dynamodb_client, wtable): + with pytest.raises(ClientError) as excinfo: + _exec(dynamodb_client, f"DELETE FROM \"{wtable}\" WHERE pk='d1'") + code, message = _error(excinfo) + assert code == "ValidationException" + assert message == "Where clause does not contain a mandatory equality on all key attributes" + + +def test_delete_return_consumed_capacity(dynamodb_client, wtable): + _seed(dynamodb_client, wtable, "d6") + resp = _exec( + dynamodb_client, + f"DELETE FROM \"{wtable}\" WHERE pk='d6' AND sk='s1'", + ReturnConsumedCapacity="TOTAL", + ) + assert resp["Items"] == [] + assert resp["ConsumedCapacity"]["TableName"] == wtable + assert resp["ConsumedCapacity"]["CapacityUnits"] == 1.0 + + +def test_delete_condfail_returnvalues_all_old_puts_item_on_error(dynamodb_client, wtable): + _seed(dynamodb_client, wtable, "rv") + with pytest.raises(ClientError) as excinfo: + _exec( + dynamodb_client, + f"DELETE FROM \"{wtable}\" WHERE pk='rv' AND sk='s1' AND attribute_exists(zzz)", + ReturnValuesOnConditionCheckFailure="ALL_OLD", + ) + response = excinfo.value.response + assert response["Error"]["Code"] == "ConditionalCheckFailedException" + assert response["Error"]["Message"] == "The conditional request failed" + assert response["Item"] == { + "a": {"N": "1"}, + "l": {"L": [{"N": "1"}, {"N": "2"}]}, + "pk": {"S": "rv"}, + "sk": {"S": "s1"}, + } + + +# --- RETURNING MODIFIED over several list indices --------------------------- +# MODIFIED projects each path the statement names against the old or the new +# item. Elements picked by index come back packed densely in ascending index +# order, whatever order the SET clauses name them in (measured 2026-11-02). + +_LIST_SETS = { + "ascending": "l[0] = 'x', l[2] = 'y'", + "descending": "l[2] = 'y', l[0] = 'x'", +} + + +def _seed_list(client, table, pk): + _exec(client, f"INSERT INTO \"{table}\" VALUE {{'pk':'{pk}','sk':'s1','l':['a','b','c']}}") + + +@pytest.mark.parametrize("order", sorted(_LIST_SETS)) +def test_update_returning_modified_new_packs_list_indices_in_index_order( + dynamodb_client, wtable, order +): + pk = f"rl-new-{order}" + _seed_list(dynamodb_client, wtable, pk) + resp = _exec( + dynamodb_client, + f"UPDATE \"{wtable}\" SET {_LIST_SETS[order]} WHERE pk='{pk}' AND sk='s1' " + "RETURNING MODIFIED NEW *", + ) + assert resp["Items"] == [{"l": {"L": [{"S": "x"}, {"S": "y"}]}}] + assert _get(dynamodb_client, wtable, pk, "s1")["l"] == { + "L": [{"S": "x"}, {"S": "b"}, {"S": "y"}] + } + + +@pytest.mark.parametrize("order", sorted(_LIST_SETS)) +def test_update_returning_modified_old_packs_list_indices_in_index_order( + dynamodb_client, wtable, order +): + pk = f"rl-old-{order}" + _seed_list(dynamodb_client, wtable, pk) + resp = _exec( + dynamodb_client, + f"UPDATE \"{wtable}\" SET {_LIST_SETS[order]} WHERE pk='{pk}' AND sk='s1' " + "RETURNING MODIFIED OLD *", + ) + assert resp["Items"] == [{"l": {"L": [{"S": "a"}, {"S": "c"}]}}] + + +@pytest.mark.parametrize( + ("mode", "expected"), + [ + # l[2] no longer resolves on the new two-element list, so NEW keeps l[0] only. + ("NEW", [{"S": "x"}]), + ("OLD", [{"S": "a"}, {"S": "c"}]), + ], +) +def test_update_returning_modified_projects_set_and_remove_list_paths( + dynamodb_client, wtable, mode, expected +): + pk = f"rl-setrem-{mode.lower()}" + _seed_list(dynamodb_client, wtable, pk) + resp = _exec( + dynamodb_client, + f"UPDATE \"{wtable}\" SET l[0] = 'x' REMOVE l[2] WHERE pk='{pk}' AND sk='s1' " + f"RETURNING MODIFIED {mode} *", + ) + assert resp["Items"] == [{"l": {"L": expected}}] + assert _get(dynamodb_client, wtable, pk, "s1")["l"] == {"L": [{"S": "x"}, {"S": "b"}]} + + +# --- DELETE on a missing key with a non-key predicate ----------------------- +# The predicate is a condition on an item that exists. A missing key is a +# silent no-op, not ConditionalCheckFailedException (measured 2026-11-02). + + +@pytest.mark.parametrize("predicate", ["a = 1", "attribute_exists(zzz)"]) +def test_delete_missing_key_with_non_key_predicate_is_silent_success( + dynamodb_client, wtable, predicate +): + resp = _exec( + dynamodb_client, + f"DELETE FROM \"{wtable}\" WHERE pk='ghost-pred' AND sk='s1' AND {predicate}", + ) + assert resp["Items"] == [] + assert _get(dynamodb_client, wtable, "ghost-pred", "s1") is None + + +def test_delete_missing_key_with_predicate_returning_all_old_is_empty(dynamodb_client, wtable): + resp = _exec( + dynamodb_client, + f"DELETE FROM \"{wtable}\" WHERE pk='ghost-pred2' AND sk='s1' AND a = 1 " + "RETURNING ALL OLD *", + ) + assert resp["Items"] == []