diff --git a/core/crates/core/src/browse/build_export.rs b/core/crates/core/src/browse/build_export.rs index 8e0049a169..bd8ad73240 100644 --- a/core/crates/core/src/browse/build_export.rs +++ b/core/crates/core/src/browse/build_export.rs @@ -16,7 +16,8 @@ use super::{BrowseSqlError, ExportRequest}; /// unchanged table come out the same. A heap address, or a key the /// engine would sort the whole table by, orders them only behind the /// user's sort: on its own it would hold every row before the first -/// could be written. +/// could be written. A table's rows hold only its columns, so SQL reads +/// the listed columns for either `ExportFields`. pub fn build_export(dialect: &dyn SqlDialect, request: &ExportRequest<'_>) -> Result { if crate::document::speaks_shell(dialect) { return crate::document::export(request); diff --git a/core/crates/core/src/browse/export_request.rs b/core/crates/core/src/browse/export_request.rs index fe050e3214..4c98486e3a 100644 --- a/core/crates/core/src/browse/export_request.rs +++ b/core/crates/core/src/browse/export_request.rs @@ -13,4 +13,16 @@ pub struct ExportRequest<'a> { pub filter: &'a FilterSet, pub sort: &'a [SortKey], pub identity: &'a RowIdentity, + pub fields: ExportFields, +} + +/// Which fields an export reads. A table's rows hold its columns either +/// way; a collection's columns come from a census of its first documents, +/// so a later document can hold a field they do not list. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ExportFields { + /// The columns `columns` lists, in its order. + Listed, + /// Every field the rows hold, as the driver finds them while it reads. + Every, } diff --git a/core/crates/core/src/browse/mod.rs b/core/crates/core/src/browse/mod.rs index fef4eeaf7f..fc31bf97dc 100644 --- a/core/crates/core/src/browse/mod.rs +++ b/core/crates/core/src/browse/mod.rs @@ -28,7 +28,7 @@ pub use build_count::build_count; pub use build_export::build_export; pub use build_page::build_page; pub use declared_precision::DeclaredPrecisionSink; -pub use export_request::ExportRequest; +pub use export_request::{ExportFields, ExportRequest}; pub use page_layout::PageLayout; pub use page_request::PageRequest; pub use page_spec::PageSpec; diff --git a/core/crates/core/src/browse/tests.rs b/core/crates/core/src/browse/tests.rs index 29f5f7e025..e8d458bf7c 100644 --- a/core/crates/core/src/browse/tests.rs +++ b/core/crates/core/src/browse/tests.rs @@ -496,19 +496,28 @@ fn build_export_projects_only_visible_columns() { ascending: true, }]; - let statement = build_export( + let table = table(); + let request = ExportRequest { + table: &table, + columns: &columns, + filter: &filter, + sort: &sort, + identity: &identity, + fields: ExportFields::Every, + }; + + let statement = build_export(&dialect, &request).expect("the export"); + let listed = build_export( &dialect, &ExportRequest { - table: &table(), - columns: &columns, - filter: &filter, - sort: &sort, - identity: &identity, + fields: ExportFields::Listed, + ..request }, ) .expect("the export"); let sql = statement.sql(); + assert_eq!(listed.sql(), sql, "a table's rows hold only its columns"); let projection = sql .strip_prefix("SELECT ") .and_then(|rest| rest.split_once(" FROM ")) @@ -538,6 +547,7 @@ fn export_sql_in(dialect: &TestDialect, identity: RowIdentity, sort: &[SortKey]) filter: &FilterSet::default(), sort, identity: &identity, + fields: ExportFields::Listed, }, ) .expect("the export"); diff --git a/core/crates/core/src/document/document_filter.rs b/core/crates/core/src/document/document_filter.rs index 94ad7c599f..e7fcaa66a5 100644 --- a/core/crates/core/src/document/document_filter.rs +++ b/core/crates/core/src/document/document_filter.rs @@ -1,4 +1,4 @@ -use crate::column::ColumnInfo; +use crate::column::{ColumnInfo, ColumnKind}; use crate::edit::parse_filter_text; use crate::filter::{BuildFilterError, Combinator, FilterOp, FilterRule, FilterSet, FilterValue, operators_for}; use crate::sql_syntax::mongo::{ShellValue, is_addressable_field, parse_value}; @@ -6,6 +6,39 @@ use crate::value::Value; use super::shell_params::ShellParams; +const JSON_OPERATORS: &[FilterOp] = &[ + FilterOp::Eq, + FilterOp::NotEq, + FilterOp::Lt, + FilterOp::LtEq, + FilterOp::Gt, + FilterOp::GtEq, + FilterOp::Between, + FilterOp::Contains, + FilterOp::StartsWith, + FilterOp::EndsWith, + FilterOp::Like, + FilterOp::NotLike, + FilterOp::Ilike, + FilterOp::In, + FilterOp::NotIn, + FilterOp::IsNull, + FilterOp::IsNotNull, +]; + +/// The operators a filter rule takes on a field, in the order the filter +/// strip lists them. A JSON column compares as well as matching text: a +/// value typed for it binds as the BSON its Extended JSON names, and the +/// server compares it with the field's values of that type, numbers of +/// every type with each other, so a field of ints and doubles keeps `<` +/// and Between. +pub fn filter_operators(column: &ColumnInfo) -> &'static [FilterOp] { + match column.column_type.kind { + ColumnKind::Json { .. } => JSON_OPERATORS, + _ => operators_for(&column.column_type.kind), + } +} + /// The filter document a collection's page, count and export read /// under: one clause per rule, then the document the user typed, joined /// with `$and` or `$or`. `None` matches every document. @@ -23,7 +56,7 @@ pub(crate) fn filter_document( .iter() .find(|column| column.name == rule.column) .ok_or_else(|| BuildFilterError::UnknownColumn(rule.column.clone()))?; - if !operators_for(&column.column_type.kind).contains(&rule.op) { + if !filter_operators(column).contains(&rule.op) { return Err(BuildFilterError::OperatorNotAllowed { column: rule.column.clone(), op: rule.op, diff --git a/core/crates/core/src/document/document_read.rs b/core/crates/core/src/document/document_read.rs index 1169b9a2a2..807ad7d896 100644 --- a/core/crates/core/src/document/document_read.rs +++ b/core/crates/core/src/document/document_read.rs @@ -1,5 +1,5 @@ use crate::browse::{ - BrowseSqlError, ExportRequest, PageLayout, PageRequest, PageSpec, PagingMode, RowKeyLayout, SortKey, + BrowseSqlError, ExportFields, ExportRequest, PageLayout, PageRequest, PageSpec, PagingMode, RowKeyLayout, SortKey, }; use crate::column::ColumnInfo; use crate::dialect::TablePaging; @@ -100,6 +100,8 @@ pub(crate) fn count(table: &TableRef, columns: &[ColumnInfo], filter: &FilterSet } /// Every document the filter matches, in the page's order, for an export. +/// `ExportFields::Every` writes no `$project`, so the server sends each +/// whole document and the driver finds the columns in them. pub(crate) fn export(request: &ExportRequest<'_>) -> Result { let key = key_column(request.identity, request.columns)?; let mut params = ShellParams::new(); @@ -114,7 +116,9 @@ pub(crate) fn export(request: &ExportRequest<'_>) -> Result { + unset.push((column.name.clone(), ShellValue::string(""))); + } + _ => set.push((column.name.clone(), params.bind(value.clone(), Some(column))?)), + } } let filter = key_filter(columns, identity, key, &mut params)?; + let operators = [("$set", set), ("$unset", unset)] + .into_iter() + .filter(|(_, fields)| !fields.is_empty()) + .map(|(name, fields)| (name.to_owned(), ShellValue::Object(fields))) + .collect(); let mut call = collection(table); call.methods.push(ShellMethod::new( "updateOne", - vec![filter, field("$set", ShellValue::Object(set))], + vec![filter, ShellValue::Object(operators)], )); Ok(params.statement(&call)) } @@ -142,12 +161,7 @@ fn key_filter( key: &[Value], params: &mut ShellParams, ) -> Result { - let names = match identity { - RowIdentity::UniqueKey { columns } => columns, - RowIdentity::SortingKey { expressions } => expressions, - RowIdentity::EngineRowId(_) => return Err(BuildSqlError::NoRowIdentity), - RowIdentity::Unordered => return Err(BuildSqlError::ReadOnlyRows), - }; + let names = key_names(identity)?; if names.len() != key.len() { return Err(BuildSqlError::LengthMismatch { expected: names.len(), @@ -167,3 +181,12 @@ fn key_filter( } Ok(ShellValue::Object(fields)) } + +fn key_names(identity: &RowIdentity) -> Result<&[String], BuildSqlError> { + match identity { + RowIdentity::UniqueKey { columns } => Ok(columns), + RowIdentity::SortingKey { expressions } => Ok(expressions), + RowIdentity::EngineRowId(_) => Err(BuildSqlError::NoRowIdentity), + RowIdentity::Unordered => Err(BuildSqlError::ReadOnlyRows), + } +} diff --git a/core/crates/core/src/document/mod.rs b/core/crates/core/src/document/mod.rs index b85ba3cc80..c9aebfd6a2 100644 --- a/core/crates/core/src/document/mod.rs +++ b/core/crates/core/src/document/mod.rs @@ -17,6 +17,7 @@ mod tests; mod value_order; pub(crate) use document_catalog::{create_database, delete_all_documents, drop_database}; +pub use document_filter::filter_operators; pub(crate) use document_filter::{filter_document, typed_document}; pub(crate) use document_read::{count, export, page}; pub(crate) use document_write::{delete, insert, insert_many, key_probe, update}; diff --git a/core/crates/core/src/document/tests.rs b/core/crates/core/src/document/tests.rs index 7e68ee9871..c5f438c3ab 100644 --- a/core/crates/core/src/document/tests.rs +++ b/core/crates/core/src/document/tests.rs @@ -1,4 +1,6 @@ -use crate::browse::{ExportRequest, PageRequest, RowKeyLayout, SortKey, build_count, build_export, build_page}; +use crate::browse::{ + ExportFields, ExportRequest, PageRequest, RowKeyLayout, SortKey, build_count, build_export, build_page, +}; use crate::column::{ ColumnDefault, ColumnInfo, ColumnKind, ColumnType, IntegerKind, IntegerWidth, ReadForm, TextLength, }; @@ -10,7 +12,7 @@ use crate::meta::{PagePosition, PageStability, RowIdentity, TableRef}; use crate::sql_syntax::SqlGrammar; use crate::sql_syntax::mongo::parse_bound_statement; use crate::statement::{ParamType, Statement, StatementBuilder}; -use crate::value::{OtherValue, Value}; +use crate::value::{JsonText, OtherValue, Value}; fn shell() -> TestDialect { TestDialect::exact().parsing_as(SqlGrammar::Mongo) @@ -215,6 +217,77 @@ fn a_filter_binds_what_was_typed_and_keeps_the_typed_document() { assert_eq!(statement.params()[1].param_type(), &ParamType::Int32); } +#[test] +fn a_json_field_compares_as_well_as_matching_text() { + let mut columns = columns(); + columns.push(column("price", "mixed", ColumnKind::Json { binary: false })); + let offered = crate::document::filter_operators(&columns[4]); + for op in [ + FilterOp::Eq, + FilterOp::Lt, + FilterOp::LtEq, + FilterOp::Gt, + FilterOp::GtEq, + FilterOp::Between, + FilterOp::In, + FilterOp::NotIn, + FilterOp::Contains, + FilterOp::Like, + FilterOp::IsNull, + FilterOp::IsNotNull, + ] { + assert!(offered.contains(&op), "{op:?} in {offered:?}"); + } + assert_eq!( + crate::document::filter_operators(&columns[2]), + crate::filter::operators_for(&columns[2].column_type.kind) + ); + + let rule = |op: FilterOp, value: FilterValue| FilterRule { + column: "price".to_owned(), + op, + value: Some(value), + }; + let filter = FilterSet { + combinator: Combinator::And, + rules: vec![ + rule(FilterOp::Between, FilterValue::Pair("1".to_owned(), "9.5".to_owned())), + rule(FilterOp::Lt, FilterValue::Single(r#"{"$numberLong": "7"}"#.to_owned())), + ], + extra_sql: None, + }; + let statement = build_count(&shell(), &users(), &columns, &filter).expect("a count"); + + assert_eq!( + checked(&statement), + r#"db.getSiblingDB("shop").getCollection("users").countDocuments({"$and": [{"price": {"$gte": $1, "$lte": $2}}, {"price": {"$lt": $3}}]})"# + ); + assert!( + statement + .params() + .iter() + .all(|param| param.param_type() == &ParamType::Json), + "{:?}", + statement.params() + ); + let text = FilterSet { + rules: vec![FilterRule { + column: "name".to_owned(), + op: FilterOp::Gt, + value: Some(FilterValue::Single("a".to_owned())), + }], + ..FilterSet::default() + }; + assert_eq!( + build_filter(&mut StatementBuilder::new(), &shell(), &columns, &text), + Err(BuildFilterError::OperatorNotAllowed { + column: "name".to_owned(), + op: FilterOp::Gt, + }), + "a text field still only matches" + ); +} + #[test] fn a_filter_that_is_not_one_document_or_names_a_dotted_field_is_refused() { let mut builder = StatementBuilder::new(); @@ -329,14 +402,13 @@ fn an_alias_never_takes_the_name_of_a_field() { assert!(text.contains(r#""__field_3": 1"#), "{text}"); } -#[test] -fn an_export_reads_every_document_in_page_order() { +fn export(fields: ExportFields) -> Statement { let columns = columns(); let sort = [SortKey { column: 2, ascending: true, }]; - let statement = build_export( + build_export( &shell(), &ExportRequest { table: &TableRef::unqualified("users"), @@ -344,9 +416,15 @@ fn an_export_reads_every_document_in_page_order() { filter: &FilterSet::default(), sort: &sort, identity: &by_id(), + fields, }, ) - .expect("an export"); + .expect("an export") +} + +#[test] +fn an_export_reads_every_document_in_page_order() { + let statement = export(ExportFields::Listed); assert_eq!( checked(&statement), @@ -356,6 +434,16 @@ fn an_export_reads_every_document_in_page_order() { ); } +#[test] +fn an_export_of_every_field_reads_whole_documents_in_page_order() { + let statement = export(ExportFields::Every); + + assert_eq!( + checked(&statement), + r#"db.getCollection("users").aggregate([{"$sort": {"qty": 1, "_id": 1}}], {"allowDiskUse": true})"# + ); +} + #[test] fn a_change_names_its_document_by_id_and_sets_only_the_fields_changed() { let dialect = shell(); @@ -368,7 +456,7 @@ fn a_change_names_its_document_by_id_and_sets_only_the_fields_changed() { &[object_id()], &[ (1, CellInput::Value(Value::Text("ann".to_owned()))), - (2, CellInput::Value(Value::Null)), + (2, CellInput::Value(Value::Int(3))), ], ) .expect("an update"); @@ -379,7 +467,7 @@ fn a_change_names_its_document_by_id_and_sets_only_the_fields_changed() { assert_eq!( update.params()[1].param_type(), &ParamType::Int32, - "a null keeps its field's type" + "a value takes its field's type" ); let delete = build_delete(&dialect, &table, &columns(), &by_id(), &[object_id()]).expect("a delete"); @@ -422,6 +510,106 @@ fn a_change_names_its_document_by_id_and_sets_only_the_fields_changed() { ); } +#[test] +fn a_change_to_null_removes_the_field_and_a_key_keeps_set() { + let change = |assignments: &[(usize, CellInput)]| { + build_update(&shell(), &users(), &columns(), &by_id(), &[object_id()], assignments).expect("an update") + }; + + let both = change(&[ + (1, CellInput::Value(Value::Text("ann".to_owned()))), + (2, CellInput::Value(Value::Null)), + ]); + assert_eq!( + checked(&both), + r#"db.getSiblingDB("shop").getCollection("users").updateOne({"_id": $2}, {"$set": {"name": $1}, "$unset": {"qty": ""}})"# + ); + assert_eq!(both.params().len(), 2); + + let removed = change(&[(2, CellInput::Value(Value::Null))]); + assert_eq!( + checked(&removed), + r#"db.getSiblingDB("shop").getCollection("users").updateOne({"_id": $1}, {"$unset": {"qty": ""}})"# + ); + assert_eq!(removed.params().len(), 1); + + let key = change(&[(0, CellInput::Value(Value::Null))]); + assert_eq!( + checked(&key), + r#"db.getSiblingDB("shop").getCollection("users").updateOne({"_id": $2}, {"$set": {"_id": $1}})"# + ); + assert_eq!(key.params()[0].value(), &Value::Null); + assert!( + matches!(key.params()[0].param_type(), ParamType::ServerText(_)), + "a null keeps its field's type" + ); + + let json_null = change(&[( + 1, + CellInput::Value(Value::Json(JsonText::parse("null".to_owned()).expect("JSON"))), + )]); + assert_eq!( + checked(&json_null), + r#"db.getSiblingDB("shop").getCollection("users").updateOne({"_id": $2}, {"$set": {"name": $1}})"#, + "a typed null is a value" + ); +} + +#[test] +fn a_new_document_leaves_out_every_null_cell() { + let dialect = shell(); + let table = users(); + let columns = columns(); + let null = || CellInput::Value(Value::Null); + + let insert = build_insert( + &dialect, + &table, + &columns, + &[ + CellInput::Default, + CellInput::Value(Value::Text("ann".to_owned())), + null(), + null(), + ], + ) + .expect("an insert"); + assert_eq!( + checked(&insert), + r#"db.getSiblingDB("shop").getCollection("users").insertOne({"name": $1})"# + ); + assert_eq!(insert.params().len(), 1); + + let empty = build_insert( + &dialect, + &table, + &columns, + &[CellInput::Default, null(), null(), null()], + ) + .expect("an insert of no field"); + assert_eq!( + checked(&empty), + r#"db.getSiblingDB("shop").getCollection("users").insertOne({})"# + ); + + let many = build_multi_row_insert( + &dialect, + &table, + &[&columns[1], &columns[2]], + vec![ + vec![Value::Text("a".to_owned()), Value::Null], + vec![Value::Null, Value::Int(2)], + vec![Value::Null, Value::Null], + ], + ) + .expect("an insert of three"); + assert_eq!( + checked(&many), + r#"db.getSiblingDB("shop").getCollection("users").insertMany([{"name": $1}, {"qty": $2}, {}])"# + ); + assert_eq!(many.params().len(), 2); +} + #[test] fn a_dotted_field_cannot_be_updated_by_name() { let refused = build_update( diff --git a/core/crates/core/src/dump/dump_writer.rs b/core/crates/core/src/dump/dump_writer.rs index 4f690c600c..c0227d0ee0 100644 --- a/core/crates/core/src/dump/dump_writer.rs +++ b/core/crates/core/src/dump/dump_writer.rs @@ -4,7 +4,7 @@ use async_trait::async_trait; use thiserror::Error; use tokio::sync::mpsc; -use crate::browse::{BrowseSqlError, DeclaredPrecisionSink, ExportRequest, build_export}; +use crate::browse::{BrowseSqlError, DeclaredPrecisionSink, ExportFields, ExportRequest, build_export}; use crate::call_options::CallOptions; use crate::column::{ColumnInfo, ResultColumn}; use crate::connection::Connection; @@ -266,6 +266,7 @@ impl Writer<'_> { filter: &FilterSet::default(), sort: &[], identity: &identity, + fields: ExportFields::Listed, }, )?; let written_as = match self.dialect.database_layout() { diff --git a/core/crates/core/src/export/sql.rs b/core/crates/core/src/export/sql.rs index 9f831016ae..5c57444dac 100644 --- a/core/crates/core/src/export/sql.rs +++ b/core/crates/core/src/export/sql.rs @@ -128,7 +128,9 @@ impl InsertWriter { Some((column, written.as_ref()?, cell)) }) .collect(); - let leaves_out_nulls = !dialect.insert_names_null_cells() && cells.iter().any(|(_, _, cell)| !is_null(*cell)); + // A document of no field is `{}`, while a SQL row needs a column. + let leaves_out_nulls = + !dialect.insert_names_null_cells() && (self.document || cells.iter().any(|(_, _, cell)| !is_null(*cell))); let cells: Vec<(&ResultColumn, &WrittenColumn, Option<&Value>)> = cells .into_iter() .filter(|(_, _, cell)| !(leaves_out_nulls && is_null(*cell))) @@ -289,6 +291,7 @@ fn document_fields( for (index, column) in columns.iter().enumerate() { let value = match cells.get(index) { Some(CellInput::Default | CellInput::Expression(_)) | None => continue, + Some(CellInput::Value(Value::Null)) if !dialect.insert_names_null_cells() => continue, Some(CellInput::Value(value)) => dialect.literal(value, &column.column_type)?, }; fields.push((column.name.clone(), value)); @@ -803,6 +806,41 @@ mod tests { ); } + #[test] + fn a_document_of_only_nulls_is_empty_and_a_row_of_only_nulls_names_them() { + let rows = [vec![Value::Int(1), Value::Null], vec![Value::Null, Value::Null]]; + let render = |dialect: &dyn SqlDialect| { + render_sql_insert( + dialect, + &TableRef::unqualified("t"), + &cols(&["a", "b"]), + &[], + &[], + &rows, + ) + }; + let documents = Numbering { + inner: TestDialect::default().parsing_as(SqlGrammar::Mongo), + null_cells_left_out: true, + ..Numbering::default() + }; + let rows_of_sql = Numbering { + null_cells_left_out: true, + ..Numbering::default() + }; + + assert_eq!( + render(&documents), + "db.getCollection(\"t\").insertOne({\"a\": 1});\n\ + db.getCollection(\"t\").insertOne({});\n" + ); + assert_eq!( + render(&rows_of_sql), + "INSERT INTO \"t\" (\"a\") VALUES (1);\n\ + INSERT INTO \"t\" (\"a\", \"b\") VALUES (NULL, NULL);\n" + ); + } + /// Every call's output taken as soon as it is written, as an export /// sink takes it. fn streamed(mut encoder: SqlInsertEncoder, rows: &[Vec]) -> String { diff --git a/core/crates/core/src/sql_syntax/mongo/shell_access.rs b/core/crates/core/src/sql_syntax/mongo/shell_access.rs index f7fca979ac..5817f38533 100644 --- a/core/crates/core/src/sql_syntax/mongo/shell_access.rs +++ b/core/crates/core/src/sql_syntax/mongo/shell_access.rs @@ -118,11 +118,39 @@ const COMMAND_READS: [&str; 34] = [ "getdefaultrwconcern", ]; -/// Commands that drop a collection or a database. -const COMMAND_DROPS: [&str; 2] = ["drop", "dropdatabase"]; +/// Collection methods that drop the collection or its indexes, as SQL's +/// DROP does. +const COLLECTION_DROPS: [&str; 4] = ["drop", "dropIndex", "dropIndexes", "dropSearchIndex"]; -/// Aggregation stages that write the results to a collection. -const WRITING_STAGES: [&str; 2] = ["$out", "$merge"]; +/// Database methods that drop the database, a user or a role. +const DATABASE_DROPS: [&str; 5] = ["dropDatabase", "dropUser", "dropRole", "dropAllUsers", "dropAllRoles"]; + +/// Commands that drop a collection, a database, an index, a user or a +/// role. `deleteIndexes` is the server's old name for `dropIndexes`. +const COMMAND_DROPS: [&str; 9] = [ + "drop", + "dropdatabase", + "dropindexes", + "deleteindexes", + "dropsearchindex", + "dropuser", + "droprole", + "dropallusersfromdatabase", + "dropallrolesfromdatabase", +]; + +/// `validate` options that write when true: `repair` inserts and removes +/// index keys and removes invalid documents, `fixMultikey` rewrites index +/// metadata. +const VALIDATE_WRITES: [&str; 2] = ["repair", "fixMultikey"]; + +/// The aggregation stage that replaces its target collection with the +/// results, as a DROP and a fill would. +const REPLACING_STAGE: &str = "$out"; + +/// The aggregation stage that writes the results into a collection, +/// keeping the documents it does not match. +const MERGING_STAGE: &str = "$merge"; /// The strictest access of the statements in `text`. A statement that /// does not parse is a write, since what it would do is a guess. @@ -170,19 +198,21 @@ fn call_access(call: &ShellCall) -> StatementAccess { fn collection_access(method: &str, args: &[ShellValue]) -> StatementAccess { match method { - "aggregate" if args.first().is_some_and(pipeline_writes) => StatementAccess::Write, + "aggregate" => args.first().map_or(StatementAccess::Read, pipeline_access), _ if COLLECTION_READS.contains(&method) => StatementAccess::Read, - "drop" => StatementAccess::Destructive, + _ if COLLECTION_DROPS.contains(&method) => StatementAccess::Destructive, + // `dropTarget` drops the collection the rename replaces. + "renameCollection" if args.get(1).is_some_and(drops_target) => StatementAccess::Destructive, // Every document, as a DELETE or UPDATE with no WHERE. "deleteMany" | "remove" | "updateMany" if args.first().is_none_or(matches_every_document) => { StatementAccess::Destructive } "update" if args.first().is_none_or(matches_every_document) - && matches!( - args.get(2).and_then(|options| options.field("multi")), - Some(ShellValue::Bool(true)) - ) => + && args + .get(2) + .and_then(|options| options.field("multi")) + .is_some_and(is_true) => { StatementAccess::Destructive } @@ -195,12 +225,32 @@ fn matches_every_document(filter: &ShellValue) -> bool { filter.is_empty_object() || matches!(filter, ShellValue::Null) } +/// `renameCollection`'s second argument, `true` or `{dropTarget: true}`. +fn drops_target(argument: &ShellValue) -> bool { + match argument.as_object() { + Some(_) => argument.field("dropTarget").is_some_and(is_true), + None => is_true(argument), + } +} + +/// Whether the server reads a flag as set, as BSON's `trueValue` does: +/// anything but `false`, zero and `null`. A bound value or a helper such +/// as `NumberInt(0)` counts as set, the stricter reading. +fn is_true(value: &ShellValue) -> bool { + match value { + ShellValue::Bool(flag) => *flag, + ShellValue::Null => false, + ShellValue::Number(number) => number.as_f64() != Some(0.0), + _ => true, + } +} + fn database_access(method: &str, args: &[ShellValue]) -> StatementAccess { match method { "runCommand" | "adminCommand" => args.first().map_or(StatementAccess::Write, command_access), - "aggregate" if args.first().is_some_and(pipeline_writes) => StatementAccess::Write, + "aggregate" => args.first().map_or(StatementAccess::Read, pipeline_access), _ if DATABASE_READS.contains(&method) => StatementAccess::Read, - "dropDatabase" => StatementAccess::Destructive, + _ if DATABASE_DROPS.contains(&method) => StatementAccess::Destructive, _ => StatementAccess::Write, } } @@ -216,11 +266,25 @@ fn command_access(command: &ShellValue) -> StatementAccess { return StatementAccess::Write; }; let name = name.to_ascii_lowercase(); - let writes = command.field("pipeline").is_some_and(pipeline_writes) + let set = |option: &str| command.field(option).is_some_and(is_true); + let pipeline = command.field("pipeline").map_or(StatementAccess::Read, pipeline_access); + let writes = pipeline > StatementAccess::Read || command .field("explain") - .is_some_and(|inner| command_access(inner) > StatementAccess::Read); - if COMMAND_DROPS.contains(&name.as_str()) || reaches_every_document(&name, command) { + .is_some_and(|inner| command_access(inner) > StatementAccess::Read) + || (name == "validate" && VALIDATE_WRITES.into_iter().any(set)); + let destructive = pipeline == StatementAccess::Destructive + || COMMAND_DROPS.contains(&name.as_str()) + || match name.as_str() { + "renamecollection" => set("dropTarget"), + "mapreduce" => command.field("out").is_some_and(replaces_output), + // Oplog entries, applied unchecked: any one can drop, empty or overwrite a collection. + "applyops" => true, + // Rebuilds the collection capped, dropping its other indexes and the oldest documents past the size. + "converttocapped" => true, + _ => reaches_every_document(&name, command), + }; + if destructive { return StatementAccess::Destructive; } match COMMAND_READS.contains(&name.as_str()) && !writes { @@ -229,18 +293,22 @@ fn command_access(command: &ShellValue) -> StatementAccess { } } -/// A raw `delete` or `update` command with an entry that reaches every -/// document, as `deleteMany({})` and `updateMany({}, ...)` do: an empty -/// filter, and no limit of one or a `multi` update. +/// A raw `delete`, `update` or `bulkWrite` command with an entry that +/// reaches every document, as `deleteMany({})` and `updateMany({}, ...)` +/// do: no filter, `null` or `{}`, and no limit of one or a `multi` +/// update. A `bulkWrite` op is a delete or an update by its key, and +/// `multi` takes either past one document. fn reaches_every_document(name: &str, command: &ShellValue) -> bool { - let (list, every): (&str, fn(&ShellValue) -> bool) = match name { - "delete" => ("deletes", |entry| { + let (list, filter, every): (&str, &str, fn(&ShellValue) -> bool) = match name { + "delete" => ("deletes", "q", |entry| { entry .field("limit") .is_none_or(|limit| !matches!(limit, ShellValue::Number(number) if number.as_f64() == Some(1.0))) }), - "update" => ("updates", |entry| { - matches!(entry.field("multi"), Some(ShellValue::Bool(true))) + "update" => ("updates", "q", |entry| entry.field("multi").is_some_and(is_true)), + "bulkwrite" => ("ops", "filter", |entry| { + (entry.field("delete").is_some() || entry.field("update").is_some()) + && entry.field("multi").is_some_and(is_true) }), _ => return false, }; @@ -249,13 +317,31 @@ fn reaches_every_document(name: &str, command: &ShellValue) -> bool { }; entries .iter() - .any(|entry| every(entry) && entry.field("q").is_none_or(ShellValue::is_empty_object)) + .any(|entry| every(entry) && entry.field(filter).is_none_or(matches_every_document)) } -/// Whether a pipeline holds a stage that writes, at any depth: a +/// What a pipeline does, by the stages it holds at any depth: a /// `$lookup` or `$facet` holds pipelines of its own. -fn pipeline_writes(pipeline: &ShellValue) -> bool { - pipeline.keys().iter().any(|key| WRITING_STAGES.contains(key)) +fn pipeline_access(pipeline: &ShellValue) -> StatementAccess { + let stages = pipeline.keys(); + if stages.contains(&REPLACING_STAGE) { + StatementAccess::Destructive + } else if stages.contains(&MERGING_STAGE) { + StatementAccess::Write + } else { + StatementAccess::Read + } +} + +/// Whether a `mapReduce` output replaces a collection: a collection's +/// name or `{replace: ...}`. `{merge: ...}` and `{reduce: ...}` write into +/// one, and `{inline: 1}` writes nowhere. Anything else but an object +/// counts as a name, the stricter reading. +fn replaces_output(out: &ShellValue) -> bool { + match out.as_object() { + Some(_) => out.field("replace").is_some(), + None => true, + } } /// Whether running `text` can add, rename or remove a collection or a @@ -294,9 +380,13 @@ mod tests { ("db.killOp(1)", StatementAccess::Write), ("db.users.find(", StatementAccess::Write), ( - "db.runCommand({aggregate: 'a', pipeline: [], pipeline: [{$out: 'b'}], cursor: {}})", + "db.runCommand({aggregate: 'a', pipeline: [], pipeline: [{$merge: 'b'}], cursor: {}})", StatementAccess::Write, ), + ( + "db.runCommand({aggregate: 'a', pipeline: [], pipeline: [{$out: 'b'}], cursor: {}})", + StatementAccess::Destructive, + ), ( "db.runCommand({delete: 'users', deletes: [{q: {}, limit: 0}]})", StatementAccess::Destructive, @@ -330,17 +420,118 @@ mod tests { } #[test] - fn an_aggregate_that_writes_its_results_is_a_write() { + fn an_aggregate_that_merges_its_results_is_a_write() { for text in [ - "db.a.aggregate([{$match: {}}, {$out: 'b'}])", "db.a.aggregate([{$merge: {into: 'b'}}])", + "db.a.aggregate([{$match: {}}, {$merge: 'b'}])", + "db.runCommand({aggregate: 'a', pipeline: [{$merge: 'b'}], cursor: {}})", + "db.runCommand({explain: {aggregate: 'a', pipeline: [{$merge: 'b'}]}})", + "db.aggregate([{$documents: [{}]}, {$merge: 'b'}])", + ] { + assert_eq!(script_access(text), StatementAccess::Write, "{text}"); + } + } + + #[test] + fn an_aggregate_that_replaces_a_collection_is_destructive() { + for text in [ + "db.a.aggregate([{$match: {}}, {$out: 'b'}])", + "db.a.aggregate([{$out: {db: 'archive', coll: 'b'}}])", "db.a.aggregate([{$facet: {x: [{$out: 'b'}]}}])", + "db.a.aggregate([{$merge: 'b'}, {$out: 'c'}])", + "db.getSiblingDB('shop').getCollection('a').aggregate([{$out: 'b'}]).toArray()", "db.runCommand({aggregate: 'a', pipeline: [{$out: 'b'}], cursor: {}})", - "db.runCommand({explain: {aggregate: 'a', pipeline: [{$merge: 'b'}]}})", + "db.adminCommand({aggregate: 1, pipeline: [{$documents: [{}]}, {$out: 'b'}], cursor: {}})", "db.aggregate([{$documents: [{}]}, {$out: 'b'}])", ] { - assert_eq!(script_access(text), StatementAccess::Write, "{text}"); + assert_eq!(script_access(text), StatementAccess::Destructive, "{text}"); + } + } + + #[test] + fn a_map_reduce_that_replaces_its_output_is_destructive() { + let map_reduce = |out: &str| { + format!( + "db.runCommand({{mapReduce: 'orders', map: 'function() {{}}', reduce: 'function() {{}}', out: {out}}})" + ) + }; + let cases = [ + (map_reduce("'totals'"), StatementAccess::Destructive), + (map_reduce("{replace: 'totals'}"), StatementAccess::Destructive), + ( + map_reduce("{replace: 'totals', db: 'archive'}"), + StatementAccess::Destructive, + ), + (map_reduce("{merge: 'totals'}"), StatementAccess::Write), + (map_reduce("{reduce: 'totals'}"), StatementAccess::Write), + (map_reduce("{inline: 1}"), StatementAccess::Write), + (map_reduce("{inline: 1}, out: 'totals'"), StatementAccess::Destructive), + (map_reduce("'totals', out: {inline: 1}"), StatementAccess::Write), + ( + "db.runCommand({mapreduce: 'orders', map: 'm', reduce: 'r', out: 'totals'})".to_owned(), + StatementAccess::Destructive, + ), + ]; + for (text, expected) in cases { + assert_eq!(script_access(&text), expected, "{text}"); + } + } + + #[test] + fn a_bulk_write_that_reaches_every_document_is_destructive() { + let bulk = + |ops: &str| format!("db.adminCommand({{bulkWrite: 1, ops: [{ops}], nsInfo: [{{ns: 'shop.users'}}]}})"); + let cases = [ + ( + bulk("{delete: 0, filter: {}, multi: true}"), + StatementAccess::Destructive, + ), + ( + bulk("{delete: 0, filter: null, multi: true}"), + StatementAccess::Destructive, + ), + (bulk("{delete: 0, multi: 1}"), StatementAccess::Destructive), + ( + bulk("{update: 0, filter: {}, updateMods: {$set: {a: 1}}, multi: true}"), + StatementAccess::Destructive, + ), + ( + bulk("{insert: 0, document: {a: 1}}, {delete: 0, filter: {}, multi: true}"), + StatementAccess::Destructive, + ), + (bulk("{delete: 0, filter: {}}"), StatementAccess::Write), + (bulk("{delete: 0, filter: {}, multi: false}"), StatementAccess::Write), + (bulk("{delete: 0, filter: {a: 1}, multi: true}"), StatementAccess::Write), + ( + bulk("{update: 0, filter: {}, updateMods: {$set: {a: 1}}}"), + StatementAccess::Write, + ), + ( + bulk("{update: 0, filter: {a: 1}, updateMods: {$set: {a: 1}}, multi: true}"), + StatementAccess::Write, + ), + (bulk("{insert: 0, document: {}}"), StatementAccess::Write), + ]; + for (text, expected) in cases { + assert_eq!(script_access(&text), expected, "{text}"); + } + } + + #[test] + fn apply_ops_is_destructive_and_multi_is_read_as_the_server_reads_a_flag() { + for text in [ + "db.adminCommand({applyOps: [{op: 'd', ns: 'shop.users', o: {_id: 1}}]})", + "db.runCommand({applyOps: []})", + "db.users.update({}, {$set: {a: 1}}, {multi: 1})", + "db.runCommand({update: 'users', updates: [{q: {}, u: {$set: {a: 1}}, multi: 1}]})", + "db.runCommand({delete: 'users', deletes: [{q: null, limit: 0}]})", + ] { + assert_eq!(script_access(text), StatementAccess::Destructive, "{text}"); } + assert_eq!( + script_access("db.users.update({}, {$set: {a: 1}}, {multi: 0})"), + StatementAccess::Write + ); } #[test] @@ -361,6 +552,123 @@ mod tests { assert_eq!(script_access(""), StatementAccess::Read); } + #[test] + fn every_drop_is_destructive_as_sql_drop_is() { + for text in [ + "db.users.dropIndex('a_1')", + "db.users.dropIndex({a: 1})", + "db.users.dropIndexes()", + "db.users.dropIndexes(['a_1', 'b_1'])", + "db.users.dropSearchIndex('s')", + "db.getSiblingDB('shop').getCollection('users').dropIndexes()", + "db.dropUser('ann')", + "db.dropRole('auditor')", + "db.dropAllUsers()", + "db.dropAllRoles()", + "db.runCommand({dropIndexes: 'users', index: '*'})", + "db.runCommand({deleteIndexes: 'users', index: 'a_1'})", + "db.runCommand({dropSearchIndex: 'users', name: 's'})", + "db.runCommand({dropUser: 'ann'})", + "db.runCommand({dropRole: 'auditor'})", + "db.runCommand({dropAllUsersFromDatabase: 1})", + "db.adminCommand({dropAllRolesFromDatabase: 1})", + "db.runCommand({convertToCapped: 'orders', size: 4096})", + ] { + assert_eq!(script_access(text), StatementAccess::Destructive, "{text}"); + } + } + + #[test] + fn a_rename_that_drops_its_target_is_destructive() { + let cases = [ + ( + "db.orders_new.renameCollection('orders', true)", + StatementAccess::Destructive, + ), + ( + "db.orders_new.renameCollection('orders', {dropTarget: true})", + StatementAccess::Destructive, + ), + ( + "db.orders_new.renameCollection('orders', 1)", + StatementAccess::Destructive, + ), + ("db.orders_new.renameCollection('orders')", StatementAccess::Write), + ( + "db.orders_new.renameCollection('orders', false)", + StatementAccess::Write, + ), + ( + "db.orders_new.renameCollection('orders', {dropTarget: false})", + StatementAccess::Write, + ), + ( + "db.adminCommand({renameCollection: 'shop.a', to: 'shop.b', dropTarget: true})", + StatementAccess::Destructive, + ), + ( + "db.adminCommand({renameCollection: 'shop.a', to: 'shop.b', dropTarget: 1})", + StatementAccess::Destructive, + ), + ( + "db.adminCommand({renameCollection: 'shop.a', to: 'shop.b'})", + StatementAccess::Write, + ), + ( + "db.adminCommand({renameCollection: 'shop.a', to: 'shop.b', dropTarget: 0})", + StatementAccess::Write, + ), + // A key written twice is sent with its last value. + ( + "db.adminCommand({renameCollection: 'shop.a', to: 'shop.b', dropTarget: false, dropTarget: true})", + StatementAccess::Destructive, + ), + ( + "db.adminCommand({renameCollection: 'shop.a', to: 'shop.b', dropTarget: true, dropTarget: false})", + StatementAccess::Write, + ), + ]; + for (text, expected) in cases { + assert_eq!(script_access(text), expected, "{text}"); + } + } + + #[test] + fn a_validate_that_repairs_is_a_write() { + let cases = [ + ("db.runCommand({validate: 'orders'})", StatementAccess::Read), + ("db.runCommand({validate: 'orders', full: true})", StatementAccess::Read), + ( + "db.runCommand({validate: 'orders', repair: false})", + StatementAccess::Read, + ), + ( + "db.runCommand({validate: 'orders', repair: true})", + StatementAccess::Write, + ), + ( + "db.runCommand({validate: 'orders', repair: true, fixMultikey: true})", + StatementAccess::Write, + ), + ( + "db.runCommand({validate: 'orders', fixMultikey: true})", + StatementAccess::Write, + ), + ("db.runCommand({validate: 'orders', repair: 1})", StatementAccess::Write), + ( + "db.runCommand({validate: 'orders', repair: false, repair: true})", + StatementAccess::Write, + ), + ( + "db.runCommand({validate: 'orders', repair: true, repair: false})", + StatementAccess::Read, + ), + ]; + for (text, expected) in cases { + assert_eq!(script_access(text), expected, "{text}"); + } + } + #[test] fn a_write_may_change_the_catalog() { assert!(may_change_catalog("db.newcoll.insertOne({})")); diff --git a/core/crates/core/tests/script_corpus.rs b/core/crates/core/tests/script_corpus.rs index d214e72847..d938bd78aa 100644 --- a/core/crates/core/tests/script_corpus.rs +++ b/core/crates/core/tests/script_corpus.rs @@ -6,10 +6,10 @@ use sqlparser::dialect::{ SQLiteDialect, }; use sqlparser::tokenizer::{Token, Tokenizer, TokenizerError}; -use tablepro_core::sql_syntax::SqlGrammar; use tablepro_core::sql_syntax::script::{ BatchErrorPolicy, LexicalSettings, OpenConstruct, ScriptDiagnostic, ScriptPlan, ScriptTokenKind, text_offsets, }; +use tablepro_core::sql_syntax::{SqlGrammar, StatementAccess, statement_access}; fn plan(grammar: SqlGrammar, text: &str) -> ScriptPlan { ScriptPlan::build(text, grammar, LexicalSettings::default_for(grammar)) @@ -838,6 +838,113 @@ fn mongo_each_statement_is_its_own_batch_and_a_failure_stops_the_script() { assert_eq!(plan.batch_error_policy(), BatchErrorPolicy::StopScript); } +fn mongo_access(text: &str) -> StatementAccess { + statement_access(text, SqlGrammar::Mongo) +} + +#[test] +fn mongo_a_rename_that_drops_its_target_is_destructive() { + for text in [ + "db.orders_new.renameCollection('orders', true)", + "db.orders_new.renameCollection('orders', {dropTarget: true})", + "db.adminCommand({renameCollection: 'shop.orders_new', to: 'shop.orders', dropTarget: true})", + "db.adminCommand({renameCollection: 'shop.a', to: 'shop.b', dropTarget: false, dropTarget: true})", + ] { + assert_eq!(mongo_access(text), StatementAccess::Destructive, "{text}"); + } + for text in [ + "db.orders_new.renameCollection('orders')", + "db.adminCommand({renameCollection: 'shop.a', to: 'shop.b', dropTarget: true, dropTarget: false})", + ] { + assert_eq!(mongo_access(text), StatementAccess::Write, "{text}"); + } +} + +#[test] +fn mongo_dropping_an_index_is_destructive() { + for text in [ + "db.orders.dropIndex('qty_1')", + "db.orders.dropIndexes()", + "db.orders.dropSearchIndex('default')", + "db.runCommand({dropIndexes: 'orders', index: '*'})", + ] { + assert_eq!(mongo_access(text), StatementAccess::Destructive, "{text}"); + } +} + +#[test] +fn mongo_dropping_a_user_or_a_role_is_destructive() { + for text in [ + "db.dropUser('ann')", + "db.dropRole('auditor')", + "db.dropAllUsers()", + "db.dropAllRoles()", + "db.runCommand({dropUser: 'ann'})", + "db.runCommand({dropRole: 'auditor'})", + "db.runCommand({dropAllUsersFromDatabase: 1})", + "db.adminCommand({dropAllRolesFromDatabase: 1})", + ] { + assert_eq!(mongo_access(text), StatementAccess::Destructive, "{text}"); + } +} + +#[test] +fn mongo_a_validate_that_repairs_is_a_write() { + for text in [ + "db.runCommand({validate: 'orders', repair: true})", + "db.runCommand({validate: 'orders', repair: true, fixMultikey: true})", + "db.runCommand({validate: 'orders', fixMultikey: 1})", + "db.runCommand({validate: 'orders', repair: false, repair: true})", + ] { + assert_eq!(mongo_access(text), StatementAccess::Write, "{text}"); + } + for text in [ + "db.runCommand({validate: 'orders', full: true})", + "db.runCommand({validate: 'orders', repair: true, repair: false})", + ] { + assert_eq!(mongo_access(text), StatementAccess::Read, "{text}"); + } +} + +#[test] +fn mongo_what_replaces_a_collection_is_destructive() { + for text in [ + "db.orders.aggregate([{$match: {}}, {$out: 'orders_copy'}])", + "db.runCommand({aggregate: 'orders', pipeline: [{$out: 'orders_copy'}], cursor: {}})", + "db.runCommand({mapReduce: 'orders', map: 'm', reduce: 'r', out: 'totals'})", + "db.runCommand({mapReduce: 'orders', map: 'm', reduce: 'r', out: {replace: 'totals'}})", + "db.adminCommand({applyOps: [{op: 'd', ns: 'shop.orders', o: {_id: 1}}]})", + ] { + assert_eq!(mongo_access(text), StatementAccess::Destructive, "{text}"); + } + for text in [ + "db.orders.aggregate([{$merge: {into: 'totals'}}])", + "db.runCommand({mapReduce: 'orders', map: 'm', reduce: 'r', out: {merge: 'totals'}})", + "db.runCommand({mapReduce: 'orders', map: 'm', reduce: 'r', out: {reduce: 'totals'}})", + "db.runCommand({mapReduce: 'orders', map: 'm', reduce: 'r', out: {inline: 1}})", + ] { + assert_eq!(mongo_access(text), StatementAccess::Write, "{text}"); + } +} + +#[test] +fn mongo_a_bulk_write_that_reaches_every_document_is_destructive() { + for ops in [ + "{delete: 0, filter: {}, multi: true}", + "{update: 0, filter: {}, updateMods: {$set: {a: 1}}, multi: true}", + ] { + let text = format!("db.adminCommand({{bulkWrite: 1, ops: [{ops}], nsInfo: [{{ns: 'shop.orders'}}]}})"); + assert_eq!(mongo_access(&text), StatementAccess::Destructive, "{text}"); + } + for ops in [ + "{delete: 0, filter: {}}", + "{update: 0, filter: {_id: 1}, updateMods: {$set: {a: 1}}, multi: true}", + ] { + let text = format!("db.adminCommand({{bulkWrite: 1, ops: [{ops}], nsInfo: [{{ns: 'shop.orders'}}]}})"); + assert_eq!(mongo_access(&text), StatementAccess::Write, "{text}"); + } +} + const AGREED_CASES: &[(SqlGrammar, &str)] = &[ ( SqlGrammar::PostgreSql, diff --git a/core/crates/drivers/mongodb/src/call.rs b/core/crates/drivers/mongodb/src/call.rs index eb5ea5a501..0ad58d12f0 100644 --- a/core/crates/drivers/mongodb/src/call.rs +++ b/core/crates/drivers/mongodb/src/call.rs @@ -1,14 +1,17 @@ -use std::sync::atomic::AtomicBool; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex, PoisonError}; use std::time::Duration; use async_trait::async_trait; -use bson::{Document, doc}; -use mongodb::{Client, ClientSession}; +use bson::{Bson, Document, doc}; +use mongodb::{Client, ClientSession, Database}; use tablepro_core::sql_syntax::mongo::{ShellStatement, parse_bound_statement}; use tablepro_core::statement::Statement; -use tablepro_core::{CallResource, DriverError, EngineStop, LossPhase, StopDelivery, StopScope}; +use tablepro_core::{ + CallResource, DriverError, EngineStop, LossPhase, StopDelivery, StopReason, StopRequest, StopScope, TimeoutPhase, +}; use tokio::time::Instant; +use tokio_util::sync::CancellationToken; use crate::endpoint::MongoEndpoint; use crate::errors::stopped_by_kill; @@ -16,13 +19,19 @@ use crate::shell_command::{Plan, plan}; /// The stop of one call: `killSessions` for the logical session the call /// runs in, which ends the operation that session has on the server and -/// closes its cursors. Every call has a session of its own, so the stop -/// names the call and nothing else. +/// closes its cursors. A server that refuses it, as Atlas M0 and Flex do, +/// gets `killOp` for each of the login's own operations in that session +/// instead. Every call has a session of its own, so the stop names the +/// call and nothing else. pub(crate) struct SessionKill { /// The client the session belongs to, which sends the kill on a /// connection of its own, and the session's id. armed: Mutex>, sent: Mutex, + /// Set once `killSessions` has its first answer, or once the kill + /// sends nothing more; a stopped read waits for it. + answered: CancellationToken, + refused: AtomicBool, } /// How far a kill got, which says whether the route to the server still @@ -32,6 +41,9 @@ pub(crate) enum KillState { NotSent, /// Sent with no answer yet, or none before the call was given up. Unanswered, + /// `killSessions` came back refused and the `killOp` fallback has not + /// finished. The route answered. + Refused, Answered, } @@ -40,6 +52,8 @@ impl SessionKill { Arc::new(Self { armed: Mutex::new(None), sent: Mutex::new(KillState::NotSent), + answered: CancellationToken::new(), + refused: AtomicBool::new(false), }) } @@ -51,32 +65,111 @@ impl SessionKill { *self.sent.lock().unwrap_or_else(PoisonError::into_inner) = state; } + /// What a read of the call checks before it asks the server for more. + pub(crate) fn follow(self: &Arc, call: &StopRequest) -> ReadStop { + ReadStop { + call: call.clone(), + kill: self.clone(), + } + } + /// Name the session the stop ends, once the call has one. pub(crate) fn arm(&self, client: &Client, session: &ClientSession) { *self.armed.lock().unwrap_or_else(PoisonError::into_inner) = Some((client.clone(), session.id().clone())); } pub(crate) async fn kill(&self) -> Result<(), DriverError> { + // A stop that sends nothing, or is given up unanswered, lets a + // waiting read go on. + let _answered = self.answered.clone().drop_guard(); let armed = self.armed.lock().unwrap_or_else(PoisonError::into_inner).clone(); let Some((client, session)) = armed else { return Ok(()); }; self.mark(KillState::Unanswered); - let answered = client - .database("admin") - .run_command(doc! { "killSessions": [session] }) - .await; - // Any reply, a refusal included, came back over the route. + let admin = client.database("admin"); + let answered = match admin.run_command(doc! { "killSessions": [session.clone()] }).await { + Ok(_) => Ok(()), + Err(refused) if refused_by_server(&refused) => { + tracing::debug!(error = %refused, "killSessions refused, stopping the session's operations by id"); + // The state first, so a read that ends here finds the + // route answered. + self.mark(KillState::Refused); + self.refused.store(true, Ordering::SeqCst); + self.answered.cancel(); + kill_operations(&admin, &session).await + } + Err(error) => Err(error), + }; + // Any reply, a refusal included, came back over the route. Until + // the last command of the stop is answered, one could still land. let replied = match &answered { - Ok(_) => true, - Err(error) => matches!(error.kind.as_ref(), mongodb::error::ErrorKind::Command(_)), + Ok(()) => true, + Err(error) => refused_by_server(error), }; - if replied { - self.mark(KillState::Answered); + self.mark(match replied { + true => KillState::Answered, + false => KillState::Unanswered, + }); + answered.map_err(|error| DriverError::Protocol(format!("the stop did not reach the server: {error}"))) + } +} + +/// What a read of one call checks before it asks the server for more: +/// the call's stop and the kill that follows it. +#[derive(Clone)] +pub(crate) struct ReadStop { + call: StopRequest, + kill: Arc, +} + +impl ReadStop { + /// Once the call is stopped, wait for the server's answer to the + /// stop, so nothing is sent that the stop could miss. After a refusal + /// the read ends here, since `killOp` reaches no cursor waiting between + /// batches; after any other answer the killed cursor's error ends it. + pub(crate) async fn before_more(&self) -> Result<(), DriverError> { + if !self.call.is_requested() { + return Ok(()); } - answered - .map(|_| ()) - .map_err(|error| DriverError::Protocol(format!("the stop did not reach the server: {error}"))) + self.kill.answered.cancelled().await; + match self.kill.refused.load(Ordering::SeqCst) { + true => Err(stopped(&self.call)), + false => Ok(()), + } + } +} + +fn refused_by_server(error: &mongodb::error::Error) -> bool { + matches!(error.kind.as_ref(), mongodb::error::ErrorKind::Command(_)) +} + +/// `killOp` for each operation the login runs in `session`. Any user may +/// list and kill their own operations, and the session's id names only +/// this call's. +async fn kill_operations(admin: &Database, session: &Document) -> Result<(), mongodb::error::Error> { + let Some(id) = session.get("id") else { + return Ok(()); + }; + // The command, not the `$currentOp` stage: Atlas documents it on every tier. + let listed = admin + .run_command(doc! { "currentOp": 1_i32, "$ownOps": true, "lsid.id": id.clone() }) + .await?; + let operations = listed.get_array("inprog").map(Vec::as_slice).unwrap_or_default(); + for opid in operations.iter().filter_map(|operation| in_session(operation, id)) { + admin.run_command(doc! { "killOp": 1_i32, "op": opid.clone() }).await?; + } + Ok(()) +} + +/// The `opid` of a `currentOp` entry that runs in the session `id` +/// names. A server that ignores the filter lists the login's other +/// operations too, and those are not this call's to end. +fn in_session<'a>(operation: &'a Bson, id: &Bson) -> Option<&'a Bson> { + let operation = operation.as_document()?; + match operation.get_document("lsid").ok()?.get("id") { + Some(session) if session == id => operation.get("opid"), + _ => None, } } @@ -139,7 +232,8 @@ pub(crate) async fn session( /// Give a call's session back. A dropped session goes back to the /// client's pool for the next call, so one whose `killSessions` is still /// on its way is kept out of the pool until it could land, or the stop -/// would end whichever call took the session next. +/// would end whichever call took the session next. A `killOp` names an +/// operation, not the session, and the fallback ends with the call. pub(crate) fn release(session: ClientSession, stop: &SessionKill) { if stop.state() != KillState::Unanswered { return; @@ -152,6 +246,20 @@ pub(crate) fn release(session: ClientSession, stop: &SessionKill) { } } +/// A call that ends at its own stop, before it asks the server for more, +/// reads as `OwnedCalls` reports a stop the server acknowledged, a +/// shutdown as a cancel. The server was not told, so a deadline is never +/// a statement the server cancelled. +fn stopped(request: &StopRequest) -> DriverError { + match request.reason() { + Some(StopReason::DeadlineElapsed) => DriverError::Timeout { + phase: TimeoutPhase::Statement, + server_cancelled: false, + }, + Some(StopReason::Cancelled | StopReason::Shutdown) | None => DriverError::Cancelled, + } +} + /// How long the server may run what is left of a call, in the /// milliseconds `maxTimeMS` takes. pub(crate) fn remaining_millis(deadline: Option) -> Option { @@ -171,3 +279,86 @@ pub(crate) fn generated(statement: &Statement, database: &str) -> Result<(ShellS let planned = plan(&parsed, database, statement.params())?; Ok((parsed, planned)) } + +#[cfg(test)] +mod tests { + use bson::Binary; + use bson::spec::BinarySubtype; + + use super::*; + + #[test] + fn a_read_that_ends_at_its_stop_reads_as_what_stopped_it() { + let stop = |reason: Option| { + let request = StopRequest::new(); + if let Some(reason) = reason { + request.request(reason); + } + stopped(&request) + }; + + assert_eq!( + stop(Some(StopReason::DeadlineElapsed)), + DriverError::Timeout { + phase: TimeoutPhase::Statement, + server_cancelled: false + } + ); + assert_eq!(stop(Some(StopReason::Shutdown)), DriverError::Cancelled); + assert_eq!(stop(Some(StopReason::Cancelled)), DriverError::Cancelled); + assert_eq!(stop(None), DriverError::Cancelled); + } + + #[tokio::test(start_paused = true)] + async fn a_stopped_read_waits_for_the_answer_and_ends_at_a_refusal() { + let (kill, call) = (SessionKill::new(), StopRequest::new()); + let read = kill.follow(&call); + assert_eq!(read.before_more().await, Ok(())); + + call.request(StopReason::DeadlineElapsed); + let waited = tokio::time::timeout(Duration::from_secs(1), read.before_more()).await; + assert!(waited.is_err(), "the read went on before the answer"); + + kill.refused.store(true, Ordering::SeqCst); + kill.answered.cancel(); + assert_eq!( + read.before_more().await, + Err(DriverError::Timeout { + phase: TimeoutPhase::Statement, + server_cancelled: false + }) + ); + } + + #[tokio::test(start_paused = true)] + async fn a_stop_that_sends_nothing_lets_the_read_go_on() { + let (kill, call) = (SessionKill::new(), StopRequest::new()); + let read = kill.follow(&call); + call.request(StopReason::Cancelled); + + assert_eq!(kill.kill().await, Ok(())); + assert_eq!(read.before_more().await, Ok(())); + assert_eq!(kill.state(), KillState::NotSent); + } + + #[test] + fn only_an_operation_of_the_session_is_ended() { + let id = |byte: u8| { + Bson::Binary(Binary { + subtype: BinarySubtype::Uuid, + bytes: vec![byte; 16], + }) + }; + let entry = |session: Option, opid: i32| { + let mut operation = doc! { "opid": opid, "op": "getmore" }; + if let Some(session) = session { + operation.insert("lsid", doc! { "id": session }); + } + Bson::Document(operation) + }; + + assert_eq!(in_session(&entry(Some(id(1)), 7), &id(1)), Some(&Bson::Int32(7))); + assert_eq!(in_session(&entry(Some(id(2)), 8), &id(1)), None, "another session's"); + assert_eq!(in_session(&entry(None, 9), &id(1)), None, "no session's"); + } +} diff --git a/core/crates/drivers/mongodb/src/catalog.rs b/core/crates/drivers/mongodb/src/catalog.rs index 1ca24818e2..85ac67f322 100644 --- a/core/crates/drivers/mongodb/src/catalog.rs +++ b/core/crates/drivers/mongodb/src/catalog.rs @@ -1,12 +1,16 @@ use std::collections::{BTreeMap, BTreeSet}; +use std::sync::{Mutex, PoisonError}; -use bson::{Bson, Document, doc}; +use bson::spec::BinarySubtype; +use bson::{Binary, Bson, Document, doc}; use mongodb::{Client, ClientSession}; +use tablepro_core::column::ColumnKind; use tablepro_core::meta::{IndexInfo, IndexKey, ObjectInfo, ObjectKind, RowCountEstimate, SchemaInfo, TableEstimate}; use tablepro_core::{ColumnInfo, DriverError, LossPhase}; use crate::endpoint::MongoEndpoint; use crate::field_types::{field_column, type_alias}; +use crate::shell_command::Plan; /// How many documents a collection's fields are found from, in their /// natural order. Enough to meet the fields a collection's documents @@ -145,22 +149,15 @@ impl Reader<'_> { /// in the order the documents place them: `_id` first. The server /// tallies them, so no document crosses the wire. pub(crate) async fn fields(&mut self, database: &str, collection: &str) -> Result, DriverError> { - let pipeline = vec![ - doc! { "$limit": CENSUS_DOCUMENTS }, - doc! { "$project": { "_id": 0_i32, "field": { "$objectToArray": "$$ROOT" } } }, - doc! { "$unwind": { "path": "$field", "includeArrayIndex": "position" } }, - doc! { "$group": { - "_id": { "name": "$field.k", "type": { "$type": "$field.v" } }, - "position": { "$min": "$position" }, - } }, - ]; + let mut pipeline = vec![doc! { "$limit": CENSUS_DOCUMENTS }]; + pipeline.extend(census_stages()); let tallies = self .documents( database, doc! { "aggregate": collection, "pipeline": pipeline, "cursor": {} }, ) .await?; - Ok(columns_from(&tallies)) + Ok(collection_columns(&tallies)) } /// The collection's indexes, `_id`'s first. @@ -190,6 +187,50 @@ impl Reader<'_> { } } +/// Each collection's fields that its columns show as JSON, from the last +/// census of its fields, so a page or an export of it shows them as +/// Extended JSON straight from the BSON, which keeps an `int` apart from +/// a `long`. +#[derive(Debug, Default)] +pub(crate) struct JsonFields(Mutex>>); + +impl JsonFields { + /// Keep the fields among a collection's columns that are JSON. + pub(crate) fn remember(&self, database: &str, collection: &str, columns: &[ColumnInfo]) -> BTreeSet { + let fields: BTreeSet = columns + .iter() + .filter(|column| matches!(column.column_type.kind, ColumnKind::Json { .. })) + .map(|column| column.name.clone()) + .collect(); + self.0 + .lock() + .unwrap_or_else(PoisonError::into_inner) + .insert((database.to_owned(), collection.to_owned()), fields.clone()); + fields + } + + /// The JSON fields of the collection `planned` reads. A connection + /// that has not read them, as a new one after a reconnect has not, + /// takes the census first in the same call, so it shows the fields the + /// columns it was handed show as JSON. + pub(crate) async fn of(&self, planned: &Plan, reader: &mut Reader<'_>) -> Result, DriverError> { + let Some((database, collection)) = planned.collection() else { + return Ok(BTreeSet::new()); + }; + let known = self + .0 + .lock() + .unwrap_or_else(PoisonError::into_inner) + .get(&(database.to_owned(), collection.to_owned())) + .cloned(); + if let Some(fields) = known { + return Ok(fields); + } + let columns = reader.fields(database, collection).await?; + Ok(self.remember(database, collection, &columns)) + } +} + /// What a collection is, as `listCollections` says. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum CollectionType { @@ -209,8 +250,55 @@ impl CollectionType { } } -/// The columns a tally of fields and types makes. -fn columns_from(tallies: &[Document]) -> Vec { +/// The stages that tally the fields of the documents before them, each +/// with the types it holds and the least position a document puts it at, +/// for `columns_from`. +pub(crate) fn census_stages() -> [Document; 3] { + [ + doc! { "$project": { "_id": 0_i32, "field": { "$objectToArray": "$$ROOT" } } }, + doc! { "$unwind": { "path": "$field", "includeArrayIndex": "position" } }, + doc! { "$group": { + "_id": { "name": "$field.k", "type": census_type() }, + "position": { "$min": "$position" }, + } }, + ] +} + +/// A field value's `$type`, or `uuid` for a UUID. `$type` names every +/// binary subtype `binData` and `$subtype` is new in MongoDB 8.3, but BSON +/// orders binary data by length, then subtype, then bytes, so a UUID is +/// a value between the least and the greatest 16-byte subtype 4 values. +fn census_type() -> Document { + let uuid = |byte: u8| { + Bson::Binary(Binary { + subtype: BinarySubtype::Uuid, + bytes: vec![byte; 16], + }) + }; + doc! { "$cond": { + "if": { "$and": [ + { "$eq": [{ "$type": "$field.v" }, "binData"] }, + { "$gte": ["$field.v", uuid(0x00)] }, + { "$lte": ["$field.v", uuid(0xff)] }, + ] }, + "then": "uuid", + "else": { "$type": "$field.v" }, + } } +} + +/// The columns of a collection whose first documents tallied `tallies`, +/// `_id` among them even when they hold none, as every document has one. +fn collection_columns(tallies: &[Document]) -> Vec { + let mut columns = columns_from(tallies); + if !columns.iter().any(|column| column.name == "_id") { + columns.insert(0, field_column("_id", &["objectId"].into_iter().collect())); + } + columns +} + +/// The columns the fields a census tallied make, in the order the +/// documents place them, `_id` first where they hold one. +pub(crate) fn columns_from(tallies: &[Document]) -> Vec { let mut fields: BTreeMap)> = BTreeMap::new(); for tally in tallies { let Ok(key) = tally.get_document("_id") else { @@ -233,14 +321,10 @@ fn columns_from(tallies: &[Document]) -> Vec { ordered.sort_by(|(left_name, (left, _)), (right_name, (right, _))| { (left_name != "_id", left, left_name).cmp(&(right_name != "_id", right, right_name)) }); - let mut columns: Vec = ordered + ordered .iter() .map(|(name, (_, aliases))| field_column(name, aliases)) - .collect(); - if !columns.iter().any(|column| column.name == "_id") { - columns.insert(0, field_column("_id", &["objectId"].into_iter().collect())); - } - columns + .collect() } /// A `$type` name the server reported, as one of the names the types @@ -258,6 +342,7 @@ fn alias_named(name: &str) -> &'static str { "int" => Bson::Int32(0), "long" => Bson::Int64(0), "decimal" => Bson::Decimal128(bson::Decimal128::from_bytes([0; 16])), + "uuid" => return "uuid", "binData" => return "binData", "timestamp" => return "timestamp", "regex" => return "regex", @@ -307,8 +392,6 @@ fn index_info(entry: &Document) -> Option { #[cfg(test)] mod tests { - use tablepro_core::column::ColumnKind; - use super::*; fn tally(name: &str, kind: &str, position: i64) -> Document { @@ -330,22 +413,50 @@ mod tests { let names: Vec<&str> = columns.iter().map(|column| column.name.as_str()).collect(); assert_eq!(names, ["_id", "a", "c", "b"]); - assert_eq!(columns[1].column_type.name, "long"); + assert!( + matches!(columns[1].column_type.kind, ColumnKind::Json { .. }), + "an int and a long" + ); assert!( matches!(columns[2].column_type.kind, ColumnKind::Json { .. }), "a mixed field" ); + assert_eq!(columns[3].column_type.name, "string"); assert!(columns[0].primary_key); } + #[test] + fn a_uuid_tally_is_a_uuid_column_unless_other_binary_data_shares_it() { + let tallies = [ + tally("_id", "uuid", 0), + tally("blob", "uuid", 1), + tally("blob", "binData", 1), + ]; + + let columns = columns_from(&tallies); + + assert_eq!(columns[0].column_type.kind, ColumnKind::Uuid); + assert!(matches!(columns[1].column_type.kind, ColumnKind::Json { .. })); + } + #[test] fn an_empty_collection_still_has_its_id() { - let columns = columns_from(&[]); + let columns = collection_columns(&[]); assert_eq!(columns.len(), 1); assert_eq!(columns[0].name, "_id"); } + #[test] + fn a_census_of_documents_without_an_id_names_only_their_fields() { + let names = |tallies: &[Document]| -> Vec { + columns_from(tallies).into_iter().map(|column| column.name).collect() + }; + + assert!(names(&[]).is_empty()); + assert_eq!(names(&[tally("total", "int", 0)]), ["total"]); + } + #[test] fn an_index_lists_its_keys_and_uniqueness() { let index = diff --git a/core/crates/drivers/mongodb/src/connection.rs b/core/crates/drivers/mongodb/src/connection.rs index a5d55fccf7..1207e95cfe 100644 --- a/core/crates/drivers/mongodb/src/connection.rs +++ b/core/crates/drivers/mongodb/src/connection.rs @@ -1,14 +1,13 @@ -use std::collections::{BTreeMap, BTreeSet}; use std::future::Future; use std::pin::Pin; +use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, Mutex, PoisonError}; use std::time::Duration; use async_trait::async_trait; use bson::doc; use mongodb::{Client, ClientSession}; -use tablepro_core::column::{ColumnInfo, ColumnKind}; +use tablepro_core::column::ColumnInfo; use tablepro_core::ddl::{ ColumnAttributes, ColumnDefinition, InboundForeignKey, OutboundForeignKey, PrimaryKeyConstraint, TableDefinition, }; @@ -26,24 +25,27 @@ use tablepro_core::{ use tokio::time::Instant; use tokio_util::sync::{CancellationToken, WaitForCancellationFutureOwned}; -use crate::call::{ClientCall, KillState, SessionKill, generated, release, remaining_millis, session}; -use crate::catalog::{CollectionType, Reader}; +use crate::call::{ClientCall, KillState, ReadStop, SessionKill, generated, release, remaining_millis, session}; +use crate::catalog::{CollectionType, JsonFields, Reader}; use crate::dialect::MongoDialect; use crate::editor_session::MongoEditorSession; use crate::endpoint::MongoEndpoint; use crate::execute::{Source, start}; use crate::rows::collect; +use crate::stream_read::ExportScope; use crate::write_batch::{WritePlan, perform, precheck}; /// A workspace call's work, which borrows the call's session. type Work<'s, T> = Pin> + Send + 's>>; /// What a workspace call works with: the client, the endpoint that -/// explains its failures, and the server's time limit for the call. +/// explains its failures, the server's time limit for the call and the +/// stop its reads check before they ask for more. pub(crate) struct CallContext { pub(crate) endpoint: Arc, pub(crate) client: Client, pub(crate) max_time: Option, + pub(crate) stop: ReadStop, } /// A workspace's link to one server: a client whose pool serves the @@ -60,11 +62,7 @@ pub(crate) struct MongoConnection { dialect: Arc, link: LinkWatch, closed: AtomicBool, - /// Each collection's fields that its columns show as JSON, from the - /// last read of its fields, so a page or an export of it shows them - /// as Extended JSON straight from the BSON, which keeps an `int` - /// apart from a `long`. - json_fields: Mutex>>, + json_fields: Arc, } impl MongoConnection { @@ -87,35 +85,10 @@ impl MongoConnection { dialect: Arc::new(MongoDialect), link: LinkWatch::new(), closed: AtomicBool::new(false), - json_fields: Mutex::new(BTreeMap::new()), + json_fields: Arc::new(JsonFields::default()), } } - fn remember_json_fields(&self, database: String, collection: String, columns: &[ColumnInfo]) { - let fields = columns - .iter() - .filter(|column| matches!(column.column_type.kind, ColumnKind::Json { .. })) - .map(|column| column.name.clone()) - .collect(); - self.json_fields - .lock() - .unwrap_or_else(PoisonError::into_inner) - .insert((database, collection), fields); - } - - /// The JSON fields of the collection a plan reads. - fn json_fields_of(&self, planned: &crate::shell_command::Plan) -> BTreeSet { - let Some((database, collection)) = planned.collection() else { - return BTreeSet::new(); - }; - self.json_fields - .lock() - .unwrap_or_else(PoisonError::into_inner) - .get(&(database.to_owned(), collection.to_owned())) - .cloned() - .unwrap_or_default() - } - fn ensure_open(&self) -> Result<(), DriverError> { if self.closed.load(Ordering::SeqCst) { return Err(DriverError::ConnectionLost { @@ -138,16 +111,19 @@ impl MongoConnection { { self.ensure_open()?; let kill = SessionKill::new(); - let context = CallContext { - endpoint: self.endpoint.clone(), - client: self.client.clone(), - max_time: remaining_millis(options.deadline()), - }; + let (endpoint, client) = (self.endpoint.clone(), self.client.clone()); + let max_time = remaining_millis(options.deadline()); let armed = kill.clone(); let stop = kill.clone(); let outcome = self .calls .run_read(options, ClientCall::new(), kill, move |call, request| async move { + let context = CallContext { + endpoint, + client, + max_time, + stop: armed.follow(&request), + }; let mut session = match session(&context.endpoint, &context.client, &armed).await { Ok(session) => session, Err(error) => return (call, Err(error)), @@ -165,8 +141,9 @@ impl MongoConnection { /// Decide whether a failed call means the link itself is gone. A kill /// that never came back means the route is gone, and one the server - /// answered means it is not; with no kill sent, the probe's own - /// connection answers within the cancel grace, or the route is gone. + /// answered, if only to refuse it, means it is not; with no kill sent, + /// the probe's own connection answers within the cancel grace, or the + /// route is gone. async fn settle(&self, outcome: Result, stop: &SessionKill) -> Result { let error = match outcome { Ok(value) => return Ok(value), @@ -177,7 +154,7 @@ impl MongoConnection { } let lost = match stop.state() { KillState::Unanswered => true, - KillState::Answered => false, + KillState::Refused | KillState::Answered => false, KillState::NotSent => self .check(self.endpoint.cancel_grace()) .await @@ -213,12 +190,10 @@ impl MongoConnection { /// A generated statement's plan, refused when it is not a read on a /// connection held read-only. fn generated_read(&self, statement: &Statement) -> Result { - let (parsed, mut planned) = generated(statement, &self.database)?; + let (parsed, planned) = generated(statement, &self.database)?; if self.settings.read_only && access_of(&parsed) != StatementAccess::Read { return Err(DriverError::ReadOnly(ReadOnlyRefusal::ClientGuard)); } - let json_fields = self.json_fields_of(&planned); - planned.show_as_json(&json_fields); Ok(planned) } } @@ -346,7 +321,7 @@ impl Connection for MongoConnection { }) }) .await?; - self.remember_json_fields(key.0, key.1, &columns); + self.json_fields.remember(&key.0, &key.1, &columns); Ok(columns) } @@ -475,11 +450,15 @@ impl Connection for MongoConnection { options: CallOptions, ) -> Result { let mut planned = self.generated_read(statement)?; - planned.limit_time(remaining_millis(options.deadline())); + let (json_fields, deadline) = (self.json_fields.clone(), options.deadline()); self.read(options, move |context, session| { Box::pin(async move { + let shown = json_fields.of(&planned, &mut reader(&context, session)).await?; + planned.show_as_json(&shown); + // After the census, which the page's deadline also bounds. + planned.limit_time(remaining_millis(deadline)); let source = start(&context.endpoint, &context.client, session, planned).await?; - collect(&context.endpoint, source, session, limit).await + collect(&context.endpoint, source, session, limit, &context.stop).await }) }) .await @@ -508,12 +487,7 @@ impl Connection for MongoConnection { ) -> Result { self.ensure_open()?; let (read, endpoint, database) = (read.clone(), self.endpoint.clone(), self.database.clone()); - let json_fields = match &read { - ReadStatement::Generated(statement) => generated(statement, &database) - .map(|(_, planned)| self.json_fields_of(&planned)) - .unwrap_or_default(), - ReadStatement::User(_) => BTreeSet::new(), - }; + let (json_fields, max_time) = (self.json_fields.clone(), remaining_millis(options.deadline())); let kill = SessionKill::new(); let armed = kill.clone(); let stop = kill.clone(); @@ -524,16 +498,13 @@ impl Connection for MongoConnection { ClientCall::new(), kill, move |call, request: StopRequest| async move { - let result = crate::stream_read::stream( - &endpoint, - &database, - &read, - &json_fields, - sink.as_mut(), - &request, - &armed, - ) - .await; + let scope = ExportScope { + database: &database, + json_fields: &json_fields, + max_time, + }; + let result = + crate::stream_read::stream(&endpoint, &scope, &read, sink.as_mut(), &request, &armed).await; (call, result) }, ) diff --git a/core/crates/drivers/mongodb/src/cursor_rows.rs b/core/crates/drivers/mongodb/src/cursor_rows.rs index 07b716758e..801044669a 100644 --- a/core/crates/drivers/mongodb/src/cursor_rows.rs +++ b/core/crates/drivers/mongodb/src/cursor_rows.rs @@ -2,54 +2,109 @@ use bson::Document; use mongodb::{ClientSession, SessionCursor}; use tablepro_core::{DriverError, LossPhase, ResultEvent, ResultSink, RowLimit, SinkFlow, Truncation}; +use crate::call::ReadStop; use crate::endpoint::MongoEndpoint; use crate::rows::{BATCH_ROWS, Layout}; use crate::shell_command::Columns; -/// Stream a cursor into `sink`. With a `limit`, as an editor reads, the -/// documents kept are all read before the first row is shown, so the -/// columns hold every field they have. With none, as an export reads, -/// each batch is accepted before the next is read, under columns fixed -/// before the first row. +/// Stream a cursor into `sink` as the result set `set_index`. With a +/// `limit`, as an editor reads, at most that many documents are kept; with +/// none, as an export reads, every one, under columns fixed before the +/// first row. pub(crate) async fn stream( - endpoint: &MongoEndpoint, - cursor: SessionCursor, + read: Read<'_>, columns: Columns, - session: &mut ClientSession, sink: &mut dyn ResultSink, limit: Option, + set_index: usize, ) -> Result<(SinkFlow, u64), DriverError> { - match limit { - Some(limit) => kept(endpoint, cursor, columns, session, sink, limit).await, - None => every(endpoint, cursor, columns, session, sink).await, + let discovered = matches!(columns, Columns::Discovered); + match (limit, discovered) { + (Some(limit), true) => discovered_kept(read, columns, sink, limit).await, + (Some(limit), false) => named_kept(read, columns, sink, limit, set_index).await, + (None, _) => every(read, columns, sink).await, } } -/// At most `limit` documents. One more says whether the result was cut, -/// and dropping the cursor closes it on the server, so the rest is never -/// read. -async fn kept( - endpoint: &MongoEndpoint, - mut cursor: SessionCursor, +/// A cursor, the session it reads in, and the stop of the call that +/// reads it. +pub(crate) struct Read<'a> { + endpoint: &'a MongoEndpoint, + cursor: SessionCursor, + session: &'a mut ClientSession, + stop: &'a ReadStop, +} + +impl<'a> Read<'a> { + pub(crate) fn new( + endpoint: &'a MongoEndpoint, + cursor: SessionCursor, + session: &'a mut ClientSession, + stop: &'a ReadStop, + ) -> Self { + Self { + endpoint, + cursor, + session, + stop, + } + } + + /// The next document, `None` once the cursor is spent. A stopped read + /// asks for no more until the server answers the stop, and one the + /// server refused ends here, where dropping the cursor closes it. + pub(crate) async fn next(&mut self) -> Result, DriverError> { + self.stop.before_more().await?; + match self.cursor.advance(self.session).await { + Ok(true) => crate::nested_document::current_document(&self.cursor).map(Some), + Ok(false) => Ok(None), + Err(error) => Err(self.endpoint.explain(&error, LossPhase::Statement).await), + } + } + + /// Up to a batch of the `room` documents left to keep, and how the set + /// ends when it does: a full `room` reads one more to say whether the + /// result was cut. + async fn batch(&mut self, room: u64) -> Result<(Vec, Option), DriverError> { + let take = usize::try_from(room).unwrap_or(usize::MAX).min(BATCH_ROWS); + let mut documents = Vec::with_capacity(take); + while documents.len() < take { + match self.next().await? { + Some(document) => documents.push(document), + None => return Ok((documents, Some(Truncation::Complete))), + } + } + if (documents.len() as u64) < room { + return Ok((documents, None)); + } + let end = match self.next().await? { + Some(_) => Truncation::ReadPastLimit, + None => Truncation::Complete, + }; + Ok((documents, Some(end))) + } +} + +/// At most `limit` documents under columns found from them. A later +/// document can add a column, so the documents kept are all read before +/// the first row is shown. One more says whether the result was cut, and +/// dropping the cursor closes it on the server, so the rest is never read. +async fn discovered_kept( + mut read: Read<'_>, columns: Columns, - session: &mut ClientSession, sink: &mut dyn ResultSink, limit: RowLimit, ) -> Result<(SinkFlow, u64), DriverError> { let mut documents = Vec::new(); let mut more = false; - loop { - match cursor.advance(session).await { - Ok(true) if (documents.len() as u64) < limit.get() => documents.push(current(&cursor)?), - Ok(true) => { - more = true; - break; - } - Ok(false) => break, - Err(error) => return Err(endpoint.explain(&error, LossPhase::Statement).await), + while let Some(document) = read.next().await? { + if documents.len() as u64 == limit.get() { + more = true; + break; } + documents.push(document); } - drop(cursor); + drop(read); let mut layout = Layout::new(&columns); for document in &documents { layout.learn(document); @@ -67,16 +122,77 @@ async fn kept( return Ok((SinkFlow::ConsumerGone, retained)); } } - if !more { - return Ok(( - finish(sink, retained, Some(retained), Truncation::Complete).await, - retained, - )); + let end = match more { + true => Truncation::ReadPastLimit, + false => Truncation::Complete, + }; + Ok((close(sink, retained, end).await, retained)) +} + +/// At most `limit` documents under columns a projection names, a batch at +/// a time, each accepted before the next is read. The set starts with the +/// types its first batch holds, and ends with `ColumnsResolved` when the +/// rows after it hold others. One more document says whether the result +/// was cut, and dropping the cursor closes it on the server, so the rest is +/// never read. +async fn named_kept( + mut read: Read<'_>, + columns: Columns, + sink: &mut dyn ResultSink, + limit: RowLimit, + set_index: usize, +) -> Result<(SinkFlow, u64), DriverError> { + let mut layout = Layout::new(&columns); + let (mut documents, mut ending) = read.batch(limit.get()).await?; + for document in &documents { + layout.learn(document); + } + let announced = layout.columns(); + let started = ResultEvent::ResultSetStarted { + columns: announced.clone(), + }; + if sink.accept(started).await == SinkFlow::ConsumerGone { + return Ok((SinkFlow::ConsumerGone, 0)); } - if sink.accept(ResultEvent::RowLimitReached { retained }).await == SinkFlow::ConsumerGone { - return Ok((SinkFlow::ConsumerGone, retained)); + let mut retained: u64 = 0; + let end = loop { + if !documents.is_empty() { + retained += documents.len() as u64; + let rows = documents.iter().map(|document| layout.row(document)).collect(); + if sink.accept(ResultEvent::Rows(rows)).await == SinkFlow::ConsumerGone { + return Ok((SinkFlow::ConsumerGone, retained)); + } + } + if let Some(end) = ending { + break end; + } + (documents, ending) = read.batch(limit.get() - retained).await?; + for document in &documents { + layout.learn(document); + } + }; + drop(read); + let columns = layout.columns(); + if columns != announced { + let resolved = ResultEvent::ColumnsResolved { set_index, columns }; + if sink.accept(resolved).await == SinkFlow::ConsumerGone { + return Ok((SinkFlow::ConsumerGone, retained)); + } + } + Ok((close(sink, retained, end).await, retained)) +} + +/// End a kept set: `RowLimitReached` first when the result was cut. +async fn close(sink: &mut dyn ResultSink, retained: u64, end: Truncation) -> SinkFlow { + match end { + Truncation::Complete => finish(sink, retained, Some(retained), Truncation::Complete).await, + cut => { + if sink.accept(ResultEvent::RowLimitReached { retained }).await == SinkFlow::ConsumerGone { + return SinkFlow::ConsumerGone; + } + finish(sink, retained, None, cut).await + } } - Ok((finish(sink, retained, None, Truncation::ReadPastLimit).await, retained)) } /// Every document, a batch at a time. The columns are set before the @@ -84,10 +200,8 @@ async fn kept( /// a write between the export's two reads can bring, fails the read /// rather than losing the field. async fn every( - endpoint: &MongoEndpoint, - mut cursor: SessionCursor, + mut read: Read<'_>, columns: Columns, - session: &mut ClientSession, sink: &mut dyn ResultSink, ) -> Result<(SinkFlow, u64), DriverError> { let found = matches!(columns, Columns::Discovered); @@ -98,13 +212,12 @@ async fn every( let mut documents = Vec::new(); let mut exhausted = false; while documents.len() < BATCH_ROWS { - match cursor.advance(session).await { - Ok(true) => documents.push(current(&cursor)?), - Ok(false) => { + match read.next().await? { + Some(document) => documents.push(document), + None => { exhausted = true; break; } - Err(error) => return Err(endpoint.explain(&error, LossPhase::Statement).await), } } for document in &documents { @@ -139,10 +252,6 @@ async fn every( } } -fn current(cursor: &SessionCursor) -> Result { - crate::nested_document::current_document(cursor) -} - async fn finish(sink: &mut dyn ResultSink, retained: u64, total_rows: Option, truncation: Truncation) -> SinkFlow { sink.accept(ResultEvent::ResultSetFinished { retained, diff --git a/core/crates/drivers/mongodb/src/dialect.rs b/core/crates/drivers/mongodb/src/dialect.rs index 9fa2d50bd4..88fa529e5e 100644 --- a/core/crates/drivers/mongodb/src/dialect.rs +++ b/core/crates/drivers/mongodb/src/dialect.rs @@ -1,8 +1,9 @@ -use tablepro_core::column::{ColumnType, SqlExpression}; +use tablepro_core::column::{ColumnInfo, ColumnType, SqlExpression}; use tablepro_core::dialect::{ BindError, BindTarget, DatabaseLayout, DialectCapabilities, InsertBatchLimits, InsertConflict, KeysetDirection, LiteralError, Placeholder, SqlDialect, }; +use tablepro_core::filter::FilterOp; use tablepro_core::meta::{EngineRowId, EngineRowIdPart}; use tablepro_core::sql_syntax::{ExpressionSyntaxError, HistoryRecording, SqlGrammar, TypeSyntaxError}; use tablepro_core::value::Value; @@ -59,6 +60,12 @@ impl SqlDialect for MongoDialect { text.to_owned() } + /// The operators `tablepro_core::document` builds a rule with, so the + /// filter strip offers no rule the filter would refuse. + fn filter_operators(&self, column: &ColumnInfo) -> &'static [FilterOp] { + tablepro_core::document::filter_operators(column) + } + fn row_window(&self, _order_by: Option<&str>, _limit: u64, _offset: u64) -> String { String::new() } @@ -101,10 +108,19 @@ impl SqlDialect for MongoDialect { fn insert_default_row(&self, _table_sql: &str) -> Option { None } + + /// A page reads a missing field as NULL, so a NULL cell is a field the + /// document leaves out, as `tablepro_core::document` inserts it. + fn insert_names_null_cells(&self) -> bool { + false + } } #[cfg(test)] mod tests { + use tablepro_core::column::ResultColumn; + use tablepro_core::export::render_sql_insert; + use tablepro_core::meta::TableRef; use tablepro_core::sql_syntax::WithholdReason; use tablepro_core::statement::ParamType; @@ -115,6 +131,47 @@ mod tests { assert_eq!(MongoDialect.insert_batch_limits().rows, 1); } + #[test] + fn an_exported_insert_leaves_a_null_field_out() { + let columns = ["_id", "name", "email"].map(|name| ResultColumn::new(name, ColumnType::unknown())); + let rows = [vec![Value::Int(1), Value::Text("ann".to_owned()), Value::Null]]; + + let sql = render_sql_insert( + &MongoDialect, + &TableRef::new(Some("shop".to_owned()), "users"), + &columns, + &[], + &[], + &rows, + ); + + assert_eq!( + sql, + "db.getSiblingDB(\"shop\").getCollection(\"users\").insertOne({\"_id\": 1, \"name\": \"ann\"});\n" + ); + } + + #[test] + fn a_field_of_ints_and_doubles_keeps_the_comparisons() { + let price = crate::field_types::field_column("price", &["int", "double"].into_iter().collect()); + let qty = crate::field_types::field_column("qty", &["int"].into_iter().collect()); + + let offered = MongoDialect.filter_operators(&price); + for op in [ + FilterOp::Lt, + FilterOp::GtEq, + FilterOp::Between, + FilterOp::Contains, + FilterOp::IsNull, + ] { + assert!(offered.contains(&op), "{op:?} in {offered:?}"); + } + assert_eq!( + MongoDialect.filter_operators(&qty), + tablepro_core::filter::operators_for(&qty.column_type.kind) + ); + } + #[test] fn a_name_is_a_shell_string_and_a_value_binds_by_its_ordinal() { let dialect = MongoDialect; diff --git a/core/crates/drivers/mongodb/src/editor_run.rs b/core/crates/drivers/mongodb/src/editor_run.rs index ac737cc1c6..bc84b37e2a 100644 --- a/core/crates/drivers/mongodb/src/editor_run.rs +++ b/core/crates/drivers/mongodb/src/editor_run.rs @@ -8,8 +8,8 @@ use mongodb::Client; use tablepro_core::sql_syntax::StatementAccess; use tablepro_core::sql_syntax::mongo::{access_of, parse_statement, statement_ranges}; use tablepro_core::{ - DriverError, ErrorCategory, ReadOnlyRefusal, ResultSink, RunEnd, RunOptions, RunSummary, SinkFlow, TimeoutPhase, - TransactionState, UserSql, + DriverError, ErrorCategory, ReadOnlyRefusal, ResultSink, RunEnd, RunOptions, RunSummary, SinkFlow, StopReason, + StopRequest, TimeoutPhase, TransactionState, UserSql, }; use tokio::time::Instant; use tokio_util::sync::CancellationToken; @@ -35,6 +35,13 @@ impl Interrupt { Self::DeadlineElapsed => RunEnd::TimedOut { server_cancelled }, } } + + fn reason(self) -> StopReason { + match self { + Self::Cancelled => StopReason::Cancelled, + Self::DeadlineElapsed => StopReason::DeadlineElapsed, + } + } } /// What became of one statement. @@ -70,6 +77,7 @@ impl EditorRun { let started = Instant::now(); let run_deadline = options.deadline.map(|bound| started + bound); let text = sql.as_str(); + let mut sets = 0; for range in statement_ranges(text) { let statement = match parse_statement(&text[range.clone()]) { Ok(statement) => statement, @@ -99,7 +107,9 @@ impl EditorRun { if let Some(interrupt) = self.interrupted(&options.cancel, deadline) { return Ok(summary(started, interrupt.end(false))); } - let ran = self.statement(planned, sink.as_mut(), &options, deadline).await; + let ran = self + .statement(planned, sink.as_mut(), &options, deadline, &mut sets) + .await; match ran { Ran::Done(SinkFlow::ConsumerGone) => return Ok(summary(started, RunEnd::Cancelled)), Ran::Done(SinkFlow::Continue) => {} @@ -120,33 +130,44 @@ impl EditorRun { /// Send one statement in a session of its own and stream what it /// returns, or stop it: `killSessions` ends what that session runs on - /// the server, cursors included. + /// the server, cursors included. The read asks for no more until the + /// server answers it, and where it is refused and `killOp` stands in, + /// ends there. fn statement<'a>( &'a self, planned: crate::shell_command::Plan, sink: &'a mut dyn ResultSink, options: &'a RunOptions, deadline: Option, + sets: &'a mut usize, ) -> Pin + Send + 'a>> { Box::pin(async move { let kill = SessionKill::new(); + let stop = StopRequest::new(); let mut session = match session(&self.endpoint, &self.client, &kill).await { Ok(session) => session, Err(error) => return Ran::Failed(error), }; let ran = { let work = async { - match start(&self.endpoint, &self.client, &mut session, planned).await? { + let (source, limit) = match start(&self.endpoint, &self.client, &mut session, planned).await? { Source::Used(next) => { *self.database.lock().unwrap_or_else(PoisonError::into_inner) = next.clone(); - emit(&self.endpoint, Source::Used(next), &mut session, sink, None) - .await - .map(|(flow, _)| flow) + (Source::Used(next), None) } - source => emit(&self.endpoint, source, &mut session, sink, Some(options.row_limit)) - .await - .map(|(flow, _)| flow), - } + source => (source, Some(options.row_limit)), + }; + emit( + &self.endpoint, + source, + &mut session, + sink, + limit, + sets, + &kill.follow(&stop), + ) + .await + .map(|(flow, _)| flow) }; let mut work = std::pin::pin!(work); let interrupt = tokio::select! { @@ -162,6 +183,7 @@ impl EditorRun { match interrupt { ControlFlow::Break(ran) => ran, ControlFlow::Continue(interrupt) => { + stop.request(interrupt.reason()); let grace_ends = Instant::now() + self.endpoint.cancel_grace(); let _ = tokio::time::timeout_at(grace_ends, kill.kill()).await; let server_cancelled = match tokio::time::timeout_at(grace_ends, &mut work).await { diff --git a/core/crates/drivers/mongodb/src/endpoint.rs b/core/crates/drivers/mongodb/src/endpoint.rs index 2c35bf439e..bd695a1d23 100644 --- a/core/crates/drivers/mongodb/src/endpoint.rs +++ b/core/crates/drivers/mongodb/src/endpoint.rs @@ -78,7 +78,7 @@ impl MongoEndpoint { let service = target.transport.service_endpoint().clone(); let database = target.database.clone().filter(|name| !name.trim().is_empty()); let mut template = match (target.options.dns_seed_list, target.transport.route()) { - (true, TransportRoute::Tcp(_)) => seed_list(&service).await?, + (true, TransportRoute::Tcp(_)) => seed_list(&service, &target.credentials).await?, (true, TransportRoute::UnixSocket(_) | TransportRoute::InProcess) => { return Err(ConfigError::SeedListOverSsh.into()); } @@ -337,13 +337,24 @@ fn replica_set(first: &NetworkEndpoint, options: &NetworkOptions) -> Result Result { - let uri = format!("mongodb+srv://{}/", percent_encoded(service.host())); +async fn seed_list(service: &NetworkEndpoint, credentials: &Credentials) -> Result { + let uri = seed_list_uri(service.host(), credentials); ClientOptions::parse(uri.as_str()) .await .map_err(|error| map_error(&error, service, LossPhase::Connect)) } +// The crate puts a TXT record's authSource only on a login the string names, so the user goes in, never the password. +fn seed_list_uri(host: &str, credentials: &Credentials) -> String { + let user = match credentials { + Credentials::Password { username, .. } if !username.trim().is_empty() => { + format!("{}@", percent_encoded(username.trim())) + } + _ => String::new(), + }; + format!("mongodb+srv://{user}{}/", percent_encoded(host)) +} + /// The login, checked in the database the user is defined in: the one /// named for it, else for a seed list the one its TXT record names or /// `admin`, else the connection's database, else `admin`, as the macOS @@ -399,8 +410,8 @@ pub(crate) fn setup_error(error: TlsSetupError) -> DriverError { } } -/// A host as it goes into a connection string, every byte outside the -/// unreserved set escaped. +/// A host or a user name as it goes into a connection string, every +/// byte outside the unreserved set escaped. fn percent_encoded(text: &str) -> String { let mut out = String::with_capacity(text.len()); for byte in text.bytes() { @@ -548,6 +559,29 @@ mod tests { ); } + #[test] + fn a_seed_list_names_the_user_and_never_the_password() { + let login = |username: &str| Credentials::Password { + username: username.to_owned(), + password: SecretString::from("p@ss:word"), + }; + let host = "cluster0.example.net"; + + let named = seed_list_uri(host, &login(" a@b:c ")); + + assert_eq!(named, "mongodb+srv://a%40b%3Ac@cluster0.example.net/"); + assert!(!named.contains("p%40ss") && !named.contains("p@ss"), "{named}"); + let parsed = mongodb::options::ConnectionString::parse(&named).expect("a connection string"); + let credential = parsed.credential.expect("a login"); + assert_eq!(credential.username.as_deref(), Some("a@b:c")); + assert!(credential.password.is_none()); + assert_eq!(seed_list_uri(host, &login(" ")), "mongodb+srv://cluster0.example.net/"); + assert_eq!( + seed_list_uri(host, &Credentials::Integrated), + "mongodb+srv://cluster0.example.net/" + ); + } + #[tokio::test] async fn an_in_process_route_dials_the_relay() { let endpoint = MongoEndpoint::new( diff --git a/core/crates/drivers/mongodb/src/field_types.rs b/core/crates/drivers/mongodb/src/field_types.rs index 4f66a56195..d1551c8dc9 100644 --- a/core/crates/drivers/mongodb/src/field_types.rs +++ b/core/crates/drivers/mongodb/src/field_types.rs @@ -37,22 +37,19 @@ pub(crate) fn type_alias(value: &Bson) -> &'static str { } } -/// The type a field's values share, as the grid treats it. Numbers that -/// differ in width share the widest; a field of several other types, or -/// of documents, arrays or binary data, is JSON, which writes each value -/// as the type it is, a UUID's subtype included. A null names no type: -/// it fits any. +/// The type a field's values share, as the grid treats it. A field of +/// more than one type, numbers of two BSON types among them, or of +/// documents, arrays or binary data other than UUIDs, is JSON, which +/// writes each value as the type it is, so an edit leaves an `int` an +/// `int` beside a `long`. A null names no type: it fits any. pub(crate) fn shared_type(aliases: &BTreeSet<&str>) -> &'static str { let typed: BTreeSet<&str> = aliases.iter().copied().filter(|alias| *alias != "null").collect(); - let numbers = ["int", "long", "double", "decimal"]; - if !typed.is_empty() && typed.iter().all(|alias| numbers.contains(alias)) { - return ["decimal", "double", "long", "int"] - .into_iter() - .find(|widest| typed.contains(widest)) - .unwrap_or("int"); - } match typed.iter().next() { Some(&alias) if typed.len() == 1 => match alias { + "int" => "int", + "long" => "long", + "double" => "double", + "decimal" => "decimal", "string" => "string", "bool" => "bool", "date" => "date", @@ -138,9 +135,12 @@ mod tests { #[test] fn a_field_takes_the_type_its_values_share() { assert_eq!(shared(&["int"]), "int"); - assert_eq!(shared(&["int", "long"]), "long"); - assert_eq!(shared(&["int", "double", "null"]), "double"); - assert_eq!(shared(&["decimal", "int"]), "decimal"); + assert_eq!(shared(&["long", "null"]), "long"); + assert_eq!(shared(&["int", "long"]), "mixed"); + assert_eq!(shared(&["int", "double", "null"]), "mixed"); + assert_eq!(shared(&["decimal", "int"]), "mixed"); + assert_eq!(shared(&["uuid", "null"]), "uuid"); + assert_eq!(shared(&["uuid", "binData"]), "mixed"); assert_eq!(shared(&["string", "null"]), "string"); assert_eq!(shared(&["string", "int"]), "mixed"); assert_eq!(shared(&["object"]), "object"); diff --git a/core/crates/drivers/mongodb/src/rows.rs b/core/crates/drivers/mongodb/src/rows.rs index 0024d16a53..008022f7a9 100644 --- a/core/crates/drivers/mongodb/src/rows.rs +++ b/core/crates/drivers/mongodb/src/rows.rs @@ -5,11 +5,11 @@ use bson::{Bson, Document}; use mongodb::ClientSession; use tablepro_core::column::{ColumnType, ResultColumn}; use tablepro_core::value::Value; -use tablepro_core::{ - DriverError, LossPhase, QueryResult, ResultEvent, ResultSink, RowLimit, ServerNotice, SinkFlow, Truncation, -}; +use tablepro_core::{DriverError, QueryResult, ResultEvent, ResultSink, RowLimit, ServerNotice, SinkFlow, Truncation}; +use crate::call::ReadStop; use crate::cells::cell; +use crate::cursor_rows::Read; use crate::endpoint::MongoEndpoint; use crate::execute::Source; use crate::field_types::{column_type, shared_type, type_alias}; @@ -116,23 +116,19 @@ pub(crate) async fn collect( source: Source, session: &mut ClientSession, limit: RowLimit, + stop: &ReadStop, ) -> Result { let (documents, columns, truncated) = match source { - Source::Cursor { mut cursor, columns } => { + Source::Cursor { cursor, columns } => { + let mut read = Read::new(endpoint, *cursor, session, stop); let mut documents = Vec::new(); let mut truncated = false; - loop { - match cursor.advance(session).await { - Ok(true) if (documents.len() as u64) < limit.get() => { - documents.push(crate::nested_document::current_document(&cursor)?); - } - Ok(true) => { - truncated = true; - break; - } - Ok(false) => break, - Err(error) => return Err(endpoint.explain(&error, LossPhase::Statement).await), + while let Some(document) = read.next().await? { + if documents.len() as u64 == limit.get() { + truncated = true; + break; } + documents.push(document); } (documents, columns, truncated) } @@ -159,14 +155,28 @@ pub(crate) async fn collect( } /// Stream a source into `sink`, keeping at most `limit` rows when there -/// is one. +/// is one. `sets` counts the result sets the run has opened, which names +/// the `set_index` of the one this source opens. pub(crate) async fn emit( endpoint: &MongoEndpoint, source: Source, session: &mut ClientSession, sink: &mut dyn ResultSink, limit: Option, + sets: &mut usize, + stop: &ReadStop, ) -> Result<(SinkFlow, u64), DriverError> { + let set_index = *sets; + if !matches!( + source, + Source::Used(_) + | Source::Write { + unconfirmed: Some(_), + .. + } + ) { + *sets += 1; + } match source { Source::Used(database) => { let notice = ServerNotice { @@ -204,7 +214,8 @@ pub(crate) async fn emit( } Source::Documents { documents, columns } => Ok(emit_documents(sink, documents, &columns, limit).await), Source::Cursor { cursor, columns } => { - crate::cursor_rows::stream(endpoint, *cursor, columns, session, sink, limit).await + let read = Read::new(endpoint, *cursor, session, stop); + crate::cursor_rows::stream(read, columns, sink, limit, set_index).await } } } diff --git a/core/crates/drivers/mongodb/src/shell_command.rs b/core/crates/drivers/mongodb/src/shell_command.rs index 91cf43cb7c..0a577f78fe 100644 --- a/core/crates/drivers/mongodb/src/shell_command.rs +++ b/core/crates/drivers/mongodb/src/shell_command.rs @@ -91,9 +91,16 @@ pub(crate) enum Plan { } impl Plan { - /// The collection a cursor reads, in its database. + /// The collection whose documents a cursor shows as rows, in its + /// database. A count shows none. pub(crate) fn collection(&self) -> Option<(&str, &str)> { - let Self::Cursor { database, command, .. } = self else { + let Self::Cursor { + database, + command, + counted: false, + .. + } = self + else { return None; }; let collection = command.get_str("aggregate").or_else(|_| command.get_str("find")).ok()?; @@ -126,10 +133,41 @@ impl Plan { } /// Fix the columns a first read of the cursor found. - pub(crate) fn name_columns(&mut self, names: Vec) { + pub(crate) fn name_columns(&mut self, named: Vec) { if let Self::Cursor { columns, .. } = self { - *columns = Columns::Named(names.into_iter().map(NamedColumn::field).collect()); + *columns = Columns::Named(named); + } + } + + /// The aggregate with `stages` after its pipeline, its columns found + /// from what they return. `None` for a cursor that is no aggregate, or + /// one whose pipeline ends in a write, which no stage may follow. + pub(crate) fn followed_by(&self, stages: impl IntoIterator) -> Option { + let Self::Cursor { + database, + command, + counted: false, + .. + } = self + else { + return None; + }; + if !command.contains_key("aggregate") { + return None; + } + let mut pipeline = command.get_array("pipeline").ok()?.clone(); + if ends_in_write(&pipeline) { + return None; } + pipeline.extend(stages.into_iter().map(Bson::Document)); + let mut command = command.clone(); + command.insert("pipeline", pipeline); + Some(Self::Cursor { + database: database.clone(), + command, + columns: Columns::Discovered, + counted: false, + }) } /// Bound the command by the server's own time limit, which stops it @@ -528,14 +566,8 @@ fn aggregate(target: Bson, mut pipeline: Vec, options: Document, database: .and_then(|stage| stage.get_document("$project").ok()) .map_or(Columns::Discovered, crate::shell_find::projected_columns); if counted && !explained { - // The server takes no stage after one that writes, and such a - // pipeline returns no documents to count. - let writes = pipeline - .last() - .and_then(Bson::as_document) - .and_then(|stage| stage.keys().next()) - .is_some_and(|stage| stage == "$out" || stage == "$merge"); - if writes { + // Such a pipeline also returns no documents to count. + if ends_in_write(&pipeline) { return Err(unsupported("itcount() after $out or $merge")); } pipeline.push(Bson::Document( @@ -560,6 +592,16 @@ fn aggregate(target: Bson, mut pipeline: Vec, options: Document, database: }) } +/// Whether a pipeline ends in `$out` or `$merge`, after which the server +/// takes no stage. +fn ends_in_write(pipeline: &[Bson]) -> bool { + pipeline + .last() + .and_then(Bson::as_document) + .and_then(|stage| stage.keys().next()) + .is_some_and(|stage| stage == "$out" || stage == "$merge") +} + #[cfg(test)] mod tests { use tablepro_core::sql_syntax::mongo::parse_statement; @@ -614,6 +656,50 @@ mod tests { )); } + #[test] + fn a_count_shows_no_collection_rows() { + let collection = |text: &str| { + planned(text).ok().and_then(|plan| { + plan.collection() + .map(|(database, name)| (database.to_owned(), name.to_owned())) + }) + }; + + assert_eq!(collection("db.a.find()"), Some(("shop".to_owned(), "a".to_owned()))); + assert_eq!( + collection("db.a.aggregate([])"), + Some(("shop".to_owned(), "a".to_owned())) + ); + assert_eq!(collection("db.a.countDocuments({})"), None); + assert_eq!(collection("db.a.aggregate([]).itcount()"), None); + } + + #[test] + fn stages_follow_an_aggregate_and_keep_its_options() { + let followed = |text: &str| { + planned(text) + .ok() + .and_then(|plan| plan.followed_by([doc! { "$count": "n" }])) + }; + + let Some(Plan::Cursor { command, columns, .. }) = + followed("db.a.aggregate([{$match: {x: 1}}, {$project: {x: 1}}], {allowDiskUse: true})") + else { + panic!("not an aggregate"); + }; + assert_eq!(columns, Columns::Discovered); + assert_eq!(command.get_bool("allowDiskUse").ok(), Some(true)); + let stages: Vec<&str> = command + .get_array("pipeline") + .expect("a pipeline") + .iter() + .filter_map(|stage| stage.as_document()?.keys().next().map(String::as_str)) + .collect(); + assert_eq!(stages, ["$match", "$project", "$count"]); + assert_eq!(followed("db.a.find()"), None); + assert_eq!(followed("db.a.aggregate([]).itcount()"), None); + } + #[test] fn a_page_of_aliased_fields_shows_their_own_names() { let Ok(Plan::Cursor { columns, .. }) = planned( diff --git a/core/crates/drivers/mongodb/src/stream_read.rs b/core/crates/drivers/mongodb/src/stream_read.rs index fc11685ecb..0f84914dd2 100644 --- a/core/crates/drivers/mongodb/src/stream_read.rs +++ b/core/crates/drivers/mongodb/src/stream_read.rs @@ -3,37 +3,50 @@ use std::future::Future; use std::pin::Pin; use std::sync::Arc; +use bson::Document; use mongodb::{Client, ClientSession}; +use tablepro_core::column::ColumnKind; use tablepro_core::sql_syntax::StatementAccess; use tablepro_core::sql_syntax::mongo::{ShellStatement, access_of, parse_statement, statement_ranges}; use tablepro_core::{ - DriverError, LossPhase, ReadOnlyRefusal, ReadStatement, ResultSink, SinkFlow, StopRequest, StreamSummary, + ColumnInfo, DriverError, ReadOnlyRefusal, ReadStatement, ResultSink, SinkFlow, StopRequest, StreamSummary, }; -use crate::call::{SessionKill, generated, session}; +use crate::call::{ReadStop, SessionKill, generated, session}; +use crate::catalog::{JsonFields, Reader, census_stages, columns_from}; +use crate::cursor_rows::Read; use crate::endpoint::{ClientUse, MongoEndpoint}; use crate::errors::syntax_error; use crate::execute::{Source, start}; use crate::rows::emit; -use crate::shell_command::{Plan, plan}; +use crate::shell_command::{NamedColumn, Plan, plan}; + +/// Where an export reads: the connection's database, and the +/// collections' JSON fields with the server's time limit for a census of +/// them. +pub(crate) struct ExportScope<'a> { + pub(crate) database: &'a str, + pub(crate) json_fields: &'a JsonFields, + pub(crate) max_time: Option, +} /// Every row of what `read` reads, for an export, on a client of its own /// that closes with it. MongoDB has no read-only session, so a statement /// that is not a read is refused before anything is sent. A generated -/// read shows `json_fields` as Extended JSON, as the grid does. +/// read shows the collection's JSON fields as Extended JSON, as the grid +/// does. pub(crate) fn stream<'a>( endpoint: &'a MongoEndpoint, - database: &'a str, + scope: &'a ExportScope<'a>, read: &'a ReadStatement, - json_fields: &'a BTreeSet, sink: &'a mut dyn ResultSink, stop: &'a StopRequest, kill: &'a Arc, ) -> Pin> + Send + 'a>> { Box::pin(async move { - let plans = plans(read, database, json_fields)?; + let plans = plans(read, scope.database)?; let client = endpoint.open(ClientUse::Own).await?; - let streamed = stream_on(endpoint, &client, database, plans, sink, stop, kill).await; + let streamed = stream_on(endpoint, &client, scope, plans, sink, stop, kill).await; client.shutdown().immediate(streamed.is_err()).await; streamed }) @@ -42,7 +55,7 @@ pub(crate) fn stream<'a>( async fn stream_on( endpoint: &MongoEndpoint, client: &Client, - database: &str, + scope: &ExportScope<'_>, plans: Vec, sink: &mut dyn ResultSink, stop: &StopRequest, @@ -51,7 +64,7 @@ async fn stream_on( let mut session = session(endpoint, client, kill).await?; let streamed = match stop.is_requested() { true => Err(DriverError::Cancelled), - false => stream_in(endpoint, client, database, plans, sink, &mut session).await, + false => stream_in(endpoint, client, scope, plans, sink, &mut session, &kill.follow(stop)).await, }; // The export's client closes with it, so no later call takes this // session from its pool, and a graceful close waits for it. @@ -62,21 +75,48 @@ async fn stream_on( async fn stream_in( endpoint: &MongoEndpoint, client: &Client, - database: &str, + scope: &ExportScope<'_>, plans: Vec, sink: &mut dyn ResultSink, session: &mut ClientSession, + stop: &ReadStop, ) -> Result { let mut rows = 0; - let mut current = database.to_owned(); + let mut sets = 0; + let mut current = scope.database.to_owned(); for planned in plans { - let mut planned = match planned { - Planned::Ready(planned) => planned, - Planned::Typed(statement) => plan(&statement, ¤t, &[])?, + let (mut planned, from_tab) = match planned { + Planned::Ready(planned) => (planned, true), + Planned::Typed(statement) => (plan(&statement, ¤t, &[])?, false), }; if planned.discovers_columns() { - let fields = census(endpoint, client, session, planned.clone()).await?; - planned.name_columns(fields); + let mut columns = census(endpoint, client, session, &planned, stop).await?; + if columns.is_empty() + && from_tab + && let Some((database, collection)) = planned.collection() + { + // No document matched: the collection's columns still head the export. + let mut reader = Reader { + endpoint, + client, + session: &mut *session, + max_time: scope.max_time, + }; + columns = reader.fields(database, collection).await?.iter().map(named).collect(); + } + planned.name_columns(columns); + } + // A column the tab shows as JSON stays JSON where the documents + // the export matches hold one type. + if from_tab { + let mut reader = Reader { + endpoint, + client, + session: &mut *session, + max_time: scope.max_time, + }; + let shown = scope.json_fields.of(&planned, &mut reader).await?; + planned.show_as_json(&shown); } let source = match start(endpoint, client, session, planned).await? { Source::Used(next) => { @@ -85,7 +125,7 @@ async fn stream_in( } source => source, }; - let (flow, emitted) = emit(endpoint, source, session, sink, None).await?; + let (flow, emitted) = emit(endpoint, source, session, sink, None, &mut sets, stop).await?; rows += emitted; if flow == SinkFlow::ConsumerGone { return Err(DriverError::Cancelled); @@ -94,33 +134,61 @@ async fn stream_in( Ok(StreamSummary::new(rows)) } -/// The fields a cursor's documents hold, in the order they first show, -/// from a read of its own. An export writes its columns before its first -/// row, so it has to know every field before it reads one to write. +/// The columns of what `planned` reads, found before its first row, as +/// an export writes its columns first. The server tallies an aggregate's +/// documents with the census a collection's columns come from, over every +/// document, so a field a late document brings is a column and a field +/// of mixed values is JSON. A `find` has no pipeline to tally in, so its +/// documents are read here, for the fields in the order they first show. async fn census( endpoint: &MongoEndpoint, client: &Client, session: &mut ClientSession, - planned: Plan, -) -> Result, DriverError> { - let Source::Cursor { mut cursor, .. } = start(endpoint, client, session, planned).await? else { - return Ok(Vec::new()); - }; - let mut fields: Vec = Vec::new(); + planned: &Plan, + stop: &ReadStop, +) -> Result, DriverError> { + if let Some(tally) = planned.followed_by(census_stages()) { + let mut tallies = Vec::new(); + each(endpoint, client, session, tally, stop, |tally| tallies.push(tally)).await?; + return Ok(columns_from(&tallies).iter().map(named).collect()); + } + let mut fields = Vec::new(); let mut seen = BTreeSet::new(); - loop { - match cursor.advance(session).await { - Ok(true) => { - let document = crate::nested_document::current_document(&cursor)?; - for key in document.keys() { - if seen.insert(key.clone()) { - fields.push(key.clone()); - } - } + each(endpoint, client, session, planned.clone(), stop, |document| { + for key in document.keys() { + if seen.insert(key.clone()) { + fields.push(NamedColumn::field(key.clone())); } - Ok(false) => return Ok(fields), - Err(error) => return Err(endpoint.explain(&error, LossPhase::Statement).await), } + }) + .await?; + Ok(fields) +} + +/// Hand each document a plan's cursor returns to `take`. +async fn each( + endpoint: &MongoEndpoint, + client: &Client, + session: &mut ClientSession, + planned: Plan, + stop: &ReadStop, + mut take: impl FnMut(Document) + Send, +) -> Result<(), DriverError> { + let Source::Cursor { cursor, .. } = start(endpoint, client, session, planned).await? else { + return Ok(()); + }; + let mut read = Read::new(endpoint, *cursor, session, stop); + while let Some(document) = read.next().await? { + take(document); + } + Ok(()) +} + +/// A census column as an export writes it: Extended JSON for a JSON one. +fn named(column: &ColumnInfo) -> NamedColumn { + NamedColumn { + json: matches!(column.column_type.kind, ColumnKind::Json { .. }), + ..NamedColumn::field(column.name.clone()) } } @@ -132,12 +200,11 @@ enum Planned { } /// Each statement of the read, every one of them a read. -fn plans(read: &ReadStatement, database: &str, json_fields: &BTreeSet) -> Result, DriverError> { +fn plans(read: &ReadStatement, database: &str) -> Result, DriverError> { let refuse = || DriverError::ReadOnly(ReadOnlyRefusal::ClientGuard); match read { ReadStatement::Generated(statement) => { - let (parsed, mut planned) = generated(statement, database)?; - planned.show_as_json(json_fields); + let (parsed, planned) = generated(statement, database)?; match access_of(&parsed) { StatementAccess::Read => Ok(vec![Planned::Ready(planned)]), _ => Err(refuse()), diff --git a/core/crates/drivers/mongodb/tests/integration/catalog.rs b/core/crates/drivers/mongodb/tests/integration/catalog.rs index 6258a40a0e..215b21842d 100644 --- a/core/crates/drivers/mongodb/tests/integration/catalog.rs +++ b/core/crates/drivers/mongodb/tests/integration/catalog.rs @@ -1,15 +1,82 @@ -//! Emptying a collection and creating and dropping a database through -//! the plans the sidebar runs with `write_batch`. Each names a database -//! other than the connection's own, which has to be the one it changes. +//! The catalogue: the columns a collection's fields make, and emptying a +//! collection and creating and dropping a database through the plans the +//! sidebar runs with `write_batch`, each in a database other than the +//! connection's own, which has to be the one it changes. +use tablepro_core::browse::{PageRead, PageRequest, read_page}; +use tablepro_core::column::ColumnKind; use tablepro_core::ddl::{ DatabaseDraft, DropDatabaseOptions, TruncateTableOptions, plan_create_database, plan_drop_database, plan_truncate_table, }; -use tablepro_core::meta::{ObjectScope, SchemaInfo, TableRef}; -use tablepro_core::{Connection, RunEnd, Value}; +use tablepro_core::filter::FilterSet; +use tablepro_core::meta::{ObjectScope, PagePosition, SchemaInfo, TableRef}; +use tablepro_core::{ColumnInfo, Connection, RunEnd, Value}; -use crate::support::{CALL, connect, editor, rows, run, start, unique, within}; +use crate::support::{CALL, DATABASE, connect, editor, rows, run, start, unique, within}; + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn uuid_ids_open_as_a_uuid_column_and_page_as_uuids() { + let fixture = start().await; + let connection = connect(&fixture).await; + let session = editor(connection.as_ref()).await; + let collection = unique("uuids"); + let ids = [ + "c4d5e6f7-0a1b-4c2d-9e3f-4a5b6c7d8e9f", + "0e3f6f0e-8a3b-4b8e-9a7c-2f1d1b3c4d5e", + "5b2c1a7e-3d4f-4a6b-8c9d-0e1f2a3b4c5d", + ]; + let [with_uuid, with_bytes, without] = ids; + let (filled, _) = run( + session.as_ref(), + &format!( + "db.getCollection('{collection}').insertMany([{{_id: UUID('{with_uuid}'), blob: UUID('{with_uuid}')}}, \ + {{_id: UUID('{with_bytes}'), blob: BinData(0, 'AAEC')}}, {{_id: UUID('{without}')}}])" + ), + ) + .await; + assert_eq!(filled.end, RunEnd::Completed); + let table = TableRef::new(Some(DATABASE.to_owned()), collection.clone()); + + let columns = connection + .fetch_columns(&table, within(CALL)) + .await + .expect("the fields"); + + assert_eq!(columns[0].name, "_id"); + assert_eq!(columns[0].column_type.kind, ColumnKind::Uuid, "{columns:?}"); + let blob = columns + .iter() + .position(|column| column.name == "blob") + .expect("the blob field"); + assert!( + matches!(columns[blob].column_type.kind, ColumnKind::Json { .. }), + "UUIDs and other binary data: {:?}", + columns[blob] + ); + let front = page_at(connection.as_ref(), &table, &columns, &PagePosition::First, 2).await; + let next = front.next.clone().expect("a page after the first"); + let back = page_at(connection.as_ref(), &table, &columns, &next, 2).await; + let paged: Vec = front + .result + .rows + .iter() + .chain(&back.result.rows) + .map(|row| row[0].clone()) + .collect(); + let mut expected: Vec = ids + .iter() + .map(|id| uuid::Uuid::parse_str(id).expect("a UUID")) + .collect(); + expected.sort(); + assert_eq!(paged, expected.into_iter().map(Value::Uuid).collect::>()); + assert!( + matches!(front.result.rows[0][blob], Value::Json(_)), + "binary data in a JSON column: {:?}", + front.result.rows[0] + ); +} #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[ignore = "requires docker"] @@ -141,3 +208,26 @@ async fn lists(connection: &dyn Connection, database: &str) -> bool { let databases: Vec = connection.list_databases(within(CALL)).await.expect("the databases"); databases.iter().any(|listed| listed.name == database) } + +/// One page of `table` under `columns`, from `position`. +async fn page_at( + connection: &dyn Connection, + table: &TableRef, + columns: &[ColumnInfo], + position: &PagePosition, + limit: u64, +) -> PageRead { + let identity = connection.row_identity(table, within(CALL)).await.expect("an identity"); + let request = PageRequest { + table, + columns, + filter: &FilterSet::default(), + sort: &[], + identity: &identity, + position, + limit, + }; + read_page::>(connection, &request, &within(CALL)) + .await + .expect("a page") +} diff --git a/core/crates/drivers/mongodb/tests/integration/conformance_engine.rs b/core/crates/drivers/mongodb/tests/integration/conformance_engine.rs index b9b49b3295..92a2c42630 100644 --- a/core/crates/drivers/mongodb/tests/integration/conformance_engine.rs +++ b/core/crates/drivers/mongodb/tests/integration/conformance_engine.rs @@ -1,10 +1,14 @@ -//! The document conformance suite against a MongoDB replica set. +//! The document conformance suite against a MongoDB replica set, once +//! over a direct route, where the client library dials the server, and +//! once over an SSH-style forward, where the driver's relay carries every +//! read, save and stop. use std::sync::Arc; use async_trait::async_trait; use tablepro_core::{ - ConnectOptions, DatabaseDriver, LivenessPolicy, NetworkEndpoint, RunEnd, TlsConfig, Value, WorkspaceSettings, + ConnectOptions, DatabaseDriver, LivenessPolicy, NetworkEndpoint, RunEnd, TlsConfig, Transport, TransportError, + TransportKind, TransportRoute, TransportStream, Value, WorkspaceSettings, }; use tablepro_driver_mongodb::MongoDriver; use tablepro_test_fixtures::MongoFixture; @@ -21,6 +25,31 @@ pub struct MongoEngine { fixture: MongoFixture, } +impl MongoEngine { + fn options( + &self, + transport: Arc, + settings: WorkspaceSettings, + liveness: LivenessPolicy, + ) -> ConnectOptions { + options_over( + transport, + password(&self.fixture.password_credentials()), + TlsConfig::disabled(), + settings, + liveness, + ) + } + + fn fault(&self, liveness: &LivenessPolicy) -> Result, ConformanceFailure> { + let endpoint = + NetworkEndpoint::new(self.fixture.host(), self.fixture.port()).map_err(ConformanceFailure::setup)?; + Ok(Arc::new( + FaultTransport::direct(endpoint, liveness).map_err(ConformanceFailure::setup)?, + )) + } +} + #[async_trait] impl DocumentEngine for MongoEngine { fn driver() -> Arc { @@ -33,13 +62,8 @@ impl DocumentEngine for MongoEngine { } fn connect_options(&self, settings: WorkspaceSettings, liveness: LivenessPolicy) -> ConnectOptions { - options_over( - direct(self.fixture.host(), self.fixture.port(), &liveness), - password(&self.fixture.password_credentials()), - TlsConfig::disabled(), - settings, - liveness, - ) + let transport = direct(self.fixture.host(), self.fixture.port(), &liveness); + self.options(transport, settings, liveness) } async fn faulty_connect_options( @@ -47,17 +71,8 @@ impl DocumentEngine for MongoEngine { settings: WorkspaceSettings, liveness: LivenessPolicy, ) -> Result<(ConnectOptions, Arc), ConformanceFailure> { - let endpoint = - NetworkEndpoint::new(self.fixture.host(), self.fixture.port()).map_err(ConformanceFailure::setup)?; - let fault = Arc::new(FaultTransport::direct(endpoint, &liveness).map_err(ConformanceFailure::setup)?); - let options = options_over( - fault.clone(), - password(&self.fixture.password_credentials()), - TlsConfig::disabled(), - settings, - liveness, - ); - Ok((options, fault)) + let fault = self.fault(&liveness)?; + Ok((self.options(fault.clone(), settings, liveness), fault)) } fn database(&self) -> &str { @@ -118,6 +133,92 @@ impl DocumentEngine for MongoEngine { } } +/// An SSH forward as the driver sees one: a socket route, so the driver +/// relays, and streams it opens itself, through a `FaultTransport` a case +/// can drop. +#[derive(Debug)] +struct Forward(Arc); + +#[async_trait] +impl Transport for Forward { + fn service_endpoint(&self) -> &NetworkEndpoint { + self.0.service_endpoint() + } + + fn kind(&self) -> TransportKind { + TransportKind::SshForward { + destination: "bastion.tablepro.test".to_owned(), + } + } + + fn route(&self) -> TransportRoute { + self.0.route() + } + + async fn open(&self) -> Result, TransportError> { + self.0.open().await + } + + async fn reroute(&self, endpoint: NetworkEndpoint) -> Result, TransportError> { + self.0.reroute(endpoint).await + } + + async fn explain_closed_stream(&self) -> Option { + self.0.explain_closed_stream().await + } +} + +/// The same server through a forward, as a connection through an SSH +/// tunnel reaches it. The server's own list of operations is still read +/// over the direct route. +pub struct MongoForwardEngine(MongoEngine); + +#[async_trait] +impl DocumentEngine for MongoForwardEngine { + fn driver() -> Arc { + MongoEngine::driver() + } + + async fn start() -> Result { + MongoEngine::start().await.map(Self) + } + + fn connect_options(&self, settings: WorkspaceSettings, liveness: LivenessPolicy) -> ConnectOptions { + let fault = self.0.fault(&liveness).expect("a forward to the fixture"); + self.0.options(Arc::new(Forward(fault)), settings, liveness) + } + + async fn faulty_connect_options( + &self, + settings: WorkspaceSettings, + liveness: LivenessPolicy, + ) -> Result<(ConnectOptions, Arc), ConformanceFailure> { + let fault = self.0.fault(&liveness)?; + let options = self.0.options(Arc::new(Forward(fault.clone())), settings, liveness); + Ok((options, fault)) + } + + fn database(&self) -> &str { + self.0.database() + } + + fn fill_statements(&self, collection: &str, documents: u32) -> String { + self.0.fill_statements(collection, documents) + } + + fn unbounded_statement(&self, collection: &str, marker: &str) -> String { + self.0.unbounded_statement(collection, marker) + } + + fn writing_read(&self, collection: &str, target: &str) -> String { + self.0.writing_read(collection, target) + } + + async fn running_operations(&self, marker: &str) -> Result { + self.0.running_operations(marker).await + } +} + mod suite { use super::MongoEngine; @@ -125,3 +226,11 @@ mod suite { engine: MongoEngine, } } + +mod forwarded { + use super::MongoForwardEngine; + + tablepro_test_support::document_conformance_suite! { + engine: MongoForwardEngine, + } +} diff --git a/core/crates/drivers/mongodb/tests/integration/export.rs b/core/crates/drivers/mongodb/tests/integration/export.rs new file mode 100644 index 0000000000..0a09b15208 --- /dev/null +++ b/core/crates/drivers/mongodb/tests/integration/export.rs @@ -0,0 +1,111 @@ +//! A table export as the export dialog's Entire Table reads it: the +//! statement `build_export` writes for every field, then `stream_read`. + +use std::sync::{Arc, Mutex, PoisonError}; + +use async_trait::async_trait; +use tablepro_core::browse::{ExportFields, ExportRequest, build_export}; +use tablepro_core::filter::{Combinator, FilterOp, FilterRule, FilterSet, FilterValue}; +use tablepro_core::meta::TableRef; +use tablepro_core::{Connection, ReadStatement, ResultEvent, ResultSink, SinkFlow, Value}; + +use crate::support::{CALL, DATABASE, column_names, connect, rows, start, unique, within}; + +struct Keep(Arc>>); + +#[async_trait] +impl ResultSink for Keep { + async fn accept(&mut self, event: ResultEvent) -> SinkFlow { + self.0.lock().unwrap_or_else(PoisonError::into_inner).push(event); + SinkFlow::Continue + } +} + +/// Every event an export of `table` under `filter` sent, with the +/// columns and identity the tab reads. +async fn export(connection: &dyn Connection, table: &TableRef, filter: &FilterSet) -> Vec { + let columns = connection.fetch_columns(table, within(CALL)).await.expect("the fields"); + let identity = connection.row_identity(table, within(CALL)).await.expect("an identity"); + let request = ExportRequest { + table, + columns: &columns, + filter, + sort: &[], + identity: &identity, + fields: ExportFields::Every, + }; + let statement = build_export(&*connection.dialect(), &request).expect("an export statement"); + let kept = Arc::new(Mutex::new(Vec::new())); + connection + .stream_read( + &ReadStatement::Generated(statement), + Box::new(Keep(kept.clone())), + within(CALL), + ) + .await + .expect("the export"); + std::mem::take(&mut *kept.lock().unwrap_or_else(PoisonError::into_inner)) +} + +fn assert_json(cell: &Value, expected: &str) { + let read = match cell { + Value::Json(json) => serde_json::from_str::(json.as_str()).ok(), + _ => None, + }; + let expected = serde_json::from_str::(expected).expect("JSON"); + assert_eq!(read.as_ref(), Some(&expected), "the cell is {cell:?}"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn a_table_export_writes_the_fields_and_types_past_the_first_thousand_documents() { + let fixture = start().await; + let connection = connect(&fixture).await; + let collection = unique("late_fields"); + // Natural order is insertion order: `discount` and `n` first show in + // the 1,001st document, past the census the tab's columns come from. + let filled = fixture + .shell(&format!( + "const target = db.getSiblingDB('{DATABASE}').getCollection('{collection}');\n\ + target.insertMany(Array.from({{length: 1000}}, (_, id) => ({{_id: id, a: id}})));\n\ + target.insertMany([{{_id: 1000, a: 1000, discount: 0.25, n: NumberLong(5)}}, {{_id: 1001, a: 1001, n: 'five'}}]).acknowledged" + )) + .await + .expect("mongosh ran"); + assert!(filled.succeeded(), "{}", filled.stderr_text()); + let table = TableRef::new(Some(DATABASE.to_owned()), collection.clone()); + let tab = connection + .fetch_columns(&table, within(CALL)) + .await + .expect("the fields"); + let tab: Vec<&str> = tab.iter().map(|column| column.name.as_str()).collect(); + assert_eq!(tab, ["_id", "a"], "the tab's census reads the first 1,000 documents"); + + let events = export(connection.as_ref(), &table, &FilterSet::default()).await; + + assert_eq!(column_names(&events), ["_id", "a", "discount", "n"]); + let written = rows(&events); + assert_eq!(written.len(), 1002); + assert_eq!(written[999][2], Value::Null); + assert_eq!(written[1000][2], Value::Float64(0.25)); + assert_json(&written[1000][3], r#"{"$numberLong": "5"}"#); + assert_json(&written[1001][3], r#""five""#); + + let nothing = FilterSet { + combinator: Combinator::And, + rules: vec![FilterRule { + column: "a".to_owned(), + op: FilterOp::Eq, + value: Some(FilterValue::Single("-1".to_owned())), + }], + extra_sql: None, + }; + let empty = export(connection.as_ref(), &table, ¬hing).await; + + assert!(rows(&empty).is_empty(), "{empty:?}"); + assert_eq!( + column_names(&empty), + ["_id", "a"], + "the collection's columns head an empty export" + ); +} diff --git a/core/crates/drivers/mongodb/tests/integration/main.rs b/core/crates/drivers/mongodb/tests/integration/main.rs index 154c4a3c02..25475b0c4d 100644 --- a/core/crates/drivers/mongodb/tests/integration/main.rs +++ b/core/crates/drivers/mongodb/tests/integration/main.rs @@ -8,6 +8,10 @@ mod catalog; mod conformance_engine; +mod export; +mod replica_set; +mod seed_list; mod statements; +mod stop; mod support; mod tls; diff --git a/core/crates/drivers/mongodb/tests/integration/replica_set.rs b/core/crates/drivers/mongodb/tests/integration/replica_set.rs new file mode 100644 index 0000000000..7bfb860304 --- /dev/null +++ b/core/crates/drivers/mongodb/tests/integration/replica_set.rs @@ -0,0 +1,67 @@ +//! A replica set found from its seeds: the client takes the names the +//! members give themselves and works through the primary, so the member +//! is named by an address the host reaches, and a seed that does not +//! answer is passed over. + +use tablepro_core::meta::{ObjectScope, TableRef}; +use tablepro_core::{NetworkOptions, NetworkTarget, RunEnd, TlsConfig, Value}; +use tablepro_test_fixtures::MongoFixture; +use tablepro_test_support::conformance::test_liveness; + +use crate::support::{CALL, DATABASE, connect_to, direct, editor, first_page, password, run, unique, within}; + +/// The name the set's one member gives itself. +const MEMBER: &str = "localhost.test.build.10gen.cc:27017"; + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn a_replica_set_is_found_from_its_seeds_and_read_through_its_primary() { + let fixture = MongoFixture::start_on_test_records() + .await + .expect("a set whose member the host reaches by name"); + let liveness = test_liveness(); + // Only the second seed reaches the set, so the set is found from the host list or not at all. + let target = NetworkTarget { + transport: direct("127.0.0.1", closed_port(), &liveness), + credentials: password(&fixture.password_credentials()), + database: Some(DATABASE.to_owned()), + tls: TlsConfig::disabled(), + options: NetworkOptions { + auth_database: Some("admin".to_owned()), + replica_set: Some(MongoFixture::TEST_RECORDS_SET.to_owned()), + more_hosts: vec![MEMBER.to_owned()], + ..NetworkOptions::default() + }, + }; + + let connection = connect_to(target, liveness).await; + + let session = editor(connection.as_ref()).await; + let collection = unique("discovered"); + let (filled, _) = run( + session.as_ref(), + &format!("db.getCollection('{collection}').insertMany([{{_id: 1, label: 'a'}}, {{_id: 2, label: 'b'}}])"), + ) + .await; + assert_eq!(filled.end, RunEnd::Completed, "a write reaches the primary"); + let objects = connection + .list_objects(&ObjectScope::Schema(DATABASE.to_owned()), within(CALL)) + .await + .expect("the database's collections"); + assert!(objects.iter().any(|object| object.name == collection), "{objects:?}"); + let read = first_page( + connection.as_ref(), + &TableRef::new(Some(DATABASE.to_owned()), collection.clone()), + ) + .await; + let ids: Vec<&Value> = read.result.rows.iter().filter_map(|row| row.first()).collect(); + assert_eq!(ids, [&Value::Int(1), &Value::Int(2)]); + session.close().await; + connection.close().await.expect("a clean close"); +} + +/// A port on this host that nothing listens on. +fn closed_port() -> u16 { + let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("a free port"); + listener.local_addr().expect("the port bound").port() +} diff --git a/core/crates/drivers/mongodb/tests/integration/seed_list.rs b/core/crates/drivers/mongodb/tests/integration/seed_list.rs new file mode 100644 index 0000000000..7ee5697748 --- /dev/null +++ b/core/crates/drivers/mongodb/tests/integration/seed_list.rs @@ -0,0 +1,75 @@ +//! A DNS seed list end to end, through the public records the MongoDB +//! specification keeps for drivers' tests, against the set they name. + +use tablepro_core::meta::{ObjectScope, TableRef}; +use tablepro_core::{NetworkOptions, NetworkTarget, RunEnd, TlsConfig, Value}; +use tablepro_test_fixtures::MongoFixture; +use tablepro_test_support::conformance::test_liveness; + +use crate::support::{CALL, connect_to, direct, editor, first_page, password, run, save_field, unique, within}; + +/// SRV `localhost.test.build.10gen.cc:27017`, TXT `replicaSet=repl0&authSource=thisDB`. +const SEED_LIST: &str = "test5.test.build.10gen.cc"; + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn a_seed_list_logs_in_where_its_txt_record_says_then_pages_and_saves() { + let fixture = MongoFixture::start_on_test_records() + .await + .expect("the set the records name"); + let liveness = test_liveness(); + // No database and no Authentication Database: only the TXT record names thisDB, the user's one database. + let target = NetworkTarget { + transport: direct(SEED_LIST, MongoFixture::PORT, &liveness), + credentials: password(&fixture.this_db_credentials().expect("a user defined in thisDB")), + database: None, + tls: TlsConfig::disabled(), + options: NetworkOptions { + dns_seed_list: true, + ..NetworkOptions::default() + }, + }; + + let connection = connect_to(target, liveness).await; + + let session = editor(connection.as_ref()).await; + let collection = unique("listed"); + let (filled, _) = run( + session.as_ref(), + &format!( + "db.getSiblingDB('{}').getCollection('{collection}').insertMany([{{_id: 1, label: 'a'}}, \ + {{_id: 2, label: 'b'}}, {{_id: 3, label: 'c'}}])", + MongoFixture::THIS_DB + ), + ) + .await; + assert_eq!(filled.end, RunEnd::Completed); + let objects = connection + .list_objects(&ObjectScope::Schema(MongoFixture::THIS_DB.to_owned()), within(CALL)) + .await + .expect("the database's collections"); + assert!(objects.iter().any(|object| object.name == collection), "{objects:?}"); + let table = TableRef::new(Some(MongoFixture::THIS_DB.to_owned()), collection.clone()); + let read = first_page(connection.as_ref(), &table).await; + let ids: Vec<&Value> = read.result.rows.iter().filter_map(|row| row.first()).collect(); + assert_eq!(ids, [&Value::Int(1), &Value::Int(2), &Value::Int(3)]); + + let report = save_field( + connection.as_ref(), + &table, + Value::Int(2), + "label", + Value::Text("renamed".to_owned()), + ) + .await; + + assert_eq!(report.rows_affected, [Some(1)]); + let read = first_page(connection.as_ref(), &table).await; + let edited = read.result.rows.iter().find(|row| row.first() == Some(&Value::Int(2))); + assert_eq!( + edited.and_then(|row| row.get(1)), + Some(&Value::Text("renamed".to_owned())) + ); + session.close().await; + connection.close().await.expect("a clean close"); +} diff --git a/core/crates/drivers/mongodb/tests/integration/statements.rs b/core/crates/drivers/mongodb/tests/integration/statements.rs index 6e7076dc9e..c7bf0246de 100644 --- a/core/crates/drivers/mongodb/tests/integration/statements.rs +++ b/core/crates/drivers/mongodb/tests/integration/statements.rs @@ -6,20 +6,22 @@ use std::str::FromStr; use secrecy::SecretString; use tablepro_core::browse::{PageRead, PageRequest, read_page}; +use tablepro_core::column::ColumnKind; use tablepro_core::edit::CellInput; use tablepro_core::filter::FilterSet; use tablepro_core::meta::{ObjectKind, ObjectScope, PagePosition, RowIdentity, TableRef}; use tablepro_core::save::{ChangeSet, RowChange, RowUpdate, plan_save}; use tablepro_core::value::{JsonText, OtherValue, SqlDecimal, Temporal}; use tablepro_core::{ - Connection, Credentials, DatabaseDriver, DriverError, ErrorCategory, ResultEvent, RunEnd, ServerCode, Value, - WorkspaceSettings, + ColumnInfo, Connection, Credentials, DatabaseDriver, DriverError, ErrorCategory, ResultEvent, RowLimit, RunEnd, + RunOptions, ServerCode, Truncation, Value, WorkspaceSettings, }; use tablepro_driver_mongodb::MongoDriver; use tokio_util::sync::CancellationToken; use crate::support::{ - CALL, DATABASE, column_names, connect, editor, fixture_options, options_over, rows, run, start, unique, within, + CALL, DATABASE, column_names, connect, editor, fixture_options, options_over, rows, run, run_with, start, unique, + within, }; #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -401,6 +403,200 @@ async fn a_result_set_event_names_the_fields_a_later_document_brings() { ); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn a_field_of_two_number_types_is_json_and_an_edit_keeps_its_type() { + let fixture = start().await; + let connection = connect(&fixture).await; + let session = editor(connection.as_ref()).await; + let collection = unique("quantities"); + let (inserted, _) = run( + session.as_ref(), + &format!("db.getCollection('{collection}').insertMany([{{_id: 1, qty: 3}}, {{_id: 2, qty: NumberLong(7)}}])"), + ) + .await; + assert_eq!(inserted.end, RunEnd::Completed); + let table = TableRef::new(Some(DATABASE.to_owned()), collection.clone()); + let columns = connection + .fetch_columns(&table, within(CALL)) + .await + .expect("the fields"); + let qty = columns + .iter() + .position(|column| column.name == "qty") + .expect("the qty field"); + assert!( + matches!(columns[qty].column_type.kind, ColumnKind::Json { .. }), + "an int and a long: {:?}", + columns[qty] + ); + let read = page_with(connection.as_ref(), &table, &columns).await; + assert_json(&read.result.rows[0][qty], "3"); + assert_json(&read.result.rows[1][qty], r#"{"$numberLong": "7"}"#); + let identity = connection + .row_identity(&table, within(CALL)) + .await + .expect("an identity"); + + let plan = plan_save( + &*connection.dialect(), + &table, + &columns, + &identity, + &ChangeSet { + changes: vec![RowChange::Update(RowUpdate { + key: vec![Value::Int(1)], + assignments: vec![( + qty, + CellInput::Value(Value::Json(JsonText::parse("4".to_owned()).expect("JSON"))), + )], + })], + }, + ) + .expect("a save plan"); + let report = connection + .write_batch(&plan.batch, within(CALL)) + .await + .expect("the save"); + + assert_eq!(report.rows_affected, [Some(1)]); + let (typed, events) = run( + session.as_ref(), + &format!( + "db.getCollection('{collection}').countDocuments({{_id: 1, qty: {{$eq: 4, $type: 'int'}}}})\n\ + db.getCollection('{collection}').countDocuments({{_id: 2, qty: {{$type: 'long'}}}})" + ), + ) + .await; + assert_eq!(typed.end, RunEnd::Completed); + assert_eq!( + rows(&events), + [vec![Value::Int(1)], vec![Value::Int(1)]], + "the edit kept the int and left the long a long" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn a_connection_that_never_read_the_fields_shows_json_fields_as_json() { + let fixture = start().await; + let connection = connect(&fixture).await; + let session = editor(connection.as_ref()).await; + let collection = unique("reconnected"); + let (inserted, _) = run( + session.as_ref(), + &format!("db.getCollection('{collection}').insertMany([{{_id: 1, n: 'five'}}, {{_id: 2, n: NumberLong(5)}}])"), + ) + .await; + assert_eq!(inserted.end, RunEnd::Completed); + let table = TableRef::new(Some(DATABASE.to_owned()), collection.clone()); + let columns = connection + .fetch_columns(&table, within(CALL)) + .await + .expect("the fields"); + let n = columns + .iter() + .position(|column| column.name == "n") + .expect("the n field"); + // As after a reconnect: the tab keeps its columns and pages on a + // connection that has not read them. + let reconnected = connect(&fixture).await; + + let read = page_with(reconnected.as_ref(), &table, &columns).await; + + assert_json(&read.result.rows[0][n], r#""five""#); + assert_json(&read.result.rows[1][n], r#"{"$numberLong": "5"}"#); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn a_projected_find_streams_its_rows_and_resolves_the_types_later_rows_bring() { + let fixture = start().await; + let connection = connect(&fixture).await; + let session = editor(connection.as_ref()).await; + let collection = unique("streamed"); + // Past the first batch of rows, `n` holds strings too. + let documents: Vec = (0..300) + .map(|id| match id < 280 { + true => format!("{{_id: {id}, n: {id}}}"), + false => format!("{{_id: {id}, n: 'n{id}'}}"), + }) + .collect(); + let (inserted, _) = run( + session.as_ref(), + &format!( + "db.getCollection('{collection}').insertMany([{}])", + documents.join(", ") + ), + ) + .await; + assert_eq!(inserted.end, RunEnd::Completed); + let find = format!("db.getCollection('{collection}').find({{}}, {{n: 1}}).sort({{_id: 1}})"); + + let (read, events) = run( + session.as_ref(), + &format!("db.getCollection('{collection}').countDocuments({{}})\n{find}"), + ) + .await; + + assert_eq!(read.end, RunEnd::Completed); + let at = events + .iter() + .enumerate() + .filter(|(_, event)| matches!(event, ResultEvent::ResultSetStarted { .. })) + .nth(1) + .map(|(index, _)| index) + .expect("the find's result set"); + let [ + ResultEvent::ResultSetStarted { columns: started }, + ResultEvent::Rows(first), + ResultEvent::Rows(rest), + ResultEvent::ColumnsResolved { + set_index, + columns: resolved, + }, + ResultEvent::ResultSetFinished { + retained: 300, + total_rows: Some(300), + truncation: Truncation::Complete, + }, + ] = &events[at..] + else { + panic!("the find sent {:?}", &events[at..]); + }; + assert_eq!((first.len(), rest.len()), (256, 44)); + assert_eq!(*set_index, 1, "the count's result set came first"); + assert_eq!(started[1].column_type.name, "int"); + assert_eq!(resolved[1].column_type.name, "mixed"); + + let limit = RowLimit::new(270).expect("a limit"); + let (cut, events) = run_with( + session.as_ref(), + &find, + RunOptions::new(CancellationToken::new()).with_row_limit(limit), + ) + .await; + + assert_eq!(cut.end, RunEnd::Completed); + assert!( + matches!( + events.as_slice(), + [ + ResultEvent::ResultSetStarted { .. }, + ResultEvent::Rows(first), + ResultEvent::Rows(rest), + ResultEvent::RowLimitReached { retained: 270 }, + ResultEvent::ResultSetFinished { + retained: 270, + total_rows: None, + truncation: Truncation::ReadPastLimit, + }, + ] if first.len() == 256 && rest.len() == 14 + ), + "{events:?}" + ); +} + /// A JSON cell that reads as `expected`, whatever its spacing. fn assert_json(cell: &Value, expected: &str) { let read = match cell { @@ -414,10 +610,16 @@ fn assert_json(cell: &Value, expected: &str) { /// The first page of `table` as the grid reads it. async fn page(connection: &dyn Connection, table: &TableRef) -> PageRead { let columns = connection.fetch_columns(table, within(CALL)).await.expect("the fields"); + page_with(connection, table, &columns).await +} + +/// The first page of `table` under `columns`, which another connection +/// may have read. +async fn page_with(connection: &dyn Connection, table: &TableRef, columns: &[ColumnInfo]) -> PageRead { let identity = connection.row_identity(table, within(CALL)).await.expect("an identity"); let request = PageRequest { table, - columns: &columns, + columns, filter: &FilterSet::default(), sort: &[], identity: &identity, diff --git a/core/crates/drivers/mongodb/tests/integration/stop.rs b/core/crates/drivers/mongodb/tests/integration/stop.rs new file mode 100644 index 0000000000..5637f71c79 --- /dev/null +++ b/core/crates/drivers/mongodb/tests/integration/stop.rs @@ -0,0 +1,505 @@ +//! A stop on a server that refuses `killSessions`, as Atlas M0 and Flex +//! do: the `failCommand` fail point refuses it here, and the stop ends the +//! call's operations by id instead. + +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::Duration; + +use async_trait::async_trait; +use tablepro_core::meta::TableRef; +use tablepro_core::statement::{Statement, StatementBuilder}; +use tablepro_core::{ + CallOptions, DatabaseDriver, DriverError, EditorSession, LinkState, ReadStatement, ResultEvent, ResultSink, + RowLimit, RunEnd, RunOptions, ServerCode, SinkFlow, TimeoutPhase, UserSql, Value, WorkspaceSettings, +}; +use tablepro_driver_mongodb::MongoDriver; +use tablepro_test_fixtures::MongoFixture; +use tablepro_test_support::conformance::test_liveness; +use tokio::time::Instant; +use tokio_util::sync::CancellationToken; + +use crate::support::{ + CALL, DATABASE, connect, editor, first_page, fixture_options, rows, run, run_with, sink, unique, within, +}; + +/// `CommandNotFound`, what the fail point answers `killSessions` with. +const REFUSED: i32 = 59; + +/// How long a statement may take to show on the server. +const REACHED: Duration = Duration::from_secs(5); + +/// Set the `failCommand` fail point to `data`, in place of what it held. +async fn fail_commands(fixture: &MongoFixture, data: &str) { + let set = fixture + .shell(&format!( + "db.adminCommand({{configureFailPoint: 'failCommand', mode: 'alwaysOn', data: {{{data}}}}}).ok" + )) + .await + .expect("mongosh ran"); + assert!( + set.succeeded() && set.stdout_text().trim() == "1", + "the fail point is not set: {} {}", + set.stdout_text(), + set.stderr_text() + ); +} + +/// A server whose every `killSessions` fails, and a collection whose +/// documents each take a second of server-side JavaScript to read. +async fn refusing_kill_sessions() -> (MongoFixture, String) { + let fixture = MongoFixture::start_with_test_commands() + .await + .expect("a MongoDB container with test commands"); + fail_commands( + &fixture, + &format!("failCommands: ['killSessions'], errorCode: {REFUSED}"), + ) + .await; + let collection = unique("slow"); + let filled = fixture + .shell(&format!( + "db.getSiblingDB('{DATABASE}').getCollection('{collection}').insertMany(\ + Array.from({{length: 50}}, (_, n) => ({{_id: n, label: 'row ' + n}}))).acknowledged" + )) + .await + .expect("mongosh ran"); + assert!(filled.succeeded(), "{}", filled.stderr_text()); + (fixture, collection) +} + +fn slow_find(collection: &str, marker: &str) -> String { + format!( + "db.getCollection('{collection}').find({{$where: 'sleep(1000) || true'}}, {{label: 1}}).comment('{marker}')" + ) +} + +/// How many operations tagged with `marker` the server runs, as it +/// counts them, `Int(0)` for none. +async fn running(watcher: &dyn EditorSession, marker: &str) -> Value { + let (summary, events) = run( + watcher, + &format!( + "db.getSiblingDB('admin').aggregate([{{$currentOp: {{allUsers: true}}}}, \ + {{$match: {{'command.comment': '{marker}'}}}}, {{$count: 'n'}}])" + ), + ) + .await; + assert_eq!(summary.end, RunEnd::Completed); + rows(&events) + .first() + .and_then(|row| row.first()) + .cloned() + .unwrap_or(Value::Int(0)) +} + +/// Cancel once the server runs the statement tagged with `marker`, so +/// the stop has an operation to find. +async fn cancel_once_running(watcher: &dyn EditorSession, marker: &str, cancel: &CancellationToken) -> Instant { + let give_up = Instant::now() + REACHED; + while running(watcher, marker).await == Value::Int(0) { + assert!(Instant::now() < give_up, "the statement never reached the server"); + tokio::time::sleep(Duration::from_millis(50)).await; + } + cancel.cancel(); + Instant::now() +} + +// Without this a stop would pass on a server that took killSessions, the fallback never run. +async fn assert_kill_sessions_refused(session: &dyn EditorSession) { + let (summary, _) = run( + session, + "db.adminCommand({killSessions: [{id: UUID('0e3f6f0e-8a3b-4b8e-9a7c-2f1d1b3c4d5e')}]})", + ) + .await; + assert!( + matches!( + &summary.end, + RunEnd::Failed(DriverError::Server(diagnostics)) + if matches!(diagnostics.code, Some(ServerCode::Mongo { code: REFUSED, .. })) + ), + "killSessions gave {:?}", + summary.end + ); +} + +async fn assert_ended_on_the_server(watcher: &dyn EditorSession, marker: &str) { + let give_up = Instant::now() + test_liveness().cancel_grace; + loop { + let left = running(watcher, marker).await; + if left == Value::Int(0) { + return; + } + assert!(Instant::now() < give_up, "{left:?} operations still run after the stop"); + tokio::time::sleep(Duration::from_millis(50)).await; + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 3)] +#[ignore = "requires docker"] +async fn an_editor_stop_ends_the_statement_when_killsessions_is_refused() { + let (fixture, collection) = refusing_kill_sessions().await; + let connection = connect(&fixture).await; + let (session, watcher) = (editor(connection.as_ref()).await, editor(connection.as_ref()).await); + assert_kill_sessions_refused(watcher.as_ref()).await; + let marker = unique("stopped"); + let cancel = CancellationToken::new(); + + let statement = async { + let options = RunOptions::new(cancel.clone()); + let (summary, _) = run_with(session.as_ref(), &slow_find(&collection, &marker), options).await; + (summary, Instant::now()) + }; + let ((summary, finished), cancelled) = + tokio::join!(statement, cancel_once_running(watcher.as_ref(), &marker, &cancel)); + + assert_eq!(summary.end, RunEnd::Cancelled); + let after_cancel = finished.saturating_duration_since(cancelled); + assert!( + after_cancel < test_liveness().cancel_grace, + "the run waited out the grace, {after_cancel:?} after its cancel" + ); + assert_ended_on_the_server(watcher.as_ref(), &marker).await; + let (again, _) = run(session.as_ref(), "db.runCommand({ping: 1})").await; + assert_eq!(again.end, RunEnd::Completed, "the session runs again"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 3)] +#[ignore = "requires docker"] +async fn a_workspace_stop_reads_as_cancelled_when_killsessions_is_refused() { + let (fixture, collection) = refusing_kill_sessions().await; + let connection = connect(&fixture).await; + let watcher = editor(connection.as_ref()).await; + assert_kill_sessions_refused(watcher.as_ref()).await; + let marker = unique("stopped"); + let cancel = CancellationToken::new(); + let read = ReadStatement::User(UserSql::new(slow_find(&collection, &marker))); + let options = CallOptions::new(Some(Instant::now() + CALL), cancel.clone()); + + let streaming = async { + let streamed = connection.stream_read(&read, sink(), options).await; + (streamed, Instant::now()) + }; + let ((streamed, finished), cancelled) = + tokio::join!(streaming, cancel_once_running(watcher.as_ref(), &marker, &cancel)); + + assert!(matches!(streamed, Err(DriverError::Cancelled)), "{streamed:?}"); + let after_cancel = finished.saturating_duration_since(cancelled); + assert!( + after_cancel < test_liveness().cancel_grace, + "the read waited out the grace, {after_cancel:?} after its cancel" + ); + assert_ended_on_the_server(watcher.as_ref(), &marker).await; + assert_eq!(connection.link_state(), LinkState::Live); +} + +/// How long the sink holds the first rows after it cancels, for the stop +/// to find the cursor waiting between batches. +const HOLD: Duration = Duration::from_millis(300); + +/// A collection of 300 documents of about 100 KB each, so a batch of the +/// server's holds fewer documents than one of the export's and the cursor +/// stays open. +async fn wide_collection(fixture: &MongoFixture) -> String { + let collection = unique("wide"); + let filled = fixture + .shell(&format!( + "db.getSiblingDB('{DATABASE}').getCollection('{collection}').insertMany(\ + Array.from({{length: 300}}, (_, n) => ({{_id: n, label: 'row ' + n, pad: 'x'.repeat(100000)}}))).acknowledged" + )) + .await + .expect("mongosh ran"); + assert!(filled.succeeded(), "{}", filled.stderr_text()); + collection +} + +fn wide_read(collection: &str, marker: &str) -> ReadStatement { + ReadStatement::User(UserSql::new(format!( + "db.getCollection('{collection}').find({{}}, {{label: 1, pad: 1}}).comment('{marker}')" + ))) +} + +/// Cancels the call when the first rows arrive and holds them, as a slow +/// export file does, so no operation runs on the server when the stop +/// comes. +struct CancelAtFirstRows { + cancel: CancellationToken, + rows: Arc, +} + +#[async_trait] +impl ResultSink for CancelAtFirstRows { + async fn accept(&mut self, event: ResultEvent) -> SinkFlow { + if let ResultEvent::Rows(rows) = event { + self.rows.fetch_add(rows.len(), Ordering::SeqCst); + if !self.cancel.is_cancelled() { + self.cancel.cancel(); + tokio::time::sleep(HOLD).await; + } + } + SinkFlow::Continue + } +} + +/// Holds the first rows until `until`, past the call's deadline, so the +/// deadline comes while the cursor waits between batches. +struct HoldFirstRows { + until: Instant, + rows: Arc, +} + +#[async_trait] +impl ResultSink for HoldFirstRows { + async fn accept(&mut self, event: ResultEvent) -> SinkFlow { + if let ResultEvent::Rows(rows) = event + && self.rows.fetch_add(rows.len(), Ordering::SeqCst) == 0 + { + tokio::time::sleep_until(self.until).await; + } + SinkFlow::Continue + } +} + +/// How many cursors tagged with `marker` wait on the server between +/// batches. +async fn idle_cursors(watcher: &dyn EditorSession, marker: &str) -> Value { + let (summary, events) = run( + watcher, + &format!( + "db.getSiblingDB('admin').aggregate([{{$currentOp: {{allUsers: true, idleCursors: true}}}}, \ + {{$match: {{type: 'idleCursor', 'cursor.originatingCommand.comment': '{marker}'}}}}, {{$count: 'n'}}])" + ), + ) + .await; + assert_eq!(summary.end, RunEnd::Completed); + rows(&events) + .first() + .and_then(|row| row.first()) + .cloned() + .unwrap_or(Value::Int(0)) +} + +async fn assert_cursors_closed(watcher: &dyn EditorSession, marker: &str) { + let give_up = Instant::now() + test_liveness().cancel_grace; + loop { + let left = idle_cursors(watcher, marker).await; + if left == Value::Int(0) { + return; + } + assert!(Instant::now() < give_up, "{left:?} cursors still open after the stop"); + tokio::time::sleep(Duration::from_millis(50)).await; + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 3)] +#[ignore = "requires docker"] +async fn a_stop_between_batches_ends_an_export_when_killsessions_is_refused() { + let (fixture, _) = refusing_kill_sessions().await; + let collection = wide_collection(&fixture).await; + let connection = connect(&fixture).await; + let watcher = editor(connection.as_ref()).await; + assert_kill_sessions_refused(watcher.as_ref()).await; + let marker = unique("between"); + let cancel = CancellationToken::new(); + let written = Arc::new(AtomicUsize::new(0)); + let sink = CancelAtFirstRows { + cancel: cancel.clone(), + rows: written.clone(), + }; + + let started = Instant::now(); + let streamed = connection + .stream_read( + &wide_read(&collection, &marker), + Box::new(sink), + CallOptions::new(None, cancel), + ) + .await; + + assert!(matches!(streamed, Err(DriverError::Cancelled)), "{streamed:?}"); + assert!( + started.elapsed() < HOLD + test_liveness().cancel_grace, + "the export waited out the grace" + ); + let written = written.load(Ordering::SeqCst); + assert!(written < 300, "the export wrote all {written} rows past its stop"); + assert_cursors_closed(watcher.as_ref(), &marker).await; + assert_eq!(connection.link_state(), LinkState::Live); +} + +/// Longer than the export takes to read its first batch. +const FIRST_BATCH: Duration = Duration::from_secs(2); + +/// How long `killSessions` and `currentOp` hold their answer in the +/// deadline test, so the read ends while the fallback waits on +/// `currentOp`. +const BLOCKED: Duration = Duration::from_millis(600); + +#[tokio::test(flavor = "multi_thread", worker_threads = 3)] +#[ignore = "requires docker"] +async fn a_deadline_between_batches_ends_an_export_when_killsessions_is_refused() { + let (fixture, small) = refusing_kill_sessions().await; + fail_commands( + &fixture, + &format!( + "failCommands: ['killSessions', 'currentOp'], errorCode: {REFUSED}, \ + blockConnection: true, blockTimeMS: {}", + BLOCKED.as_millis() + ), + ) + .await; + let collection = wide_collection(&fixture).await; + let mut options = fixture_options(&fixture, WorkspaceSettings::workspace(false)); + // The export's client closes once `currentOp` answers, so the call's + // grace has room for both blocked commands. + options.liveness.cancel_grace = 3 * BLOCKED + test_liveness().cancel_grace; + let connection = MongoDriver + .connect(options, CancellationToken::new()) + .await + .expect("a connection"); + let watcher = editor(connection.as_ref()).await; + assert_kill_sessions_refused(watcher.as_ref()).await; + let marker = unique("deadline"); + let deadline = Instant::now() + FIRST_BATCH; + // After the refusal and before `currentOp` answers. + let released = deadline + BLOCKED + BLOCKED / 2; + let written = Arc::new(AtomicUsize::new(0)); + let sink = HoldFirstRows { + until: released, + rows: written.clone(), + }; + + let streamed = connection + .stream_read( + &wide_read(&collection, &marker), + Box::new(sink), + CallOptions::new(Some(deadline), CancellationToken::new()), + ) + .await; + + assert!( + matches!( + streamed, + Err(DriverError::Timeout { + phase: TimeoutPhase::Statement, + server_cancelled: false + }) + ), + "{streamed:?}" + ); + let after_release = Instant::now().saturating_duration_since(released); + assert!( + after_release < test_liveness().cancel_grace, + "the export waited out the grace, {after_release:?} after the sink gave its rows back" + ); + let written = written.load(Ordering::SeqCst); + assert!( + written > 0 && written < 300, + "the deadline did not come between batches: {written} rows" + ); + assert_cursors_closed(watcher.as_ref(), &marker).await; + assert_eq!(connection.link_state(), LinkState::Live); + let page = first_page(connection.as_ref(), &TableRef::new(Some(DATABASE.to_owned()), small)).await; + assert_eq!(page.result.rows.len(), 50, "the connection reads again"); +} + +/// How long the fail point holds each `getMore`, `killSessions` and +/// `currentOp` in the page test, so each batch after a page's first takes +/// this long, and so does the fallback's `currentOp`. +const HELD: Duration = Duration::from_millis(800); + +/// A page whose first batch holds five documents, so the rest take a +/// `getMore` of at most 16 MB each. +fn page_read(collection: &str) -> Statement { + let mut builder = StatementBuilder::new(); + builder.push_sql(&format!("db.getCollection('{collection}').find({{}}).batchSize(5)")); + builder.finish() +} + +/// Refuse the call's `killSessions`, which the fail point holds, by +/// ending it with `Interrupted`: the fail point has one setting, and +/// `currentOp` has to be held without an error. +async fn refuse_held_kill_sessions(watcher: &dyn EditorSession) -> Instant { + let give_up = Instant::now() + REACHED; + loop { + let (summary, events) = run( + watcher, + "db.getSiblingDB('admin').aggregate([{$currentOp: {allUsers: true}}, \ + {$match: {'command.killSessions': {$exists: true}}}, {$project: {_id: 0, opid: 1}}])", + ) + .await; + assert_eq!(summary.end, RunEnd::Completed); + match rows(&events).first().and_then(|row| row.first()) { + Some(Value::Int(opid)) => { + let (killed, _) = run(watcher, &format!("db.adminCommand({{killOp: 1, op: {opid}}})")).await; + assert_eq!(killed.end, RunEnd::Completed); + return Instant::now(); + } + other => assert!(other.is_none(), "an opid of {other:?}"), + } + assert!(Instant::now() < give_up, "the stop never sent killSessions"); + tokio::time::sleep(Duration::from_millis(20)).await; + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 3)] +#[ignore = "requires docker"] +async fn a_deadline_ends_a_page_while_the_fallback_waits_on_currentop() { + let (fixture, small) = refusing_kill_sessions().await; + let wide = wide_collection(&fixture).await; + fail_commands( + &fixture, + &format!( + "failCommands: ['getMore', 'killSessions', 'currentOp'], blockConnection: true, blockTimeMS: {}", + HELD.as_millis() + ), + ) + .await; + let mut options = fixture_options(&fixture, WorkspaceSettings::workspace(false)); + options.liveness.cancel_grace = 3 * HELD; + let connection = MongoDriver + .connect(options, CancellationToken::new()) + .await + .expect("a connection"); + let watcher = editor(connection.as_ref()).await; + // The wide page takes two held `getMore`s, and the deadline comes + // halfway through the second. + let deadline = Instant::now() + HELD + HELD / 2; + + let page = async { + let page = connection + .query_bound( + &page_read(&wide), + RowLimit::new(1_000).expect("a limit"), + CallOptions::new(Some(deadline), CancellationToken::new()), + ) + .await; + (page, Instant::now()) + }; + let ((page, finished), refused) = tokio::join!(page, refuse_held_kill_sessions(watcher.as_ref())); + + assert!( + matches!( + page, + Err(DriverError::Timeout { + phase: TimeoutPhase::Statement, + server_cancelled: false + }) + ), + "{:?}", + page.map(|page| page.rows.len()) + ); + // Before `currentOp` answered, so `settle` found the kill refused. + assert!( + finished < refused + HELD, + "the page waited for the fallback's currentOp, {:?} after the refusal", + finished.saturating_duration_since(refused) + ); + assert_eq!(connection.link_state(), LinkState::Live); + let next = connection + .query_bound(&page_read(&small), RowLimit::new(30).expect("a limit"), within(CALL)) + .await + .expect("the next page"); + assert_eq!(next.rows.len(), 30); + assert_eq!(connection.link_state(), LinkState::Live); +} diff --git a/core/crates/drivers/mongodb/tests/integration/support.rs b/core/crates/drivers/mongodb/tests/integration/support.rs index d279eded83..ce31d71e6b 100644 --- a/core/crates/drivers/mongodb/tests/integration/support.rs +++ b/core/crates/drivers/mongodb/tests/integration/support.rs @@ -1,15 +1,21 @@ //! What the MongoDB integration modules share: targets over the fixture's -//! port, one-line runs, and a sink that keeps what it gets. +//! port, one-line runs, a page and a save as the grid makes them, and a +//! sink that keeps what it gets. use std::sync::{Arc, Mutex, PoisonError}; use std::time::Duration; use async_trait::async_trait; use secrecy::SecretString; +use tablepro_core::browse::{PageRead, PageRequest, read_page}; +use tablepro_core::edit::CellInput; +use tablepro_core::filter::FilterSet; +use tablepro_core::meta::{PagePosition, TableRef}; +use tablepro_core::save::{ChangeSet, RowChange, RowUpdate, plan_save}; use tablepro_core::{ CallOptions, ConnectEnvironment, ConnectOptions, ConnectTarget, Connection, Credentials, DatabaseDriver, EditorSession, EditorSessionOptions, LivenessPolicy, NetworkEndpoint, NetworkOptions, NetworkTarget, ResultEvent, - ResultSink, RunOptions, RunSummary, SinkFlow, TlsConfig, Transport, UserSql, Value, WorkspaceSettings, + ResultSink, RunOptions, RunSummary, SinkFlow, TlsConfig, Transport, UserSql, Value, WorkspaceSettings, WriteReport, }; use tablepro_driver_mongodb::MongoDriver; use tablepro_net::DirectTcpTransport; @@ -99,6 +105,20 @@ pub async fn connect(fixture: &MongoFixture) -> Arc { .expect("a connection") } +/// A writable connection to `target`. +pub async fn connect_to(target: NetworkTarget, liveness: LivenessPolicy) -> Arc { + let options = ConnectOptions { + target: ConnectTarget::Network(target), + workspace_settings: WorkspaceSettings::workspace(false), + liveness, + environment: ConnectEnvironment::new("test"), + }; + MongoDriver + .connect(options, CancellationToken::new()) + .await + .expect("a connection") +} + pub async fn editor(connection: &dyn Connection) -> Arc { connection .open_editor_session(EditorSessionOptions::default(), CancellationToken::new()) @@ -108,19 +128,70 @@ pub async fn editor(connection: &dyn Connection) -> Arc { /// Run `text` in the editor session and keep what it sent. pub async fn run(session: &dyn EditorSession, text: &str) -> (RunSummary, Vec) { + run_with(session, text, RunOptions::new(CancellationToken::new())).await +} + +pub async fn run_with(session: &dyn EditorSession, text: &str, options: RunOptions) -> (RunSummary, Vec) { let kept = Arc::new(Mutex::new(Vec::new())); let summary = session - .run( - UserSql::new(text), - RunOptions::new(CancellationToken::new()), - Box::new(Keep(kept.clone())), - ) + .run(UserSql::new(text), options, Box::new(Keep(kept.clone()))) .await .expect("the session ran the text"); let events = std::mem::take(&mut *kept.lock().unwrap_or_else(PoisonError::into_inner)); (summary, events) } +/// A sink that takes every event. +pub fn sink() -> Box { + Box::new(Keep(Arc::new(Mutex::new(Vec::new())))) +} + +/// The first page of `table` as the grid reads it. +pub async fn first_page(connection: &dyn Connection, table: &TableRef) -> PageRead { + let columns = connection.fetch_columns(table, within(CALL)).await.expect("the fields"); + let identity = connection.row_identity(table, within(CALL)).await.expect("an identity"); + let request = PageRequest { + table, + columns: &columns, + filter: &FilterSet::default(), + sort: &[], + identity: &identity, + position: &PagePosition::First, + limit: 100, + }; + read_page::>(connection, &request, &within(CALL)) + .await + .expect("a page") +} + +/// Save `value` into `field` of the document `key` names, as the grid's +/// save does. +pub async fn save_field( + connection: &dyn Connection, + table: &TableRef, + key: Value, + field: &str, + value: Value, +) -> WriteReport { + let columns = connection.fetch_columns(table, within(CALL)).await.expect("the fields"); + let identity = connection.row_identity(table, within(CALL)).await.expect("an identity"); + let column = columns + .iter() + .position(|column| column.name == field) + .expect("the field is a column"); + let changes = ChangeSet { + changes: vec![RowChange::Update(RowUpdate { + key: vec![key], + assignments: vec![(column, CellInput::Value(value))], + })], + }; + let plan = plan_save(&*connection.dialect(), table, &columns, &identity, &changes).expect("a save plan"); + connection + .write_batch(&plan.batch, within(CALL)) + .await + .expect("the save") +} + /// Every row the events carried. pub fn rows(events: &[ResultEvent]) -> Vec> { events diff --git a/core/crates/mcp/src/export_tools.rs b/core/crates/mcp/src/export_tools.rs index bdcf434f47..600b764481 100644 --- a/core/crates/mcp/src/export_tools.rs +++ b/core/crates/mcp/src/export_tools.rs @@ -2,7 +2,7 @@ use std::sync::Arc; use std::time::Duration; use serde::Serialize; -use tablepro_core::browse::{ExportRequest, build_export}; +use tablepro_core::browse::{ExportFields, ExportRequest, build_export}; use tablepro_core::column::ResultColumn; use tablepro_core::export::{CsvOptions, render_csv, render_json, render_sql_insert}; use tablepro_core::{ColumnInfo, ReadStatement, TableRef, Value}; @@ -120,6 +120,8 @@ async fn export_table( filter: &filter, sort: &[], identity: &identity, + // Every field would take a census of the whole collection before the first row. + fields: ExportFields::Listed, }; Ok(ReadStatement::Generated(build_export(&*lease.dialect(), &request)?)) }, diff --git a/core/crates/mcp/src/query_tools.rs b/core/crates/mcp/src/query_tools.rs index cb8b47c0a8..c1364a03c7 100644 --- a/core/crates/mcp/src/query_tools.rs +++ b/core/crates/mcp/src/query_tools.rs @@ -2,7 +2,7 @@ use std::sync::{Arc, Mutex, PoisonError}; use std::time::{Duration, Instant, SystemTime}; use serde::Serialize; -use tablepro_core::browse::{DeclaredPrecisionSink, ExportRequest, build_count, build_export}; +use tablepro_core::browse::{DeclaredPrecisionSink, ExportFields, ExportRequest, build_count, build_export}; use tablepro_core::meta::{ObjectScope, RowIdentity}; use tablepro_core::sql_syntax::script::{LexicalSettings, ScriptPlan}; use tablepro_core::sql_syntax::{SqlGrammar, StatementAccess, changes_catalog, statement_access}; @@ -313,6 +313,8 @@ pub async fn browse_table( filter: &filter, sort: &sort, identity: &identity, + // Every field would take a census of the whole collection before the first row. + fields: ExportFields::Listed, }; Ok(ReadStatement::Generated(build_export(&*lease.dialect(), &request)?)) }, diff --git a/core/crates/session/tests/session_mongodb.rs b/core/crates/session/tests/session_mongodb.rs index 8c0298b003..375a7baf63 100644 --- a/core/crates/session/tests/session_mongodb.rs +++ b/core/crates/session/tests/session_mongodb.rs @@ -1,7 +1,7 @@ //! A Session against a real MongoDB through OpenSSH: the collections it -//! lists and the statements an editor sends go through the tunnel, to a -//! replica set member that names itself by an address the app never -//! reaches. +//! lists, the pages it reads, the edits it saves and the statements an +//! editor sends go through the tunnel, to a replica set member that names +//! itself by an address the app never reaches. #![expect( clippy::expect_used, @@ -13,11 +13,15 @@ use std::sync::Arc; use std::time::Duration; use secrecy::SecretString; +use tablepro_core::browse::{PageRead, PageRequest, read_page}; use tablepro_core::credentials::CredentialInteraction; -use tablepro_core::meta::{ObjectKind, ObjectScope}; +use tablepro_core::edit::CellInput; +use tablepro_core::filter::FilterSet; +use tablepro_core::meta::{ObjectKind, ObjectScope, PagePosition, RowIdentity, TableRef}; +use tablepro_core::save::{ChangeSet, RowChange, RowUpdate, plan_save}; use tablepro_core::{ - ConnectEnvironment, Credentials, EditorSessionOptions, LivenessPolicy, NetworkEndpoint, ResultEvent, ResultSink, - RunEnd, RunOptions, SinkFlow, TlsConfig, UserSql, WorkspaceSettings, + ColumnInfo, ConnectEnvironment, Credentials, EditorSessionOptions, LivenessPolicy, NetworkEndpoint, ResultEvent, + ResultSink, RunEnd, RunOptions, SinkFlow, TlsConfig, UserSql, Value, WorkspaceSettings, WriteKind, WriteReport, }; use tablepro_driver_mongodb::MongoDriver; use tablepro_session::runtime::Tasks; @@ -80,6 +84,103 @@ impl ResultSink for Discard { } } +/// Fewer documents than the collection holds, so a second page is read. +const PAGE: u64 = 2; + +/// A collection as the browse tab holds it: the fields and the key its +/// pages and saves are built from. +#[derive(Clone)] +struct Grid { + table: TableRef, + columns: Vec, + identity: RowIdentity, +} + +impl Grid { + async fn open(session: &Session, table: TableRef) -> Self { + let read = table.clone(); + let (columns, identity) = session + .run(OperationKind::Metadata, move |lease, options| { + let table = read.clone(); + async move { + let columns = lease.fetch_columns(&table, options.clone()).await?; + let identity = lease.row_identity(&table, options).await?; + Ok((columns, identity)) + } + }) + .await + .expect("the collection's fields and key"); + Self { + table, + columns, + identity, + } + } + + fn column(&self, name: &str) -> usize { + self.columns + .iter() + .position(|column| column.name == name) + .expect("a column of that name") + } + + async fn page(&self, session: &Session, position: &PagePosition) -> PageRead { + let (grid, position) = (self.clone(), position.clone()); + session + .run(OperationKind::Page, move |lease, options| { + let (grid, position) = (grid.clone(), position.clone()); + async move { + let filter = FilterSet::default(); + let request = PageRequest { + table: &grid.table, + columns: &grid.columns, + filter: &filter, + sort: &[], + identity: &grid.identity, + position: &position, + limit: PAGE, + }; + read_page::(&*lease, &request, &options).await + } + }) + .await + .expect("a page through the tunnel") + } + + /// One field of the document whose `_id` is `id` set to `text`, saved + /// the way the grid saves an edited cell. + async fn edit(&self, session: &Session, id: i64, field: &str, text: &str) -> WriteReport { + let changes = ChangeSet { + changes: vec![RowChange::Update(RowUpdate { + key: vec![Value::Int(id)], + assignments: vec![(self.column(field), CellInput::Value(Value::Text(text.to_owned())))], + })], + }; + let grid = self.clone(); + session + .run(OperationKind::Save(WriteKind::RowChanges), move |lease, options| { + let (grid, changes) = (grid.clone(), changes.clone()); + async move { + let plan = plan_save(&*lease.dialect(), &grid.table, &grid.columns, &grid.identity, &changes)?; + Ok(lease.write_batch(&plan.batch, options).await?) + } + }) + .await + .expect("a save through the tunnel") + } +} + +fn ids(read: &PageRead) -> Vec { + read.result + .rows + .iter() + .filter_map(|row| match row.first() { + Some(Value::Int(id)) => Some(*id), + _ => None, + }) + .collect() +} + /// A connection to the replica set's member by its name on the test /// network, which only the SSH server can resolve. fn tunnelled_request(mongo: &MongoFixture, sshd: &OpenSshFixture, private_key: PathBuf) -> ConnectRequest { @@ -124,7 +225,7 @@ fn tunnelled_request(mongo: &MongoFixture, sshd: &OpenSshFixture, private_key: P #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[ignore = "requires docker"] -async fn collections_list_and_statements_run_through_openssh() { +async fn collections_list_page_and_save_through_openssh() { let network = TestNetwork::new(); let mongo = MongoFixture::start_in(&network).await.expect("MongoDB"); let key = SshKeyPair::generate("tablepro-mongodb-test").await.expect("a key pair"); @@ -164,7 +265,10 @@ async fn collections_list_and_statements_run_through_openssh() { .expect("an editor session"); let written = editor .run( - UserSql::new("db.getCollection('orders').insertOne({_id: 1, status: 'sent'})"), + UserSql::new( + "db.getCollection('orders').insertMany([{_id: 1, status: 'sent'}, {_id: 2, status: 'packed'}, \ + {_id: 3, status: 'packed'}])", + ), RunOptions::new(CancellationToken::new()).with_deadline(Some(Duration::from_secs(30))), Box::new(Discard), ) @@ -179,6 +283,13 @@ async fn collections_list_and_statements_run_through_openssh() { }) .await .expect("a listing"); + let grid = Grid::open(&session, TableRef::new(Some("tunnelled".to_owned()), "orders")).await; + let first = grid.page(&session, &PagePosition::First).await; + let second = grid + .page(&session, first.next.as_ref().expect("a page after the first")) + .await; + let saved = grid.edit(&session, 2, "status", "shipped").await; + let reread = grid.page(&session, &PagePosition::First).await; assert_eq!(written.end, RunEnd::Completed); assert!( @@ -187,11 +298,21 @@ async fn collections_list_and_statements_run_through_openssh() { .any(|object| object.name == "orders" && object.kind == ObjectKind::Table), "{listed:?}" ); + assert_eq!(ids(&first), [1, 2]); + assert_eq!(ids(&second), [3]); + assert_eq!(saved.rows_affected, [Some(1)]); + let edited = reread + .result + .rows + .iter() + .find(|row| row.first() == Some(&Value::Int(2))) + .and_then(|row| row.get(grid.column("status"))); + assert_eq!(edited, Some(&Value::Text("shipped".to_owned()))); let in_database = mongo - .shell("db.getSiblingDB('tunnelled').orders.findOne({_id: 1}).status") + .shell("db.getSiblingDB('tunnelled').orders.find().sort({_id: 1}).toArray().map(o => o.status).join(',')") .await .expect("mongosh ran"); - assert_eq!(in_database.stdout_text().trim(), "sent"); + assert_eq!(in_database.stdout_text().trim(), "sent,shipped,packed"); editor.close().await; let report = session.begin_shutdown().await.expect("the close task"); assert_eq!(report, tablepro_session::CloseReport::Closed); diff --git a/core/crates/test-fixtures/src/mongo_fixture.rs b/core/crates/test-fixtures/src/mongo_fixture.rs index 00cb13365b..717a0dc4a1 100644 --- a/core/crates/test-fixtures/src/mongo_fixture.rs +++ b/core/crates/test-fixtures/src/mongo_fixture.rs @@ -13,6 +13,9 @@ use crate::{ const PORT: u16 = MongoFixture::PORT; const CERTIFICATE_PORT: u16 = MongoFixture::CERTIFICATE_PORT; const USER: &str = "tablepro"; +/// The user `start_on_test_records` defines in `thisDB`, a name `admin` +/// has no user by. +const THIS_DB_USER: &str = "tablepro_thisdb"; const CONFIG_DIR: &str = "/etc/tablepro"; const READY_TIMEOUT: Duration = Duration::from_secs(120); /// What the start script prints once the set has a primary and the root @@ -22,8 +25,9 @@ const READY: &str = "tablepro: ready"; /// The container's command. It starts the member, and on a TLS fixture /// the certificate-only server beside it, initiates the set through the /// localhost exception, waits for the member to be primary, then makes -/// the root user, which ends the exception. Done here rather than through -/// `docker exec`, whose result a `Send` future cannot await. +/// the root user, which ends the exception, and as root any user the +/// fixture defines elsewhere. Done here rather than through `docker +/// exec`, whose result a `Send` future cannot await. const START_SCRIPT: &str = r#"set -e cp /etc/tablepro/keyfile /tmp/keyfile chmod 400 /tmp/keyfile @@ -36,15 +40,27 @@ if [ "$TABLEPRO_TLS" = 1 ]; then MEMBER_TLS="--tlsMode requireTLS $TLS --tlsAllowConnectionsWithoutCertificates" SHELL_TLS="--tls --tlsCAFile /etc/tablepro/ca.pem" fi -mongod --replSet rs0 --bind_ip_all --port 27017 --keyFile /tmp/keyfile $MEMBER_TLS & +TEST_COMMANDS="" +if [ "$TABLEPRO_TEST_COMMANDS" = 1 ]; then + TEST_COMMANDS="--setParameter enableTestCommands=1" +fi +mongod --replSet "$TABLEPRO_SET" --bind_ip_all --port 27017 --keyFile /tmp/keyfile $MEMBER_TLS $TEST_COMMANDS & member=$! ask() { mongosh --quiet --host localhost --port 27017 $SHELL_TLS --eval "$1" } +ask_as_root() { + mongosh --quiet --host localhost --port 27017 $SHELL_TLS -u tablepro -p "$TABLEPRO_PASSWORD" \ + --authenticationDatabase admin --eval "$1" +} until [ "$(ask 'db.runCommand({ping: 1}).ok' 2> /dev/null)" = 1 ]; do sleep 0.5; done -ask "rs.initiate({_id: 'rs0', members: [{_id: 0, host: 'localhost:27017'}]}).ok" +ask "rs.initiate({_id: process.env.TABLEPRO_SET, members: [{_id: 0, host: process.env.TABLEPRO_MEMBER}]}).ok" until [ "$(ask 'db.hello().isWritablePrimary' 2> /dev/null)" = true ]; do sleep 0.5; done ask "db.getSiblingDB('admin').createUser({user: 'tablepro', pwd: process.env.TABLEPRO_PASSWORD, roles: ['root']}).ok" +if [ -n "$TABLEPRO_USER_DB" ]; then + ask_as_root "db.getSiblingDB(process.env.TABLEPRO_USER_DB).createUser({user: process.env.TABLEPRO_USER, \ + pwd: process.env.TABLEPRO_PASSWORD, roles: [{role: 'readWrite', db: process.env.TABLEPRO_USER_DB}]}).ok" +fi echo "tablepro: ready" wait "$member" "#; @@ -59,12 +75,64 @@ enum Listening { Tls, } +/// How a fixture's server starts and what it holds. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct Setup { + listening: Listening, + set: &'static str, + member: &'static str, + /// A host port of its own, which a member named by an address the host + /// reaches needs, since discovery dials that name. + host_port: Option, + test_commands: bool, + /// The database a second user is defined in. + user_database: Option<&'static str>, +} + +impl Setup { + const fn listening(listening: Listening) -> Self { + Self { + listening, + set: "rs0", + member: "localhost:27017", + host_port: None, + test_commands: false, + user_database: None, + } + } + + const TEST_RECORDS: Self = Self { + listening: Listening::Plain, + set: MongoFixture::TEST_RECORDS_SET, + member: "localhost.test.build.10gen.cc:27017", + host_port: Some(PORT), + test_commands: false, + user_database: Some(MongoFixture::THIS_DB), + }; + + fn env(&self) -> [(&'static str, String); 6] { + [ + ("TABLEPRO_TLS", flag(self.listening == Listening::Tls)), + ("TABLEPRO_SET", self.set.to_owned()), + ("TABLEPRO_MEMBER", self.member.to_owned()), + ("TABLEPRO_TEST_COMMANDS", flag(self.test_commands)), + ("TABLEPRO_USER_DB", self.user_database.unwrap_or_default().to_owned()), + ("TABLEPRO_USER", THIS_DB_USER.to_owned()), + ] + } +} + +fn flag(on: bool) -> String { + String::from(if on { "1" } else { "0" }) +} + /// A one-member replica set, `rs0`, with access control on and a user, /// `tablepro`, who holds the root role. /// /// The member is named `localhost:27017` inside the container, which the /// tests never reach by that name: they connect to the one host they are -/// given, as a direct connection does. +/// given, as a direct connection does. `start_on_test_records` names it +/// so the host reaches it, for discovery. pub struct MongoFixture { container: ContainerAsync, pki: TestPki, @@ -74,35 +142,60 @@ pub struct MongoFixture { certificate_port: Option, network_alias: Option, version: MongoVersion, - listening: Listening, + setup: Setup, } impl MongoFixture { pub const PORT: u16 = 27017; pub const CERTIFICATE_PORT: u16 = 27018; pub const NETWORK_ROLE: &'static str = "mongodb"; + /// The set `start_on_test_records` makes, as the records name it. + pub const TEST_RECORDS_SET: &'static str = "repl0"; + /// The database `test5.test.build.10gen.cc`'s TXT record names as + /// `authSource`. + pub const THIS_DB: &'static str = "thisDB"; pub async fn start() -> Result { - Self::launch(TestPki::generate()?, None, Listening::Plain).await + Self::launch(TestPki::generate()?, None, Setup::listening(Listening::Plain)).await } pub async fn start_in(network: &TestNetwork) -> Result { - Self::launch(TestPki::generate()?, Some(network), Listening::Plain).await + Self::launch(TestPki::generate()?, Some(network), Setup::listening(Listening::Plain)).await } /// A replica set that takes only TLS, with a server beside it that /// takes only clients that present a certificate the CA issued. pub async fn start_tls() -> Result { - Self::launch(TestPki::generate()?, None, Listening::Tls).await + Self::launch(TestPki::generate()?, None, Setup::listening(Listening::Tls)).await } /// `start_tls` with a self-signed server certificate, which a client /// accepts only by trusting that certificate itself. pub async fn start_tls_self_signed() -> Result { - Self::launch(TestPki::generate_self_signed()?, None, Listening::Tls).await + Self::launch(TestPki::generate_self_signed()?, None, Setup::listening(Listening::Tls)).await } - async fn launch(pki: TestPki, network: Option<&TestNetwork>, listening: Listening) -> Result { + /// `start` with the server's test commands on, `configureFailPoint` + /// among them, which only a start can turn on. + pub async fn start_with_test_commands() -> Result { + let setup = Setup { + test_commands: true, + ..Setup::listening(Listening::Plain) + }; + Self::launch(TestPki::generate()?, None, setup).await + } + + /// The set the MongoDB specification's DNS records under + /// `test.build.10gen.cc` name: `repl0`, whose one member is + /// `localhost.test.build.10gen.cc:27017`, a public name for 127.0.0.1, + /// on host port 27017, with a user defined in `thisDB`. It needs + /// public DNS on the host and in the container, and the host port + /// free, so one runs at a time. + pub async fn start_on_test_records() -> Result { + Self::launch(TestPki::generate()?, None, Setup::TEST_RECORDS).await + } + + async fn launch(pki: TestPki, network: Option<&TestNetwork>, setup: Setup) -> Result { let password = FixtureCredentials::generate_password(); let version = MongoVersion::selected()?; let network_alias = network.map(|network| network.container_name(Self::NETWORK_ROLE)); @@ -117,11 +210,16 @@ impl MongoFixture { .with_entrypoint("sh") .with_cmd([format!("{CONFIG_DIR}/start.sh")]) .with_env_var("TABLEPRO_PASSWORD", &password) - .with_env_var("TABLEPRO_TLS", if listening == Listening::Tls { "1" } else { "0" }) // The image hands rseq to TCMalloc for its per-CPU caches, and with // those on MongoDB 8.0 exits on kernels 6.19 to 7.0.13 (SERVER-121912). .with_env_var("GLIBC_TUNABLES", "glibc.pthread.rseq=1") .with_startup_timeout(READY_TIMEOUT); + for (name, value) in setup.env() { + request = request.with_env_var(name, value); + } + if let Some(host_port) = setup.host_port { + request = request.with_mapped_port(host_port, PORT.tcp()); + } if let (Some(network), Some(alias)) = (network, &network_alias) { request = request.with_network(network.name()).with_container_name(alias); } @@ -130,7 +228,7 @@ impl MongoFixture { .await?; let host = container.get_host().await?.to_string(); let port = container.get_host_port_ipv4(PORT).await?; - let certificate_port = match listening { + let certificate_port = match setup.listening { Listening::Plain => None, Listening::Tls => Some(container.get_host_port_ipv4(CERTIFICATE_PORT).await?), }; @@ -143,7 +241,7 @@ impl MongoFixture { certificate_port, network_alias, version, - listening, + setup, }) } @@ -199,6 +297,14 @@ impl MongoFixture { FixtureCredentials::new(USER, &self.password) } + /// The user defined in `thisDB` with `readWrite` there, on a fixture + /// from `start_on_test_records`. + pub fn this_db_credentials(&self) -> Option { + self.setup + .user_database + .map(|_| FixtureCredentials::new(THIS_DB_USER, &self.password)) + } + pub fn container(&self) -> &ContainerAsync { &self.container } @@ -212,7 +318,7 @@ impl MongoFixture { pub async fn shell(&self, script: &str) -> Result { let mut command = vec!["mongosh", "--quiet", "--host", "localhost", "--port", "27017"]; let ca = format!("{CONFIG_DIR}/ca.pem"); - if self.listening == Listening::Tls { + if self.setup.listening == Listening::Tls { command.extend(["--tls", "--tlsCAFile", &ca]); } command.extend([ @@ -248,6 +354,24 @@ mod tests { assert_eq!(key.mode(), ContainerFile::PRIVATE); } + #[test] + fn the_test_records_set_is_named_as_the_records_name_it() { + let env = Setup::TEST_RECORDS.env(); + let value = |name: &str| { + env.iter() + .find(|(key, _)| *key == name) + .map(|(_, value)| value.as_str()) + }; + + assert_eq!(value("TABLEPRO_SET"), Some(MongoFixture::TEST_RECORDS_SET)); + assert_eq!(value("TABLEPRO_MEMBER"), Some("localhost.test.build.10gen.cc:27017")); + assert_eq!(value("TABLEPRO_USER_DB"), Some(MongoFixture::THIS_DB)); + assert_eq!(Setup::TEST_RECORDS.host_port, Some(MongoFixture::PORT)); + let plain = Setup::listening(Listening::Plain).env(); + assert!(plain.contains(&("TABLEPRO_SET", "rs0".to_owned())), "{plain:?}"); + assert!(plain.contains(&("TABLEPRO_USER_DB", String::new())), "{plain:?}"); + } + #[test] fn the_start_script_makes_the_user_only_once_the_set_has_a_primary() { let primary = START_SCRIPT.find("isWritablePrimary").unwrap(); @@ -256,5 +380,10 @@ mod tests { assert!(primary < user && user < ready, "{START_SCRIPT}"); assert!(START_SCRIPT.contains("--port 27018") && START_SCRIPT.contains("--fork")); + let second = START_SCRIPT.rfind("createUser").unwrap(); + assert!( + user < second && second < ready, + "the second user is made once root exists" + ); } } diff --git a/core/crates/test-support/src/conformance/cases/document.rs b/core/crates/test-support/src/conformance/cases/document.rs index b072921fc2..9fbe9757b4 100644 --- a/core/crates/test-support/src/conformance/cases/document.rs +++ b/core/crates/test-support/src/conformance/cases/document.rs @@ -12,16 +12,16 @@ use tablepro_core::filter::{Combinator, FilterOp, FilterRule, FilterSet, FilterV use tablepro_core::meta::{ObjectKind, ObjectScope, PagePosition, RowIdentity, TableRef}; use tablepro_core::save::{ChangeSet, RowChange, RowUpdate, plan_save}; use tablepro_core::{ - ColumnInfo, Connection, DataModel, DriverError, EditorSession, EditorSessionOptions, EditorSessionState, - ErrorCategory, LinkState, ReadStatement, RowLimit, RunEnd, RunOptions, Truncation, UserSql, Value, - WorkspaceSettings, + CallOptions, ColumnInfo, Connection, DataModel, DriverError, EditorSession, EditorSessionOptions, + EditorSessionState, ErrorCategory, LinkState, ReadStatement, RowLimit, RunEnd, RunOptions, StatementOutcome, + Truncation, UserSql, Value, WorkspaceSettings, WriteAtomicity, WriteReport, }; use tokio::time::Instant; use tokio_util::sync::CancellationToken; use crate::conformance::support::{ - CALL_DEADLINE, Collected, SCHEDULING_SLACK, dial, expect, run_completed, run_editor, run_options, unexpected, - unique, within, writable, + CALL_DEADLINE, Collected, SCHEDULING_SLACK, dial, expect, run_completed, run_editor, run_options, statement, + unexpected, unique, within, writable, }; use crate::conformance::{ConformanceFailure, DocumentEngine, test_liveness}; @@ -29,6 +29,8 @@ use crate::conformance::{ConformanceFailure, DocumentEngine, test_liveness}; /// it, so the stop is tested rather than a race with sending. const SETTLE: Duration = Duration::from_millis(500); +const SEEN_LIMIT: Duration = Duration::from_secs(10); + /// More documents than two pages hold, so paging reads a third. const MANY: u32 = 1_234; @@ -275,7 +277,7 @@ async fn save( connection: &dyn Connection, table: &TableRef, changes: Vec, -) -> Result, ConformanceFailure> { +) -> Result, ConformanceFailure> { let (columns, identity) = shape(connection, table).await?; let plan = plan_save( &*connection.dialect(), @@ -421,6 +423,121 @@ pub async fn an_insert_and_a_delete_land(engine: E) -> Result Ok(()) } +/// Whether the first step of a save that failed at its second step for +/// the reason `cause` matches was kept, as the engine's atomicity says it +/// must be, or `None` when the outcome is not what that atomicity gives. +fn first_step_kept( + atomicity: WriteAtomicity, + outcome: &Result, + cause: impl Fn(&DriverError) -> bool, +) -> Option { + match (atomicity, outcome) { + ( + WriteAtomicity::StatementByStatement, + Err(DriverError::PartiallyApplied { + applied: 1, + statement_index: 1, + failed_outcome: StatementOutcome::Failed, + source, + }), + ) if cause(source) => Some(true), + ( + WriteAtomicity::Transactional, + Err(DriverError::RolledBack { + statement_index: 1, + source, + }), + ) if cause(source) => Some(false), + // With nothing applied, a guard mismatch is its own outcome. + (WriteAtomicity::Transactional, Err(error @ DriverError::RowGuardFailed { .. })) if cause(error) => Some(false), + _ => None, + } +} + +const SECOND_STEP_CASE: &str = "a_save_that_fails_at_its_second_step_keeps_the_first"; + +/// A save whose second step the server refuses, or finds its document +/// gone, ends as the engine's atomicity says: where each step lands on +/// its own, the first step stays and the outcome says one step applied +/// and the second failed; on a transactional engine nothing stays. +pub async fn a_save_that_fails_at_its_second_step_keeps_the_first( + engine: E, +) -> Result<(), ConformanceFailure> { + let atomicity = E::driver().capabilities().dml_writes; + let connection = connect(SECOND_STEP_CASE, &engine, writable()).await?; + let table = filled(SECOND_STEP_CASE, &engine, connection.as_ref(), 3).await?; + let insert = |id: i64, label: &str| RowChange::Insert { + cells: vec![ + CellInput::Value(Value::Int(id)), + CellInput::Value(Value::Text(label.to_owned())), + CellInput::Default, + ], + }; + let rename = |id: i64, label: &str| { + RowChange::Update(RowUpdate { + key: vec![Value::Int(id)], + assignments: vec![(1, CellInput::Value(Value::Text(label.to_owned())))], + }) + }; + + let refused = save( + connection.as_ref(), + &table, + vec![insert(100, "first"), insert(1, "taken")], + ) + .await?; + let Some(kept) = first_step_kept(atomicity, &refused, |error: &DriverError| { + error.category() == ErrorCategory::Server + }) else { + return Err(unexpected( + SECOND_STEP_CASE, + format!("a {atomicity:?} engine answered a save whose second insert takes a used key with {refused:?}"), + )); + }; + let first = label_of(SECOND_STEP_CASE, connection.as_ref(), &table, 100).await?; + expect(SECOND_STEP_CASE, first.is_some() == kept, || { + format!("the first insert reads {first:?} after the save failed at the second") + })?; + let taken = label_of(SECOND_STEP_CASE, connection.as_ref(), &table, 1).await?; + expect(SECOND_STEP_CASE, taken == Some(Value::Text("row 1".to_owned())), || { + format!("the document whose key the second insert took reads {taken:?}") + })?; + + save( + connection.as_ref(), + &table, + vec![RowChange::Delete { + key: vec![Value::Int(3)], + }], + ) + .await??; + let gone = save( + connection.as_ref(), + &table, + vec![rename(2, "edited"), rename(3, "ghost")], + ) + .await?; + let Some(kept) = first_step_kept(atomicity, &gone, |error: &DriverError| { + matches!(error, DriverError::RowGuardFailed { statement_index: 1, .. }) + }) else { + return Err(unexpected( + SECOND_STEP_CASE, + format!("a {atomicity:?} engine answered a save whose second edit finds no document with {gone:?}"), + )); + }; + let edited = label_of(SECOND_STEP_CASE, connection.as_ref(), &table, 2).await?; + let expected = Value::Text(if kept { "edited" } else { "row 2" }.to_owned()); + expect(SECOND_STEP_CASE, edited.as_ref() == Some(&expected), || { + format!("the first edit reads {edited:?} after the save failed at the second, not {expected:?}") + })?; + let ghost = label_of(SECOND_STEP_CASE, connection.as_ref(), &table, 3).await?; + expect(SECOND_STEP_CASE, ghost.is_none(), || { + format!("the deleted document came back as {ghost:?}") + })?; + connection.close().await?; + Ok(()) +} + const READ_ONLY_CASE: &str = "a_read_only_connection_refuses_writes_before_sending"; /// On a read-only connection a write never reaches the server: a save, a @@ -490,30 +607,40 @@ pub async fn a_read_only_connection_refuses_writes_before_sending( +/// Wait until the number of operations tagged with `marker` the server +/// runs satisfies `done`, or `bound` passes. +async fn until_running( case: &'static str, engine: &E, marker: &str, + done: impl Fn(u64) -> bool, bound: Duration, ) -> Result<(), ConformanceFailure> { let give_up = Instant::now() + bound; loop { let running = engine.running_operations(marker).await?; - if running == 0 { + if done(running) { return Ok(()); } if Instant::now() >= give_up { return Err(unexpected( case, - format!("{running} operations were still running after {bound:?}"), + format!("{running} operations were running after {bound:?}"), )); } tokio::time::sleep(Duration::from_millis(100)).await; } } +async fn until_stopped( + case: &'static str, + engine: &E, + marker: &str, + bound: Duration, +) -> Result<(), ConformanceFailure> { + until_running(case, engine, marker, |running| running == 0, bound).await +} + const STOPPED_CASE: &str = "a_stopped_statement_ends_and_the_session_runs_again"; /// A statement running on the server ends as cancelled within the cancel @@ -631,6 +758,72 @@ pub async fn a_timed_out_statement_is_stopped_by_the_server( Ok(()) } +const WORKSPACE_STOP_CASE: &str = "a_stopped_workspace_read_ends_and_the_connection_reads_again"; + +/// A generated read the server is running ends as cancelled within the +/// cancel grace, the server lets it go, and the link stays live for the +/// next page. +pub async fn a_stopped_workspace_read_ends_and_the_connection_reads_again( + engine: E, +) -> Result<(), ConformanceFailure> { + let grace = test_liveness().cancel_grace + SCHEDULING_SLACK; + let connection = connect(WORKSPACE_STOP_CASE, &engine, writable()).await?; + let table = filled(WORKSPACE_STOP_CASE, &engine, connection.as_ref(), 50).await?; + let marker = unique("workspace_stop"); + let unbounded = statement(&engine.unbounded_statement(&table.name, &marker)); + let cancel = CancellationToken::new(); + let options = CallOptions::new(Some(Instant::now() + CALL_DEADLINE), cancel.clone()); + + let running = async { + let outcome = connection + .query_bound(&unbounded, RowLimit::EDITOR_DEFAULT, options) + .await; + (outcome, Instant::now()) + }; + let cancelling = async { + let seen = until_running( + WORKSPACE_STOP_CASE, + &engine, + &marker, + |running| running >= 1, + SEEN_LIMIT, + ) + .await; + cancel.cancel(); + (seen, Instant::now()) + }; + let ((outcome, finished), (seen, cancelled)) = tokio::join!(running, cancelling); + seen?; + let after_cancel = finished.saturating_duration_since(cancelled); + + expect( + WORKSPACE_STOP_CASE, + matches!(&outcome, Err(DriverError::Cancelled)), + || format!("the stopped read gave {outcome:?}"), + )?; + expect(WORKSPACE_STOP_CASE, after_cancel <= grace, || { + format!("the read ended {after_cancel:?} after its cancel, past the {grace:?} grace") + })?; + until_stopped(WORKSPACE_STOP_CASE, &engine, &marker, grace).await?; + expect(WORKSPACE_STOP_CASE, connection.link_state() == LinkState::Live, || { + format!("the link is {:?} after a stop", connection.link_state()) + })?; + let next = page( + WORKSPACE_STOP_CASE, + connection.as_ref(), + &table, + &FilterSet::default(), + &[], + &PagePosition::First, + ) + .await?; + expect(WORKSPACE_STOP_CASE, ids(&next).len() == 50, || { + format!("the page after the stop read {} documents, not 50", ids(&next).len()) + })?; + connection.close().await?; + Ok(()) +} + const LIMIT_CASE: &str = "an_editor_result_keeps_the_row_limit"; /// A read past the row limit keeps the limit's worth and says the result diff --git a/core/crates/test-support/src/conformance/cases/large_table.rs b/core/crates/test-support/src/conformance/cases/large_table.rs index 5dc295e763..2e6cb3a351 100644 --- a/core/crates/test-support/src/conformance/cases/large_table.rs +++ b/core/crates/test-support/src/conformance/cases/large_table.rs @@ -5,7 +5,7 @@ use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; use async_trait::async_trait; -use tablepro_core::browse::{ExportRequest, PageRead, PageRequest, PagingMode, build_export, read_page}; +use tablepro_core::browse::{ExportFields, ExportRequest, PageRead, PageRequest, PagingMode, build_export, read_page}; use tablepro_core::export::{CsvEncoder, CsvOptions, CsvQuote, RowEncoder}; use tablepro_core::meta::{PagePosition, RowIdentity, TableRef}; use tablepro_core::{ @@ -169,6 +169,7 @@ async fn export_table( filter: &browsed.filter, sort: &[], identity: &browsed.identity, + fields: ExportFields::Every, }; let read = ReadStatement::Generated( build_export(&*connection.dialect(), &request) diff --git a/core/crates/test-support/src/conformance/cases/round_trip.rs b/core/crates/test-support/src/conformance/cases/round_trip.rs index 94f0a82d79..31939d490d 100644 --- a/core/crates/test-support/src/conformance/cases/round_trip.rs +++ b/core/crates/test-support/src/conformance/cases/round_trip.rs @@ -4,7 +4,7 @@ use std::sync::{Arc, Mutex}; use std::time::Duration; use async_trait::async_trait; -use tablepro_core::browse::{DeclaredPrecisionSink, ExportRequest, build_export}; +use tablepro_core::browse::{DeclaredPrecisionSink, ExportFields, ExportRequest, build_export}; use tablepro_core::column::{ColumnInfo, ColumnType, ResultColumn}; use tablepro_core::dialect::SqlDialect; use tablepro_core::edit::{EditText, edit_text, read_only_reason}; @@ -581,6 +581,7 @@ async fn stream_table( filter: &FilterSet::default(), sort: &[], identity: &identity, + fields: ExportFields::Every, }, ) .map_err(ConformanceFailure::setup)?; diff --git a/core/crates/test-support/src/conformance/cases/value_corpus.rs b/core/crates/test-support/src/conformance/cases/value_corpus.rs index 20f9d43659..79f5d00e03 100644 --- a/core/crates/test-support/src/conformance/cases/value_corpus.rs +++ b/core/crates/test-support/src/conformance/cases/value_corpus.rs @@ -2,7 +2,7 @@ use std::collections::BTreeMap; use std::sync::Arc; use tablepro_core::browse::{ - BrowseSqlError, DeclaredPrecisionSink, ExportRequest, PageRequest, build_export, read_page, + BrowseSqlError, DeclaredPrecisionSink, ExportFields, ExportRequest, PageRequest, build_export, read_page, }; use tablepro_core::column::{ColumnInfo, ColumnType, ResultColumn}; use tablepro_core::export::value_text; @@ -175,6 +175,7 @@ async fn export(connection: &dyn Connection, table: &TableRef) -> Result ERROR`, then `CLIENT KILL ID ` past the grace, from a link of its own ([redis.md](drivers/redis.md#stops-and-timeouts)) | `InBand` | -| MongoDB | `killSessions` for the call's own logical session, from the same client | `OutOfBand(Statement)` | +| MongoDB | `killSessions` for the call's own logical session, from the same client; where it is refused, `killOp` for each operation `currentOp` lists in that session ([mongodb.md](drivers/mongodb.md#stops-and-timeouts)) | `OutOfBand(Statement)` | ### `CallResource` release rules @@ -314,7 +314,7 @@ A run streams `ResultEvent`s into a `ResultSink`: - `Notice` passes on a server notice or warning. - `NoticeLimitReached { retained }` comes once per run when the driver kept fewer notices than the server sent, after the `retained` ones it passed on. -`ResultSink::accept` is awaited before the next rows are read off the wire, so a grid that cannot keep up slows the read instead of growing a queue the size of the result. `SinkFlow::ConsumerGone` ends the run with `RunEnd::Cancelled`. PostgreSQL, MySQL, SQL Server and ClickHouse then stop the statement on the server, SQLite stops stepping it, DuckDB stops fetching its streamed result, and the Cassandra driver asks for no more pages. `stream_read` uses the same sink and events, without a row limit. +`ResultSink::accept` is awaited before the next rows are read off the wire, so a grid that cannot keep up slows the read instead of growing a queue the size of the result. One exception: an editor result whose columns a document engine finds from its documents, since a later document can add a column. It reads the documents it keeps, at most the row limit and one more, before `ResultSetStarted`, then sends them under the same backpressure and closes the cursor. `SinkFlow::ConsumerGone` ends the run with `RunEnd::Cancelled`. PostgreSQL, MySQL, SQL Server and ClickHouse then stop the statement on the server, SQLite stops stepping it, DuckDB stops fetching its streamed result, and the Cassandra driver asks for no more pages. `stream_read` uses the same sink and events, without a row limit. ## The row-limit rule @@ -417,8 +417,8 @@ What a new engine without SQL does: MongoDB holds databases of collections and takes mongosh statements. A collection opens in the table tab, so every builder the tab, an export, a save and the MCP server reach (`build_page`, `build_count`, `build_export`, `plan_save`, `build_filter`, `validate_raw_predicate`, the SQL export's inserts) has a document arm, chosen by the dialect's grammar, that writes the engine's shell call with every value bound ([ADR 0026](decisions/0026-documents-in-the-table-tab.md)). What a document engine does beyond the steps above: -1. **Capabilities.** `data_model: Document`, `object_kinds` tables and views, and `network_options` for what its connection strings carry beyond a host (MongoDB: `DnsSeedList`, `AuthDatabase`). -2. **Catalogue.** Databases from `list_schemas`, collections and views from `list_objects`. `fetch_columns` reads the top-level fields of a bounded sample on the server, the key first, with the column kind the field's values share and JSON for nested or mixed ones. `row_identity` is the document key, `Unordered` for a view. +1. **Capabilities.** `data_model: Document`, `object_kinds` tables and views, and `network_options` for what its connection strings carry beyond a host (MongoDB: `DnsSeedList`, `AuthDatabase`, `HostList`). +2. **Catalogue.** Databases from `list_schemas`, collections and views from `list_objects`. `fetch_columns` reads the top-level fields of a bounded sample on the server, the key first, with the column kind the field's values share and JSON for nested or mixed ones, numbers of more than one BSON type included. `row_identity` is the document key, `Unordered` for a view. 3. **Grammar and builders.** The `SqlGrammar` variant's module (`sql_syntax/mongo/`) parses the shell into an AST that keeps key order and prints it back, so a generated statement is runnable text, and the driver reads generated and typed statements through the same parser and the same read-only check. The document arms live in `tablepro_core::document`; a builder that meets a field a query cannot name refuses it (`UnaddressableField`). 4. **Values.** A cell comes from the value's own type, not the column's; a nested value is Extended JSON that names its types, and a JSON cell the user edits goes back as the BSON it names. 5. **Tests.** `document_conformance_suite!` through `DocumentEngine` ([testing.md](testing.md#document-conformance)), and the TLS suite on each route the driver does TLS on. @@ -431,7 +431,7 @@ MongoDB holds databases of collections and takes mongosh statements. A collectio - [ ] Every call runs under `OwnedCalls` with the engine's stop, and a connection-scoped stop never releases an unsettled connection. - [ ] `probe` never waits for a pool slot; `stream_read` never holds one. - [ ] Saves record every step in a `WriteLedger`, and the stop is held while a commit is in flight. -- [ ] Editor sessions are one connection per tab, one run at a time, with `ResultSink` backpressure and the row-limit rule. +- [ ] Editor sessions are one connection per tab, one run at a time, with `ResultSink` backpressure (or the document exception in [ResultSink and backpressure](#resultsink-and-backpressure)) and the row-limit rule. - [ ] Errors map to `DriverError` by the engine's code ([error-handling.md](error-handling.md)). - [ ] The client library's log target is in `SILENCED_LIBRARIES` in `crates/app/src/logging.rs`, so no statement text reaches the journal ([error-handling.md](error-handling.md#logging)). - [ ] Registered in `build_registry`; added to the workspace and to `crates/app/Cargo.toml`. diff --git a/linux/docs/drivers/mongodb.md b/linux/docs/drivers/mongodb.md index e7d415cd25..e6b72b5356 100644 --- a/linux/docs/drivers/mongodb.md +++ b/linux/docs/drivers/mongodb.md @@ -46,9 +46,11 @@ Databases are the sidebar's schemas, collections its tables and views its views. A collection's columns are its top-level fields, found by the server from the first 1,000 documents in their natural order: the fields, the BSON types each holds, and where each first appears. No document crosses the wire for it. `_id` comes first, then the fields in the order the documents place them. -A field's column type is its values' type: a number widens across `int`, `long`, `double` and `decimal`, a string is text, a date a timestamp with a zone, an `objectId` its own type. A field that holds documents, arrays, binary data or more than one kind of value is a JSON column, written as relaxed Extended JSON from the BSON itself, with each `long` in its canonical `{"$numberLong": "..."}` form, so the type survives an edit. A UUID (binary subtype 4) is a UUID, other binary data bytes. A field missing from a document reads as NULL. +A field's column type is its values' type: one numeric type is a number, a string is text, a date a timestamp with a zone, an `objectId` its own type, and UUIDs (binary subtype 4) a UUID. `$type` names every binary subtype `binData`, so the census tells a UUID by where it sorts: binary data orders by length, then subtype, then bytes. A field that holds documents, arrays, other binary data, numbers of more than one type (`int` and `long`, or `int` and `double`) or more than one kind of value is a JSON column, written as relaxed Extended JSON from the BSON itself, with each `long` in its canonical `{"$numberLong": "..."}` form, so the type survives an edit, a copy and an export. A field missing from a document reads as NULL. -`_id` is the primary key and names a document for every save. A new row leaves `_id` out unless one is typed, and the document gets an ObjectId, as every driver gives it. A view has no key, and neither has a time-series collection, whose `_id` the server does not keep unique: their pages are counted off in the order they give, and their rows read and never write. Indexes come from `listIndexes`, `_id`'s first. The row count is the collection's metadata count, shown as an estimate. +A connection that has not read a collection's fields reads them before its first page or export of it, so a page after a reconnect shows the same JSON columns. + +`_id` is the primary key and names a document for every save. A new row leaves `_id` out unless one is typed, and the document gets an ObjectId, as every driver gives it. A cell left NULL in a new row, a duplicated row or an imported one is a field left out, since a page reads a missing field as NULL: the document holds only the fields given a value, so a sparse or partial index and a validator's optional field see what `insertOne` in mongosh would give them. The statements an export or Copy As writes leave them out too, and an edit that sets a field to NULL removes it with `$unset`. To store an explicit `null`, type `null` in a JSON cell or write the statement in the editor. A view has no key, and neither has a time-series collection, whose `_id` the server does not keep unique: their pages are counted off in the order they give, and their rows read and never write. Indexes come from `listIndexes`, `_id`'s first. The row count is the collection's metadata count, shown as an estimate. ## Pages, filters and saves @@ -64,7 +66,7 @@ A collection opens in the table tab, and every statement the tab builds is a mon Every value is a bound parameter, never spliced into the text. A sort always ends with `_id`, so a page is stable, and paging past the first pages walks `_id` by key: across `_id`s of more than one type, in the order MongoDB compares types. -A filter rule becomes a `$match` condition: equality, comparisons, `$in`, a pattern as an anchored `$regex`, NULL as missing or null. The raw filter takes a query document, `{ qty: { $gt: 1 }, status: 'A' }`, checked before it is sent. A field whose name holds a `.` or starts with `$` is read through `$getField` and shown, but cannot be filtered, sorted or edited, since a query cannot name it. +A filter rule becomes a `$match` condition: equality, comparisons, `$in`, a pattern as an anchored `$regex`, NULL as missing or null. The raw filter takes a query document, `{ qty: { $gt: 1 }, status: 'A' }`, checked before it is sent. A field whose name holds a `.` or starts with `$` is read through `$getField` and shown, but cannot be filtered, sorted or edited, since a query cannot name it. A JSON column filters with comparisons as well as patterns: MongoDB orders any two values, by type and then by value, and a filter value is the BSON its Extended JSON names, so `price > 10` works on a field of `int` and `double`. A JSON cell is edited as JSON text, refused when it does not parse, and saved as the BSON its Extended JSON names. @@ -82,15 +84,15 @@ The editor runs mongosh statements. A statement ends at `;` or at the end of a l Anything else, a JavaScript function, a variable, `forEach`, `bulkWrite`, is refused before it is sent with a `SyntaxError` at its place, or `Unsupported`. As mongosh checks, `updateOne` and `updateMany` take update operators or a pipeline and `replaceOne` a document with none, so a forgotten `$set` never replaces a document, and `dropIndex()` has to name its index. A write's `writeConcern`, `ordered`, `comment` and `let` options go with its command. A statement's documents show one per row, a column per field: the fields a projection names, in its order, or every field the documents hold, in the order they first show it. A write shows what mongosh prints for it (`insertedId`, `matchedCount`, `modifiedCount`). -The run's row limit keeps 10,000 documents, all read before the first row is shown, so the columns hold every field they have: the cursor reads one more to say the result was cut, and closing it frees the rest on the server. A tab's `use` changes only its own database. +The run's row limit keeps 10,000 documents, and the cursor reads one more to say the result was cut; closing it frees the rest on the server. When a projection names the columns, the documents go to the grid a batch at a time, each batch read once the grid took the last, and the columns' types are settled when the last one is read. Otherwise the documents kept are all read before the first row is shown, so the columns hold every field they have. A tab's `use` changes only its own database. -An export of an editor result whose columns are not named by a projection reads the documents twice: once for the fields they hold, then to write them under those columns. A document that gains a field between the two reads fails the export rather than losing the field. +A table export reads every field the documents hold, not only the tab's columns, so a field none of the first 1,000 documents holds is still written. Before the first row, the server tallies the fields the export's documents hold and the types each holds, the same census the columns come from but over every matching document, and no document crosses the wire for it. A field of nested or mixed values is written as Extended JSON. An export that matches no document names the collection's columns. An export of an editor result whose columns are not named by a projection finds them the same way for an `aggregate`, and reads a `find`'s documents twice: once for the fields they hold, then to write them under those columns. A document that gains a field between the two reads fails the export rather than losing the field. The MCP server's `browse_table` and `export_data` read the tab's columns, since they stop at a row limit and a deadline. A history entry is withheld when a statement carries a password: `createUser`, `updateUser`, `changeUserPassword`, `auth`, `connect` and `Mongo`, by name or in brackets (`db['changeUserPassword'](...)`), a `pwd`, `password` or `passwordDigestor` key, and a connection string with a password. Text that does not close is withheld too. The editor highlights JavaScript and has no SQL completion, formatter or comment toggle. ## Stops and timeouts -A stop is `killSessions` for the call's logical session, sent from the same client on a connection of its own (`OutOfBand(Statement)`): the server ends the operation that session runs and closes its cursors, and the call ends with `Interrupted` (11601) or `CursorKilled` (237), which reads as the stop it is. A session names one call, so a late stop cannot land on anything else: a session whose `killSessions` has not been answered stays out of the client's pool for a minute, so no later call takes its id while the stop could still arrive. +A stop is `killSessions` for the call's logical session, sent from the same client on a connection of its own (`OutOfBand(Statement)`): the server ends the operation that session runs and closes its cursors, and the call ends with `Interrupted` (11601) or `CursorKilled` (237), which reads as the stop it is. A session names one call, so a late stop cannot land on anything else: a session whose `killSessions` has not been answered stays out of the client's pool for a minute, so no later call takes its id while the stop could still arrive. Once a call is stopped, its read sends nothing more until the server has answered the stop, so no `getMore` gets past it. Atlas M0 and Flex clusters refuse `killSessions`: the read then ends and closes its cursor, since a cursor waiting between batches is no operation `killOp` reaches, and reads as `Cancelled`, or for a deadline as a timeout the server did not stop. The stop then asks `currentOp` for the user's own operations under the call's session and sends `killOp` for each, which ends them with `Interrupted` too. A statement also carries `maxTimeMS` for what is left of its deadline, so the server stops it on its own; `MaxTimeMSExpired` (50) is a timeout the server stopped. `EditorSessionOptions::statement_timeout` bounds each editor statement the same way. @@ -98,9 +100,9 @@ A call that times out or loses its connection checks the server with the probe c ## Read-only -MongoDB has no read-only session, so a read-only connection rests on the app's own check (`ReadOnlyEnforcement::ClientGuard`): each statement is read before it is sent, and one that is not a read is refused with `ReadOnlyRefusal::ClientGuard`, in the editor, in a save and in an export. A statement reads only when `sql_syntax::mongo` lists its method, or its command's name, as a read, with every cursor method after it a read too; an `aggregate` with `$out` or `$merge` at any depth writes, and so does an `explain` of a write. A method it does not know counts as a write. For a guarantee, log in as a user with only the `read` role. +MongoDB has no read-only session, so a read-only connection rests on the app's own check (`ReadOnlyEnforcement::ClientGuard`): each statement is read before it is sent, and one that is not a read is refused with `ReadOnlyRefusal::ClientGuard`, in the editor, in a save and in an export. A statement reads only when `sql_syntax::mongo` lists its method, or its command's name, as a read, with every cursor method after it a read too; an `aggregate` with `$out` or `$merge` at any depth writes, and so does an `explain` of a write, and a `validate` with `repair` or `fixMultikey`, which change indexes and documents. A method it does not know counts as a write. For a guarantee, log in as a user with only the `read` role. -Safe Mode reads statements the same way: Confirm Writes asks before every statement that is not a read, and Off asks before `drop`, `dropDatabase`, a `deleteMany`, `remove` or `updateMany` with no filter, `null` or `{}`, an `update` with `multi` and such a filter, and a `delete` or `update` command through `runCommand` that reaches every document the same way. `aggregate()` takes only its own options, so none can stand in for the pipeline that was checked, and a key written twice is read as its last value, as the shell sends it. +Safe Mode reads statements the same way: Confirm Writes asks before every statement that is not a read, and Off asks before every drop, as it does before SQL's `DROP`: `drop`, `dropDatabase`, `dropIndex`, `dropIndexes`, `dropUser`, `dropRole`, `dropAllUsersFromDatabase`, `dropAllRolesFromDatabase`, a `renameCollection` that drops its target, and the same commands through `runCommand`. It asks before what replaces a collection: an `aggregate` with `$out`, a `mapReduce` whose `out` names a collection or `replace`, `convertToCapped` and `applyOps`. It also asks before a `deleteMany`, `remove` or `updateMany` with no filter, `null` or `{}`, an `update` with `multi` and such a filter, and a `delete`, `update` or `bulkWrite` command through `runCommand` that reaches every document the same way. `aggregate()` takes only its own options, so none can stand in for the pipeline that was checked, and a key written twice is read as its last value, as the shell sends it. ## Trust @@ -119,6 +121,6 @@ A server error keeps its code and name as `ServerCode::Mongo { code, name }`, su ## Limits - Sharded clusters are reached through `mongos` like any server; nothing shard-aware is shown. -- A DNS seed list resolves through the system resolver configuration; it is not tested end to end, since a test SRV record needs a DNS server the crate would ask. +- A DNS seed list resolves through the system resolver configuration. Its test uses the MongoDB driver specification's public records (`test5.test.build.10gen.cc`, whose TXT record names `repl0` and `thisDB`), since the crate takes no other DNS server, so it needs public DNS and host port 27017. - `bulkWrite`, transactions in the editor, change streams and GridFS are not offered. - The structure of a collection is not edited: there is no schema to change. Indexes are listed and created or dropped in the editor. diff --git a/linux/docs/testing.md b/linux/docs/testing.md index 14493583fe..6b81f84ac5 100644 --- a/linux/docs/testing.md +++ b/linux/docs/testing.md @@ -275,7 +275,7 @@ tablepro_test_support::document_conformance_suite! { } ``` -The cases: collections are listed in their database; fields become columns with the key first; pages read every document in order; a filter and a sort are read on the server; a field edit saves and reads back; a save on a document that is gone changes nothing; an insert and a delete land; a read-only connection refuses a save, an editor write, a read that writes and an export that writes before sending; a stopped statement ends and the session runs again; a dropped run stops the statement; a timed-out statement is stopped by the server; an editor result keeps the row limit; `stream_read` streams every document; a probe answers while a statement runs; `close` is idempotent; and a dropped route reports `ConnectionLost`. `document_suite_matches_driver_capabilities` is never ignored and fails when the driver's `data_model` is not `Document`. The engine also runs the TLS suite. +The cases: collections are listed in their database; fields become columns with the key first; pages read every document in order; a filter and a sort are read on the server; a field edit saves and reads back; a save on a document that is gone changes nothing; a save that fails at its second step keeps the first, reported as partly applied; an insert and a delete land; a read-only connection refuses a save, an editor write, a read that writes and an export that writes before sending; a stopped statement ends and the session runs again; a stopped workspace read ends, the server lets it go and the connection reads again; a dropped run stops the statement; a timed-out statement is stopped by the server; an editor result keeps the row limit; `stream_read` streams every document; a probe answers while a statement runs; `close` is idempotent; and a dropped route reports `ConnectionLost`. `document_suite_matches_driver_capabilities` is never ignored and fails when the driver's `data_model` is not `Document`. MongoDB runs the suite twice, on a direct route and on a forward, where the driver's relay carries every page and save, and also runs the TLS suite. ### TLS conformance